osborn 0.1.0

package/.env.example

@@ -0,0 +1,14 @@
+# LLM Provider: 'openai' or 'gemini'
+LLM_PROVIDER=openai
+
+# LiveKit
+LIVEKIT_URL=wss://your-project.livekit.cloud
+LIVEKIT_API_KEY=your-api-key
+LIVEKIT_API_SECRET=your-api-secret
+
+# OpenAI (for Realtime API)
+OPENAI_API_KEY=sk-...
+
+# Google AI (for Gemini Live - FREE during preview!)
+# Get your key at: https://aistudio.google.com/apikey
+GOOGLE_API_KEY=AIzaSy...

package/bin/cli.js

@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+
+import { spawn } from 'child_process'
+import { fileURLToPath } from 'url'
+import { dirname, join } from 'path'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = dirname(__filename)
+
+// Pass all args to the actual agent
+const args = process.argv.slice(2)
+
+// Check for help
+if (args.includes('--help') || args.includes('-h')) {
+  console.log(`
+Osborn Agent - Voice AI Coding Assistant
+
+Usage:
+  osborn-agent --room <code>     Connect to a specific room
+  osborn-agent dev               Run in development mode
+  osborn-agent start             Run in production mode
+
+Options:
+  --room <code>    Room code from the Osborn web interface
+  --help, -h       Show this help message
+
+Environment Variables:
+  LIVEKIT_URL          LiveKit server URL
+  LIVEKIT_API_KEY      LiveKit API key
+  LIVEKIT_API_SECRET   LiveKit API secret
+  OPENAI_API_KEY       OpenAI API key (for voice)
+  ANTHROPIC_API_KEY    Anthropic API key (for Claude Code)
+  GOOGLE_API_KEY       Google API key (for Gemini)
+
+Config File:
+  ~/.osborn/config.yaml    MCP servers and settings
+
+Example:
+  osborn-agent --room abc123
+`)
+  process.exit(0)
+}
+
+// Run the agent using tsx
+const agentPath = join(__dirname, '..', 'src', 'index.ts')
+const tsxPath = join(__dirname, '..', 'node_modules', '.bin', 'tsx')
+
+// Determine mode (default to 'dev' if no mode specified)
+let mode = 'dev'
+if (args.includes('start')) {
+  mode = 'start'
+  args.splice(args.indexOf('start'), 1)
+} else if (args.includes('dev')) {
+  args.splice(args.indexOf('dev'), 1)
+}
+
+const child = spawn(tsxPath, [agentPath, mode, ...args], {
+  stdio: 'inherit',
+  cwd: join(__dirname, '..'),
+  env: process.env,
+})
+
+child.on('error', (err) => {
+  console.error('Failed to start agent:', err.message)
+  process.exit(1)
+})
+
+child.on('exit', (code) => {
+  process.exit(code || 0)
+})

package/dist/claude-handler.d.ts

@@ -0,0 +1,78 @@
+import { type McpServerConfig } from '@anthropic-ai/claude-agent-sdk';
+import { EventEmitter } from 'events';
+interface ClaudeHandlerOptions {
+    workingDirectory?: string;
+    allowedTools?: string[];
+    permissionMode?: 'default' | 'acceptEdits' | 'bypassPermissions';
+    mcpServers?: Record<string, McpServerConfig>;
+}
+export type { McpServerConfig };
+export interface PermissionRequestEvent {
+    toolName: string;
+    description: string;
+    toolInput: any;
+    toolUseId: string;
+}
+/**
+ * Claude Handler using the official Claude Agent SDK
+ *
+ * Features:
+ * - Session persistence (reuses same session for context)
+ * - Built-in tools (Read, Write, Edit, Bash, Glob, Grep, WebSearch, WebFetch)
+ * - Permission interception for voice approval
+ * - Hooks for observability
+ */
+export type PermissionResponse = 'allow' | 'deny' | 'always_allow';
+export declare class ClaudeHandler extends EventEmitter {
+    private options;
+    private abortController;
+    private sessionId;
+    private pendingPermission;
+    private dangerousTools;
+    private alwaysAllowedTools;
+    private static readonly ALL_TOOLS;
+    constructor(options?: ClaudeHandlerOptions);
+    run(prompt: string): Promise<string>;
+    /**
+     * Request permission from user via event emission
+     * Returns a promise that resolves when user responds with allow/deny/always_allow
+     */
+    private requestPermission;
+    /**
+     * Respond to a pending permission request
+     */
+    respondToPermission(response: PermissionResponse): void;
+    /**
+     * Grant permission for pending request (shorthand)
+     */
+    grantPermission(): void;
+    /**
+     * Deny permission for pending request (shorthand)
+     */
+    denyPermission(): void;
+    /**
+     * Always allow this tool type (shorthand)
+     */
+    alwaysAllowPermission(): void;
+    /**
+     * Check if there's a pending permission request
+     */
+    hasPendingPermission(): boolean;
+    /**
+     * Get current pending permission info (for displaying to user)
+     */
+    getPendingPermission(): PermissionRequestEvent | null;
+    /**
+     * Get list of always-allowed tools
+     */
+    getAlwaysAllowedTools(): string[];
+    /**
+     * Reset always-allowed tools
+     */
+    resetAlwaysAllowed(): void;
+    getSessionId(): string | null;
+    clearSession(): void;
+    private handleMessage;
+    stop(): void;
+}
+export declare function askClaude(prompt: string, cwd?: string): Promise<string>;

package/dist/claude-handler.js

@@ -0,0 +1,320 @@
+import { query } from '@anthropic-ai/claude-agent-sdk';
+import { EventEmitter } from 'events';
+export class ClaudeHandler extends EventEmitter {
+    options;
+    abortController = null;
+    sessionId = null;
+    pendingPermission = null;
+    // Tools that require permission
+    dangerousTools = ['Bash', 'Write', 'Edit'];
+    // Tools the user has permanently approved
+    alwaysAllowedTools = new Set();
+    // All available Claude Agent SDK tools
+    static ALL_TOOLS = [
+        // File operations
+        'Read', 'Write', 'Edit', 'MultiEdit',
+        // Search
+        'Glob', 'Grep',
+        // Terminal
+        'Bash',
+        // Web
+        'WebSearch', 'WebFetch',
+        // Notebooks
+        'NotebookEdit',
+        // Task management
+        'Task', 'TodoWrite',
+        // LSP (Language Server Protocol)
+        'LSP',
+    ];
+    constructor(options = {}) {
+        super();
+        this.options = {
+            workingDirectory: options.workingDirectory || process.cwd(),
+            allowedTools: options.allowedTools || ClaudeHandler.ALL_TOOLS,
+            permissionMode: options.permissionMode || 'default',
+            mcpServers: options.mcpServers,
+        };
+        console.log(`🔧 Allowed tools: ${this.options.allowedTools?.join(', ')}`);
+        if (this.options.mcpServers) {
+            console.log(`🔌 MCP servers: ${Object.keys(this.options.mcpServers).join(', ')}`);
+        }
+    }
+    async run(prompt) {
+        this.abortController = new AbortController();
+        let fullResponse = '';
+        try {
+            // Log session state
+            if (this.sessionId) {
+                console.log(`🔄 RESUMING session: ${this.sessionId}`);
+            }
+            else {
+                console.log('🆕 STARTING new session');
+            }
+            console.log(`📁 CWD: ${this.options.workingDirectory}`);
+            console.log(`🔑 Mode: ${this.options.permissionMode}`);
+            // Build SDK options with session resume
+            const sdkOptions = {
+                allowedTools: this.options.allowedTools,
+                cwd: this.options.workingDirectory,
+                permissionMode: this.options.permissionMode,
+                abortController: this.abortController || undefined,
+                // CRITICAL: Resume existing session for context continuity
+                ...(this.sessionId && { resume: this.sessionId }),
+                // MCP servers configuration
+                ...(this.options.mcpServers && { mcpServers: this.options.mcpServers }),
+                hooks: {
+                    PreToolUse: [{
+                            matcher: '.*',
+                            hooks: [async (input, toolUseId) => {
+                                    const toolName = input?.tool_name || 'unknown';
+                                    const toolInput = input?.tool_input || {};
+                                    console.log(`🔧 Tool: ${toolName}`);
+                                    this.emit('tool_use', { name: toolName, input: toolInput });
+                                    // Check if this tool needs permission
+                                    if (this.dangerousTools.includes(toolName) && this.options.permissionMode === 'default') {
+                                        // Skip if user has permanently approved this tool
+                                        if (this.alwaysAllowedTools.has(toolName)) {
+                                            console.log(`✅ Auto-approved (always allow): ${toolName}`);
+                                        }
+                                        else {
+                                            console.log(`⚠️ Permission required for: ${toolName}`);
+                                            // Emit permission request and wait for approval
+                                            const response = await this.requestPermission(toolName, toolInput, toolUseId || 'unknown');
+                                            if (response === 'deny') {
+                                                console.log(`❌ Permission denied for: ${toolName}`);
+                                                return {
+                                                    decision: 'block',
+                                                    reason: 'User denied permission for this operation'
+                                                };
+                                            }
+                                            if (response === 'always_allow') {
+                                                this.alwaysAllowedTools.add(toolName);
+                                                console.log(`✅ Permission granted (always allow): ${toolName}`);
+                                            }
+                                            else {
+                                                console.log(`✅ Permission granted for: ${toolName}`);
+                                            }
+                                        }
+                                    }
+                                    return {}; // Allow tool to proceed
+                                }]
+                        }],
+                    PostToolUse: [{
+                            matcher: '.*',
+                            hooks: [async (input) => {
+                                    const toolName = input?.tool_name || 'unknown';
+                                    console.log(`✅ Completed: ${toolName}`);
+                                    this.emit('tool_result', { name: toolName });
+                                    return {};
+                                }]
+                        }]
+                }
+            };
+            // Run the query
+            for await (const message of query({
+                prompt,
+                options: sdkOptions,
+            })) {
+                this.handleMessage(message);
+                // Capture session ID from init
+                if (message.type === 'system' && message.subtype === 'init') {
+                    const newSessionId = message.session_id;
+                    if (newSessionId) {
+                        if (!this.sessionId) {
+                            this.sessionId = newSessionId;
+                            console.log(`📋 Session CREATED: ${this.sessionId}`);
+                        }
+                        else if (this.sessionId === newSessionId) {
+                            console.log(`📋 Session CONTINUED: ${this.sessionId}`);
+                        }
+                        else {
+                            // Session ID changed - SDK created new session despite resume
+                            console.log(`⚠️ Session CHANGED: ${this.sessionId} → ${newSessionId}`);
+                            this.sessionId = newSessionId;
+                        }
+                    }
+                }
+                // Collect text from assistant
+                if (message.type === 'assistant' && message.message?.content) {
+                    for (const block of message.message.content) {
+                        if (block.type === 'text') {
+                            fullResponse += block.text;
+                        }
+                    }
+                }
+                // Final result
+                if (message.type === 'result') {
+                    console.log(`📋 Result: ${message.subtype}`);
+                    if (message.result && !fullResponse) {
+                        fullResponse = message.result;
+                    }
+                }
+            }
+            console.log(`✅ Done. Length: ${fullResponse.length}`);
+            return fullResponse || 'Task completed.';
+        }
+        catch (error) {
+            if (error.name === 'AbortError') {
+                this.emit('aborted');
+                return 'Task was cancelled.';
+            }
+            console.error('❌ SDK error:', error);
+            this.emit('error', error);
+            // Don't clear session on error - might be transient
+            throw error;
+        }
+    }
+    /**
+     * Request permission from user via event emission
+     * Returns a promise that resolves when user responds with allow/deny/always_allow
+     */
+    requestPermission(toolName, toolInput, toolUseId) {
+        return new Promise((resolve) => {
+            // Format the permission request message
+            let description = '';
+            if (toolName === 'Bash') {
+                description = `Run command: ${toolInput.command || 'unknown command'}`;
+            }
+            else if (toolName === 'Write') {
+                description = `Create file: ${toolInput.file_path || 'unknown file'}`;
+            }
+            else if (toolName === 'Edit') {
+                description = `Edit file: ${toolInput.file_path || 'unknown file'}`;
+            }
+            this.pendingPermission = { toolName, toolInput, toolUseId, resolve: resolve };
+            // Emit event for voice handler to pick up
+            this.emit('permission_request', {
+                toolName,
+                description,
+                toolInput,
+                toolUseId,
+            });
+            // No auto-approve - wait for user response (with 2 minute timeout for safety)
+            setTimeout(() => {
+                if (this.pendingPermission?.toolUseId === toolUseId) {
+                    console.log(`⏰ Permission timeout - denying for safety: ${toolName}`);
+                    this.pendingPermission = null;
+                    resolve('deny');
+                }
+            }, 120000); // 2 minute timeout
+        });
+    }
+    /**
+     * Respond to a pending permission request
+     */
+    respondToPermission(response) {
+        if (this.pendingPermission) {
+            console.log(`📋 Permission response: ${response}`);
+            this.pendingPermission.resolve(response);
+            this.pendingPermission = null;
+        }
+    }
+    /**
+     * Grant permission for pending request (shorthand)
+     */
+    grantPermission() {
+        this.respondToPermission('allow');
+    }
+    /**
+     * Deny permission for pending request (shorthand)
+     */
+    denyPermission() {
+        this.respondToPermission('deny');
+    }
+    /**
+     * Always allow this tool type (shorthand)
+     */
+    alwaysAllowPermission() {
+        this.respondToPermission('always_allow');
+    }
+    /**
+     * Check if there's a pending permission request
+     */
+    hasPendingPermission() {
+        return this.pendingPermission !== null;
+    }
+    /**
+     * Get current pending permission info (for displaying to user)
+     */
+    getPendingPermission() {
+        if (!this.pendingPermission)
+            return null;
+        const { toolName, toolInput, toolUseId } = this.pendingPermission;
+        let description = '';
+        if (toolName === 'Bash') {
+            description = `Run command: ${toolInput.command || 'unknown command'}`;
+        }
+        else if (toolName === 'Write') {
+            description = `Create file: ${toolInput.file_path || 'unknown file'}`;
+        }
+        else if (toolName === 'Edit') {
+            description = `Edit file: ${toolInput.file_path || 'unknown file'}`;
+        }
+        return { toolName, description, toolInput, toolUseId };
+    }
+    /**
+     * Get list of always-allowed tools
+     */
+    getAlwaysAllowedTools() {
+        return Array.from(this.alwaysAllowedTools);
+    }
+    /**
+     * Reset always-allowed tools
+     */
+    resetAlwaysAllowed() {
+        this.alwaysAllowedTools.clear();
+        console.log('🔄 Reset always-allowed tools');
+    }
+    getSessionId() {
+        return this.sessionId;
+    }
+    clearSession() {
+        this.sessionId = null;
+        console.log('🗑️ Session cleared');
+    }
+    handleMessage(message) {
+        switch (message.type) {
+            case 'assistant':
+                if (message.message?.content) {
+                    for (const block of message.message.content) {
+                        if (block.type === 'text') {
+                            this.emit('text', block.text);
+                        }
+                        else if (block.type === 'tool_use') {
+                            this.emit('tool_use', {
+                                id: block.id,
+                                name: block.name,
+                                input: block.input,
+                            });
+                        }
+                    }
+                }
+                break;
+            case 'tool_result':
+                this.emit('tool_result', {
+                    id: message.tool_use_id,
+                    content: message.content,
+                });
+                break;
+            case 'system':
+                this.emit('system', message);
+                break;
+            case 'result':
+                this.emit('result', message);
+                break;
+            default:
+                this.emit('message', message);
+        }
+    }
+    stop() {
+        if (this.abortController) {
+            this.abortController.abort();
+            this.abortController = null;
+        }
+    }
+}
+// Quick utility for one-off queries
+export async function askClaude(prompt, cwd) {
+    const handler = new ClaudeHandler({ workingDirectory: cwd });
+    return handler.run(prompt);
+}

package/dist/codex-handler.d.ts

@@ -0,0 +1,38 @@
+import { EventEmitter } from 'events';
+type SandboxMode = 'workspace-write' | 'danger-full-access';
+type ApprovalPolicy = 'on-failure' | 'on-request' | 'never';
+interface CodexHandlerOptions {
+    workingDirectory?: string;
+    skipGitRepoCheck?: boolean;
+    sandboxMode?: SandboxMode;
+    approvalPolicy?: ApprovalPolicy;
+}
+/**
+ * Codex Handler using the official OpenAI Codex SDK
+ *
+ * Features:
+ * - Thread persistence (reuses same thread for context)
+ * - Built-in coding tools (file operations, terminal commands)
+ * - Hooks for observability
+ *
+ * Requirements:
+ * - Codex CLI must be installed (npm install -g @openai/codex or use npx)
+ * - Must be authenticated via: codex login (ChatGPT) or OPENAI_API_KEY env var
+ *
+ * Sandbox/Approval modes are configured via:
+ * - ~/.codex/config.yaml for global settings
+ * - CLI flags: --sandbox, --approval-policy
+ *
+ * Note: The SDK spawns the CLI binary, so authentication must be done beforehand.
+ */
+export declare class CodexHandler extends EventEmitter {
+    private codex;
+    private thread;
+    private options;
+    constructor(options?: CodexHandlerOptions);
+    run(prompt: string): Promise<string>;
+    getThreadId(): string | null;
+    clearThread(): void;
+}
+export declare function askCodex(prompt: string, cwd?: string): Promise<string>;
+export {};

package/dist/codex-handler.js

@@ -0,0 +1,83 @@
+import { Codex } from '@openai/codex-sdk';
+import { EventEmitter } from 'events';
+/**
+ * Codex Handler using the official OpenAI Codex SDK
+ *
+ * Features:
+ * - Thread persistence (reuses same thread for context)
+ * - Built-in coding tools (file operations, terminal commands)
+ * - Hooks for observability
+ *
+ * Requirements:
+ * - Codex CLI must be installed (npm install -g @openai/codex or use npx)
+ * - Must be authenticated via: codex login (ChatGPT) or OPENAI_API_KEY env var
+ *
+ * Sandbox/Approval modes are configured via:
+ * - ~/.codex/config.yaml for global settings
+ * - CLI flags: --sandbox, --approval-policy
+ *
+ * Note: The SDK spawns the CLI binary, so authentication must be done beforehand.
+ */
+export class CodexHandler extends EventEmitter {
+    codex;
+    thread = null;
+    options;
+    constructor(options = {}) {
+        super();
+        this.options = {
+            workingDirectory: options.workingDirectory || process.cwd(),
+            skipGitRepoCheck: options.skipGitRepoCheck ?? true,
+        };
+        // Initialize Codex (inherits process env by default)
+        this.codex = new Codex();
+    }
+    async run(prompt) {
+        try {
+            // Create or reuse thread
+            if (!this.thread) {
+                console.log('🆕 Starting new Codex thread');
+                this.thread = this.codex.startThread({
+                    workingDirectory: this.options.workingDirectory,
+                    skipGitRepoCheck: this.options.skipGitRepoCheck,
+                });
+            }
+            else {
+                console.log('🔄 Continuing Codex thread');
+            }
+            console.log(`📁 CWD: ${this.options.workingDirectory}`);
+            this.emit('thinking', prompt);
+            // Run the prompt and get the Turn result
+            const turn = await this.thread.run(prompt);
+            // Extract the final response text
+            const result = turn.finalResponse || '';
+            this.emit('result', result);
+            console.log(`✅ Codex done. Length: ${result.length}`);
+            // Log tool usage if any
+            if (turn.items && turn.items.length > 0) {
+                for (const item of turn.items) {
+                    this.emit('tool_use', item);
+                }
+            }
+            return result || 'Task completed.';
+        }
+        catch (error) {
+            console.error('❌ Codex SDK error:', error);
+            this.emit('error', error);
+            throw error;
+        }
+    }
+    getThreadId() {
+        // Codex threads have IDs but they may not be directly exposed
+        // Check the thread object for an id property
+        return this.thread?.id || null;
+    }
+    clearThread() {
+        this.thread = null;
+        console.log('🗑️ Codex thread cleared');
+    }
+}
+// Quick utility for one-off queries
+export async function askCodex(prompt, cwd) {
+    const handler = new CodexHandler({ workingDirectory: cwd });
+    return handler.run(prompt);
+}

package/dist/fast-llm.d.ts

@@ -0,0 +1,15 @@
+interface AcknowledgmentResult {
+    acknowledgment: string;
+    isCodingTask: boolean;
+    intent: 'code' | 'chat' | 'question' | 'permission';
+}
+/**
+ * Fast LLM for immediate acknowledgment and intent classification
+ * Target: < 200ms response time
+ */
+export declare function getAcknowledgment(userInput: string): Promise<AcknowledgmentResult>;
+/**
+ * Summarize Claude's response for voice output
+ */
+export declare function summarizeForVoice(claudeResponse: string): Promise<string>;
+export {};

package/dist/fast-llm.js

@@ -0,0 +1,81 @@
+import Groq from 'groq-sdk';
+const groq = new Groq({
+    apiKey: process.env.GROQ_API_KEY,
+});
+/**
+ * Fast LLM for immediate acknowledgment and intent classification
+ * Target: < 200ms response time
+ */
+export async function getAcknowledgment(userInput) {
+    const response = await groq.chat.completions.create({
+        model: 'llama-3.3-70b-versatile', // Fast model
+        messages: [
+            {
+                role: 'system',
+                content: `You are Osborn, a voice-enabled coding assistant. Classify user requests:
+
+CODING TASKS (isCodingTask: true) - Anything involving:
+- Files: read, write, create, edit, list, find, search
+- Directories: what directory, current folder, list files, project structure
+- Code: fix bugs, refactor, explain code, run tests
+- Terminal: run commands, install packages, git operations
+- Project: what's in this project, show me files, analyze codebase
+
+NOT CODING (isCodingTask: false):
+- General chat: hi, how are you, tell me a joke
+- Off-topic: weather, news, recipes
+- Permission responses: yes, no, approve, deny
+
+Respond in JSON:
+{
+  "acknowledgment": "brief friendly response",
+  "isCodingTask": true/false,
+  "intent": "code|chat|question|permission"
+}
+
+Examples:
+- "What directory are we in?" → {"acknowledgment": "Let me check", "isCodingTask": true, "intent": "code"}
+- "What files are in this project?" → {"acknowledgment": "I'll list them", "isCodingTask": true, "intent": "code"}
+- "Read package.json" → {"acknowledgment": "On it", "isCodingTask": true, "intent": "code"}
+- "Fix the bug" → {"acknowledgment": "Let me look", "isCodingTask": true, "intent": "code"}
+- "Hello" → {"acknowledgment": "Hi there!", "isCodingTask": false, "intent": "chat"}
+- "Yes, do it" → {"acknowledgment": "Got it", "isCodingTask": false, "intent": "permission"}`,
+            },
+            {
+                role: 'user',
+                content: userInput,
+            },
+        ],
+        temperature: 0.3,
+        max_tokens: 150,
+        response_format: { type: 'json_object' },
+    });
+    const content = response.choices[0]?.message?.content || '{}';
+    return JSON.parse(content);
+}
+/**
+ * Summarize Claude's response for voice output
+ */
+export async function summarizeForVoice(claudeResponse) {
+    if (claudeResponse.length < 200) {
+        return claudeResponse;
+    }
+    const response = await groq.chat.completions.create({
+        model: 'llama-3.3-70b-versatile',
+        messages: [
+            {
+                role: 'system',
+                content: `Summarize this coding assistant response for voice output.
+Keep it under 2 sentences. Focus on what was done and the result.
+Don't include code - just describe what happened.`,
+            },
+            {
+                role: 'user',
+                content: claudeResponse,
+            },
+        ],
+        temperature: 0.3,
+        max_tokens: 100,
+    });
+    return response.choices[0]?.message?.content || claudeResponse;
+}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+import 'dotenv/config';
+declare const _default: import("@livekit/agents").Agent;
+export default _default;

package/dist/index.js

@@ -0,0 +1,356 @@
+import { ServerOptions, cli, defineAgent, llm, voice } from '@livekit/agents';
+import * as openai from '@livekit/agents-plugin-openai';
+import * as google from '@livekit/agents-plugin-google';
+import { z } from 'zod';
+import { fileURLToPath } from 'url';
+import 'dotenv/config';
+import { ClaudeHandler } from './claude-handler.js';
+import { CodexHandler } from './codex-handler.js';
+// Global error handlers to catch silent failures
+process.on('unhandledRejection', (reason, promise) => {
+    console.error('❌ Unhandled Rejection:', reason);
+});
+process.on('uncaughtException', (error) => {
+    console.error('❌ Uncaught Exception:', error);
+});
+// Default provider (can be overridden by participant metadata)
+const DEFAULT_PROVIDER = process.env.LLM_PROVIDER || 'openai';
+// Debug mode
+const DEBUG = process.env.DEBUG_LIVEKIT === 'true';
+if (DEBUG) {
+    console.log('🐛 Debug logging enabled');
+}
+console.log(`🤖 Default LLM Provider: ${DEFAULT_PROVIDER}`);
+// Example MCP server configurations (uncomment to enable)
+const MCP_SERVERS = {
+// GitHub integration
+// 'github': {
+//   command: 'npx',
+//   args: ['@modelcontextprotocol/server-github'],
+//   env: { GITHUB_TOKEN: process.env.GITHUB_TOKEN || '' }
+// },
+// Filesystem with specific allowed paths
+// 'filesystem': {
+//   command: 'npx',
+//   args: ['@modelcontextprotocol/server-filesystem'],
+//   env: { ALLOWED_PATHS: '/Users/newupgrade/Desktop/Developer' }
+// },
+};
+// Pre-initialize Claude handler at module load (before any connections)
+console.log('🔥 Pre-initializing Claude Code...');
+const claude = new ClaudeHandler({
+    workingDirectory: '/Users/newupgrade/Desktop/Developer/osborn',
+    permissionMode: 'default', // Ask for permission on dangerous tools (Bash, Write, Edit)
+    // Uncomment to enable MCP servers:
+    // mcpServers: MCP_SERVERS,
+});
+// Listen for permission requests from Claude
+claude.on('permission_request', (req) => {
+    console.log(`\n⚠️ PERMISSION REQUIRED ⚠️`);
+    console.log(`🔧 Tool: ${req.toolName}`);
+    console.log(`📝 Action: ${req.description}`);
+    console.log(`⏳ Waiting for user response (say: allow, deny, or always allow)...`);
+    // Send to frontend for UI display
+    sendToFrontend({
+        type: 'permission_request',
+        toolName: req.toolName,
+        description: req.description,
+    });
+});
+// Pre-warm Claude immediately on server start
+claude.run('Respond with just: ready')
+    .then(() => console.log('✅ Claude pre-warmed and ready!'))
+    .catch((err) => console.log('⚠️ Pre-warm failed:', err.message));
+// Track job context and session for data channel
+let jobContext = null;
+let currentSession = null;
+// Track the current coding handler (can be Claude or Codex)
+let currentCodingAgent = 'claude';
+let codexHandler = null;
+// Helper to send data to frontend
+async function sendToFrontend(data) {
+    if (!jobContext)
+        return;
+    try {
+        const encoder = new TextEncoder();
+        const payload = encoder.encode(JSON.stringify(data));
+        await jobContext.room.localParticipant?.publishData(payload, {
+            reliable: true,
+            topic: 'osborn-updates',
+        });
+    }
+    catch (err) {
+        // Ignore send errors
+    }
+}
+// Define the run_code tool (works with both Claude and Codex)
+const runCodeTool = llm.tool({
+    description: `Execute coding tasks using the coding agent. Use for:
+- Files: read, write, create, edit, list, search
+- Directories: current directory, list contents
+- Code: fix bugs, refactor, explain, review
+- Terminal: run commands, install packages, git
+- Project: analyze codebase, make changes
+- Web: search the web for information`,
+    parameters: z.object({
+        task: z.string().describe('The coding task to execute'),
+    }),
+    execute: async ({ task }) => {
+        const agentName = currentCodingAgent === 'claude' ? 'Claude Code' : 'OpenAI Codex';
+        console.log(`\n🔨 ${agentName}: "${task}"`);
+        await sendToFrontend({ type: 'system', text: `Working on: ${task}` });
+        try {
+            let result;
+            if (currentCodingAgent === 'codex' && codexHandler) {
+                result = await codexHandler.run(task);
+            }
+            else {
+                result = await claude.run(task);
+            }
+            console.log(`✅ Done: ${result.length} chars`);
+            await sendToFrontend({ type: 'assistant_response', text: result });
+            return result;
+        }
+        catch (err) {
+            console.error('❌ Error:', err);
+            return `Error: ${err.message}`;
+        }
+    },
+});
+// Define the permission response tool
+const respondPermissionTool = llm.tool({
+    description: `Respond to a pending permission request from Claude Code.
+Use this ONLY when there is a pending permission request.
+Call this after hearing the user's response to a permission prompt.`,
+    parameters: z.object({
+        response: z.enum(['allow', 'deny', 'always_allow']).describe('The user response: "allow" for one-time approval, "deny" to reject, "always_allow" to permanently allow this tool type'),
+    }),
+    execute: async ({ response }) => {
+        if (!claude.hasPendingPermission()) {
+            return 'No pending permission request.';
+        }
+        const pending = claude.getPendingPermission();
+        claude.respondToPermission(response);
+        await sendToFrontend({
+            type: 'permission_response',
+            response,
+            toolName: pending?.toolName
+        });
+        return `Permission ${response} for ${pending?.toolName || 'tool'}.`;
+    },
+});
+// Agent instructions - dynamically includes available tools
+const OSBORN_INSTRUCTIONS = `You are Osborn, a voice-enabled AI assistant with coding superpowers.
+Keep responses under 50 words. Sound natural and human.
+
+AVAILABLE CAPABILITIES via run_code tool:
+- Read, Write, Edit, MultiEdit files
+- Glob (find files by pattern), Grep (search content)
+- Bash (run terminal commands)
+- WebSearch (search the web), WebFetch (fetch URLs)
+- NotebookEdit (edit Jupyter notebooks)
+- Task (delegate complex tasks), TodoWrite (track tasks)
+- LSP (code intelligence - go to definition, find references)
+
+WHEN TO USE run_code:
+- File operations (read, write, create, edit, list, find)
+- Code tasks (fix, refactor, explain, review, debug)
+- Terminal commands (run, install, test, build, git)
+- Web searches (look up documentation, APIs, errors)
+- Project analysis (understand codebase, find patterns)
+
+WHEN TO RESPOND DIRECTLY:
+- Greetings and small talk
+- General knowledge questions
+- Clarifying what the user wants
+
+PERMISSION HANDLING:
+When the coding agent needs permission, you MUST:
+1. Tell the user: "[Agent] wants to [action]. Allow, deny, or always allow?"
+2. When they respond, call respond_permission with their choice
+
+Be conversational and helpful. Ask follow-up questions when needed.`;
+// Voice assistant with tools
+class OsbornAssistant extends voice.Agent {
+    constructor() {
+        super({
+            instructions: OSBORN_INSTRUCTIONS,
+            tools: {
+                run_code: runCodeTool,
+                respond_permission: respondPermissionTool,
+            },
+        });
+    }
+}
+// Create the appropriate model based on provider
+function createModel(provider) {
+    if (provider === 'gemini') {
+        console.log('📱 Using Gemini Live API');
+        console.log('🔑 GOOGLE_API_KEY:', process.env.GOOGLE_API_KEY ? 'set' : 'NOT SET');
+        // From official docs: https://docs.livekit.io/agents/models/realtime/plugins/gemini/
+        // Package v1.0.31 uses google.beta.realtime (not google.realtime yet)
+        const model = new google.beta.realtime.RealtimeModel({
+            model: 'gemini-2.5-flash-native-audio-preview-12-2025', // From official docs
+            voice: 'Puck',
+            instructions: OSBORN_INSTRUCTIONS,
+        });
+        console.log('✅ Gemini model created with gemini-2.5-flash-native-audio-preview-12-2025');
+        return model;
+    }
+    else {
+        console.log('📱 Using OpenAI Realtime API');
+        console.log('🔑 OPENAI_API_KEY:', process.env.OPENAI_API_KEY ? 'set' : 'NOT SET');
+        const model = new openai.realtime.RealtimeModel({
+            voice: 'alloy',
+        });
+        console.log('✅ OpenAI model created');
+        return model;
+    }
+}
+// Helper to get provider from participant metadata
+function getProviderFromParticipant(metadata) {
+    if (!metadata)
+        return DEFAULT_PROVIDER;
+    try {
+        const data = JSON.parse(metadata);
+        return data.provider || DEFAULT_PROVIDER;
+    }
+    catch {
+        return DEFAULT_PROVIDER;
+    }
+}
+// Helper to get coding agent from participant metadata
+function getCodingAgentFromParticipant(metadata) {
+    if (!metadata)
+        return 'claude';
+    try {
+        const data = JSON.parse(metadata);
+        return data.codingAgent || 'claude';
+    }
+    catch {
+        return 'claude';
+    }
+}
+export default defineAgent({
+    entry: async (ctx) => {
+        console.log('🚀 Agent starting for room:', ctx.room.name);
+        jobContext = ctx;
+        // Claude verbose logging
+        claude.on('tool_use', (tool) => {
+            console.log(`\n🔧 Claude Tool Started: ${tool.name}`);
+            if (tool.input) {
+                const inputStr = JSON.stringify(tool.input).substring(0, 200);
+                console.log(`   Input: ${inputStr}${inputStr.length >= 200 ? '...' : ''}`);
+            }
+        });
+        claude.on('tool_result', (result) => {
+            console.log(`✅ Claude Tool Completed: ${result.name || 'unknown'}`);
+        });
+        claude.on('text', (text) => {
+            if (text.length > 0) {
+                console.log(`💬 Claude says: ${text.substring(0, 100)}${text.length > 100 ? '...' : ''}`);
+            }
+        });
+        claude.on('error', (err) => {
+            console.error(`❌ Claude Error:`, err);
+        });
+        // Connect FIRST so we can wait for participants
+        console.log('📡 Connecting to room...');
+        await ctx.connect();
+        console.log('✅ Connected to room');
+        // Wait for a participant to join using LiveKit's built-in method
+        console.log('⏳ Waiting for participant...');
+        const participant = await ctx.waitForParticipant();
+        console.log('👤 Participant joined:', participant.identity);
+        console.log('📋 Participant metadata:', participant.metadata);
+        const provider = getProviderFromParticipant(participant.metadata);
+        const codingAgent = getCodingAgentFromParticipant(participant.metadata);
+        console.log(`🎯 User selected provider: ${provider}`);
+        console.log(`🔧 User selected coding agent: ${codingAgent}`);
+        // Set the current coding agent and initialize if needed
+        currentCodingAgent = codingAgent;
+        if (codingAgent === 'codex') {
+            console.log('🔧 Initializing Codex handler...');
+            codexHandler = new CodexHandler({
+                workingDirectory: '/Users/newupgrade/Desktop/Developer/osborn',
+            });
+            console.log('✅ Codex handler ready');
+        }
+        // Create model based on user's choice
+        const model = createModel(provider);
+        const session = new voice.AgentSession({
+            llm: model,
+        });
+        currentSession = session;
+        // Add session event listeners for debugging
+        // Using string literals as AgentSessionEventTypes is not directly exported
+        session.on('user_state_changed', (ev) => {
+            console.log(`👤 User state: ${ev.oldState} → ${ev.newState}`);
+        });
+        session.on('agent_state_changed', (ev) => {
+            console.log(`🤖 Agent state: ${ev.oldState} → ${ev.newState}`);
+        });
+        session.on('user_input_transcribed', (ev) => {
+            console.log(`📝 Transcribed: "${ev.transcript}" (final: ${ev.isFinal})`);
+        });
+        session.on('error', (ev) => {
+            console.error('❌ Session error:', ev.error);
+        });
+        session.on('close', (ev) => {
+            console.log('🚪 Session closed:', ev.reason);
+        });
+        ctx.room.on('trackSubscribed', (track, publication, p) => {
+            console.log(`📥 Track subscribed: ${track.kind} from ${p.identity}`);
+        });
+        // Listen for data channel messages from frontend
+        ctx.room.on('dataReceived', async (payload, participant, kind, topic) => {
+            if (topic === 'user-input') {
+                try {
+                    const data = JSON.parse(new TextDecoder().decode(payload));
+                    console.log(`📨 Received from frontend:`, data);
+                    if (data.type === 'permission_response') {
+                        // Handle permission response from UI
+                        if (claude.hasPendingPermission()) {
+                            claude.respondToPermission(data.response);
+                            console.log(`✅ Permission ${data.response} from UI`);
+                        }
+                    }
+                    else if (data.type === 'user_text') {
+                        // Handle text input from frontend
+                        console.log(`📝 Text input: "${data.content}"`);
+                        // Inject text into the session as user input
+                        if (currentSession) {
+                            try {
+                                // Interrupt any current speech first
+                                currentSession.interrupt();
+                                // Generate a reply to the text input
+                                await currentSession.generateReply({
+                                    userInput: data.content,
+                                });
+                                console.log(`✅ Injected text to session`);
+                            }
+                            catch (err) {
+                                console.error(`❌ Failed to inject text:`, err);
+                            }
+                        }
+                    }
+                }
+                catch (e) {
+                    // Not JSON, ignore
+                }
+            }
+        });
+        // Create the agent
+        const agent = new OsbornAssistant();
+        // Start session
+        console.log('🎬 Starting voice session...');
+        const startTime = Date.now();
+        await session.start({
+            agent,
+            room: ctx.room,
+        });
+        console.log(`✅ Session started in ${Date.now() - startTime}ms with ${provider.toUpperCase()} + Claude tools`);
+        console.log('🎤 Ready for voice input! Speak to start.');
+    },
+});
+cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) }));

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "osborn",
+  "version": "0.1.0",
+  "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
+  "type": "module",
+  "bin": {
+    "osborn": "./bin/cli.js"
+  },
+  "scripts": {
+    "dev": "tsx src/index.ts dev",
+    "start": "tsx src/index.ts start",
+    "build": "tsc"
+  },
+  "keywords": [
+    "voice",
+    "ai",
+    "coding",
+    "assistant",
+    "claude",
+    "livekit"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Erriccc/osborn.git"
+  },
+  "author": "Osborn Ojure",
+  "license": "MIT",
+  "dependencies": {
+    "@anthropic-ai/claude-agent-sdk": "^0.1.74",
+    "@livekit/agents": "^1.0.0",
+    "@livekit/agents-plugin-google": "^1.0.0",
+    "@livekit/agents-plugin-openai": "^1.0.0",
+    "@openai/codex-sdk": "^0.77.0",
+    "dotenv": "^16.4.0",
+    "tsx": "^4.0.0",
+    "yaml": "^2.3.0",
+    "zod": "^3.23.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "typescript": "^5.0.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}