osborn 0.1.6 → 0.5.3

package/dist/conversation-brain.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Conversation Brain - Gemini 2.5 Pro powered conversation manager
+ *
+ * This is the "smart brain" that:
+ * 1. Keeps conversation alive with relevant questions
+ * 2. Builds context until we understand what user wants
+ * 3. Dispatches background research agents
+ * 4. Receives progress updates and decides when to execute
+ * 5. Handles direct commands immediately
+ */
+export interface ConversationMessage {
+    role: 'user' | 'assistant' | 'system';
+    content: string;
+    timestamp: Date;
+}
+export interface ResearchTask {
+    id: string;
+    query: string;
+    status: 'pending' | 'running' | 'completed' | 'failed';
+    result?: string;
+    startedAt?: Date;
+    completedAt?: Date;
+}
+export interface BrainDecision {
+    action: 'speak' | 'research' | 'execute' | 'clarify' | 'direct_command';
+    speech?: string;
+    researchQueries?: string[];
+    executeTask?: string;
+    directCommand?: string;
+    reasoning?: string;
+}
+export interface BrainState {
+    conversationHistory: ConversationMessage[];
+    userGoal: string | null;
+    userGoalConfidence: number;
+    pendingResearch: ResearchTask[];
+    completedResearch: ResearchTask[];
+    readyToExecute: boolean;
+    executionPlan: string | null;
+}
+export declare class ConversationBrain {
+    private llm;
+    private state;
+    private workingDir;
+    private onSpeak;
+    private onStateChange;
+    constructor(config: {
+        workingDir: string;
+        onSpeak: (text: string) => Promise<void>;
+        onStateChange: (state: string) => Promise<void>;
+    });
+    /**
+     * Process user input and decide what to do
+     */
+    processUserInput(input: string): Promise<BrainDecision>;
+    /**
+     * Receive research results from background agents
+     */
+    receiveResearchResult(taskId: string, result: string, success: boolean): void;
+    /**
+     * Check if we should provide a status update
+     */
+    shouldProvideUpdate(): boolean;
+    /**
+     * Generate a status update based on completed research
+     */
+    generateStatusUpdate(): Promise<string | null>;
+    /**
+     * Get current state for debugging/display
+     */
+    getState(): BrainState;
+    /**
+     * Reset conversation state
+     */
+    reset(): void;
+    private buildAnalysisPrompt;
+    private parseDecision;
+    private updateState;
+    /**
+     * Create research tasks for background agents
+     */
+    getPendingResearchTasks(): ResearchTask[];
+    /**
+     * Mark a research task as running
+     */
+    markResearchRunning(taskId: string): void;
+}
+export declare function createConversationBrain(config: {
+    workingDir: string;
+    onSpeak: (text: string) => Promise<void>;
+    onStateChange: (state: string) => Promise<void>;
+}): ConversationBrain;

package/dist/conversation-brain.js

@@ -0,0 +1,360 @@
+/**
+ * Conversation Brain - Gemini 2.5 Pro powered conversation manager
+ *
+ * This is the "smart brain" that:
+ * 1. Keeps conversation alive with relevant questions
+ * 2. Builds context until we understand what user wants
+ * 3. Dispatches background research agents
+ * 4. Receives progress updates and decides when to execute
+ * 5. Handles direct commands immediately
+ */
+import { llm } from '@livekit/agents';
+import * as google from '@livekit/agents-plugin-google';
+// ============================================================
+// Conversation Brain Class
+// ============================================================
+export class ConversationBrain {
+    llm;
+    state;
+    workingDir;
+    onSpeak;
+    onStateChange;
+    constructor(config) {
+        // Use Gemini 2.0 Flash for brain decisions - it's faster and less likely to conflict
+        // with the Gemini Realtime voice session which uses a different model
+        this.llm = new google.LLM({
+            model: 'gemini-2.0-flash',
+            // Set lower temperature for more consistent decisions
+            temperature: 0.3,
+        });
+        // Add error handler to prevent unhandled rejection crashes
+        this.llm.on('error', (err) => {
+            const errorMsg = err.error?.message || String(err);
+            // Only log non-abort errors
+            if (!errorMsg.includes('aborted') && !errorMsg.includes('AbortError')) {
+                console.error('🧠 [Brain LLM] Error:', errorMsg);
+            }
+        });
+        this.workingDir = config.workingDir;
+        this.onSpeak = config.onSpeak;
+        this.onStateChange = config.onStateChange;
+        this.state = {
+            conversationHistory: [],
+            userGoal: null,
+            userGoalConfidence: 0,
+            pendingResearch: [],
+            completedResearch: [],
+            readyToExecute: false,
+            executionPlan: null,
+        };
+    }
+    /**
+     * Process user input and decide what to do
+     */
+    async processUserInput(input) {
+        // Add to conversation history
+        this.state.conversationHistory.push({
+            role: 'user',
+            content: input,
+            timestamp: new Date(),
+        });
+        // Build the analysis prompt
+        const prompt = this.buildAnalysisPrompt(input);
+        let stream = null;
+        let streamError = null;
+        try {
+            // Call Gemini 2.5 Pro for decision
+            const chatCtx = new llm.ChatContext();
+            chatCtx.addMessage({ role: 'user', content: prompt });
+            let response = '';
+            stream = this.llm.chat({ chatCtx });
+            // Create a promise that wraps the stream iteration with proper error handling
+            const collectResponse = async () => {
+                let result = '';
+                try {
+                    for await (const chunk of stream) {
+                        if (chunk.delta?.content) {
+                            result += chunk.delta.content;
+                        }
+                    }
+                }
+                catch (e) {
+                    streamError = e;
+                    throw e;
+                }
+                return result;
+            };
+            // Race the stream against a timeout to prevent hanging
+            const timeoutPromise = new Promise((_, reject) => {
+                setTimeout(() => reject(new Error('Brain timeout after 30s')), 30000);
+            });
+            response = await Promise.race([collectResponse(), timeoutPromise]);
+            // Parse the decision
+            const decision = this.parseDecision(response);
+            // Update state based on decision
+            this.updateState(decision);
+            // Add assistant response to history
+            if (decision.speech) {
+                this.state.conversationHistory.push({
+                    role: 'assistant',
+                    content: decision.speech,
+                    timestamp: new Date(),
+                });
+            }
+            return decision;
+        }
+        catch (err) {
+            const errorMsg = err.message || String(err);
+            // Only log if not an abort (which is expected when user interrupts)
+            if (!errorMsg.includes('aborted')) {
+                console.error('🧠 [Brain] Stream error:', errorMsg);
+            }
+            // Close stream if it exists to prevent further errors
+            if (stream) {
+                try {
+                    stream.close();
+                }
+                catch { }
+            }
+            return {
+                action: 'speak',
+                speech: "I'm having trouble processing that. Could you try again?",
+            };
+        }
+    }
+    /**
+     * Receive research results from background agents
+     */
+    receiveResearchResult(taskId, result, success) {
+        const task = this.state.pendingResearch.find(t => t.id === taskId);
+        if (task) {
+            task.status = success ? 'completed' : 'failed';
+            task.result = result;
+            task.completedAt = new Date();
+            // Move to completed
+            this.state.pendingResearch = this.state.pendingResearch.filter(t => t.id !== taskId);
+            this.state.completedResearch.push(task);
+            console.log(`🧠 Research completed: ${taskId.substring(0, 8)}... (${this.state.completedResearch.length} done)`);
+        }
+    }
+    /**
+     * Check if we should provide a status update
+     */
+    shouldProvideUpdate() {
+        // Provide update if research just completed
+        return this.state.completedResearch.length > 0 &&
+            this.state.pendingResearch.length === 0 &&
+            !this.state.readyToExecute;
+    }
+    /**
+     * Generate a status update based on completed research
+     */
+    async generateStatusUpdate() {
+        if (this.state.completedResearch.length === 0)
+            return null;
+        const researchSummary = this.state.completedResearch
+            .map(r => `- ${r.query}: ${r.result?.substring(0, 200) || 'No result'}`)
+            .join('\n');
+        const prompt = `Based on completed research, provide a brief conversational status update.
+
+RESEARCH RESULTS:
+${researchSummary}
+
+USER GOAL (so far): ${this.state.userGoal || 'Still understanding...'}
+
+Generate a 1-2 sentence update that:
+1. Summarizes what you learned
+2. Either asks a clarifying question OR proposes next steps
+3. Sounds natural and conversational
+
+Just the update text, no JSON.`;
+        let stream = null;
+        try {
+            const chatCtx = new llm.ChatContext();
+            chatCtx.addMessage({ role: 'user', content: prompt });
+            let response = '';
+            stream = this.llm.chat({ chatCtx });
+            for await (const chunk of stream) {
+                if (chunk.delta?.content) {
+                    response += chunk.delta.content;
+                }
+            }
+            return response.trim();
+        }
+        catch (err) {
+            console.error('🧠 [Brain] Status update error:', err.message || err);
+            if (stream) {
+                try {
+                    stream.close();
+                }
+                catch { }
+            }
+            return null;
+        }
+    }
+    /**
+     * Get current state for debugging/display
+     */
+    getState() {
+        return { ...this.state };
+    }
+    /**
+     * Reset conversation state
+     */
+    reset() {
+        this.state = {
+            conversationHistory: [],
+            userGoal: null,
+            userGoalConfidence: 0,
+            pendingResearch: [],
+            completedResearch: [],
+            readyToExecute: false,
+            executionPlan: null,
+        };
+    }
+    // ============================================================
+    // Private Methods
+    // ============================================================
+    buildAnalysisPrompt(userInput) {
+        const recentHistory = this.state.conversationHistory.slice(-6)
+            .map(m => `${m.role.toUpperCase()}: ${m.content}`)
+            .join('\n');
+        const researchContext = this.state.completedResearch.length > 0
+            ? `\nCOMPLETED RESEARCH:\n${this.state.completedResearch.map(r => `- ${r.query}: ${r.result?.substring(0, 300)}`).join('\n')}`
+            : '';
+        const pendingContext = this.state.pendingResearch.length > 0
+            ? `\nPENDING RESEARCH: ${this.state.pendingResearch.map(r => r.query).join(', ')}`
+            : '';
+        return `You are the brain of a voice AI coding assistant. Analyze this input and decide what to do.
+
+WORKING DIRECTORY: ${this.workingDir}
+
+CAPABILITIES:
+- Full internet access (web search, fetch URLs, API calls)
+- Read/write files in the working directory
+- Run shell commands (npm, git, etc.)
+- Search and analyze codebases
+
+CONVERSATION HISTORY:
+${recentHistory}
+
+CURRENT USER INPUT: "${userInput}"
+
+CURRENT UNDERSTANDING:
+- User Goal: ${this.state.userGoal || 'Unknown - still gathering context'}
+- Confidence: ${Math.round(this.state.userGoalConfidence * 100)}%
+- Ready to Execute: ${this.state.readyToExecute}
+${researchContext}
+${pendingContext}
+
+DECIDE WHAT TO DO:
+
+1. DIRECT_COMMAND - If user gives a simple, clear command:
+   - "read file X" → direct_command
+   - "run npm test" → direct_command
+   - "show me the package.json" → direct_command
+   - "search the web for X" → direct_command
+   - "look up X online" → direct_command
+
+2. CLARIFY - If request is ambiguous, ask a specific question to understand better
+
+3. RESEARCH - If we need more info, start background research (2-3 queries max)
+   - Search codebase, read docs, explore files
+   - Search the web for information
+   - Keep conversation going while research runs
+
+4. EXECUTE - ONLY if we have HIGH confidence (>80%) about what user wants
+   - Must have clear plan
+   - Only ONE execution task at a time
+
+5. SPEAK - Just respond conversationally (greetings, status, etc.)
+
+IMPORTANT: Keep speech SHORT and conversational. No markdown formatting.
+
+Respond in JSON:
+{
+  "action": "speak" | "research" | "execute" | "clarify" | "direct_command",
+  "speech": "What to say to user RIGHT NOW (keep it short, conversational, NO markdown)",
+  "researchQueries": ["query1", "query2"],  // Only if action=research
+  "executeTask": "detailed task description",  // Only if action=execute
+  "directCommand": "simple command to run",  // Only if action=direct_command
+  "updatedGoal": "What we think user wants now",
+  "goalConfidence": 0.0-1.0,
+  "reasoning": "Brief explanation"
+}`;
+    }
+    parseDecision(response) {
+        try {
+            // Extract JSON from response
+            const jsonMatch = response.match(/\{[\s\S]*\}/);
+            if (!jsonMatch) {
+                return {
+                    action: 'speak',
+                    speech: response.substring(0, 200),
+                };
+            }
+            const parsed = JSON.parse(jsonMatch[0]);
+            // Update goal tracking
+            if (parsed.updatedGoal) {
+                this.state.userGoal = parsed.updatedGoal;
+            }
+            if (typeof parsed.goalConfidence === 'number') {
+                this.state.userGoalConfidence = parsed.goalConfidence;
+            }
+            return {
+                action: parsed.action || 'speak',
+                speech: parsed.speech,
+                researchQueries: parsed.researchQueries,
+                executeTask: parsed.executeTask,
+                directCommand: parsed.directCommand,
+                reasoning: parsed.reasoning,
+            };
+        }
+        catch (err) {
+            console.error('Failed to parse brain decision:', err);
+            return {
+                action: 'speak',
+                speech: "Let me think about that...",
+            };
+        }
+    }
+    updateState(decision) {
+        // Track research tasks
+        if (decision.action === 'research' && decision.researchQueries) {
+            for (const query of decision.researchQueries) {
+                this.state.pendingResearch.push({
+                    id: `research-${Date.now()}-${Math.random().toString(36).substring(2, 6)}`,
+                    query,
+                    status: 'pending',
+                });
+            }
+        }
+        // Mark ready to execute
+        if (decision.action === 'execute') {
+            this.state.readyToExecute = true;
+            this.state.executionPlan = decision.executeTask || null;
+        }
+    }
+    /**
+     * Create research tasks for background agents
+     */
+    getPendingResearchTasks() {
+        return this.state.pendingResearch.filter(t => t.status === 'pending');
+    }
+    /**
+     * Mark a research task as running
+     */
+    markResearchRunning(taskId) {
+        const task = this.state.pendingResearch.find(t => t.id === taskId);
+        if (task) {
+            task.status = 'running';
+            task.startedAt = new Date();
+        }
+    }
+}
+// ============================================================
+// Factory function
+// ============================================================
+export function createConversationBrain(config) {
+    return new ConversationBrain(config);
+}

package/dist/fast-brain.d.ts

@@ -0,0 +1,122 @@
+/**
+ * Fast Brain Agent — Middle-tier intelligence for the Voice AI System
+ *
+ * A fast intermediary between the realtime voice model and the Claude SDK agent.
+ * Uses direct API calls for ~2 second responses.
+ *
+ * Capabilities:
+ * - Read/write session files (spec.md + library/)
+ * - Web search for quick factual lookups
+ * - Record user decisions and preferences into spec.md
+ * - Post-research: synthesize findings into spec.md
+ * - Escalate to ask_agent when deeper research is needed
+ *
+ * Key constraint: The fast brain NEVER calls ask_agent. The realtime model is always the router.
+ *
+ * Auth chain (tried in order):
+ * 1. ANTHROPIC_API_KEY env var → Anthropic SDK (Haiku)
+ * 2. ANTHROPIC_AUTH_TOKEN env var → Anthropic SDK (Haiku)
+ * 3. GOOGLE_API_KEY env var → Gemini Flash fallback
+ *
+ * Note: Claude Code OAuth (macOS Keychain) was tested but Anthropic's Messages API
+ * rejects OAuth tokens with 401 "OAuth authentication is currently not supported."
+ */
+/**
+ * Extract useful content snippets from tool responses, truncated by tool type.
+ * Returns null for tools with no useful content (Write confirmations, etc.)
+ */
+export declare function extractToolContent(toolName: string, toolInput: any, toolResponse: any): string | null;
+/** A single voice conversation turn from the realtime LLM's chatCtx */
+export interface ConversationTurn {
+    role: 'user' | 'assistant';
+    text: string;
+}
+/** No-op — history is now sourced live from agent.chatCtx, passed per-call */
+export declare function clearFastBrainHistory(): void;
+/**
+ * Ask the fast brain a question with access to session files and web search.
+ * Returns an answer or "NEEDS_DEEPER_RESEARCH: ..." for escalation.
+ *
+ * Auth chain: Anthropic (API key → auth token → Keychain OAuth) → Gemini Flash fallback
+ *
+ * @param researchContext - Optional snapshot of the live research log.
+ * ~2 second response time for most queries.
+ */
+export declare function askHaiku(workingDir: string, sessionId: string, question: string, researchContext?: string, chatHistory?: ConversationTurn[]): Promise<string>;
+/**
+ * Process a batch of research content chunks through the fast brain.
+ * Updates spec.md and library/ files incrementally during research.
+ *
+ * @param isRefinement - true for the final post-research consolidation pass (higher token budget)
+ */
+export declare function processResearchChunk(workingDir: string, sessionId: string, task: string, contentChunks: string[], isRefinement?: boolean): Promise<{
+    spec: string | null;
+    libraryFiles: string[];
+} | null>;
+/**
+ * Augment agent SDK research results with context from spec.md.
+ * Passes ALL specific details through verbatim — only ADDS context annotations.
+ * The voice model downstream handles summarization for speech.
+ *
+ * Falls back to returning the original result if the fast brain is unavailable.
+ */
+export declare function augmentResearchResult(workingDir: string, sessionId: string, task: string, agentResult: string): Promise<string>;
+/**
+ * Update spec.md and library/ files after research completes.
+ * Reads FULL untruncated data directly from Claude Agent SDK JSONL files
+ * instead of receiving pre-truncated content chunks.
+ *
+ * Data sources:
+ * - getRecentToolResults() — last 30 full tool results (Read, Bash, WebSearch, etc.)
+ * - readSessionHistory() — last 50 assistant messages (agent reasoning/analysis)
+ * - getSubagentTranscripts() — all sub-agent findings
+ *
+ * Returns { spec, libraryFiles } or null if update failed.
+ */
+export declare function updateSpecFromJSONL(workingDir: string, sessionId: string, task: string, researchLog: string[]): Promise<{
+    spec: string | null;
+    libraryFiles: string[];
+} | null>;
+/**
+ * Fire-and-forget: Write a user question to spec.md Open Questions > From User
+ * before the agent starts researching. Ensures every escalated question is tracked.
+ *
+ * Uses a simple LLM call to fuzzy-match existing questions and avoid duplicates.
+ * Skips if spec.md doesn't exist yet or no provider is available.
+ */
+export declare function writeQuestionToSpec(workingDir: string, sessionId: string, question: string): Promise<void>;
+/**
+ * Fire-and-forget: Check if substantial agent output answers any open questions in spec.md.
+ * Debounced (3s) to prevent flooding during rapid tool_result sequences.
+ *
+ * When a question is answered, marks it with [x] and moves the answer to Findings.
+ */
+export declare function checkOutputAgainstQuestions(workingDir: string, sessionId: string, output: string, outputType: 'tool_result' | 'assistant_text'): Promise<void>;
+/**
+ * Generate a natural, contextualized voice update from raw research events.
+ * Called by scheduleResearchBatch() instead of injecting raw events directly.
+ *
+ * Returns a natural 1-2 sentence update, or null if nothing interesting to say.
+ * 3-second timeout — returns null if the LLM is too slow.
+ */
+export declare function contextualizeResearchUpdate(workingDir: string, sessionId: string, task: string, batchEvents: string[], researchLog: string[]): Promise<string | null>;
+/**
+ * Generate a proactive conversational prompt to keep the user engaged during research.
+ * Called periodically (every 15s) during active research.
+ *
+ * Can ask open questions, discuss implications of findings, or give progress with depth.
+ * Returns null/NOTHING if nothing interesting to say.
+ * 3-second timeout.
+ */
+export declare function generateProactivePrompt(workingDir: string, sessionId: string, task: string, researchLog: string[], previousPrompts: string[]): Promise<string | null>;
+/**
+ * Generate a structured visual document (comparison table, Mermaid diagram,
+ * analysis, or summary) from research findings.
+ *
+ * Reads spec.md, JSONL results, and library for context.
+ * Writes the result to library/ and returns the filename + content.
+ */
+export declare function generateVisualDocument(workingDir: string, sessionId: string, request: string, documentType: 'comparison' | 'diagram' | 'analysis' | 'summary'): Promise<{
+    fileName: string;
+    content: string;
+} | null>;