osborn 0.9.4 → 0.9.9

package/dist/claude-llm.js

@@ -547,6 +547,8 @@ export class ClaudeLLM extends llm.LLM {
      * @param callbacks - Event callbacks for the background consumer
      */
     pushMessage(userText, sdkOptions, callbacks) {
+        // Lower compaction threshold to 65% so PreCompact fires earlier and context is preserved
+        process.env.CLAUDE_AUTOCOMPACT_PCT_OVERRIDE = '60';
         const userMessage = {
             type: 'user',
             message: { role: 'user', content: [{ type: 'text', text: userText }] },
@@ -912,30 +914,136 @@ class ClaudeLLMStream extends llm.LLMStream {
                                     }
                                 }]
                         }],
-                    // ── PreCompact: inject "include behavioral learnings" instruction into the compact ──
-                    // When the SDK is about to compress the conversation, this tells Claude to
-                    // include a BEHAVIORAL_LEARNINGS section in the compact summary. The instruction
-                    // is read from disk (hot-editable like the other prompts). Also includes
-                    // existing learned skills so Claude can merge/update rather than start fresh.
+                    // ── PreCompact: read full transcript, call Anthropic API directly to extract skills ──
+                    // Fires before the SDK compresses the conversation. Reads the FULL transcript JSONL,
+                    // calls claude-haiku-4-5-20251001 directly for structured extraction, and writes
+                    // skill files to disk immediately — no relying on PostCompact for persistence.
                     PreCompact: [{
                             matcher: '.*',
                             hooks: [async (input) => {
                                     try {
+                                        const { readFileSync, existsSync, writeFileSync, mkdirSync, readdirSync } = await import('node:fs');
+                                        const { join } = await import('node:path');
+                                        // 1. Read the instruction file
                                         const instructionPath = join(__claudeLlmDir, 'prompts', 'compact-learnings-instruction.md');
-                                        const instruction = readFileSync(instructionPath, 'utf-8');
-                                        // Load existing learned skills so Claude can update them
+                                        const instruction = existsSync(instructionPath) ? readFileSync(instructionPath, 'utf-8') : '';
+                                        // 2. Read FULL transcript (no line limit)
+                                        const transcriptPath = input?.transcript_path;
+                                        let transcriptContent = '';
+                                        if (transcriptPath && existsSync(transcriptPath)) {
+                                            try {
+                                                transcriptContent = readFileSync(transcriptPath, 'utf-8');
+                                            }
+                                            catch (readErr) {
+                                                console.warn('⚠️ PreCompact: could not read transcript:', readErr instanceof Error ? readErr.message : readErr);
+                                            }
+                                        }
                                         const skillDir = this.#opts.sessionBaseDir || this.#opts.workingDirectory || process.cwd();
-                                        const skillPath = join(skillDir, '.claude', 'skills', 'learned-behaviors', 'SKILL.md');
-                                        let existingSkills = '';
-                                        try {
-                                            existingSkills = readFileSync(skillPath, 'utf-8');
+                                        const skillsRoot = join(skillDir, '.claude', 'skills');
+                                        // 3. Load existing skills for context (cap each at 500 chars)
+                                        let existingSkillsBlock = '';
+                                        if (existsSync(skillsRoot)) {
+                                            try {
+                                                const names = readdirSync(skillsRoot).slice(0, 15);
+                                                const fragments = [];
+                                                for (const name of names) {
+                                                    const p = join(skillsRoot, name, 'SKILL.md');
+                                                    if (existsSync(p)) {
+                                                        const body = readFileSync(p, 'utf-8');
+                                                        const cleaned = body.replace(/^===.*===\s*$/gm, '').substring(0, 500);
+                                                        fragments.push(`### ${name}\n${cleaned}`);
+                                                    }
+                                                }
+                                                if (fragments.length) {
+                                                    existingSkillsBlock = `EXISTING SKILLS (do NOT re-emit unchanged):\n${fragments.join('\n\n')}`;
+                                                }
+                                            }
+                                            catch (skillErr) {
+                                                console.warn('⚠️ PreCompact: skills load failed:', skillErr instanceof Error ? skillErr.message : skillErr);
+                                            }
+                                        }
+                                        // 4. If transcript available, use direct Anthropic API call to extract everything
+                                        if (transcriptContent && process.env.ANTHROPIC_API_KEY) {
+                                            try {
+                                                const Anthropic = (await import('@anthropic-ai/sdk')).default;
+                                                const client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
+                                                const extractionPrompt = [
+                                                    instruction,
+                                                    existingSkillsBlock,
+                                                    `FULL SESSION TRANSCRIPT (JSONL format — extract the four sections from this):`,
+                                                    transcriptContent,
+                                                ].filter(Boolean).join('\n\n---\n\n');
+                                                console.log(`🧠 PreCompact: calling extraction API (${extractionPrompt.length} chars, trigger=${input?.trigger || 'unknown'})`);
+                                                const response = await client.messages.create({
+                                                    model: 'claude-haiku-4-5-20251001',
+                                                    max_tokens: 4096,
+                                                    messages: [{ role: 'user', content: extractionPrompt }],
+                                                });
+                                                const extractedText = response.content[0]?.type === 'text' ? response.content[0].text : '';
+                                                if (extractedText) {
+                                                    // Write SKILL_CANDIDATES to disk directly
+                                                    const skillRegex = /--- SKILL: ([a-z][a-z0-9-]{1,39}) ---\n([\s\S]*?)\n--- END SKILL ---/g;
+                                                    let match;
+                                                    while ((match = skillRegex.exec(extractedText)) !== null) {
+                                                        const [, name, body] = match;
+                                                        if (/^[a-z][a-z0-9-]{1,39}$/.test(name)) {
+                                                            const today = new Date().toISOString().split('T')[0];
+                                                            const sessionId = this.#sessionId || 'unknown';
+                                                            const skillFolder = join(skillsRoot, name);
+                                                            mkdirSync(skillFolder, { recursive: true });
+                                                            writeFileSync(join(skillFolder, 'SKILL.md'), `# ${name}\nAuto-extracted: ${today} | Session: ${sessionId.substring(0, 8)}\n\n${body}`);
+                                                            console.log(`🧠 PreCompact: wrote skill '${name}' to ${skillFolder}`);
+                                                        }
+                                                    }
+                                                    // Write BEHAVIORAL_LEARNINGS to learned-behaviors skill
+                                                    const blMarker = '=== BEHAVIORAL_LEARNINGS ===';
+                                                    const blIdx = extractedText.indexOf(blMarker);
+                                                    if (blIdx !== -1) {
+                                                        const blContent = extractedText.substring(blIdx);
+                                                        if (blContent.length > 30) {
+                                                            const today = new Date().toISOString().split('T')[0];
+                                                            const sessionId = this.#sessionId || 'unknown';
+                                                            const learnedDir = join(skillsRoot, 'learned-behaviors');
+                                                            mkdirSync(learnedDir, { recursive: true });
+                                                            const header = `# Learned Behaviors\n\nAuto-extracted from voice sessions via PreCompact.\nLast updated: ${today} | Session: ${sessionId.substring(0, 8)}...\n\n`;
+                                                            writeFileSync(join(learnedDir, 'SKILL.md'), header + blContent);
+                                                            console.log(`🧠 PreCompact: wrote learned-behaviors (${blContent.length} chars)`);
+                                                        }
+                                                    }
+                                                    // Write project-scoped DECISIONS
+                                                    const decMarker = '=== DECISIONS ===';
+                                                    const decIdx = extractedText.indexOf(decMarker);
+                                                    if (decIdx !== -1) {
+                                                        const decEnd = extractedText.indexOf('===', decIdx + decMarker.length);
+                                                        const decContent = extractedText.substring(decIdx + decMarker.length, decEnd !== -1 ? decEnd : undefined);
+                                                        const projectDecisions = decContent.split('\n').filter((l) => l.includes('SCOPE: project')).join('\n');
+                                                        if (projectDecisions.trim()) {
+                                                            const decDir = join(skillsRoot, 'project-decisions');
+                                                            mkdirSync(decDir, { recursive: true });
+                                                            const decFile = join(decDir, 'SKILL.md');
+                                                            const existing = existsSync(decFile) ? readFileSync(decFile, 'utf-8') : '# Project Decisions\n\n';
+                                                            writeFileSync(decFile, existing + '\n' + projectDecisions);
+                                                            console.log(`🧠 PreCompact: appended project decisions`);
+                                                        }
+                                                    }
+                                                    // Return extracted content as systemMessage for compact summary
+                                                    const systemMessage = [
+                                                        '## Pre-Compaction Extraction Complete',
+                                                        'Skill files have been written to disk. Include the following in your compact summary verbatim:',
+                                                        extractedText.substring(0, 8000),
+                                                    ].join('\n\n').substring(0, 9500);
+                                                    console.log(`🧠 PreCompact: extraction complete, returning systemMessage (${systemMessage.length} chars)`);
+                                                    return { systemMessage };
+                                                }
+                                            }
+                                            catch (apiErr) {
+                                                console.error('⚠️ PreCompact: API extraction failed, falling back to basic mode:', apiErr instanceof Error ? apiErr.message : apiErr);
+                                                // Fall through to basic fallback
+                                            }
                                         }
-                                        catch { }
-                                        const fullInstruction = existingSkills
-                                            ? `${instruction}\n\nEXISTING LEARNED SKILLS (update/merge — remove outdated, add new, strengthen confirmed):\n${existingSkills}`
-                                            : instruction;
-                                        console.log(`🧠 PreCompact: injected learnings instruction (${fullInstruction.length} chars, trigger=${input?.trigger || 'unknown'})`);
-                                        return { systemMessage: fullInstruction };
+                                        // 5. Fallback: basic mode — just inject the instruction
+                                        console.log(`🧠 PreCompact: basic mode (no transcript or API key unavailable)`);
+                                        return { systemMessage: instruction.substring(0, 9500) };
                                     }
                                     catch (err) {
                                         console.error('⚠️ PreCompact hook error:', err instanceof Error ? err.message : err);
@@ -943,34 +1051,108 @@ class ClaudeLLMStream extends llm.LLMStream {
                                     }
                                 }]
                         }],
-                    // ── PostCompact: extract BEHAVIORAL_LEARNINGS from summary and write to skill file ──
+                    // ── PostCompact: extract all four sections from summary and persist to disk ──
                     PostCompact: [{
                             matcher: '.*',
                             hooks: [async (input) => {
                                     try {
                                         const summary = input?.compact_summary || '';
-                                        const marker = '=== BEHAVIORAL_LEARNINGS ===';
-                                        const idx = summary.indexOf(marker);
-                                        if (idx === -1) {
-                                            console.log('🧠 PostCompact: no BEHAVIORAL_LEARNINGS section found in summary — skipping');
-                                            return {};
-                                        }
-                                        const learnings = summary.substring(idx + marker.length).trim();
-                                        if (learnings.length < 30) {
-                                            console.log('🧠 PostCompact: BEHAVIORAL_LEARNINGS section too short — skipping');
-                                            return {};
-                                        }
-                                        // Write the skill file
-                                        const skillDir = this.#opts.sessionBaseDir || this.#opts.workingDirectory || process.cwd();
-                                        const skillFolder = join(skillDir, '.claude', 'skills', 'learned-behaviors');
-                                        const skillPath = join(skillFolder, 'SKILL.md');
                                         const { mkdirSync, writeFileSync: writeSyncFs } = await import('fs');
-                                        mkdirSync(skillFolder, { recursive: true });
+                                        const skillDir = this.#opts.sessionBaseDir || this.#opts.workingDirectory || process.cwd();
                                         const today = new Date().toISOString().split('T')[0];
                                         const sessionId = this.#sessionId || 'unknown';
-                                        const header = `# Learned Behaviors\n\nAuto-extracted from voice sessions via PreCompact.\nLast updated: ${today} | Session: ${sessionId.substring(0, 8)}...\n\n`;
-                                        writeSyncFs(skillPath, header + learnings + '\n', 'utf-8');
-                                        console.log(`🧠 PostCompact: wrote learned behaviors to ${skillPath} (${learnings.length} chars)`);
+                                        // Helper: extract text between a marker and the next === marker (or end of string)
+                                        const extractSection = (marker) => {
+                                            const idx = summary.indexOf(marker);
+                                            if (idx === -1)
+                                                return '';
+                                            const start = idx + marker.length;
+                                            const nextMarker = summary.indexOf('=== ', start);
+                                            return (nextMarker === -1 ? summary.substring(start) : summary.substring(start, nextMarker)).trim();
+                                        };
+                                        // ── Section 1: HANDOFF_STATE — log only, not written to disk ──
+                                        const handoff = extractSection('=== HANDOFF_STATE ===');
+                                        if (handoff) {
+                                            console.log(`🧠 PostCompact: HANDOFF_STATE present (${handoff.length} chars) — not written to disk`);
+                                        }
+                                        // ── Section 2: DECISIONS — append project-scoped decisions ──
+                                        try {
+                                            const decisions = extractSection('=== DECISIONS ===');
+                                            if (decisions) {
+                                                const projectLines = decisions
+                                                    .split('\n')
+                                                    .filter(l => /DECISION:.*SCOPE:\s*project/i.test(l));
+                                                if (projectLines.length) {
+                                                    const decFolder = join(skillDir, '.claude', 'skills', 'project-decisions');
+                                                    const decPath = join(decFolder, 'SKILL.md');
+                                                    mkdirSync(decFolder, { recursive: true });
+                                                    const existing = (() => { try {
+                                                        return require('fs').readFileSync(decPath, 'utf-8');
+                                                    }
+                                                    catch {
+                                                        return '';
+                                                    } })();
+                                                    const header = existing ? '' : `# Project Decisions\n\nAuto-extracted from compact summaries.\n\n`;
+                                                    const entry = `\n## ${today} (session ${sessionId.substring(0, 8)})\n${projectLines.join('\n')}\n`;
+                                                    writeSyncFs(decPath, header + existing + entry, 'utf-8');
+                                                    console.log(`🧠 PostCompact: appended ${projectLines.length} decision(s) to ${decPath}`);
+                                                }
+                                            }
+                                        }
+                                        catch (decErr) {
+                                            console.error('⚠️ PostCompact: DECISIONS write failed:', decErr instanceof Error ? decErr.message : decErr);
+                                        }
+                                        // ── Section 3: SKILL_CANDIDATES — parse and write each skill ──
+                                        try {
+                                            const skillsSection = extractSection('=== SKILL_CANDIDATES ===');
+                                            if (skillsSection) {
+                                                const skillBlockRe = /---\s*SKILL:\s*([^\n-]+?)\s*---\n([\s\S]*?)---\s*END SKILL\s*---/g;
+                                                const nameRe = /^[a-z][a-z0-9-]{1,39}$/;
+                                                let match;
+                                                while ((match = skillBlockRe.exec(skillsSection)) !== null) {
+                                                    const name = match[1].trim();
+                                                    const body = match[2].trim();
+                                                    if (!nameRe.test(name)) {
+                                                        console.warn(`⚠️ PostCompact: skipping skill with invalid name "${name}" (must match /^[a-z][a-z0-9-]{1,39}$/)`);
+                                                        continue;
+                                                    }
+                                                    const skillFolder = join(skillDir, '.claude', 'skills', name);
+                                                    const skillPath = join(skillFolder, 'SKILL.md');
+                                                    mkdirSync(skillFolder, { recursive: true });
+                                                    const header = `# ${name}\nAuto-extracted: ${today} | Session: ${sessionId.substring(0, 8)}\n\n`;
+                                                    writeSyncFs(skillPath, header + body + '\n', 'utf-8');
+                                                    console.log(`🧠 PostCompact: wrote skill '${name}' to ${skillPath}`);
+                                                }
+                                            }
+                                        }
+                                        catch (skillErr) {
+                                            console.error('⚠️ PostCompact: SKILL_CANDIDATES write failed:', skillErr instanceof Error ? skillErr.message : skillErr);
+                                        }
+                                        // ── Section 4: BEHAVIORAL_LEARNINGS — write to learned-behaviors skill file ──
+                                        try {
+                                            const marker = '=== BEHAVIORAL_LEARNINGS ===';
+                                            const idx = summary.indexOf(marker);
+                                            if (idx !== -1) {
+                                                const learnings = summary.substring(idx + marker.length).trim();
+                                                if (learnings.length >= 30) {
+                                                    const skillFolder = join(skillDir, '.claude', 'skills', 'learned-behaviors');
+                                                    const skillPath = join(skillFolder, 'SKILL.md');
+                                                    mkdirSync(skillFolder, { recursive: true });
+                                                    const header = `# Learned Behaviors\n\nAuto-extracted from voice sessions via PostCompact.\nLast updated: ${today} | Session: ${sessionId.substring(0, 8)}...\n\n`;
+                                                    writeSyncFs(skillPath, header + learnings + '\n', 'utf-8');
+                                                    console.log(`🧠 PostCompact: wrote learned behaviors to ${skillPath} (${learnings.length} chars)`);
+                                                }
+                                                else {
+                                                    console.log('🧠 PostCompact: BEHAVIORAL_LEARNINGS section too short — skipping');
+                                                }
+                                            }
+                                            else {
+                                                console.log('🧠 PostCompact: no BEHAVIORAL_LEARNINGS section found in summary — skipping');
+                                            }
+                                        }
+                                        catch (blErr) {
+                                            console.error('⚠️ PostCompact: BEHAVIORAL_LEARNINGS write failed:', blErr instanceof Error ? blErr.message : blErr);
+                                        }
                                     }
                                     catch (err) {
                                         console.error('⚠️ PostCompact hook error:', err instanceof Error ? err.message : err);

package/dist/conversation-brain.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Conversation Brain - Gemini 2.5 Pro powered conversation manager
+ *
+ * This is the "smart brain" that:
+ * 1. Keeps conversation alive with relevant questions
+ * 2. Builds context until we understand what user wants
+ * 3. Dispatches background research agents
+ * 4. Receives progress updates and decides when to execute
+ * 5. Handles direct commands immediately
+ */
+export interface ConversationMessage {
+    role: 'user' | 'assistant' | 'system';
+    content: string;
+    timestamp: Date;
+}
+export interface ResearchTask {
+    id: string;
+    query: string;
+    status: 'pending' | 'running' | 'completed' | 'failed';
+    result?: string;
+    startedAt?: Date;
+    completedAt?: Date;
+}
+export interface BrainDecision {
+    action: 'speak' | 'research' | 'execute' | 'clarify' | 'direct_command';
+    speech?: string;
+    researchQueries?: string[];
+    executeTask?: string;
+    directCommand?: string;
+    reasoning?: string;
+}
+export interface BrainState {
+    conversationHistory: ConversationMessage[];
+    userGoal: string | null;
+    userGoalConfidence: number;
+    pendingResearch: ResearchTask[];
+    completedResearch: ResearchTask[];
+    readyToExecute: boolean;
+    executionPlan: string | null;
+}
+export declare class ConversationBrain {
+    private llm;
+    private state;
+    private workingDir;
+    private onSpeak;
+    private onStateChange;
+    constructor(config: {
+        workingDir: string;
+        onSpeak: (text: string) => Promise<void>;
+        onStateChange: (state: string) => Promise<void>;
+    });
+    /**
+     * Process user input and decide what to do
+     */
+    processUserInput(input: string): Promise<BrainDecision>;
+    /**
+     * Receive research results from background agents
+     */
+    receiveResearchResult(taskId: string, result: string, success: boolean): void;
+    /**
+     * Check if we should provide a status update
+     */
+    shouldProvideUpdate(): boolean;
+    /**
+     * Generate a status update based on completed research
+     */
+    generateStatusUpdate(): Promise<string | null>;
+    /**
+     * Get current state for debugging/display
+     */
+    getState(): BrainState;
+    /**
+     * Reset conversation state
+     */
+    reset(): void;
+    private buildAnalysisPrompt;
+    private parseDecision;
+    private updateState;
+    /**
+     * Create research tasks for background agents
+     */
+    getPendingResearchTasks(): ResearchTask[];
+    /**
+     * Mark a research task as running
+     */
+    markResearchRunning(taskId: string): void;
+}
+export declare function createConversationBrain(config: {
+    workingDir: string;
+    onSpeak: (text: string) => Promise<void>;
+    onStateChange: (state: string) => Promise<void>;
+}): ConversationBrain;