osborn 0.9.6 → 0.9.9

package/dist/claude-llm.js

@@ -547,6 +547,8 @@ export class ClaudeLLM extends llm.LLM {
      * @param callbacks - Event callbacks for the background consumer
      */
     pushMessage(userText, sdkOptions, callbacks) {
+        // Lower compaction threshold to 65% so PreCompact fires earlier and context is preserved
+        process.env.CLAUDE_AUTOCOMPACT_PCT_OVERRIDE = '60';
         const userMessage = {
             type: 'user',
             message: { role: 'user', content: [{ type: 'text', text: userText }] },
@@ -912,30 +914,136 @@ class ClaudeLLMStream extends llm.LLMStream {
                                     }
                                 }]
                         }],
-                    // ── PreCompact: inject "include behavioral learnings" instruction into the compact ──
-                    // When the SDK is about to compress the conversation, this tells Claude to
-                    // include a BEHAVIORAL_LEARNINGS section in the compact summary. The instruction
-                    // is read from disk (hot-editable like the other prompts). Also includes
-                    // existing learned skills so Claude can merge/update rather than start fresh.
+                    // ── PreCompact: read full transcript, call Anthropic API directly to extract skills ──
+                    // Fires before the SDK compresses the conversation. Reads the FULL transcript JSONL,
+                    // calls claude-haiku-4-5-20251001 directly for structured extraction, and writes
+                    // skill files to disk immediately — no relying on PostCompact for persistence.
                     PreCompact: [{
                             matcher: '.*',
                             hooks: [async (input) => {
                                     try {
+                                        const { readFileSync, existsSync, writeFileSync, mkdirSync, readdirSync } = await import('node:fs');
+                                        const { join } = await import('node:path');
+                                        // 1. Read the instruction file
                                         const instructionPath = join(__claudeLlmDir, 'prompts', 'compact-learnings-instruction.md');
-                                        const instruction = readFileSync(instructionPath, 'utf-8');
-                                        // Load existing learned skills so Claude can update them
+                                        const instruction = existsSync(instructionPath) ? readFileSync(instructionPath, 'utf-8') : '';
+                                        // 2. Read FULL transcript (no line limit)
+                                        const transcriptPath = input?.transcript_path;
+                                        let transcriptContent = '';
+                                        if (transcriptPath && existsSync(transcriptPath)) {
+                                            try {
+                                                transcriptContent = readFileSync(transcriptPath, 'utf-8');
+                                            }
+                                            catch (readErr) {
+                                                console.warn('⚠️ PreCompact: could not read transcript:', readErr instanceof Error ? readErr.message : readErr);
+                                            }
+                                        }
                                         const skillDir = this.#opts.sessionBaseDir || this.#opts.workingDirectory || process.cwd();
-                                        const skillPath = join(skillDir, '.claude', 'skills', 'learned-behaviors', 'SKILL.md');
-                                        let existingSkills = '';
-                                        try {
-                                            existingSkills = readFileSync(skillPath, 'utf-8');
+                                        const skillsRoot = join(skillDir, '.claude', 'skills');
+                                        // 3. Load existing skills for context (cap each at 500 chars)
+                                        let existingSkillsBlock = '';
+                                        if (existsSync(skillsRoot)) {
+                                            try {
+                                                const names = readdirSync(skillsRoot).slice(0, 15);
+                                                const fragments = [];
+                                                for (const name of names) {
+                                                    const p = join(skillsRoot, name, 'SKILL.md');
+                                                    if (existsSync(p)) {
+                                                        const body = readFileSync(p, 'utf-8');
+                                                        const cleaned = body.replace(/^===.*===\s*$/gm, '').substring(0, 500);
+                                                        fragments.push(`### ${name}\n${cleaned}`);
+                                                    }
+                                                }
+                                                if (fragments.length) {
+                                                    existingSkillsBlock = `EXISTING SKILLS (do NOT re-emit unchanged):\n${fragments.join('\n\n')}`;
+                                                }
+                                            }
+                                            catch (skillErr) {
+                                                console.warn('⚠️ PreCompact: skills load failed:', skillErr instanceof Error ? skillErr.message : skillErr);
+                                            }
+                                        }
+                                        // 4. If transcript available, use direct Anthropic API call to extract everything
+                                        if (transcriptContent && process.env.ANTHROPIC_API_KEY) {
+                                            try {
+                                                const Anthropic = (await import('@anthropic-ai/sdk')).default;
+                                                const client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
+                                                const extractionPrompt = [
+                                                    instruction,
+                                                    existingSkillsBlock,
+                                                    `FULL SESSION TRANSCRIPT (JSONL format — extract the four sections from this):`,
+                                                    transcriptContent,
+                                                ].filter(Boolean).join('\n\n---\n\n');
+                                                console.log(`🧠 PreCompact: calling extraction API (${extractionPrompt.length} chars, trigger=${input?.trigger || 'unknown'})`);
+                                                const response = await client.messages.create({
+                                                    model: 'claude-haiku-4-5-20251001',
+                                                    max_tokens: 4096,
+                                                    messages: [{ role: 'user', content: extractionPrompt }],
+                                                });
+                                                const extractedText = response.content[0]?.type === 'text' ? response.content[0].text : '';
+                                                if (extractedText) {
+                                                    // Write SKILL_CANDIDATES to disk directly
+                                                    const skillRegex = /--- SKILL: ([a-z][a-z0-9-]{1,39}) ---\n([\s\S]*?)\n--- END SKILL ---/g;
+                                                    let match;
+                                                    while ((match = skillRegex.exec(extractedText)) !== null) {
+                                                        const [, name, body] = match;
+                                                        if (/^[a-z][a-z0-9-]{1,39}$/.test(name)) {
+                                                            const today = new Date().toISOString().split('T')[0];
+                                                            const sessionId = this.#sessionId || 'unknown';
+                                                            const skillFolder = join(skillsRoot, name);
+                                                            mkdirSync(skillFolder, { recursive: true });
+                                                            writeFileSync(join(skillFolder, 'SKILL.md'), `# ${name}\nAuto-extracted: ${today} | Session: ${sessionId.substring(0, 8)}\n\n${body}`);
+                                                            console.log(`🧠 PreCompact: wrote skill '${name}' to ${skillFolder}`);
+                                                        }
+                                                    }
+                                                    // Write BEHAVIORAL_LEARNINGS to learned-behaviors skill
+                                                    const blMarker = '=== BEHAVIORAL_LEARNINGS ===';
+                                                    const blIdx = extractedText.indexOf(blMarker);
+                                                    if (blIdx !== -1) {
+                                                        const blContent = extractedText.substring(blIdx);
+                                                        if (blContent.length > 30) {
+                                                            const today = new Date().toISOString().split('T')[0];
+                                                            const sessionId = this.#sessionId || 'unknown';
+                                                            const learnedDir = join(skillsRoot, 'learned-behaviors');
+                                                            mkdirSync(learnedDir, { recursive: true });
+                                                            const header = `# Learned Behaviors\n\nAuto-extracted from voice sessions via PreCompact.\nLast updated: ${today} | Session: ${sessionId.substring(0, 8)}...\n\n`;
+                                                            writeFileSync(join(learnedDir, 'SKILL.md'), header + blContent);
+                                                            console.log(`🧠 PreCompact: wrote learned-behaviors (${blContent.length} chars)`);
+                                                        }
+                                                    }
+                                                    // Write project-scoped DECISIONS
+                                                    const decMarker = '=== DECISIONS ===';
+                                                    const decIdx = extractedText.indexOf(decMarker);
+                                                    if (decIdx !== -1) {
+                                                        const decEnd = extractedText.indexOf('===', decIdx + decMarker.length);
+                                                        const decContent = extractedText.substring(decIdx + decMarker.length, decEnd !== -1 ? decEnd : undefined);
+                                                        const projectDecisions = decContent.split('\n').filter((l) => l.includes('SCOPE: project')).join('\n');
+                                                        if (projectDecisions.trim()) {
+                                                            const decDir = join(skillsRoot, 'project-decisions');
+                                                            mkdirSync(decDir, { recursive: true });
+                                                            const decFile = join(decDir, 'SKILL.md');
+                                                            const existing = existsSync(decFile) ? readFileSync(decFile, 'utf-8') : '# Project Decisions\n\n';
+                                                            writeFileSync(decFile, existing + '\n' + projectDecisions);
+                                                            console.log(`🧠 PreCompact: appended project decisions`);
+                                                        }
+                                                    }
+                                                    // Return extracted content as systemMessage for compact summary
+                                                    const systemMessage = [
+                                                        '## Pre-Compaction Extraction Complete',
+                                                        'Skill files have been written to disk. Include the following in your compact summary verbatim:',
+                                                        extractedText.substring(0, 8000),
+                                                    ].join('\n\n').substring(0, 9500);
+                                                    console.log(`🧠 PreCompact: extraction complete, returning systemMessage (${systemMessage.length} chars)`);
+                                                    return { systemMessage };
+                                                }
+                                            }
+                                            catch (apiErr) {
+                                                console.error('⚠️ PreCompact: API extraction failed, falling back to basic mode:', apiErr instanceof Error ? apiErr.message : apiErr);
+                                                // Fall through to basic fallback
+                                            }
                                         }
-                                        catch { }
-                                        const fullInstruction = existingSkills
-                                            ? `${instruction}\n\nEXISTING LEARNED SKILLS (update/merge — remove outdated, add new, strengthen confirmed):\n${existingSkills}`
-                                            : instruction;
-                                        console.log(`🧠 PreCompact: injected learnings instruction (${fullInstruction.length} chars, trigger=${input?.trigger || 'unknown'})`);
-                                        return { systemMessage: fullInstruction };
+                                        // 5. Fallback: basic mode — just inject the instruction
+                                        console.log(`🧠 PreCompact: basic mode (no transcript or API key unavailable)`);
+                                        return { systemMessage: instruction.substring(0, 9500) };
                                     }
                                     catch (err) {
                                         console.error('⚠️ PreCompact hook error:', err instanceof Error ? err.message : err);
@@ -943,34 +1051,108 @@ class ClaudeLLMStream extends llm.LLMStream {
                                     }
                                 }]
                         }],
-                    // ── PostCompact: extract BEHAVIORAL_LEARNINGS from summary and write to skill file ──
+                    // ── PostCompact: extract all four sections from summary and persist to disk ──
                     PostCompact: [{
                             matcher: '.*',
                             hooks: [async (input) => {
                                     try {
                                         const summary = input?.compact_summary || '';
-                                        const marker = '=== BEHAVIORAL_LEARNINGS ===';
-                                        const idx = summary.indexOf(marker);
-                                        if (idx === -1) {
-                                            console.log('🧠 PostCompact: no BEHAVIORAL_LEARNINGS section found in summary — skipping');
-                                            return {};
-                                        }
-                                        const learnings = summary.substring(idx + marker.length).trim();
-                                        if (learnings.length < 30) {
-                                            console.log('🧠 PostCompact: BEHAVIORAL_LEARNINGS section too short — skipping');
-                                            return {};
-                                        }
-                                        // Write the skill file
-                                        const skillDir = this.#opts.sessionBaseDir || this.#opts.workingDirectory || process.cwd();
-                                        const skillFolder = join(skillDir, '.claude', 'skills', 'learned-behaviors');
-                                        const skillPath = join(skillFolder, 'SKILL.md');
                                         const { mkdirSync, writeFileSync: writeSyncFs } = await import('fs');
-                                        mkdirSync(skillFolder, { recursive: true });
+                                        const skillDir = this.#opts.sessionBaseDir || this.#opts.workingDirectory || process.cwd();
                                         const today = new Date().toISOString().split('T')[0];
                                         const sessionId = this.#sessionId || 'unknown';
-                                        const header = `# Learned Behaviors\n\nAuto-extracted from voice sessions via PreCompact.\nLast updated: ${today} | Session: ${sessionId.substring(0, 8)}...\n\n`;
-                                        writeSyncFs(skillPath, header + learnings + '\n', 'utf-8');
-                                        console.log(`🧠 PostCompact: wrote learned behaviors to ${skillPath} (${learnings.length} chars)`);
+                                        // Helper: extract text between a marker and the next === marker (or end of string)
+                                        const extractSection = (marker) => {
+                                            const idx = summary.indexOf(marker);
+                                            if (idx === -1)
+                                                return '';
+                                            const start = idx + marker.length;
+                                            const nextMarker = summary.indexOf('=== ', start);
+                                            return (nextMarker === -1 ? summary.substring(start) : summary.substring(start, nextMarker)).trim();
+                                        };
+                                        // ── Section 1: HANDOFF_STATE — log only, not written to disk ──
+                                        const handoff = extractSection('=== HANDOFF_STATE ===');
+                                        if (handoff) {
+                                            console.log(`🧠 PostCompact: HANDOFF_STATE present (${handoff.length} chars) — not written to disk`);
+                                        }
+                                        // ── Section 2: DECISIONS — append project-scoped decisions ──
+                                        try {
+                                            const decisions = extractSection('=== DECISIONS ===');
+                                            if (decisions) {
+                                                const projectLines = decisions
+                                                    .split('\n')
+                                                    .filter(l => /DECISION:.*SCOPE:\s*project/i.test(l));
+                                                if (projectLines.length) {
+                                                    const decFolder = join(skillDir, '.claude', 'skills', 'project-decisions');
+                                                    const decPath = join(decFolder, 'SKILL.md');
+                                                    mkdirSync(decFolder, { recursive: true });
+                                                    const existing = (() => { try {
+                                                        return require('fs').readFileSync(decPath, 'utf-8');
+                                                    }
+                                                    catch {
+                                                        return '';
+                                                    } })();
+                                                    const header = existing ? '' : `# Project Decisions\n\nAuto-extracted from compact summaries.\n\n`;
+                                                    const entry = `\n## ${today} (session ${sessionId.substring(0, 8)})\n${projectLines.join('\n')}\n`;
+                                                    writeSyncFs(decPath, header + existing + entry, 'utf-8');
+                                                    console.log(`🧠 PostCompact: appended ${projectLines.length} decision(s) to ${decPath}`);
+                                                }
+                                            }
+                                        }
+                                        catch (decErr) {
+                                            console.error('⚠️ PostCompact: DECISIONS write failed:', decErr instanceof Error ? decErr.message : decErr);
+                                        }
+                                        // ── Section 3: SKILL_CANDIDATES — parse and write each skill ──
+                                        try {
+                                            const skillsSection = extractSection('=== SKILL_CANDIDATES ===');
+                                            if (skillsSection) {
+                                                const skillBlockRe = /---\s*SKILL:\s*([^\n-]+?)\s*---\n([\s\S]*?)---\s*END SKILL\s*---/g;
+                                                const nameRe = /^[a-z][a-z0-9-]{1,39}$/;
+                                                let match;
+                                                while ((match = skillBlockRe.exec(skillsSection)) !== null) {
+                                                    const name = match[1].trim();
+                                                    const body = match[2].trim();
+                                                    if (!nameRe.test(name)) {
+                                                        console.warn(`⚠️ PostCompact: skipping skill with invalid name "${name}" (must match /^[a-z][a-z0-9-]{1,39}$/)`);
+                                                        continue;
+                                                    }
+                                                    const skillFolder = join(skillDir, '.claude', 'skills', name);
+                                                    const skillPath = join(skillFolder, 'SKILL.md');
+                                                    mkdirSync(skillFolder, { recursive: true });
+                                                    const header = `# ${name}\nAuto-extracted: ${today} | Session: ${sessionId.substring(0, 8)}\n\n`;
+                                                    writeSyncFs(skillPath, header + body + '\n', 'utf-8');
+                                                    console.log(`🧠 PostCompact: wrote skill '${name}' to ${skillPath}`);
+                                                }
+                                            }
+                                        }
+                                        catch (skillErr) {
+                                            console.error('⚠️ PostCompact: SKILL_CANDIDATES write failed:', skillErr instanceof Error ? skillErr.message : skillErr);
+                                        }
+                                        // ── Section 4: BEHAVIORAL_LEARNINGS — write to learned-behaviors skill file ──
+                                        try {
+                                            const marker = '=== BEHAVIORAL_LEARNINGS ===';
+                                            const idx = summary.indexOf(marker);
+                                            if (idx !== -1) {
+                                                const learnings = summary.substring(idx + marker.length).trim();
+                                                if (learnings.length >= 30) {
+                                                    const skillFolder = join(skillDir, '.claude', 'skills', 'learned-behaviors');
+                                                    const skillPath = join(skillFolder, 'SKILL.md');
+                                                    mkdirSync(skillFolder, { recursive: true });
+                                                    const header = `# Learned Behaviors\n\nAuto-extracted from voice sessions via PostCompact.\nLast updated: ${today} | Session: ${sessionId.substring(0, 8)}...\n\n`;
+                                                    writeSyncFs(skillPath, header + learnings + '\n', 'utf-8');
+                                                    console.log(`🧠 PostCompact: wrote learned behaviors to ${skillPath} (${learnings.length} chars)`);
+                                                }
+                                                else {
+                                                    console.log('🧠 PostCompact: BEHAVIORAL_LEARNINGS section too short — skipping');
+                                                }
+                                            }
+                                            else {
+                                                console.log('🧠 PostCompact: no BEHAVIORAL_LEARNINGS section found in summary — skipping');
+                                            }
+                                        }
+                                        catch (blErr) {
+                                            console.error('⚠️ PostCompact: BEHAVIORAL_LEARNINGS write failed:', blErr instanceof Error ? blErr.message : blErr);
+                                        }
                                     }
                                     catch (err) {
                                         console.error('⚠️ PostCompact hook error:', err instanceof Error ? err.message : err);

package/dist/index.js

@@ -622,6 +622,34 @@ function startApiServer(workingDir, port) {
         console.log(`🌐 API server listening on http://${host}:${port}`);
         console.log(`   Sessions: http://${host}:${port}/sessions`);
     });
+    // Stale upload-chunk cleanup: remove osborn-upload-* dirs older than 30 minutes
+    const cleanStaleUploadDirs = () => {
+        const tmp = tmpdir();
+        const cutoff = Date.now() - 30 * 60 * 1000;
+        try {
+            const entries = readdirSync(tmp);
+            for (const entry of entries) {
+                if (!entry.startsWith('osborn-upload-'))
+                    continue;
+                const full = `${tmp}/${entry}`;
+                try {
+                    const st = statSync(full);
+                    if (st.isDirectory() && st.mtimeMs < cutoff) {
+                        rmSync(full, { recursive: true, force: true });
+                        console.log(`🧹 Removed stale upload dir: ${full}`);
+                    }
+                }
+                catch {
+                    // ignore per-entry errors
+                }
+            }
+        }
+        catch {
+            // ignore if /tmp is unreadable
+        }
+    };
+    cleanStaleUploadDirs();
+    setInterval(cleanStaleUploadDirs, 10 * 60 * 1000);
     server.on('error', (err) => {
         if (err.code === 'EADDRINUSE') {
             console.warn(`⚠️ API port ${port} in use, trying ${port + 1}...`);

package/dist/prompts/compact-learnings-instruction.md

@@ -1,14 +1,57 @@
-IMPORTANT — BEFORE COMPACTING: Include a section at the END of your compact summary titled exactly:
+# Pre-Compaction Instructions
 
-=== BEHAVIORAL_LEARNINGS ===
+Before Claude Code compacts this conversation, you MUST preserve critical context by including the following four sections at the very END of your compact summary. Be selective and specific — vague summaries are useless.
 
-In this section, extract and list:
-1. USER CORRECTIONS — things the user explicitly told you to stop/start doing (e.g., "stop patching scripts", "use step-by-step not autonomous", "don't name sub-agents")
-2. USER PREFERENCES — recurring patterns in how the user works (e.g., "prefers yahoo email for new accounts", "wants visible browser not headless", "expects score-based salary calculation")
-3. DOMAIN KNOWLEDGE — specific technical facts verified during this session (e.g., "Workday sign-in is inline switch not modal", "CareSource hibernation is 30s confirmed")
-4. EFFECTIVE PATTERNS — approaches that worked and the user confirmed (e.g., "co-direction questions during steerable work", "grounding with 2 reads before speaking")
-5. ANTI-PATTERNS — approaches that failed or frustrated the user (e.g., "reporting exit code 0 as success without verification", "multiple browser windows from competing automation approaches")
+---
 
-Be SELECTIVE — only include items that are generalizable, actionable, and non-obvious. Skip task-specific details. Each item should be one line.
+## Section 1: HANDOFF_STATE
+Format: === HANDOFF_STATE ===
+Include:
+- **Current goal**: What are we building/fixing and WHY (the big-picture reason, not just the immediate task)
+- **Progress**: What is done, what is in-progress, what is the very next step
+- **Active facts**: Environment details mentioned in this session (API keys referenced, service URLs, version numbers confirmed, file paths that matter)
+- **Test results**: What was tried, what it showed, what it ruled out
 
-This section will be extracted after compaction and saved as a persistent skill for future sessions.
+Keep this under 800 characters. This stays in the compact summary — it is NOT written to disk.
+
+---
+
+## Section 2: DECISIONS
+Format: === DECISIONS ===
+List each architectural or project decision made in this session, one per line:
+- DECISION: <choice made> | RATIONALE: <why> | SCOPE: project
+
+Only include decisions that would matter in a future session on the same project. Skip trivial choices.
+
+---
+
+## Section 3: SKILL_CANDIDATES
+Format: === SKILL_CANDIDATES ===
+For each reusable how-to procedure confirmed working in this session, emit:
+
+--- SKILL: <kebab-case-name> ---
+WHEN: <one line: when this skill applies>
+STEPS:
+1. ...
+2. ...
+VERIFIED: <exact command or observation that confirmed it works>
+--- END SKILL ---
+
+A skill is worth extracting only if: (1) it was confirmed working in this session AND (2) it would apply to future sessions on different tasks. Do NOT re-emit skills already shown in the EXISTING SKILLS section unless they need substantive updates.
+
+---
+
+## Section 4: BEHAVIORAL_LEARNINGS
+Format: === BEHAVIORAL_LEARNINGS ===
+Capture user corrections, preferences, and anti-patterns in these subsections:
+USER CORRECTIONS:
+USER PREFERENCES:
+DOMAIN KNOWLEDGE:
+EFFECTIVE PATTERNS:
+ANTI-PATTERNS:
+
+If the EXISTING LEARNED SKILLS section is shown below, merge — update outdated items, add new ones, keep confirmed ones.
+
+---
+
+IMPORTANT: All four sections MUST appear at the end of the compact summary even if some are empty. Empty sections should have a single line: (none this session)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.6",
+  "version": "0.9.9",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {