escribano 0.4.5 → 0.5.0

package/README.md

@@ -95,19 +95,49 @@ Good for retrospectives or blog drafts.
 
 ## Benchmarks
 
-Ran the full pipeline on 11 real screen recordings:
+### Architecture Benefits (MLX Migration)
+
+| Improvement | Impact |
+|-------------|--------|
+| **Zero dependencies** | No external daemons required |
+| **Unified backend** | VLM + LLM use same MLX infrastructure |
+| **Native Metal** | Optimized for Apple Silicon |
+| **Memory efficient** | Sequential model loading (no OOM) |
+| **Auto-detection** | RAM-based model selection |
+
+### Production Run (March 2026)
+
+Processed **17 real screen recordings** with MLX backend:
 
 | Metric | Result |
 |--------|--------|
-| Videos processed | 11 |
-| Artifacts generated | 33 (3 formats × 11 videos) |
-| Success rate | 100% |
-| Total time | 1h 41m |
-| Avg per video | **~9 min** (pipeline + all 3 formats) |
+| Videos processed | 17 |
+| Successful | 15 (88%) |
+| Total video duration | 25.6 hours |
+| Artifacts generated | 45 (3 formats × 15 videos) |
+| **LLM generation** | **~2.2 min per video** |
+| Subject grouping | 78.7s avg |
+| Artifact generation | 53.6s avg |
+| LLM success rate | 100% (92 calls) |
 | Hardware | MacBook Pro M4 Max, 128GB |
+| Backend | MLX (Qwen3-VL-2B + Qwen3.5-27B) |
 
 Everything runs locally. No API keys. Nothing leaves your machine.
 
+### Hardware Tiers (March 2026)
+
+Performance varies by hardware:
+
+| Hardware | RAM | VLM Speed | LLM Model | LLM Speed | Total (1min video) |
+|----------|-----|-----------|-----------|-----------|-------------------|
+| **M4 Max** | 128GB | 0.7s/frame | Qwen3.5-27B | 53s avg | **~2.2 min** |
+| **M1/M2/M3 Pro** | 16-32GB | 1.5-3s/frame | Qwen3.5-9B | 80-120s | ~5-8 min |
+| **M1/M2 Air** | 16GB | 7-9s/frame | Qwen3.5-9B | 150-250s | ~12-15 min |
+
+**Minimum viable**: 16GB unified memory (slower but functional)
+
+**Recommended**: 32GB+ for comfortable use, 64GB+ for best quality
+
 ---
 
 ## Why this exists
@@ -141,7 +171,7 @@ Screen recording
 Activity segmentation → temporal audio alignment → TopicBlocks
      │
      ▼
-LLM summary (Ollama, auto-detected) → Markdown artifact
+LLM summary (MLX-LM, auto-detected) → Markdown artifact
 ```
 
 Uses VLM-first visual understanding, not OCR + text clustering. OCR fails for developer work because all code screens produce similar tokens. VLMs understand the *activity*, not just the text.
@@ -154,32 +184,22 @@ Uses VLM-first visual understanding, not OCR + text clustering. OCR fails for de
 
 ```bash
 # macOS (Homebrew)
-brew install ollama whisper-cpp ffmpeg
+brew install whisper-cpp ffmpeg
 
-# MLX-VLM for frame analysis (Apple Silicon)
-# Using uv (recommended, faster)
-uv pip install mlx-vlm
-
-# Or using pip
-pip install mlx-vlm
+# MLX for inference (Apple Silicon) - auto-installed on first run
+# Or pre-install with:
+pip install mlx-vlm mlx-lm
 ```
 
-### LLM Model Setup
+That's it. No external daemons required. MLX-VLM and MLX-LM run in-process.
 
-Escribano auto-detects the best model for your hardware:
+### (Optional) Ollama Backend
 
-| Your RAM | Auto-selected | Install command |
-|----------|---------------|-----------------|
-| 16GB | `qwen3:8b` | `ollama pull qwen3:8b` |
-| 32GB | `qwen3:14b` | `ollama pull qwen3:14b` |
-| 64GB+ | `qwen3.5:27b` | `ollama pull qwen3.5:27b` |
+If you prefer Ollama, set `ESCRIBANO_LLM_BACKEND=ollama`:
 
 ```bash
-# Minimum (16GB)
-ollama pull qwen3:8b
-
-# Or best quality (64GB+)
-ollama pull qwen3.5:27b
+brew install ollama
+ollama pull qwen3:8b  # or qwen3.5:27b for 64GB+ RAM
 ```
 
 ### Run

package/dist/actions/generate-artifact-v3.js

@@ -228,9 +228,11 @@ async function generateLlmArtifact(subjects, groupingResult, format, recording,
         .replace('{{SUBJECT_COUNT}}', String(subjects.length))
         .replace('{{SUBJECTS_DATA}}', subjectsData)
         .replace('{{WORK_SUBJECTS}}', subjectsData);
-    return intelligence.generateText(prompt, {
-        expectJson: false,
-        think: ARTIFACT_THINK,
+    return step('llm_artifact_generation', async () => {
+        return intelligence.generateText(prompt, {
+            expectJson: false,
+            think: ARTIFACT_THINK,
+        });
     });
 }
 function buildSubjectsDataForPrompt(subjects, allTopicBlocks) {

package/dist/actions/generate-summary-v3.js

@@ -8,7 +8,7 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import { homedir } from 'node:os';
 import path, { dirname, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { log } from '../pipeline/context.js';
+import { log, step } from '../pipeline/context.js';
 import { groupTopicBlocksIntoSubjects, saveSubjectsToDatabase, } from '../services/subject-grouping.js';
 const __dirname = dirname(fileURLToPath(import.meta.url));
 /**
@@ -225,10 +225,35 @@ ${section.transcript ? `**Audio Transcript:**\n${section.transcript}` : '*No aud
         .replace('{{APPS_LIST}}', appsList)
         .replace('{{URLS_LIST}}', urlsList);
     // Call LLM
-    const result = await intelligence.generateText(prompt, {
-        expectJson: false,
+    const result = await step('llm_artifact_generation', async () => {
+        return intelligence.generateText(prompt, {
+            expectJson: false,
+            debugContext: {
+                recordingId: recording.id,
+                callType: 'artifact_generation',
+            },
+        });
     });
-    return result;
+    // Strip thinking leakage if present
+    let cleaned = result.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+    if (cleaned.includes('</think>')) {
+        // Handle orphan </think> tag (Qwen3.5 behavior)
+        cleaned = cleaned.split('</think>')[1].trim();
+    }
+    // Strip "Thinking Process:" prose (Qwen3.5-OptiQ format)
+    const tpMatch = cleaned.match(/(?:^|\n)Thinking Process:/);
+    if (tpMatch !== null) {
+        const after = cleaned.slice((tpMatch.index ?? 0) + tpMatch[0].length);
+        const heading = after.match(/\n(#\s|\*\*)/);
+        cleaned =
+            heading?.index !== undefined ? after.slice(heading.index).trim() : '';
+    }
+    // If cleaning leaves nothing usable, fall back to template
+    if (cleaned.length > 50) {
+        return cleaned;
+    }
+    console.warn('[artifact-generation] Thinking leakage detected or response too short — falling back to template');
+    return formatSummary(sections, recording.duration, recording.id);
 }
 /**
  * Format sections into a readable markdown summary (template fallback).

package/dist/adapters/cap.adapter.js

@@ -0,0 +1,94 @@
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { capConfigSchema } from '../0_types.js';
+function expandPath(path) {
+    if (path.startsWith('~/')) {
+        return join(homedir(), path.slice(2));
+    }
+    return path;
+}
+async function parseCapRecording(capDirPath) {
+    try {
+        const metaPath = join(capDirPath, 'recording-meta.json');
+        const metaContent = await readFile(metaPath, 'utf-8');
+        const meta = JSON.parse(metaContent);
+        if (!meta.segments ||
+            !Array.isArray(meta.segments) ||
+            meta.segments.length === 0) {
+            throw new Error(`Invalid metadata in ${capDirPath}: missing or empty segments array`);
+        }
+        const firstSegment = meta.segments[0];
+        const videoPath = firstSegment.display?.path
+            ? join(capDirPath, firstSegment.display.path)
+            : null;
+        // we fked up cuz we have mic but also system_audio.ogg
+        const micAudio = firstSegment.mic?.path
+            ? join(capDirPath, firstSegment.mic.path)
+            : null;
+        const systemAudio = firstSegment.system_audio?.path
+            ? join(capDirPath, firstSegment.system_audio.path)
+            : null;
+        const audioToStat = micAudio || systemAudio;
+        if (!audioToStat) {
+            console.log(`Skipping ${capDirPath}: none audio track found`);
+            return null;
+        }
+        const stats = await stat(audioToStat);
+        const capturedAt = stats.mtime;
+        const recordingId = capDirPath.split('/').pop() || 'unknown';
+        return {
+            id: recordingId,
+            source: {
+                type: 'cap',
+                originalPath: capDirPath,
+                metadata: meta,
+            },
+            videoPath,
+            audioMicPath: micAudio ? micAudio : null,
+            audioSystemPath: systemAudio ? systemAudio : null,
+            duration: 0,
+            capturedAt,
+        };
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            throw new Error(`Recording directory or files not found: ${capDirPath}`);
+        }
+        if (error.name === 'SyntaxError') {
+            throw new Error(`Invalid JSON in recording-meta.json at ${capDirPath}`);
+        }
+        throw new Error(`Failed to parse recording at ${capDirPath}: ${error.message}`);
+    }
+}
+export function createCapSource(config = {}) {
+    const parsedConfig = capConfigSchema.parse(config);
+    const recordingsPath = expandPath(parsedConfig.recordingsPath);
+    const innerList = async (limit = 10) => {
+        try {
+            //
+            // 7 directories, 5 files
+            const entries = await readdir(recordingsPath, { withFileTypes: true });
+            const capDirs = entries.filter((entry) => entry.isDirectory() && entry.name.endsWith('.cap'));
+            const recordings = await Promise.allSettled(capDirs.map(async (dir) => parseCapRecording(join(recordingsPath, dir.name))));
+            // logging errors
+            console.log(recordings
+                .filter((p) => p.status === 'rejected')
+                .map((p) => p.reason + '\n'));
+            return recordings
+                .filter((p) => p.status === 'fulfilled')
+                .map((x) => x.value)
+                .filter((r) => r !== null)
+                .sort((a, b) => b.capturedAt.getTime() - a.capturedAt.getTime())
+                .slice(0, limit);
+        }
+        catch (error) {
+            console.error('Failed to list Cap recordings:', error);
+            return [];
+        }
+    };
+    return {
+        getLatestRecording: () => innerList(1).then((recordings) => recordings[0] ?? null),
+        listRecordings: innerList,
+    };
+}

package/dist/adapters/intelligence.adapter.js

@@ -0,0 +1,202 @@
+/**
+ * Escribano - Intelligence Adapter (Ollama)
+ *
+ * Implements IntelligenceService using Ollama REST API
+ */
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+export function createIntelligenceService(config) {
+    return {
+        classify: (transcript) => classifyWithOllama(transcript, config),
+        extractMetadata: (transcript, classification) => extractMetadata(transcript, classification, config),
+        generate: (artifactType, context) => generateArtifact(artifactType, context, config),
+    };
+}
+async function checkOllamaHealth() {
+    try {
+        const response = await fetch('http://localhost:11434/api/tags');
+        if (!response.ok) {
+            throw new Error('Ollama API not accessible');
+        }
+        const data = await response.json();
+        console.log('✓ Ollama is running and accessible');
+        console.log(`  Available models: ${data.models?.length || 0}`);
+    }
+    catch (error) {
+        console.error('✗ Ollama is not running or not accessible');
+        console.error('  Error:', error.message);
+        console.error('');
+        console.error('Please start Ollama:');
+        console.error('  brew install ollama');
+        console.error('  ollama pull qwen3:32b');
+        console.error('  ollama serve');
+        console.error('');
+        throw new Error('Ollama service required for classification');
+    }
+}
+async function classifyWithOllama(transcript, config) {
+    console.log('Classifying transcript with Ollama...');
+    const tick = setInterval(() => {
+        process.stdout.write('.');
+    }, 1000);
+    await checkOllamaHealth();
+    const prompt = loadClassifyPrompt(transcript);
+    const raw = await callOllama(prompt, config, { expectJson: true });
+    clearInterval(tick);
+    console.log('\nClassification completed.');
+    const classification = {
+        meeting: raw.meeting * (raw.meeting <= 1 ? 100 : 1) || 0,
+        debugging: raw.debugging * (raw.debugging <= 1 ? 100 : 1) || 0,
+        tutorial: raw.tutorial * (raw.tutorial <= 1 ? 100 : 1) || 0,
+        learning: raw.learning * (raw.learning <= 1 ? 100 : 1) || 0,
+        working: raw.working * (raw.working <= 1 ? 100 : 1) || 0,
+    };
+    return classification;
+}
+function loadClassifyPrompt(transcript) {
+    const promptPath = join(process.cwd(), 'prompts', 'classify.md');
+    let prompt = readFileSync(promptPath, 'utf-8');
+    const segmentsText = transcript.segments
+        .map((seg) => `[seg-${seg.id}] [${seg.start}s - ${seg.end}s] ${seg.text}`)
+        .join('\n');
+    prompt = prompt.replace('{{TRANSCRIPT_ALL}}', transcript.fullText);
+    prompt = prompt.replace('{{TRANSCRIPT_SEGMENTS}}', segmentsText);
+    return prompt;
+}
+async function callOllama(prompt, config, options = { expectJson: true }) {
+    const { endpoint, model, maxRetries, timeout } = config;
+    let lastError = null;
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+        try {
+            const controller = new AbortController();
+            const timeoutId = setTimeout(() => controller.abort(), timeout);
+            const response = await fetch(endpoint, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({
+                    model,
+                    messages: [
+                        {
+                            role: 'system',
+                            content: options.expectJson
+                                ? 'You are a JSON-only output system. Output ONLY valid JSON, no other text.'
+                                : 'You are a helpful assistant that generates high-quality markdown documentation.',
+                        },
+                        {
+                            role: 'user',
+                            content: prompt,
+                        },
+                    ],
+                    stream: false,
+                    ...(options.expectJson && { format: 'json' }),
+                }),
+                signal: controller.signal,
+            });
+            clearTimeout(timeoutId);
+            if (!response.ok) {
+                throw new Error(`Ollama API error: ${response.status} ${response.statusText}`);
+            }
+            const data = await response.json();
+            if (!data.done || data.done_reason !== 'stop') {
+                throw new Error(`Incomplete response: done=${data.done}, reason=${data.done_reason}`);
+            }
+            const content = data.message.content;
+            if (options.expectJson) {
+                const jsonMatch = content.match(/\{[\s\S]*\}/);
+                if (!jsonMatch)
+                    throw new Error('No JSON found in response');
+                return JSON.parse(jsonMatch[0]);
+            }
+            return content;
+        }
+        catch (error) {
+            lastError = error;
+            if (error instanceof Error && error.name === 'AbortError') {
+                console.log(`Attempt ${attempt}/${maxRetries}: Request timed out, retrying...`);
+            }
+            else {
+                console.log(`Attempt ${attempt}/${maxRetries}: Request failed, retrying...`);
+            }
+            if (attempt < maxRetries) {
+                await new Promise((resolve) => setTimeout(resolve, 1000 * attempt));
+            }
+        }
+    }
+    throw new Error(`Request failed after ${maxRetries} retries: ${lastError?.message}`);
+}
+async function extractMetadata(transcript, classification, config) {
+    const prompt = loadMetadataPrompt(transcript, classification);
+    const raw = await callOllama(prompt, config, { expectJson: true });
+    return {
+        speakers: raw.speakers || [],
+        keyMoments: raw.keyMoments || [],
+        actionItems: raw.actionItems || [],
+        technicalTerms: raw.technicalTerms || [],
+        codeSnippets: raw.codeSnippets || [],
+    };
+}
+function loadMetadataPrompt(transcript, classification) {
+    const promptPath = join(process.cwd(), 'prompts', 'extract-metadata.md');
+    let prompt = readFileSync(promptPath, 'utf-8');
+    const classificationSummary = Object.entries(classification)
+        .filter(([_, score]) => score >= 25)
+        .map(([type, score]) => `${type}: ${score}%`)
+        .join(', ');
+    const segmentsText = transcript.segments
+        .map((seg) => `[${seg.start}s - ${seg.end}s] ${seg.text}`)
+        .join('\n');
+    prompt = prompt.replace('{{CLASSIFICATION_SUMMARY}}', classificationSummary);
+    prompt = prompt.replace('{{TRANSCRIPT_SEGMENTS}}', segmentsText);
+    prompt = prompt.replace('{{TRANSCRIPT_ALL}}', transcript.fullText);
+    return prompt;
+}
+function parseMetadataJson(content) {
+    const jsonMatch = content.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) {
+        throw new Error('No JSON object found in metadata extraction response');
+    }
+    const parsed = JSON.parse(jsonMatch[0]);
+    return {
+        speakers: parsed.speakers || [],
+        keyMoments: parsed.keyMoments || [],
+        actionItems: parsed.actionItems || [],
+        technicalTerms: parsed.technicalTerms || [],
+        codeSnippets: parsed.codeSnippets || [],
+    };
+}
+async function generateArtifact(artifactType, context, config) {
+    const prompt = loadArtifactPrompt(artifactType, context);
+    const response = await callOllama(prompt, config, { expectJson: false });
+    return response;
+}
+function loadArtifactPrompt(artifactType, context) {
+    const promptPath = join(process.cwd(), 'prompts', `${artifactType}.md`);
+    let prompt = readFileSync(promptPath, 'utf-8');
+    prompt = prompt.replace('{{TRANSCRIPT_ALL}}', context.transcript.fullText);
+    const segmentsText = context.transcript.segments
+        .map((seg) => `[${seg.start}s - ${seg.end}s] ${seg.text}`)
+        .join('\n');
+    prompt = prompt.replace('{{TRANSCRIPT_SEGMENTS}}', segmentsText);
+    const classificationSummary = Object.entries(context.classification)
+        .filter(([_, score]) => score >= 25)
+        .map(([type, score]) => `${type}: ${score}%`)
+        .join(', ');
+    prompt = prompt.replace('{{CLASSIFICATION_SUMMARY}}', classificationSummary);
+    if (context.metadata) {
+        prompt = prompt.replace('{{SPEAKERS}}', JSON.stringify(context.metadata.speakers || [], null, 2));
+        prompt = prompt.replace('{{KEY_MOMENTS}}', JSON.stringify(context.metadata.keyMoments || [], null, 2));
+        prompt = prompt.replace('{{ACTION_ITEMS}}', JSON.stringify(context.metadata.actionItems || [], null, 2));
+        prompt = prompt.replace('{{TECHNICAL_TERMS}}', JSON.stringify(context.metadata.technicalTerms || [], null, 2));
+        prompt = prompt.replace('{{CODE_SNIPPETS}}', JSON.stringify(context.metadata.codeSnippets || [], null, 2));
+    }
+    else {
+        prompt = prompt.replace('{{SPEAKERS}}', 'N/A');
+        prompt = prompt.replace('{{KEY_MOMENTS}}', 'N/A');
+        prompt = prompt.replace('{{ACTION_ITEMS}}', 'N/A');
+        prompt = prompt.replace('{{TECHNICAL_TERMS}}', 'N/A');
+        prompt = prompt.replace('{{CODE_SNIPPETS}}', 'N/A');
+    }
+    return prompt;
+}