escribano 0.4.3 → 0.4.5

package/dist/0_types.js

@@ -262,7 +262,7 @@ export const intelligenceConfigSchema = z.object({
         similarityThreshold: 0.75,
     }),
     // MLX-VLM specific config
-    vlmBatchSize: z.number().default(4),
+    vlmBatchSize: z.number().default(2),
     vlmMaxTokens: z.number().default(2000),
     mlxSocketPath: z.string().default('/tmp/escribano-mlx.sock'),
 });

package/dist/actions/generate-summary-v3.js

@@ -3,11 +3,13 @@
  *
  * Generates a work session summary from V3 processed TopicBlocks using LLM.
  */
+import { execSync } from 'node:child_process';
 import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import { homedir } from 'node:os';
 import path, { dirname, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { log } from '../pipeline/context.js';
+import { groupTopicBlocksIntoSubjects, saveSubjectsToDatabase, } from '../services/subject-grouping.js';
 const __dirname = dirname(fileURLToPath(import.meta.url));
 /**
  * Generate a work session summary artifact from processed TopicBlocks.
@@ -19,21 +21,54 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
  * @returns Generated artifact
  */
 export async function generateSummaryV3(recordingId, repos, intelligence, options) {
-    log('info', `[Summary V3] Generating summary for recording ${recordingId}...`);
+    log('info', `[Summary V3] Generating narrative for recording ${recordingId}...`);
     // Get the recording
     const recording = repos.recordings.findById(recordingId);
     if (!recording) {
         throw new Error(`Recording ${recordingId} not found`);
     }
     // Get TopicBlocks for this recording
-    const topicBlocks = repos.topicBlocks.findByRecording(recordingId);
-    if (topicBlocks.length === 0) {
+    const allTopicBlocks = repos.topicBlocks.findByRecording(recordingId);
+    if (allTopicBlocks.length === 0) {
         throw new Error(`No TopicBlocks found for recording ${recordingId}. Run process-v3 first.`);
     }
-    log('info', `[Summary V3] Found ${topicBlocks.length} TopicBlocks`);
+    log('info', `[Summary V3] Found ${allTopicBlocks.length} TopicBlocks`);
+    // Check if subjects already exist for this recording
+    const existingSubjects = repos.subjects.findByRecording(recordingId);
+    let subjects;
+    let personalDuration;
+    let workDuration;
+    if (existingSubjects.length > 0) {
+        log('info', `[Summary V3] Reusing ${existingSubjects.length} existing subjects (no re-grouping needed)`);
+        const loaded = loadExistingSubjects(existingSubjects, repos);
+        subjects = loaded.subjects;
+        personalDuration = loaded.personalDuration;
+        workDuration = loaded.workDuration;
+    }
+    else {
+        // Group TopicBlocks into subjects
+        log('info', '[Summary V3] Grouping TopicBlocks into subjects...');
+        const groupingResult = await groupTopicBlocksIntoSubjects(allTopicBlocks, intelligence, recordingId);
+        log('info', `[Summary V3] Saving ${groupingResult.subjects.length} subjects to database...`);
+        saveSubjectsToDatabase(groupingResult.subjects, recordingId, repos);
+        subjects = groupingResult.subjects;
+        personalDuration = groupingResult.personalDuration;
+        workDuration = groupingResult.workDuration;
+    }
+    // Filter TopicBlocks based on personal/work classification
+    let topicBlocksToUse = allTopicBlocks;
+    if (!options.includePersonal) {
+        // Filter out blocks from personal subjects
+        const personalSubjectIds = new Set(subjects.filter((s) => s.isPersonal).map((s) => s.id));
+        topicBlocksToUse = allTopicBlocks.filter((block) => {
+            const subjectForBlock = subjects.find((s) => s.topicBlockIds.includes(block.id));
+            // Use the collected personalSubjectIds set for filtering
+            return !personalSubjectIds.has(subjectForBlock?.id ?? '');
+        });
+    }
     // Build sections from TopicBlocks
     const sections = [];
-    for (const block of topicBlocks) {
+    for (const block of topicBlocksToUse) {
         const classification = JSON.parse(block.classification || '{}');
         sections.push({
             activity: classification.activity_type || 'unknown',
@@ -65,16 +100,48 @@ export async function generateSummaryV3(recordingId, repos, intelligence, option
     await mkdir(outputDir, { recursive: true });
     // Generate filename
     const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
-    const fileName = `${recordingId}-summary-${timestamp}.md`;
+    const fileName = `${recordingId}-narrative-${timestamp}.md`;
     const filePath = path.join(outputDir, fileName);
     // Write to file
     await writeFile(filePath, summaryContent, 'utf-8');
     log('info', `[Summary V3] Summary saved to: ${filePath}`);
+    // Save to database
+    const artifactId = `artifact-${recordingId}-narrative-${Date.now()}`;
+    repos.artifacts.save({
+        id: artifactId,
+        recording_id: recordingId,
+        type: 'narrative',
+        content: summaryContent,
+        format: 'markdown',
+        source_block_ids: JSON.stringify(subjects.flatMap((s) => s.topicBlockIds)),
+        source_context_ids: null,
+    });
+    log('info', `[Summary V3] Saved to database: ${artifactId}`);
+    // Link subjects to artifact
+    repos.artifacts.linkSubjects(artifactId, subjects.map((s) => s.id));
+    log('info', `[Summary V3] Linked ${subjects.length} subjects to artifact`);
+    // Handle stdout/clipboard
+    if (options.printToStdout) {
+        console.log(`\n${summaryContent}\n`);
+    }
+    if (options.copyToClipboard && process.platform === 'darwin') {
+        try {
+            execSync('pbcopy', { input: summaryContent, encoding: 'utf-8' });
+            log('info', '[Summary V3] Copied to clipboard');
+        }
+        catch (error) {
+            log('warn', `[Summary V3] Failed to copy to clipboard: ${error}`);
+        }
+    }
     return {
-        id: `summary-${recordingId}-${Date.now()}`,
+        id: artifactId,
         recordingId,
+        format: 'narrative',
         content: summaryContent,
         filePath,
+        subjects,
+        personalDuration,
+        workDuration,
         createdAt: new Date(),
     };
 }
@@ -260,3 +327,31 @@ ${section.transcript}
 `;
     return summary;
 }
+function loadExistingSubjects(existingSubjects, repos) {
+    const subjects = [];
+    for (const dbSubject of existingSubjects) {
+        const topicBlocks = repos.subjects.getTopicBlocks(dbSubject.id);
+        const activityBreakdown = dbSubject.activity_breakdown
+            ? JSON.parse(dbSubject.activity_breakdown)
+            : {};
+        const metadata = dbSubject.metadata ? JSON.parse(dbSubject.metadata) : {};
+        const apps = metadata.apps || [];
+        subjects.push({
+            id: dbSubject.id,
+            recordingId: topicBlocks[0]?.recording_id || '',
+            label: dbSubject.label,
+            topicBlockIds: topicBlocks.map((b) => b.id),
+            totalDuration: dbSubject.duration,
+            activityBreakdown,
+            apps,
+            isPersonal: dbSubject.is_personal === 1,
+        });
+    }
+    const personalDuration = subjects
+        .filter((s) => s.isPersonal)
+        .reduce((sum, s) => sum + s.totalDuration, 0);
+    const workDuration = subjects
+        .filter((s) => !s.isPersonal)
+        .reduce((sum, s) => sum + s.totalDuration, 0);
+    return { subjects, personalDuration, workDuration };
+}

package/dist/adapters/intelligence.mlx.adapter.js

@@ -15,22 +15,14 @@ import { createConnection } from 'node:net';
 import { dirname, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 const __dirname = dirname(fileURLToPath(import.meta.url));
+import { loadConfig } from '../config.js';
 import { ESCRIBANO_HOME, ESCRIBANO_VENV, ESCRIBANO_VENV_PYTHON, getPythonPath, } from '../python-utils.js';
-const DEBUG_MLX = process.env.ESCRIBANO_VERBOSE === 'true';
 function debugLog(...args) {
-    if (DEBUG_MLX) {
+    const config = loadConfig();
+    if (config.verbose) {
         console.log('[VLM] [MLX]', ...args);
     }
 }
-const DEFAULT_CONFIG = {
-    model: process.env.ESCRIBANO_VLM_MODEL ??
-        'mlx-community/Qwen3-VL-2B-Instruct-bf16',
-    batchSize: Number(process.env.ESCRIBANO_VLM_BATCH_SIZE) || 4,
-    maxTokens: Number(process.env.ESCRIBANO_VLM_MAX_TOKENS) || 2000,
-    socketPath: process.env.ESCRIBANO_MLX_SOCKET_PATH ?? '/tmp/escribano-mlx.sock',
-    bridgeScript: resolve(__dirname, '../../scripts/mlx_bridge.py'),
-    startupTimeout: Number(process.env.ESCRIBANO_MLX_STARTUP_TIMEOUT) || 120000,
-};
 /** pip binary inside Escribano's managed venv. */
 const _ESCRIBANO_VENV_PIP = resolve(ESCRIBANO_VENV, 'bin', 'pip');
 /**
@@ -129,7 +121,16 @@ export function cleanupMlxBridge() {
  * Other methods (classify, generate, etc.) are not implemented and will throw.
  */
 export function createMlxIntelligenceService(_config = {}) {
-    const mlxConfig = { ...DEFAULT_CONFIG };
+    // Load unified config (respects env vars, config file, and RAM-aware defaults)
+    const config = loadConfig();
+    const mlxConfig = {
+        model: config.vlmModel,
+        batchSize: config.vlmBatchSize,
+        maxTokens: config.vlmMaxTokens,
+        socketPath: config.mlxSocketPath,
+        bridgeScript: resolve(__dirname, '../../scripts/mlx_bridge.py'),
+        startupTimeout: config.mlxStartupTimeout,
+    };
     const bridge = {
         process: null,
         socket: null,

package/dist/adapters/intelligence.ollama.adapter.js

@@ -132,6 +132,43 @@ async function doModelWarmup(modelName, config) {
         warmedModels.add(modelName); // Mark as warmed to avoid repeated attempts
     }
 }
+/**
+ * Unload an Ollama model from memory.
+ * Uses keep_alive: 0 to tell Ollama to release the model immediately.
+ */
+export async function unloadOllamaModel(modelName, config) {
+    try {
+        debugLog(`Unloading model: ${modelName}...`);
+        const response = await fetch(`${config.endpoint.replace('/chat', '').replace('/generate', '')}/generate`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                model: modelName,
+                prompt: '',
+                keep_alive: 0, // Unload immediately
+            }),
+        });
+        if (response.ok) {
+            warmedModels.delete(modelName);
+            debugLog(`Model ${modelName} unloaded.`);
+        }
+        else {
+            let bodyText = '';
+            try {
+                bodyText = await response.text();
+            }
+            catch {
+                // Ignore errors while reading response body for logging
+            }
+            debugLog(`Failed to unload model ${modelName}: HTTP ${response.status} ${response.statusText}` +
+                (bodyText ? ` - Response body: ${bodyText}` : ''));
+        }
+    }
+    catch (error) {
+        // Unload is best-effort - don't throw
+        debugLog(`Failed to unload model ${modelName}: ${error.message}`);
+    }
+}
 async function checkOllamaHealth() {
     try {
         const response = await fetch('http://localhost:11434/api/tags');

package/dist/batch-context.js

@@ -14,17 +14,18 @@ import { execSync } from 'node:child_process';
 import { homedir } from 'node:os';
 import path from 'node:path';
 import { generateArtifactV3, } from './actions/generate-artifact-v3.js';
+import { generateSummaryV3 } from './actions/generate-summary-v3.js';
 import { updateGlobalIndex } from './actions/outline-index.js';
 import { processRecordingV3 } from './actions/process-recording-v3.js';
 import { hasContentChanged, publishSummaryV3, updateRecordingOutlineMetadata, } from './actions/publish-summary-v3.js';
 import { createSileroPreprocessor } from './adapters/audio.silero.adapter.js';
 import { createFilesystemCaptureSource } from './adapters/capture.filesystem.adapter.js';
 import { cleanupMlxBridge, createMlxIntelligenceService, } from './adapters/intelligence.mlx.adapter.js';
-import { createOllamaIntelligenceService } from './adapters/intelligence.ollama.adapter.js';
+import { createOllamaIntelligenceService, unloadOllamaModel, } from './adapters/intelligence.ollama.adapter.js';
 import { createOutlinePublishingService } from './adapters/publishing.outline.adapter.js';
 import { createWhisperTranscriptionService } from './adapters/transcription.whisper.adapter.js';
 import { createFfmpegVideoService } from './adapters/video.ffmpeg.adapter.js';
-import { createDefaultConfig } from './config.js';
+import { createDefaultConfig, loadConfig, logConfig } from './config.js';
 import { getDbPath, getRepositories } from './db/index.js';
 import { log, setResourceTracker, step, withPipeline, } from './pipeline/context.js';
 import { ResourceTracker, setupStatsObserver, } from './stats/index.js';
@@ -39,6 +40,10 @@ const MODEL_PATH = path.join(MODELS_DIR, MODEL_FILE);
 export async function initializeSystem() {
     // Create default config file if it doesn't exist
     createDefaultConfig();
+    // Load and log unified configuration
+    const config = loadConfig();
+    logConfig();
+    console.log('');
     console.log('Initializing database...');
     const repos = getRepositories();
     console.log(`Database ready: ${getDbPath()}`);
@@ -49,11 +54,11 @@ export async function initializeSystem() {
     const modelSelection = await selectBestLLMModel();
     console.log(formatModelSelection(modelSelection));
     console.log('');
-    // Initialize adapters ONCE
+    // Initialize adapters ONCE (config is now used by adapters)
     console.log('[VLM] Using MLX-VLM for image processing');
-    const vlm = createMlxIntelligenceService();
+    const vlm = createMlxIntelligenceService(config);
     console.log('[LLM] Using Ollama for text generation');
-    const llm = createOllamaIntelligenceService();
+    const llm = createOllamaIntelligenceService(config);
     const video = createFfmpegVideoService();
     const preprocessor = createSileroPreprocessor();
     const transcription = createWhisperTranscriptionService({
@@ -101,6 +106,8 @@ export async function processVideo(videoPath, ctx, options = {}) {
     const { force = false, skipSummary = false, micAudioPath, systemAudioPath, format = 'card', includePersonal = false, copyToClipboard = false, printToStdout = false, } = options;
     const { repos, adapters, outlineConfig } = ctx;
     const { vlm, llm, video, preprocessor, transcription } = adapters;
+    // Load unified config for lifecycle management
+    const config = loadConfig();
     try {
         // Create capture source for this specific file
         // Note: Hardcoded to filesystem source, not Cap recordings
@@ -160,6 +167,9 @@ export async function processVideo(videoPath, ctx, options = {}) {
             await withPipeline(recording.id, runType, runMetadata, async () => {
                 await processRecordingV3(recording.id, repos, { preprocessor, transcription, video, intelligence: vlm }, { force });
             });
+            // Free VLM memory after processing (good hygiene for all RAM tiers)
+            console.log('[VLM] Freeing VLM memory...');
+            cleanupMlxBridge();
         }
         // Generate artifact and publish (unless skipped), tracked as a pipeline run
         let artifact = null;
@@ -168,13 +178,28 @@ export async function processVideo(videoPath, ctx, options = {}) {
             const artifactRunMetadata = collectRunMetadata(ctx.resourceTracker);
             const pipelineResult = await withPipeline(recording.id, 'artifact', artifactRunMetadata, async () => {
                 console.log(`\nGenerating ${format} artifact...`);
-                const generatedArtifact = await generateArtifactV3(recording.id, repos, llm, {
-                    recordingId: recording.id,
-                    format,
-                    includePersonal,
-                    copyToClipboard,
-                    printToStdout,
-                });
+                let generatedArtifact;
+                if (format === 'narrative') {
+                    // Route narrative through the corrected path
+                    generatedArtifact = await generateSummaryV3(recording.id, repos, llm, {
+                        recordingId: recording.id,
+                        outputDir: options.outputDir,
+                        useTemplate: false,
+                        includePersonal,
+                        copyToClipboard,
+                        printToStdout,
+                    });
+                }
+                else {
+                    // Card and standup use the original path
+                    generatedArtifact = await generateArtifactV3(recording.id, repos, llm, {
+                        recordingId: recording.id,
+                        format,
+                        includePersonal,
+                        copyToClipboard,
+                        printToStdout,
+                    });
+                }
                 console.log(`Artifact saved: ${generatedArtifact.filePath}`);
                 if (generatedArtifact.workDuration > 0) {
                     const workMins = Math.round(generatedArtifact.workDuration / 60);
@@ -256,6 +281,26 @@ export async function processVideo(videoPath, ctx, options = {}) {
             });
             artifact = pipelineResult.artifact;
             outlineUrl = pipelineResult.outlineUrl;
+            // Unload LLM after artifact generation to free memory (good hygiene for all RAM tiers)
+            if (config.llmModel) {
+                console.log('[LLM] Unloading model to free memory...');
+                const intelConfig = {
+                    provider: 'ollama',
+                    endpoint: 'http://localhost:11434/api/chat',
+                    model: config.llmModel,
+                    generationModel: config.llmModel,
+                    visionModel: config.vlmModel,
+                    maxRetries: 3,
+                    timeout: 600000,
+                    keepAlive: '10m',
+                    maxContextSize: 131072,
+                    embedding: { model: 'nomic-embed-text', similarityThreshold: 0.75 },
+                    vlmBatchSize: config.vlmBatchSize,
+                    vlmMaxTokens: config.vlmMaxTokens,
+                    mlxSocketPath: config.mlxSocketPath,
+                };
+                await unloadOllamaModel(config.llmModel, intelConfig);
+            }
         }
         console.log('\n✓ Complete!');
         return {