moflo 4.9.36 → 4.10.0

package/dist/src/cli/embeddings/persistent-cache.js

@@ -1,28 +1,31 @@
 /**
- * SQLite-backed Persistent Cache for Embeddings (sql.js)
+ * SQLite-backed Persistent Cache for Embeddings (node:sqlite)
  *
  * Features:
- * - Cross-platform support (pure JavaScript/WASM, no native compilation)
- * - Disk persistence across sessions
+ * - Built-in node:sqlite (Node 22+) — no native compile, no WASM
+ * - Disk persistence via WAL — writes are incremental, no whole-file dumps
  * - LRU eviction with configurable max size
  * - Automatic schema creation
  * - TTL support for cache entries
  * - Lazy initialization (no startup cost if not used)
+ *
+ * Phase 5 (#1084) migrated this from sql.js to node:sqlite via the unified
+ * `openDaemonDatabase` factory. The sql.js whole-file-export pattern was the
+ * source of the multi-writer clobber class fixed in epic #1078.
  */
-import { existsSync, mkdirSync, readFileSync } from 'fs';
+import { existsSync, mkdirSync, statSync } from 'fs';
 import { dirname } from 'path';
-import { atomicWriteFileSync } from '../shared/utils/atomic-file-write.js';
+import { openDaemonDatabase } from '../memory/daemon-backend.js';
 /**
- * SQLite-backed persistent embedding cache using sql.js (pure JS/WASM)
+ * SQLite-backed persistent embedding cache using node:sqlite via the
+ * unified daemon-backend factory.
  */
 export class PersistentEmbeddingCache {
     db = null;
-    SQL = null;
     initialized = false;
     dirty = false;
     hits = 0;
     misses = 0;
-    autoSaveTimer = null;
     dbPath;
     maxSize;
     ttlMs;
@@ -31,33 +34,29 @@ export class PersistentEmbeddingCache {
         this.dbPath = config.dbPath;
         this.maxSize = config.maxSize ?? 10000;
         this.ttlMs = config.ttlMs ?? 7 * 24 * 60 * 60 * 1000; // 7 days
-        this.autoSaveInterval = config.autoSaveInterval ?? 30000; // 30 seconds
+        // Kept for API compatibility; node:sqlite WAL persists incrementally so
+        // there's no auto-save timer to drive any more.
+        this.autoSaveInterval = config.autoSaveInterval ?? 30000;
     }
     /**
-     * Lazily initialize database connection
+     * Lazily initialize database connection.
+     *
+     * Phase 5 (#1084): swapped the sql.js readFileSync + new SQL.Database
+     * round-trip for openDaemonDatabase(dbPath). WAL writes incrementally so
+     * the auto-save timer + saveToFile() that used to live here are gone.
      */
     async ensureInitialized() {
         if (this.initialized)
             return;
         try {
-            // Dynamically import sql.js
-            const initSqlJs = (await import('sql.js')).default;
-            // Initialize sql.js (loads WASM)
-            this.SQL = await initSqlJs();
-            // Ensure directory exists
+            // Ensure directory exists (openDaemonDatabase also does this, but the
+            // dbExisted probe below needs the path to be stable first).
             const dir = dirname(this.dbPath);
             if (!existsSync(dir)) {
                 mkdirSync(dir, { recursive: true });
             }
-            // Load existing database or create new
             const dbExisted = existsSync(this.dbPath);
-            if (dbExisted) {
-                const fileBuffer = readFileSync(this.dbPath);
-                this.db = new this.SQL.Database(fileBuffer);
-            }
-            else {
-                this.db = new this.SQL.Database();
-            }
+            this.db = openDaemonDatabase(this.dbPath);
             // Create schema
             this.db.run(`
         CREATE TABLE IF NOT EXISTS embeddings (
@@ -88,53 +87,16 @@ export class PersistentEmbeddingCache {
             }
             // Clean expired entries on startup
             this.cleanExpired();
-            // Save after initialization to persist schema
-            this.saveToFile();
-            // Start auto-save timer
-            this.startAutoSave();
             this.initialized = true;
         }
         catch (error) {
-            // If sql.js not available, fall back gracefully
-            console.warn('[persistent-cache] sql.js not available, cache disabled:', error instanceof Error ? error.message : error);
+            // node:sqlite is built into Node 22+, so failure here is a real fault
+            // (corrupt DB, permission error, etc.) rather than missing dep. Surface
+            // and disable the cache so the embedding pipeline keeps working.
+            console.warn('[persistent-cache] disabled:', error instanceof Error ? error.message : error);
             this.initialized = true; // Mark as initialized to prevent retry
         }
     }
-    /**
-     * Start auto-save timer
-     */
-    startAutoSave() {
-        if (this.autoSaveTimer)
-            return;
-        this.autoSaveTimer = setInterval(() => {
-            if (this.dirty && this.db) {
-                this.saveToFile();
-            }
-        }, this.autoSaveInterval);
-    }
-    /**
-     * Stop auto-save timer
-     */
-    stopAutoSave() {
-        if (this.autoSaveTimer) {
-            clearInterval(this.autoSaveTimer);
-            this.autoSaveTimer = null;
-        }
-    }
-    /**
-     * Save database to file
-     */
-    saveToFile() {
-        if (!this.db)
-            return;
-        try {
-            atomicWriteFileSync(this.dbPath, this.db.export());
-            this.dirty = false;
-        }
-        catch (error) {
-            console.error('[persistent-cache] Save error:', error);
-        }
-    }
     /**
      * Generate cache key from text
      */
@@ -284,11 +246,12 @@ export class PersistentEmbeddingCache {
         if (this.db) {
             const result = this.db.exec('SELECT COUNT(*) as count FROM embeddings');
             stats.size = result[0]?.values[0]?.[0] ?? 0;
-            // Get file size if exists
+            // Get file size if exists. node:sqlite leaves the file on disk via WAL
+            // so statSync is enough — no whole-file read needed (sql.js used to
+            // readFileSync the entire DB to compute size).
             if (existsSync(this.dbPath)) {
                 try {
-                    const buffer = readFileSync(this.dbPath);
-                    stats.dbSizeBytes = buffer.length;
+                    stats.dbSizeBytes = statSync(this.dbPath).size;
                 }
                 catch {
                     // Ignore
@@ -298,51 +261,49 @@ export class PersistentEmbeddingCache {
         return stats;
     }
     /**
-     * Clear all cached entries
+     * Clear all cached entries. WAL persists the DELETE incrementally so
+     * there's no explicit flush — Phase 5 (#1084) removed the sql.js
+     * whole-file save here.
      */
     async clear() {
         await this.ensureInitialized();
         if (!this.db)
             return;
         this.db.run('DELETE FROM embeddings');
-        this.dirty = true;
         this.hits = 0;
         this.misses = 0;
-        this.saveToFile();
+        this.dirty = false;
     }
     /**
-     * Force save to disk
+     * Force save to disk. node:sqlite + WAL persists each `db.run` immediately,
+     * so flush is a no-op kept for API compatibility.
      */
     async flush() {
         await this.ensureInitialized();
-        if (this.db && this.dirty) {
-            this.saveToFile();
-        }
+        this.dirty = false;
     }
     /**
      * Close database connection
      */
     async close() {
-        this.stopAutoSave();
         if (this.db) {
-            // Save before closing
-            if (this.dirty) {
-                this.saveToFile();
-            }
             this.db.close();
             this.db = null;
-            this.SQL = null;
             this.initialized = false;
         }
     }
 }
 /**
- * Check if persistent cache is available (sql.js installed)
+ * Check if persistent cache is available. node:sqlite is built into Node 22+
+ * (moflo's minimum) so this always succeeds; kept for API compatibility.
+ *
+ * Loads the warning-suppression side-effect BEFORE the probe import so the
+ * once-per-process ExperimentalWarning doesn't leak to stderr (#1098).
  */
 export async function isPersistentCacheAvailable() {
     try {
-        const initSqlJs = (await import('sql.js')).default;
-        await initSqlJs();
+        await import('../memory/suppress-sqlite-warning.js');
+        await import('node:sqlite');
         return true;
     }
     catch {

package/dist/src/cli/init/claudemd-generator.js

@@ -31,6 +31,10 @@ function mofloSection() {
 
 Your first tool call MUST be \`mcp__moflo__memory_search\` — before any Glob/Grep/Read. Search \`guidance\`, \`patterns\`, and \`learnings\` every prompt; add \`code-map\` when navigating code, \`tests\` when looking for test inventory or coverage. When the user says "remember this", call \`mcp__moflo__memory_store\` with namespace \`learnings\`.
 
+### Traverse chunks, don't bulk-retrieve
+
+Search results carry a compact \`navigation\` crumb (parentDoc, prev/next, chunkTitle). For adjacent/sibling/hierarchical context use \`mcp__moflo__memory_get_neighbors\`; for full chunk content use \`mcp__moflo__memory_retrieve\`; \`Read\` the source doc only via \`parentPath\` when truly needed. Full protocol: \`.claude/guidance/moflo-memory-protocol.md\`.
+
 ### Auto-enforced gates
 
 - **TaskCreate-first**: Call \`TaskCreate\` before spawning the Agent tool

package/dist/src/cli/init/moflo-init.js

@@ -480,11 +480,51 @@ function syncScripts(root, force) {
             copied++;
         }
     }
+    // Sync bin/lib/ and bin/migrations/ recursively. The top-level scripts
+    // import `./lib/moflo-resolve.mjs` etc., so omitting these subtrees leaves
+    // every synced script unable to load (#1090). The upgrade path in
+    // executor.ts and the post-install bootstrap both sync these trees — init
+    // had drifted out of step.
+    copied += syncTree(path.join(binDir, 'lib'), path.join(scriptsDir, 'lib'), force);
+    copied += syncTree(path.join(binDir, 'migrations'), path.join(scriptsDir, 'migrations'), force);
     if (copied === 0) {
         return { name: '.claude/scripts/', status: 'skipped', detail: 'Scripts already up to date' };
     }
     return { name: '.claude/scripts/', status: 'updated', detail: `${copied} scripts synced from moflo` };
 }
+function syncTree(srcRoot, destRoot, force) {
+    if (!fs.existsSync(srcRoot))
+        return 0;
+    let entries;
+    try {
+        entries = fs.readdirSync(srcRoot, { recursive: true, withFileTypes: true });
+    }
+    catch {
+        return 0;
+    }
+    let copied = 0;
+    for (const entry of entries) {
+        if (!entry.isFile())
+            continue;
+        const parent = entry.parentPath
+            ?? entry.path
+            ?? srcRoot;
+        const absSrc = path.join(parent, entry.name);
+        const rel = path.relative(srcRoot, absSrc).split(path.sep).join('/');
+        const absDest = path.join(destRoot, rel);
+        try {
+            fs.mkdirSync(path.dirname(absDest), { recursive: true });
+            if (!fs.existsSync(absDest) || force || isStale(absSrc, absDest)) {
+                fs.copyFileSync(absSrc, absDest);
+                copied++;
+            }
+        }
+        catch {
+            // Non-fatal — skip individual file on error
+        }
+    }
+    return copied;
+}
 function isStale(srcPath, destPath) {
     try {
         return fs.statSync(srcPath).mtimeMs > fs.statSync(destPath).mtimeMs;

package/dist/src/cli/mcp-tools/memory-tools.js

@@ -54,6 +54,63 @@ function notifyMemoryGate() {
 const MAX_KEY_LENGTH = 1024;
 const MAX_VALUE_SIZE = 1024 * 1024; // 1MB
 const MAX_QUERY_LENGTH = 4096;
+function parseNavigation(metadataJson, mode) {
+    if (!metadataJson)
+        return null;
+    let meta;
+    try {
+        meta = JSON.parse(metadataJson);
+    }
+    catch {
+        return null;
+    }
+    if (!meta || typeof meta !== 'object')
+        return null;
+    // Discriminator: only `type === 'chunk'` entries carry the nav fields.
+    if (meta.type !== 'chunk')
+        return null;
+    if (mode === 'compact') {
+        return {
+            parentDoc: meta.parentDoc,
+            prevChunk: meta.prevChunk ?? null,
+            nextChunk: meta.nextChunk ?? null,
+            chunkTitle: meta.chunkTitle,
+        };
+    }
+    return {
+        parentDoc: meta.parentDoc,
+        parentPath: meta.parentPath,
+        prevChunk: meta.prevChunk ?? null,
+        nextChunk: meta.nextChunk ?? null,
+        siblings: meta.siblings,
+        chunkIndex: meta.chunkIndex,
+        totalChunks: meta.totalChunks,
+        hierarchicalParent: meta.hierarchicalParent ?? null,
+        hierarchicalChildren: meta.hierarchicalChildren ?? null,
+        chunkTitle: meta.chunkTitle,
+        headerLevel: meta.headerLevel,
+    };
+}
+function shapeRetrievedEntry(entry) {
+    let value = entry.content;
+    try {
+        value = JSON.parse(entry.content);
+    }
+    catch { /* keep string */ }
+    return {
+        key: entry.key,
+        namespace: entry.namespace,
+        value,
+        tags: entry.tags,
+        storedAt: entry.createdAt,
+        updatedAt: entry.updatedAt,
+        accessCount: entry.accessCount,
+        hasEmbedding: entry.hasEmbedding,
+        navigation: parseNavigation(entry.metadata, 'full'),
+        found: true,
+        backend: 'sql.js + HNSW',
+    };
+}
 function validateMemoryInput(key, value, query) {
     if (key && key.length > MAX_KEY_LENGTH) {
         throw new Error(`Key exceeds maximum length of ${MAX_KEY_LENGTH} characters`);
@@ -154,7 +211,7 @@ async function ensureInitialized() {
 export const memoryTools = [
     {
         name: 'memory_store',
-        description: 'Store a value in memory with vector embedding for semantic search (sql.js + HNSW backend). Upserts by default — pass upsert:false to fail on duplicate keys.',
+        description: 'Store a value in memory with vector embedding for semantic search (sql.js + HNSW backend). Upserts by default — pass upsert:false to fail on duplicate keys. Optional `metadata` lets chunk-row producers set the navigation fields (parentDoc, prevChunk, nextChunk, siblings, …) that `memory_get_neighbors` reads.',
         category: 'memory',
         inputSchema: {
             type: 'object',
@@ -169,6 +226,11 @@ export const memoryTools = [
                 },
                 ttl: { type: 'number', description: 'Time-to-live in seconds (optional)' },
                 upsert: { type: 'boolean', description: 'If false, fail on duplicate keys instead of replacing (default: true)' },
+                metadata: {
+                    type: 'object',
+                    additionalProperties: true,
+                    description: 'Optional per-row metadata persisted to the `metadata` TEXT column. For chunk entries, include `type: "chunk"` plus the navigation fields (parentDoc, parentPath, chunkIndex, totalChunks, prevChunk, nextChunk, siblings, hierarchicalParent, hierarchicalChildren, chunkTitle, headerLevel) so `memory_get_neighbors` can traverse. Capped at 64KB serialised.',
+                },
             },
             required: ['key', 'value'],
         },
@@ -180,6 +242,7 @@ export const memoryTools = [
             const value = typeof input.value === 'string' ? input.value : JSON.stringify(input.value);
             const tags = input.tags || [];
             const ttl = input.ttl;
+            const metadata = input.metadata;
             // #962: default upsert=true — silent UNIQUE-constraint failures on update
             // were dropping schedule cancels and similar updates on the floor.
             const upsert = input.upsert === false ? false : true;
@@ -193,6 +256,7 @@ export const memoryTools = [
                     generateEmbeddingFlag: true,
                     tags,
                     ttl,
+                    metadata,
                     upsert,
                 });
                 const duration = performance.now() - startTime;
@@ -220,7 +284,7 @@ export const memoryTools = [
     },
     {
         name: 'memory_retrieve',
-        description: 'Retrieve a value from memory by key',
+        description: 'Retrieve the full value for a SPECIFIC key. For chunk entries, prefer `memory_get_neighbors` for traversal — bulk-retrieving search hits is a protocol violation. The returned `navigation` object lets you keep traversing. See `.claude/guidance/moflo-memory-protocol.md`.',
         category: 'memory',
         inputSchema: {
             type: 'object',
@@ -238,27 +302,9 @@ export const memoryTools = [
             try {
                 const result = await getEntry({ key, namespace });
                 if (result.found && result.entry) {
-                    // Try to parse JSON value
-                    let value = result.entry.content;
-                    try {
-                        value = JSON.parse(result.entry.content);
-                    }
-                    catch {
-                        // Keep as string
-                    }
                     notifyMemoryGate();
-                    return {
-                        key,
-                        namespace,
-                        value,
-                        tags: result.entry.tags,
-                        storedAt: result.entry.createdAt,
-                        updatedAt: result.entry.updatedAt,
-                        accessCount: result.entry.accessCount,
-                        hasEmbedding: result.entry.hasEmbedding,
-                        found: true,
-                        backend: 'sql.js + HNSW',
-                    };
+                    // #1053 S1: surface RAG navigation for chunked guidance entries.
+                    return shapeRetrievedEntry(result.entry);
                 }
                 return {
                     key,
@@ -280,15 +326,15 @@ export const memoryTools = [
     },
     {
         name: 'memory_search',
-        description: 'Semantic vector search using HNSW index (150x-12,500x faster than keyword search)',
+        description: 'Semantic vector search using HNSW index (150x-12,500x faster than keyword search). When a result has a non-null `navigation` crumb, you MUST traverse via `memory_get_neighbors` — bulk `memory_retrieve` per hit is a protocol violation. See `.claude/guidance/moflo-memory-protocol.md`.',
         category: 'memory',
         inputSchema: {
             type: 'object',
             properties: {
                 query: { type: 'string', description: 'Search query (semantic similarity)' },
                 namespace: { type: 'string', description: 'Namespace to search (default: all namespaces)' },
-                limit: { type: 'number', description: 'Maximum results (default: 10)' },
-                threshold: { type: 'number', description: 'Minimum similarity threshold 0-1 (default: 0.3)' },
+                limit: { type: 'number', description: 'Maximum results (default: 8)' },
+                threshold: { type: 'number', description: 'Minimum similarity threshold 0-1 (default: 0.5)' },
             },
             required: ['query'],
         },
@@ -297,13 +343,13 @@ export const memoryTools = [
             const { searchEntries } = await getMemoryFunctions();
             const query = input.query;
             const namespace = input.namespace || 'all';
-            const limit = input.limit || 10;
-            // Falsiness check would coerce a caller-supplied 0 to 0.3 and silently
+            // #1053 S6: tighter defaults — fewer hits, higher relevance bar.
+            const limit = input.limit || 8;
+            // Falsiness check would coerce a caller-supplied 0 to default and silently
             // filter low-similarity matches; use a typeof guard so explicit zero
             // means "no threshold" (#837).
-            const threshold = typeof input.threshold === 'number' ? input.threshold : 0.3;
+            const threshold = typeof input.threshold === 'number' ? input.threshold : 0.5;
             validateMemoryInput(undefined, undefined, query);
-            const startTime = performance.now();
             try {
                 const result = await searchEntries({
                     query,
@@ -311,7 +357,6 @@ export const memoryTools = [
                     limit,
                     threshold,
                 });
-                const duration = performance.now() - startTime;
                 // Parse JSON values in results
                 const results = result.results.map(r => {
                     let value = r.content;
@@ -321,19 +366,27 @@ export const memoryTools = [
                     catch {
                         // Keep as string
                     }
+                    // #1053 S1: compact RAG navigation crumb per result.
+                    // Compact subset is small enough to always include — keeps the
+                    // result envelope navigable without ballooning per-hit size.
+                    const navigation = parseNavigation(r.metadata, 'compact');
                     return {
                         key: r.key,
                         namespace: r.namespace,
                         value,
-                        similarity: r.score,
+                        // #1053 S6: 2dp keeps signal, drops noise (8-decimal floats add ~6
+                        // bytes per hit and don't help any caller).
+                        similarity: Math.round(r.score * 100) / 100,
+                        navigation,
                     };
                 });
                 notifyMemoryGate();
+                // #1053 S6: searchTime dropped from MCP envelope (CLI keeps it for
+                // human reading); `backend` retained — doctor reads it (#1053 epic).
                 return {
                     query,
                     results,
                     total: results.length,
-                    searchTime: `${duration.toFixed(2)}ms`,
                     backend: 'HNSW + sql.js',
                 };
             }
@@ -347,6 +400,98 @@ export const memoryTools = [
             }
         },
     },
+    {
+        name: 'memory_get_neighbors',
+        description: 'Traverse the chunk graph in one call: fetch the requested neighbors (prev/next/siblings/parent/children) of a chunk key. Returns success:false if the source is not a chunk.',
+        category: 'memory',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                key: { type: 'string', description: 'Source chunk key (must be a chunk-* entry)' },
+                namespace: { type: 'string', description: 'Namespace (default: "default")' },
+                include: {
+                    type: 'array',
+                    items: { type: 'string', enum: ['prev', 'next', 'siblings', 'parent', 'children'] },
+                    description: "Which neighbors to fetch. Default: ['prev','next']. parent/children = hierarchical (h2→h3) chunk neighbors; siblings = same-doc chunk peers.",
+                },
+            },
+            required: ['key'],
+        },
+        handler: async (input) => {
+            await ensureInitialized();
+            const { getEntry } = await getMemoryFunctions();
+            const key = input.key;
+            const namespace = input.namespace || 'default';
+            const includeRaw = input.include;
+            const include = Array.isArray(includeRaw) && includeRaw.length > 0 ? includeRaw : ['prev', 'next'];
+            validateMemoryInput(key);
+            try {
+                const sourceResult = await getEntry({ key, namespace });
+                if (!sourceResult.found || !sourceResult.entry) {
+                    return {
+                        success: false,
+                        key,
+                        namespace,
+                        error: `Source key '${key}' not found in namespace '${namespace}'`,
+                    };
+                }
+                const sourceMeta = sourceResult.entry.metadata;
+                const nav = parseNavigation(sourceMeta, 'full');
+                if (!nav) {
+                    return {
+                        success: false,
+                        key,
+                        namespace,
+                        error: `Source key '${key}' has no chunk metadata; only chunk-* entries are navigable`,
+                    };
+                }
+                // Resolve requested neighbor keys, dedup, exclude the source key itself.
+                const neighborKeys = new Set();
+                const addIfChunkKey = (k) => {
+                    if (k && k !== key)
+                        neighborKeys.add(k);
+                };
+                for (const inc of include) {
+                    if (inc === 'prev')
+                        addIfChunkKey(nav.prevChunk);
+                    else if (inc === 'next')
+                        addIfChunkKey(nav.nextChunk);
+                    else if (inc === 'siblings')
+                        (nav.siblings ?? []).forEach(addIfChunkKey);
+                    else if (inc === 'parent')
+                        addIfChunkKey(nav.hierarchicalParent);
+                    else if (inc === 'children')
+                        (nav.hierarchicalChildren ?? []).forEach(addIfChunkKey);
+                }
+                // Parallel fetch — one round-trip from the caller's perspective.
+                // Missing neighbors (deleted/renamed) are silently skipped rather
+                // than failing the whole call; the response.total reflects what
+                // we actually returned.
+                const fetched = await Promise.all(Array.from(neighborKeys).map(async (k) => {
+                    const res = await getEntry({ key: k, namespace });
+                    return res.found && res.entry ? shapeRetrievedEntry(res.entry) : null;
+                }));
+                notifyMemoryGate();
+                const neighbors = fetched.filter((e) => e !== null);
+                return {
+                    success: true,
+                    source: { key, namespace },
+                    include,
+                    neighbors,
+                    total: neighbors.length,
+                    backend: 'sql.js + HNSW',
+                };
+            }
+            catch (error) {
+                return {
+                    success: false,
+                    key,
+                    namespace,
+                    error: error instanceof Error ? error.message : 'Unknown error',
+                };
+            }
+        },
+    },
     {
         name: 'memory_delete',
         description: 'Delete a memory entry by key',