moflo 4.9.37 → 4.10.0

package/dist/src/cli/embeddings/persistent-cache.js

@@ -1,28 +1,31 @@
 /**
- * SQLite-backed Persistent Cache for Embeddings (sql.js)
+ * SQLite-backed Persistent Cache for Embeddings (node:sqlite)
  *
  * Features:
- * - Cross-platform support (pure JavaScript/WASM, no native compilation)
- * - Disk persistence across sessions
+ * - Built-in node:sqlite (Node 22+) — no native compile, no WASM
+ * - Disk persistence via WAL — writes are incremental, no whole-file dumps
  * - LRU eviction with configurable max size
  * - Automatic schema creation
  * - TTL support for cache entries
  * - Lazy initialization (no startup cost if not used)
+ *
+ * Phase 5 (#1084) migrated this from sql.js to node:sqlite via the unified
+ * `openDaemonDatabase` factory. The sql.js whole-file-export pattern was the
+ * source of the multi-writer clobber class fixed in epic #1078.
  */
-import { existsSync, mkdirSync, readFileSync } from 'fs';
+import { existsSync, mkdirSync, statSync } from 'fs';
 import { dirname } from 'path';
-import { atomicWriteFileSync } from '../shared/utils/atomic-file-write.js';
+import { openDaemonDatabase } from '../memory/daemon-backend.js';
 /**
- * SQLite-backed persistent embedding cache using sql.js (pure JS/WASM)
+ * SQLite-backed persistent embedding cache using node:sqlite via the
+ * unified daemon-backend factory.
  */
 export class PersistentEmbeddingCache {
     db = null;
-    SQL = null;
     initialized = false;
     dirty = false;
     hits = 0;
     misses = 0;
-    autoSaveTimer = null;
     dbPath;
     maxSize;
     ttlMs;
@@ -31,33 +34,29 @@ export class PersistentEmbeddingCache {
         this.dbPath = config.dbPath;
         this.maxSize = config.maxSize ?? 10000;
         this.ttlMs = config.ttlMs ?? 7 * 24 * 60 * 60 * 1000; // 7 days
-        this.autoSaveInterval = config.autoSaveInterval ?? 30000; // 30 seconds
+        // Kept for API compatibility; node:sqlite WAL persists incrementally so
+        // there's no auto-save timer to drive any more.
+        this.autoSaveInterval = config.autoSaveInterval ?? 30000;
     }
     /**
-     * Lazily initialize database connection
+     * Lazily initialize database connection.
+     *
+     * Phase 5 (#1084): swapped the sql.js readFileSync + new SQL.Database
+     * round-trip for openDaemonDatabase(dbPath). WAL writes incrementally so
+     * the auto-save timer + saveToFile() that used to live here are gone.
      */
     async ensureInitialized() {
         if (this.initialized)
             return;
         try {
-            // Dynamically import sql.js
-            const initSqlJs = (await import('sql.js')).default;
-            // Initialize sql.js (loads WASM)
-            this.SQL = await initSqlJs();
-            // Ensure directory exists
+            // Ensure directory exists (openDaemonDatabase also does this, but the
+            // dbExisted probe below needs the path to be stable first).
             const dir = dirname(this.dbPath);
             if (!existsSync(dir)) {
                 mkdirSync(dir, { recursive: true });
             }
-            // Load existing database or create new
             const dbExisted = existsSync(this.dbPath);
-            if (dbExisted) {
-                const fileBuffer = readFileSync(this.dbPath);
-                this.db = new this.SQL.Database(fileBuffer);
-            }
-            else {
-                this.db = new this.SQL.Database();
-            }
+            this.db = openDaemonDatabase(this.dbPath);
             // Create schema
             this.db.run(`
         CREATE TABLE IF NOT EXISTS embeddings (
@@ -88,53 +87,16 @@ export class PersistentEmbeddingCache {
             }
             // Clean expired entries on startup
             this.cleanExpired();
-            // Save after initialization to persist schema
-            this.saveToFile();
-            // Start auto-save timer
-            this.startAutoSave();
             this.initialized = true;
         }
         catch (error) {
-            // If sql.js not available, fall back gracefully
-            console.warn('[persistent-cache] sql.js not available, cache disabled:', error instanceof Error ? error.message : error);
+            // node:sqlite is built into Node 22+, so failure here is a real fault
+            // (corrupt DB, permission error, etc.) rather than missing dep. Surface
+            // and disable the cache so the embedding pipeline keeps working.
+            console.warn('[persistent-cache] disabled:', error instanceof Error ? error.message : error);
             this.initialized = true; // Mark as initialized to prevent retry
         }
     }
-    /**
-     * Start auto-save timer
-     */
-    startAutoSave() {
-        if (this.autoSaveTimer)
-            return;
-        this.autoSaveTimer = setInterval(() => {
-            if (this.dirty && this.db) {
-                this.saveToFile();
-            }
-        }, this.autoSaveInterval);
-    }
-    /**
-     * Stop auto-save timer
-     */
-    stopAutoSave() {
-        if (this.autoSaveTimer) {
-            clearInterval(this.autoSaveTimer);
-            this.autoSaveTimer = null;
-        }
-    }
-    /**
-     * Save database to file
-     */
-    saveToFile() {
-        if (!this.db)
-            return;
-        try {
-            atomicWriteFileSync(this.dbPath, this.db.export());
-            this.dirty = false;
-        }
-        catch (error) {
-            console.error('[persistent-cache] Save error:', error);
-        }
-    }
     /**
      * Generate cache key from text
      */
@@ -284,11 +246,12 @@ export class PersistentEmbeddingCache {
         if (this.db) {
             const result = this.db.exec('SELECT COUNT(*) as count FROM embeddings');
             stats.size = result[0]?.values[0]?.[0] ?? 0;
-            // Get file size if exists
+            // Get file size if exists. node:sqlite leaves the file on disk via WAL
+            // so statSync is enough — no whole-file read needed (sql.js used to
+            // readFileSync the entire DB to compute size).
             if (existsSync(this.dbPath)) {
                 try {
-                    const buffer = readFileSync(this.dbPath);
-                    stats.dbSizeBytes = buffer.length;
+                    stats.dbSizeBytes = statSync(this.dbPath).size;
                 }
                 catch {
                     // Ignore
@@ -298,51 +261,49 @@ export class PersistentEmbeddingCache {
         return stats;
     }
     /**
-     * Clear all cached entries
+     * Clear all cached entries. WAL persists the DELETE incrementally so
+     * there's no explicit flush — Phase 5 (#1084) removed the sql.js
+     * whole-file save here.
      */
     async clear() {
         await this.ensureInitialized();
         if (!this.db)
             return;
         this.db.run('DELETE FROM embeddings');
-        this.dirty = true;
         this.hits = 0;
         this.misses = 0;
-        this.saveToFile();
+        this.dirty = false;
     }
     /**
-     * Force save to disk
+     * Force save to disk. node:sqlite + WAL persists each `db.run` immediately,
+     * so flush is a no-op kept for API compatibility.
      */
     async flush() {
         await this.ensureInitialized();
-        if (this.db && this.dirty) {
-            this.saveToFile();
-        }
+        this.dirty = false;
     }
     /**
      * Close database connection
      */
     async close() {
-        this.stopAutoSave();
         if (this.db) {
-            // Save before closing
-            if (this.dirty) {
-                this.saveToFile();
-            }
             this.db.close();
             this.db = null;
-            this.SQL = null;
             this.initialized = false;
         }
     }
 }
 /**
- * Check if persistent cache is available (sql.js installed)
+ * Check if persistent cache is available. node:sqlite is built into Node 22+
+ * (moflo's minimum) so this always succeeds; kept for API compatibility.
+ *
+ * Loads the warning-suppression side-effect BEFORE the probe import so the
+ * once-per-process ExperimentalWarning doesn't leak to stderr (#1098).
  */
 export async function isPersistentCacheAvailable() {
     try {
-        const initSqlJs = (await import('sql.js')).default;
-        await initSqlJs();
+        await import('../memory/suppress-sqlite-warning.js');
+        await import('node:sqlite');
         return true;
     }
     catch {

package/dist/src/cli/init/moflo-init.js

@@ -480,11 +480,51 @@ function syncScripts(root, force) {
             copied++;
         }
     }
+    // Sync bin/lib/ and bin/migrations/ recursively. The top-level scripts
+    // import `./lib/moflo-resolve.mjs` etc., so omitting these subtrees leaves
+    // every synced script unable to load (#1090). The upgrade path in
+    // executor.ts and the post-install bootstrap both sync these trees — init
+    // had drifted out of step.
+    copied += syncTree(path.join(binDir, 'lib'), path.join(scriptsDir, 'lib'), force);
+    copied += syncTree(path.join(binDir, 'migrations'), path.join(scriptsDir, 'migrations'), force);
     if (copied === 0) {
         return { name: '.claude/scripts/', status: 'skipped', detail: 'Scripts already up to date' };
     }
     return { name: '.claude/scripts/', status: 'updated', detail: `${copied} scripts synced from moflo` };
 }
+function syncTree(srcRoot, destRoot, force) {
+    if (!fs.existsSync(srcRoot))
+        return 0;
+    let entries;
+    try {
+        entries = fs.readdirSync(srcRoot, { recursive: true, withFileTypes: true });
+    }
+    catch {
+        return 0;
+    }
+    let copied = 0;
+    for (const entry of entries) {
+        if (!entry.isFile())
+            continue;
+        const parent = entry.parentPath
+            ?? entry.path
+            ?? srcRoot;
+        const absSrc = path.join(parent, entry.name);
+        const rel = path.relative(srcRoot, absSrc).split(path.sep).join('/');
+        const absDest = path.join(destRoot, rel);
+        try {
+            fs.mkdirSync(path.dirname(absDest), { recursive: true });
+            if (!fs.existsSync(absDest) || force || isStale(absSrc, absDest)) {
+                fs.copyFileSync(absSrc, absDest);
+                copied++;
+            }
+        }
+        catch {
+            // Non-fatal — skip individual file on error
+        }
+    }
+    return copied;
+}
 function isStale(srcPath, destPath) {
     try {
         return fs.statSync(srcPath).mtimeMs > fs.statSync(destPath).mtimeMs;

package/dist/src/cli/mcp-tools/memory-tools.js

@@ -211,7 +211,7 @@ async function ensureInitialized() {
 export const memoryTools = [
     {
         name: 'memory_store',
-        description: 'Store a value in memory with vector embedding for semantic search (sql.js + HNSW backend). Upserts by default — pass upsert:false to fail on duplicate keys.',
+        description: 'Store a value in memory with vector embedding for semantic search (sql.js + HNSW backend). Upserts by default — pass upsert:false to fail on duplicate keys. Optional `metadata` lets chunk-row producers set the navigation fields (parentDoc, prevChunk, nextChunk, siblings, …) that `memory_get_neighbors` reads.',
         category: 'memory',
         inputSchema: {
             type: 'object',
@@ -226,6 +226,11 @@ export const memoryTools = [
                 },
                 ttl: { type: 'number', description: 'Time-to-live in seconds (optional)' },
                 upsert: { type: 'boolean', description: 'If false, fail on duplicate keys instead of replacing (default: true)' },
+                metadata: {
+                    type: 'object',
+                    additionalProperties: true,
+                    description: 'Optional per-row metadata persisted to the `metadata` TEXT column. For chunk entries, include `type: "chunk"` plus the navigation fields (parentDoc, parentPath, chunkIndex, totalChunks, prevChunk, nextChunk, siblings, hierarchicalParent, hierarchicalChildren, chunkTitle, headerLevel) so `memory_get_neighbors` can traverse. Capped at 64KB serialised.',
+                },
             },
             required: ['key', 'value'],
         },
@@ -237,6 +242,7 @@ export const memoryTools = [
             const value = typeof input.value === 'string' ? input.value : JSON.stringify(input.value);
             const tags = input.tags || [];
             const ttl = input.ttl;
+            const metadata = input.metadata;
             // #962: default upsert=true — silent UNIQUE-constraint failures on update
             // were dropping schedule cancels and similar updates on the floor.
             const upsert = input.upsert === false ? false : true;
@@ -250,6 +256,7 @@ export const memoryTools = [
                     generateEmbeddingFlag: true,
                     tags,
                     ttl,
+                    metadata,
                     upsert,
                 });
                 const duration = performance.now() - startTime;
@@ -277,7 +284,7 @@ export const memoryTools = [
     },
     {
         name: 'memory_retrieve',
-        description: 'Retrieve a value from memory by key. Chunk entries also return a full `navigation` object — use it with memory_get_neighbors for traversal instead of bulk-retrieving every search hit.',
+        description: 'Retrieve the full value for a SPECIFIC key. For chunk entries, prefer `memory_get_neighbors` for traversal — bulk-retrieving search hits is a protocol violation. The returned `navigation` object lets you keep traversing. See `.claude/guidance/moflo-memory-protocol.md`.',
         category: 'memory',
         inputSchema: {
             type: 'object',
@@ -319,7 +326,7 @@ export const memoryTools = [
     },
     {
         name: 'memory_search',
-        description: 'Semantic vector search using HNSW index (150x-12,500x faster than keyword search). Results include a compact `navigation` crumb on chunk hits — traverse via memory_get_neighbors rather than retrieving every hit.',
+        description: 'Semantic vector search using HNSW index (150x-12,500x faster than keyword search). When a result has a non-null `navigation` crumb, you MUST traverse via `memory_get_neighbors` — bulk `memory_retrieve` per hit is a protocol violation. See `.claude/guidance/moflo-memory-protocol.md`.',
         category: 'memory',
         inputSchema: {
             type: 'object',