moflo 4.9.37 → 4.10.1

package/dist/src/cli/memory/daemon-write-client.js

@@ -1,22 +1,38 @@
 /**
- * Daemon write client (#981 / #984 — single-writer architecture).
+ * Daemon RPC client (#981 / #984 / #1058 — single-writer architecture and
+ * its read-side symmetry).
  *
- * HTTP client for the `POST /api/memory/{store,delete,batch}` RPC added by
- * Story #983. Lets short-lived CLI processes and the long-lived MCP server
- * route their `.moflo/moflo.db` writes through the daemon, which owns the
- * authoritative sql.js handle. Avoids the multi-process clobber from #981.
+ * HTTP client for the `POST /api/memory/{store,delete,batch,get,search,list}`
+ * RPC added by Stories #983 and #1058. Lets short-lived CLI processes and
+ * the long-lived MCP server route their `.moflo/moflo.db` operations through
+ * the daemon, which owns the authoritative sql.js handle. Avoids the
+ * multi-process write clobber from #981 AND the stale read snapshot from
+ * #1058 (sql.js never re-reads disk after init).
  *
  * Contract — every function in this module:
- *   - Never throws. Any error path returns `{ routed: false }`.
+ *   - Never throws. Outcomes are reported via the result envelope.
  *   - Returns within ≤100ms even if the daemon is dead/slow (HTTP timeout).
- *   - Caches daemon health for 5s to keep the hot write path cheap.
+ *   - Caches daemon health for 5s to keep the hot path cheap.
  *   - Short-circuits without HTTP when:
  *       (a) `process.env.MOFLO_IS_DAEMON === '1'` (daemon's own process)
  *       (b) `moflo.yaml` has `daemon.auto_start: false`
  *
- * Story #984 ships the client without any consumer wiring — Story #985 / #986
- * add the routing preamble inside `storeEntry` / `deleteEntry` (see
- * `docs/internal/981-writer-audit.md`).
+ * Failure-shape contract (#1101 — surface 4xx as a real error):
+ *   - 4xx response                 → `routed: true, ok: false, error: <msg>` (writes)
+ *                                  → `routed: true, error: <msg>` (reads)
+ *                                  → caller PROPAGATES the error; does NOT fall back.
+ *   - 5xx, 503, timeout,           → `routed: false`
+ *     ECONNREFUSED, malformed JSON,  → caller falls back to bridge-direct.
+ *     socket destroyed mid-stream
+ *
+ * The 4xx codes only fire on daemon-side payload validation (see
+ * daemon-memory-rpc.ts). Bridge-direct has the same validation and would
+ * fail the same way — falling back silently loses the daemon's actionable
+ * error message. 5xx and transport faults are transient/daemon-side bugs;
+ * bridge-direct is the right next step.
+ *
+ * Naming note: the module is named `daemon-write-client` for compat with
+ * existing importers, but as of #1058 it also covers reads.
  *
  * @module cli/memory/daemon-write-client
  */
@@ -136,6 +152,7 @@ export async function tryDaemonStore(opts) {
         value: opts.value,
         tags: opts.tags,
         ttl: opts.ttl,
+        metadata: opts.metadata,
     });
 }
 /**
@@ -150,9 +167,90 @@ export async function tryDaemonDelete(opts) {
         key: opts.key,
     });
 }
+/**
+ * Route a single-entry retrieve through the daemon (#1058). Returns
+ * `{ routed: false }` if the daemon is unavailable; otherwise
+ * `{ routed: true, data: { found, entry? } }`. The `entry` field is the
+ * same shape as `getEntry`'s in-process return.
+ */
+export async function tryDaemonGet(opts) {
+    if (!(await isDaemonAvailable()))
+        return { routed: false };
+    return postReadJson('/api/memory/get', { namespace: opts.namespace, key: opts.key }, (data) => ({
+        found: !!data?.found,
+        entry: data?.entry,
+    }));
+}
+/**
+ * Route a semantic search through the daemon (#1058).
+ */
+export async function tryDaemonSearch(opts) {
+    if (!(await isDaemonAvailable()))
+        return { routed: false };
+    return postReadJson('/api/memory/search', {
+        query: opts.query,
+        namespace: opts.namespace,
+        limit: opts.limit,
+        threshold: opts.threshold,
+    }, (data) => ({
+        results: Array.isArray(data?.results) ? data.results : [],
+        searchTime: typeof data?.searchTime === 'number' ? data.searchTime : undefined,
+    }));
+}
+/**
+ * Route a paginated list through the daemon (#1058).
+ */
+export async function tryDaemonList(opts) {
+    if (!(await isDaemonAvailable()))
+        return { routed: false };
+    return postReadJson('/api/memory/list', {
+        namespace: opts.namespace,
+        limit: opts.limit,
+        offset: opts.offset,
+    }, (data) => ({
+        entries: Array.isArray(data?.entries) ? data.entries : [],
+        total: typeof data?.total === 'number' ? data.total : 0,
+    }));
+}
 // ============================================================================
 // Internal HTTP poster — never throws, bounded timeout
 // ============================================================================
+/**
+ * Extract a human-readable error message from a daemon 4xx response body.
+ * Prefers `message` (the daemon's specific reason — e.g. "invalid namespace"),
+ * falls back to `error` (the daemon's error category), then to a generic
+ * status-code string when the body is non-JSON.
+ */
+function parse4xxError(buf, status) {
+    try {
+        const data = JSON.parse(buf);
+        const detail = typeof data?.message === 'string' ? data.message
+            : typeof data?.error === 'string' ? data.error
+                : undefined;
+        if (detail)
+            return detail;
+    }
+    catch {
+        // Non-JSON 4xx body — fall through to the generic message.
+    }
+    return `daemon returned ${status}`;
+}
+/**
+ * Narrow a parsed JSON value to the `{ dimensions, model }` embedding-response
+ * shape (#1065). Returns `undefined` when the field is missing or malformed —
+ * a malformed field is treated as "no embedding info" rather than failing the
+ * whole response, so an older daemon that hasn't been updated still works.
+ */
+function parseEmbeddingField(value) {
+    if (!value || typeof value !== 'object')
+        return undefined;
+    const v = value;
+    if (typeof v.dimensions !== 'number' || !Number.isFinite(v.dimensions))
+        return undefined;
+    if (typeof v.model !== 'string' || v.model.length === 0)
+        return undefined;
+    return { dimensions: v.dimensions, model: v.model };
+}
 function postJson(path, body) {
     return new Promise((resolve) => {
         let done = false;
@@ -183,15 +281,23 @@ function postJson(path, body) {
             res.setEncoding('utf8');
             res.on('data', (chunk) => { buf += chunk; });
             res.on('end', () => {
-                // Status >=500 is a daemon-side fault; treat as unrouted so the
-                // caller falls back. Status 4xx (validation) is ALSO unrouted —
-                // we don't want a malformed payload silently lost just because
-                // the HTTP delivery succeeded.
+                // #1101 — per-shape failure contract:
+                //   2xx  → routed:true,  ok:true   (caller uses data)
+                //   4xx  → routed:true,  ok:false  (caller propagates daemon error)
+                //   5xx  → routed:false           (caller falls back to bridge)
+                //   parse fail → routed:false (fall back)
                 const status = res.statusCode ?? 0;
-                if (status < 200 || status >= 300) {
+                if (status >= 500 || status < 200) {
                     finish({ routed: false });
                     return;
                 }
+                if (status >= 400) {
+                    // Daemon validated the payload and rejected it. Bridge-direct
+                    // has the same validation; falling back loses the actionable
+                    // error. Surface it to the caller instead.
+                    finish({ routed: true, ok: false, error: parse4xxError(buf, status) });
+                    return;
+                }
                 try {
                     const data = JSON.parse(buf);
                     finish({
@@ -199,6 +305,7 @@ function postJson(path, body) {
                         ok: !!data?.ok,
                         id: typeof data?.id === 'string' ? data.id : undefined,
                         deleted: typeof data?.deleted === 'boolean' ? data.deleted : undefined,
+                        embedding: parseEmbeddingField(data?.embedding),
                         error: typeof data?.error === 'string' ? data.error : undefined,
                     });
                 }
@@ -214,4 +321,74 @@ function postJson(path, body) {
         req.end();
     });
 }
+/**
+ * Generic JSON POST that returns a daemon-read envelope. Same transport
+ * guarantees as `postJson`: never throws, bounded timeout, invalidates health
+ * cache on routed-failure.
+ *
+ * The `shape` callback maps the daemon's parsed JSON payload to the typed
+ * data shape the caller expects. Returning `null` from `shape` (or a parse
+ * failure) downgrades to `{ routed: false }` so the caller falls back.
+ */
+function postReadJson(path, body, shape) {
+    return new Promise((resolve) => {
+        let done = false;
+        const finish = (result) => {
+            if (done)
+                return;
+            done = true;
+            if (result.routed === false)
+                healthCache = null;
+            resolve(result);
+        };
+        const payload = JSON.stringify(body);
+        const req = http.request({
+            host: '127.0.0.1',
+            port: getDaemonPort(),
+            path,
+            method: 'POST',
+            timeout: DAEMON_HTTP_TIMEOUT_MS,
+            headers: {
+                'Content-Type': 'application/json',
+                'Content-Length': Buffer.byteLength(payload),
+            },
+        }, (res) => {
+            let buf = '';
+            res.setEncoding('utf8');
+            res.on('data', (chunk) => { buf += chunk; });
+            res.on('end', () => {
+                // #1101 — mirror postJson contract for reads:
+                //   2xx  → routed:true with shaped data
+                //   4xx  → routed:true with error (no data) — caller propagates
+                //   5xx  → routed:false (caller falls back)
+                const status = res.statusCode ?? 0;
+                if (status >= 500 || status < 200) {
+                    finish({ routed: false });
+                    return;
+                }
+                if (status >= 400) {
+                    finish({ routed: true, error: parse4xxError(buf, status) });
+                    return;
+                }
+                try {
+                    const parsed = JSON.parse(buf);
+                    const shaped = shape(parsed);
+                    if (shaped === null) {
+                        finish({ routed: false });
+                        return;
+                    }
+                    finish({ routed: true, data: shaped });
+                }
+                catch {
+                    finish({ routed: false });
+                }
+            });
+            res.on('error', () => finish({ routed: false }));
+        });
+        req.on('error', () => finish({ routed: false }));
+        req.on('timeout', () => { req.destroy(); finish({ routed: false }); });
+        req.write(payload);
+        req.end();
+    });
+}
 //# sourceMappingURL=daemon-write-client.js.map

package/dist/src/cli/memory/database-provider.js

@@ -1,16 +1,16 @@
 /**
- * DatabaseProvider - Platform-aware database selection
+ * DatabaseProvider — moflo's IMemoryBackend factory.
  *
- * Automatically selects best backend:
- * - All platforms: sql.js (WASM, no native deps)
- * - Windows: sql.js (WASM, universal) when native fails
- * - Fallback: JSON file storage
+ * Phase 5 (#1084) removed the sql.js backend. Selection now collapses to:
+ * - `node-sqlite` (Node 22+ built-in, the only SQLite backend)
+ * - `rvf` (pure-TS fallback when node:sqlite is somehow unavailable)
+ * - `json` (last-resort file storage when nothing else works)
  *
  * @module v3/memory/database-provider
  */
 import { platform } from 'node:os';
 import { existsSync } from 'node:fs';
-import { SqlJsBackend } from './sqljs-backend.js';
+import { SqliteBackend } from './sqlite-backend.js';
 /**
  * Detect platform and recommend provider
  */
@@ -19,8 +19,9 @@ function detectPlatform() {
     const isWindows = os === 'win32';
     const isMacOS = os === 'darwin';
     const isLinux = os === 'linux';
-    // Recommend better-sqlite3 for Unix-like systems, sql.js for Windows
-    const recommendedProvider = 'sql.js';
+    // Phase 4 (#1083) flipped the default to node:sqlite (built into Node 22+,
+    // moflo's minimum). Phase 5 (#1084) deletes the sql.js backend + dep.
+    const recommendedProvider = 'node-sqlite';
     return {
         os,
         isWindows,
@@ -35,55 +36,60 @@ function detectPlatform() {
 async function testRvf() {
     return true;
 }
-/** better-sqlite3 removed — sql.js is the only SQLite backend */
 /**
- * Test if sql.js is available and working
+ * Test if the built-in node:sqlite engine is available (Node 22+).
+ *
+ * Loads the suppress-sqlite-warning side-effect BEFORE the probe import so
+ * the once-per-process ExperimentalWarning never fires (#1098). A probe
+ * that prints the warning to stderr defeats every consumer's "clean
+ * startup" expectation even when the rest of the run is healthy.
  */
-async function testSqlJs() {
+async function testNodeSqlite() {
     try {
-        const { initSqlJsForNode } = await import('./sqljs-backend.js');
-        const SQL = await initSqlJsForNode();
-        const testDb = new SQL.Database();
-        testDb.close();
+        await import('./suppress-sqlite-warning.js');
+        await import('node:sqlite');
         return true;
     }
-    catch (error) {
+    catch {
         return false;
     }
 }
 /**
- * Select best available provider
+ * Select best available provider.
+ *
+ * Phase 5 (#1084) collapsed the chain — sql.js is gone, so the order is
+ * just: explicit override → node:sqlite → RVF → JSON. Passing `'sql.js'`
+ * explicitly is a hard error.
  */
 async function selectProvider(preferred, verbose = false) {
+    if (preferred === 'sql.js') {
+        throw new Error(`DatabaseProvider: sql.js was removed in Phase 5 (#1084). ` +
+            `Use 'node-sqlite' (the new default) or omit the provider entirely.`);
+    }
     if (preferred && preferred !== 'auto') {
         if (verbose) {
             console.log(`[DatabaseProvider] Using explicitly specified provider: ${preferred}`);
         }
         return preferred;
     }
-    const platformInfo = detectPlatform();
+    if (await testNodeSqlite()) {
+        if (verbose) {
+            console.log('[DatabaseProvider] node:sqlite available — using new default');
+        }
+        return 'node-sqlite';
+    }
+    // node:sqlite missing is the "broken install" signal — surface it whenever
+    // verbose is on so the fallback chain doesn't silently regress consumers
+    // to a slower backend without anyone noticing.
     if (verbose) {
-        console.log(`[DatabaseProvider] Platform detected: ${platformInfo.os}`);
-        console.log(`[DatabaseProvider] Recommended provider: ${platformInfo.recommendedProvider}`);
+        console.warn('[DatabaseProvider] node:sqlite unavailable — check Node version (22+ required); falling back to RVF');
     }
-    // Try RVF first (always available via pure-TS fallback)
     if (await testRvf()) {
-        if (verbose) {
-            console.log('[DatabaseProvider] RVF backend available');
-        }
         return 'rvf';
     }
-    // Try sql.js (moflo: sql.js is the only SQLite backend)
-    if (await testSqlJs()) {
-        if (verbose) {
-            console.log('[DatabaseProvider] sql.js available and working');
-        }
-        return 'sql.js';
-    }
-    else if (verbose) {
-        console.log('[DatabaseProvider] sql.js not available, using JSON fallback');
+    if (verbose) {
+        console.warn('[DatabaseProvider] node:sqlite + RVF unavailable — falling back to JSON');
     }
-    // Final fallback to JSON
     return 'json';
 }
 /**
@@ -112,7 +118,7 @@ async function selectProvider(preferred, verbose = false) {
  * ```
  */
 export async function createDatabase(path, options = {}) {
-    const { provider = 'auto', verbose = false, walMode = true, optimize = true, defaultNamespace = 'default', maxEntries = 1000000, autoPersistInterval = 5000, wasmPath, } = options;
+    const { provider = 'auto', verbose = false, walMode: _walMode = true, optimize = true, defaultNamespace = 'default', maxEntries = 1000000, autoPersistInterval = 5000, wasmPath: _wasmPath, } = options;
     // Select provider
     const selectedProvider = await selectProvider(provider, verbose);
     if (verbose) {
@@ -122,6 +128,13 @@ export async function createDatabase(path, options = {}) {
     let backend;
     switch (selectedProvider) {
         case 'sql.js': {
+            // selectProvider() guards against 'sql.js' as an explicit preference,
+            // but a stale caller could still land here if `auto` resolution drifted
+            // (it can't — left only for exhaustive-check safety).
+            throw new Error(`DatabaseProvider: sql.js was removed in Phase 5 (#1084). ` +
+                `This case is unreachable; if you see this error, file a bug.`);
+        }
+        case 'node-sqlite': {
             const config = {
                 databasePath: path,
                 optimize,
@@ -129,9 +142,8 @@ export async function createDatabase(path, options = {}) {
                 maxEntries,
                 verbose,
                 autoPersistInterval,
-                wasmPath,
             };
-            backend = new SqlJsBackend(config);
+            backend = new SqliteBackend(config);
             break;
         }
         case 'rvf': {
@@ -167,13 +179,18 @@ export function getPlatformInfo() {
     return detectPlatform();
 }
 /**
- * Check which providers are available
+ * Check which providers are available.
+ *
+ * `sqlJs` / `betterSqlite3` are retained for API stability but always
+ * report `false` — Phase 5 (#1084) deleted the sql.js backend and
+ * better-sqlite3 was never wired.
  */
 export async function getAvailableProviders() {
     return {
         rvf: true,
-        betterSqlite3: false, // Removed — sql.js is the only SQLite backend
-        sqlJs: await testSqlJs(),
+        betterSqlite3: false,
+        sqlJs: false,
+        nodeSqlite: await testNodeSqlite(),
         json: true,
     };
 }

package/dist/src/cli/memory/hnsw-persistence.js

@@ -16,11 +16,11 @@
  */
 import * as fs from 'node:fs';
 import * as path from 'node:path';
-import { mofloImport } from '../services/moflo-require.js';
 import { atomicWriteFileSync } from '../services/atomic-file-write.js';
 import { HnswLite } from './hnsw-lite.js';
 import { parseEmbeddingJson } from './controllers/_shared.js';
 import { hnswIndexPath } from '../services/moflo-paths.js';
+import { openDaemonDatabase } from './daemon-backend.js';
 /**
  * Build an HnswLite from every active row in `dbPath` that has an embedding,
  * then atomically write the sidecar to `<projectRoot>/.moflo/hnsw.index`.
@@ -37,13 +37,11 @@ export async function buildAndWriteHnswSidecar(dbPath, projectRoot, options = {}
     if (!fs.existsSync(dbPath)) {
         throw new Error(`buildAndWriteHnswSidecar: db not found at ${dbPath}`);
     }
-    const sqlJsModule = await mofloImport('sql.js');
-    if (!sqlJsModule) {
-        throw new Error(`buildAndWriteHnswSidecar: sql.js not available`);
-    }
-    const SQL = await sqlJsModule.default();
-    const buf = fs.readFileSync(dbPath);
-    const db = new SQL.Database(buf);
+    // node:sqlite via the unified factory — Phase 5 (#1084) replaced the
+    // sql.js readFileSync + new SQL.Database round-trip with a direct open
+    // through openDaemonDatabase. WAL writes incrementally so there's nothing
+    // to flush back here; the sidecar persistence below is unaffected.
+    const db = openDaemonDatabase(dbPath);
     const hnsw = new HnswLite(dimensions, m, efConstruction, metric);
     let skipped = 0;
     try {

package/dist/src/cli/memory/index.js

@@ -61,7 +61,6 @@ export { ControllerRegistry, INIT_LEVELS } from './controller-registry.js';
 export { CONTROLLER_SPECS } from './controller-specs.js';
 // ===== Core Components =====
 export { MofloDbAdapter } from './moflo-db-adapter.js';
-export { SqlJsBackend } from './sqljs-backend.js';
 export { RvfBackend } from './rvf-backend.js';
 export { HnswLite, cosineSimilarity } from './hnsw-lite.js';
 export { buildAndWriteHnswSidecar, tryLoadHnswSidecar } from './hnsw-persistence.js';

package/dist/src/cli/memory/memory-bridge.js

@@ -119,17 +119,49 @@ export async function bridgeAddToHNSW(id, embedding, entry, dbPath) {
         // Bridge-produced vectors come from getBridgeEmbedder() (fastembed, 384-dim).
         // Pre-#648 this column was hardcoded to 'Xenova/all-MiniLM-L6-v2' which
         // misrepresented the producing model — fixed to the canonical bridge label.
-        ctx.db.prepare(`
-      INSERT OR REPLACE INTO memory_entries (
-        id, key, namespace, content, type,
-        embedding, embedding_dimensions, embedding_model,
-        created_at, updated_at, status
-      ) VALUES (?, ?, ?, ?, 'semantic', ?, ?, ?, ?, ?, 'active')
+        //
+        // Pre-#1067: this issued `INSERT OR REPLACE INTO memory_entries (...)` with
+        // only the embedding-related columns. When `storeEntry`'s direct-write
+        // fallback first inserted a row carrying `metadata`/`tags`/`expires_at`
+        // and then called `addToHNSWIndex(...)` for the same id, REPLACE clobbered
+        // the row — wiping the metadata column to NULL and re-introducing the
+        // "first chunk has no navigation" smoke failure even though serialiseMetadata
+        // produced the right JSON one statement earlier. UPDATE-by-id preserves the
+        // pre-existing row's metadata/tags/expires_at/access_count while still
+        // back-filling embedding columns for callers that supplied them.
+        const updated = ctx.db.prepare(`
+      UPDATE memory_entries
+         SET embedding = ?,
+             embedding_dimensions = ?,
+             embedding_model = ?,
+             updated_at = ?
+       WHERE id = ?
     `).run([
-            id, entry.key, entry.namespace, entry.content,
             embeddingJson, embedding.length, BRIDGE_EMBEDDING_MODEL,
-            now, now,
+            now, id,
         ]);
+        // If no row was matched by id (rare: a rebuild path passing an embedding
+        // for a key that never went through storeEntry first) fall back to an
+        // INSERT carrying every column the bridge can populate, so the row still
+        // becomes searchable. Metadata/tags/expires_at stay NULL because the
+        // caller didn't supply them — same shape `bridgeStoreEntry` would write
+        // for a no-metadata caller.
+        const rowCount = typeof updated.changes === 'number'
+            ? updated.changes
+            : (ctx.db.getRowsModified?.() ?? 0);
+        if (rowCount === 0) {
+            ctx.db.prepare(`
+        INSERT INTO memory_entries (
+          id, key, namespace, content, type,
+          embedding, embedding_dimensions, embedding_model,
+          created_at, updated_at, status
+        ) VALUES (?, ?, ?, ?, 'semantic', ?, ?, ?, ?, ?, 'active')
+      `).run([
+                id, entry.key, entry.namespace, entry.content,
+                embeddingJson, embedding.length, BRIDGE_EMBEDDING_MODEL,
+                now, now,
+            ]);
+        }
         persistBridgeDb(ctx.db, dbPath);
         return true;
     });