moflo 4.9.37 → 4.10.1

package/dist/src/cli/memory/controllers/hierarchical-memory.js

@@ -108,7 +108,10 @@ export class HierarchicalMemory {
      * transaction. Rolls back on exception and rethrows.
      */
     transaction(fn) {
-        this.db.run('BEGIN TRANSACTION');
+        // BEGIN IMMEDIATE — busy_handler engages on lock acquisition. Plain
+        // `BEGIN` (= deferred) would fail-fast if the inner fn reads-then-
+        // writes while another process holds RESERVED (#1099 / #1098 trap).
+        this.db.run('BEGIN IMMEDIATE');
         return Promise.resolve()
             .then(() => fn())
             .then((result) => {
@@ -187,7 +190,9 @@ export class HierarchicalMemory {
         if (ids.length === 0)
             return;
         const now = Date.now();
-        this.db.run('BEGIN TRANSACTION');
+        // BEGIN IMMEDIATE — pure write batch. See transaction() above for the
+        // #1099 rationale; converting consistently keeps the policy uniform.
+        this.db.run('BEGIN IMMEDIATE');
         try {
             for (const id of ids) {
                 this.db.run(`UPDATE ${TABLE} SET access_count = access_count + 1, accessed_at = ? WHERE id = ?`, [now, id]);

package/dist/src/cli/memory/controllers/mutation-guard.js

@@ -49,8 +49,28 @@ export class MutationGuard {
         if (!input.bypassDedupe && entries.some((e) => e.hash === hash)) {
             return { allowed: false, reason: 'duplicate mutation within dedupe window' };
         }
-        this.record(op, entries, hash, ts);
-        return { allowed: true };
+        // #1098: defer the recording — return a token instead of fire-and-
+        // forget recording at validate-time. Callers commit() after the
+        // write succeeds; failed writes leave the buffer clean so retries
+        // can re-validate without seeing themselves as duplicates.
+        return { allowed: true, token: { op, hash, ts } };
+    }
+    /**
+     * Record a previously-validated mutation in the dedupe buffer. Call
+     * exactly once per token, only after the corresponding write has
+     * succeeded. No-op for tokens that don't match a watched op — those
+     * never produce a real recording in the first place.
+     *
+     * Silent no-op if `token` is null/undefined so callers can do
+     * `guard.commit(result.token)` without a null check.
+     */
+    commit(token) {
+        if (!token)
+            return;
+        if (!this.isWatched(token.op))
+            return;
+        const entries = this.pruneAndFetch(token.op, token.ts);
+        this.record(token.op, entries, token.hash, token.ts);
     }
     reset() {
         this.recent.clear();

package/dist/src/cli/memory/daemon-backend.js

@@ -0,0 +1,400 @@
+/**
+ * Daemon-side memory DB factory — TS twin of `bin/lib/get-backend.mjs`.
+ *
+ * Returns a handle whose surface matches the sql.js `Database` type that the
+ * controller registry + bridge code expect (prepare → Statement, run, exec,
+ * close, plus a `save()` that maps to the engine's preferred persistence).
+ * Currently always returns a node:sqlite-backed adapter — Phase 4 (#1083)
+ * flipped the SQLite default; Phase 5 (#1084) deletes the remaining sql.js
+ * paths in the bridge + memory-initializer.
+ *
+ * The sql.js Statement API the bridge code relies on:
+ *   - db.prepare(sql) → stmt
+ *   - db.run(sql, params?)
+ *   - db.exec(sql) → [{ columns, values }]
+ *   - db.close()
+ *   - stmt.bind(params)
+ *   - stmt.step() → boolean
+ *   - stmt.getAsObject() → row object
+ *   - stmt.run(params?) → boolean
+ *   - stmt.free()
+ *
+ * node:sqlite's `StatementSync` is stateless (each call takes params), so we
+ * shim a stateful wrapper via `stmt.iterate(...)` opened on first `step()`.
+ * This is the same shape implemented in bin/lib/get-backend.mjs — keep the
+ * two in lockstep until Phase 5 collapses them.
+ *
+ * @module v3/memory/daemon-backend
+ */
+// MUST come before `import 'node:sqlite'` below — that module fires
+// `ExperimentalWarning: SQLite is an experimental feature` exactly once per
+// process on first load. Once it fires, there's no way to scrub it from
+// stderr, and consumer-smoke's 200-char stderr tails get filled with it
+// (hiding the real error message; #1098).
+import './suppress-sqlite-warning.js';
+import { existsSync, mkdirSync, readFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+/**
+ * Coerce a caller-supplied parameter set into node:sqlite's `SQLInputValue[]`
+ * shape. Callers pass `any` (sql.js parity) so we just pass through after
+ * normalising null/undefined/array.
+ */
+function toParamsArray(params) {
+    if (params === undefined || params === null)
+        return [];
+    if (Array.isArray(params))
+        return params;
+    // sql.js's `Statement.bind(obj)` named-param shape isn't reachable from
+    // moflo's bridge code today — every caller passes an array. Tolerate
+    // anyway by wrapping the lone value.
+    return [params];
+}
+function wrapStatement(stmt) {
+    let pendingParams = [];
+    let iter = null;
+    let currentRow = null;
+    let columnNamesCache = null;
+    const ensureIter = () => {
+        if (!iter) {
+            iter = (pendingParams.length > 0
+                ? stmt.iterate(...pendingParams)
+                : stmt.iterate());
+        }
+    };
+    return {
+        bind(params) {
+            pendingParams = toParamsArray(params);
+            iter = null;
+            currentRow = null;
+            return true;
+        },
+        step() {
+            ensureIter();
+            const next = iter.next();
+            if (next.done) {
+                currentRow = null;
+                return false;
+            }
+            currentRow = next.value;
+            return true;
+        },
+        getAsObject(params) {
+            // sql.js semantics: with params it's a one-shot bind+step+return;
+            // without params it returns whatever the last step() materialised. The
+            // bridge uses both shapes (one-shot in bridge-entries.ts:bridgeGetEntry,
+            // iterator in list/search). Returning {} from the one-shot form when
+            // there's no row is correct (caller checks for nullish primary key).
+            if (params !== undefined) {
+                pendingParams = toParamsArray(params);
+                iter = null;
+                ensureIter();
+                const next = iter.next();
+                if (next.done) {
+                    currentRow = null;
+                    return {};
+                }
+                currentRow = next.value;
+                return currentRow;
+            }
+            return currentRow ?? {};
+        },
+        get(params) {
+            // sql.js `Statement.get()` semantics:
+            //   - With params: one-shot bind+step, returns positional values of the
+            //     first row (or [] if no rows).
+            //   - Without params: returns positional values of the CURRENT row —
+            //     the row that the most-recent `step()` landed on. Callers use it
+            //     in a `while (stmt.step()) { const row = stmt.get(); ... }` loop;
+            //     calling iter.next() again here would skip every other row.
+            if (params !== undefined) {
+                pendingParams = toParamsArray(params);
+                iter = null;
+                ensureIter();
+                const next = iter.next();
+                if (next.done) {
+                    currentRow = null;
+                    return [];
+                }
+                currentRow = next.value;
+            }
+            if (!currentRow)
+                return [];
+            const cols = columnNamesCache ?? Object.keys(currentRow);
+            columnNamesCache = cols;
+            return cols.map((c) => currentRow[c]);
+        },
+        getColumnNames() {
+            if (columnNamesCache)
+                return columnNamesCache;
+            // Force one step to materialise a row so column names are knowable.
+            ensureIter();
+            const next = iter.next();
+            if (next.done) {
+                columnNamesCache = [];
+                currentRow = null;
+                return [];
+            }
+            currentRow = next.value;
+            columnNamesCache = Object.keys(currentRow);
+            return columnNamesCache;
+        },
+        run(params) {
+            const arr = toParamsArray(params);
+            if (arr.length > 0)
+                stmt.run(...arr);
+            else
+                stmt.run();
+            return true;
+        },
+        reset() {
+            iter = null;
+            currentRow = null;
+        },
+        free() {
+            iter = null;
+            currentRow = null;
+            pendingParams = [];
+            columnNamesCache = null;
+        },
+    };
+}
+/**
+ * Per-process dedupe of network-FS warnings — emit once per (dbPath, process).
+ * Matches the JS twin's `_networkFsWarnedPaths` set so a session that opens
+ * both the daemon adapter and a bin/ writer on the same path only logs once.
+ */
+const _networkFsWarnedPaths = new Set();
+/**
+ * Read `journal_mode` back after we requested WAL. If the engine returned a
+ * different mode (`delete`, `truncate`, `persist`, `memory`, `off`), the
+ * underlying filesystem doesn't support WAL's shared-memory sidecar — a
+ * strong signal that POSIX advisory locks are also unreliable. Surface a
+ * one-line stderr warning naming the path so the user knows to move the
+ * project off the network mount. Deduped per (path, process).
+ *
+ * Twin: `bin/lib/get-backend.mjs:warnIfNotWal`. Must stay in lockstep until
+ * Phase 5 (#1084) extracts a shared module.
+ */
+function warnIfNotWal(db, dbPath) {
+    if (_networkFsWarnedPaths.has(dbPath))
+        return;
+    let mode;
+    try {
+        const stmt = db.prepare('PRAGMA journal_mode');
+        const row = stmt.get();
+        mode = String(row?.journal_mode ?? '').toLowerCase();
+    }
+    catch {
+        return;
+    }
+    if (mode && mode !== 'wal') {
+        _networkFsWarnedPaths.add(dbPath);
+        process.stderr.write(`[moflo] WARNING: SQLite journal_mode=${mode} on ${dbPath} (WAL not active). ` +
+            `If this directory is on NFS/SMB or another network filesystem, POSIX ` +
+            `advisory locks are unreliable and concurrent moflo processes can corrupt ` +
+            `the database. Move the project to a local disk to restore multi-process safety.\n`);
+    }
+}
+/** @internal — test hook only (resets the dedupe set). */
+export function _resetDaemonNetworkFsWarnings() {
+    _networkFsWarnedPaths.clear();
+}
+/**
+ * Heuristic: SQL is multi-statement if there's a `;` followed by anything
+ * substantive. node:sqlite's `db.prepare()` does NOT throw on multi-statement
+ * input — it silently parses only the first statement and discards the rest,
+ * which is the worst possible failure mode for DDL batches like
+ * `CREATE TABLE a; CREATE INDEX i; CREATE TABLE b;`. Detect explicitly and
+ * route multi-stmt SQL to `db.exec()` (which runs every statement).
+ *
+ * The bridge + controller schema strings don't embed literal `;` inside
+ * string literals, so the naive index-of check is sound. If that ever
+ * changes, this needs a real tokeniser.
+ */
+function isMultiStatement(sql) {
+    const trimmed = sql.trimEnd();
+    const semi = trimmed.indexOf(';');
+    if (semi === -1)
+        return false;
+    return /\S/.test(trimmed.slice(semi + 1));
+}
+function execAsRowsNodeSqlite(db, sql, params) {
+    // Multi-statement DDL: route to db.exec() so every statement runs.
+    // (sql.js's exec() runs every statement and returns row sets from any
+    // that produce rows; the bridge code only reads [0]?.values, so DDL
+    // batches correctly return []. Match that contract.)
+    if (isMultiStatement(sql)) {
+        db.exec(sql);
+        return [];
+    }
+    let stmt;
+    try {
+        stmt = db.prepare(sql);
+    }
+    catch {
+        // Last resort for any single-statement SQL that prepare rejects.
+        db.exec(sql);
+        return [];
+    }
+    const args = toParamsArray(params);
+    const rows = (args.length > 0 ? stmt.all(...args) : stmt.all());
+    if (rows.length === 0)
+        return [];
+    const columns = Object.keys(rows[0]);
+    const values = rows.map((r) => columns.map((c) => r[c]));
+    return [{ columns, values }];
+}
+/**
+ * Open the daemon's memory DB handle. Always returns a node:sqlite-backed
+ * adapter shaped like sql.js's Database so the existing bridge + controller
+ * surface keeps working.
+ *
+ * @param dbPath disk path or `:memory:`
+ */
+export function openDaemonDatabase(dbPath) {
+    // node:sqlite opens forgivingly even on non-SQLite files. Keep parity with
+    // openSqlJsDatabase's "create if missing" semantic — DatabaseSync handles
+    // file creation for us, BUT does NOT auto-create parent directories. The
+    // bridge's first-init path commonly lands on a path whose parent .moflo/
+    // doesn't exist yet (fresh consumer install, test fixtures with temp
+    // project roots) — without the mkdir below, DatabaseSync throws ENOENT,
+    // the controller-registry sets mofloDb=null, and the bridge silently
+    // falls back to a raw-sql.js write rooted at process.cwd() (catastrophic
+    // path drift bug; #1057 was about exactly this class of issue).
+    if (dbPath !== ':memory:') {
+        try {
+            mkdirSync(dirname(dbPath), { recursive: true });
+        }
+        catch { /* tolerate — DatabaseSync's ENOENT below is the real signal */ }
+    }
+    const db = new DatabaseSync(dbPath);
+    if (dbPath !== ':memory:') {
+        try {
+            // busy_timeout MUST be set BEFORE journal_mode=WAL, because the WAL
+            // pragma briefly takes an EXCLUSIVE lock and concurrent openers race
+            // on it. Without busy_timeout in place, parallel doctor probes /
+            // bridge initializations / indexer subprocess opens hit "database is
+            // locked" and the bridge tears down (CI #1097). Order matters:
+            //   1. busy_timeout — gives every subsequent pragma a retry budget
+            //   2. journal_mode = WAL — needs the budget on contention
+            //   3. synchronous — purely advisory, can come anytime
+            //
+            // Budget: 15000ms. The consumer-smoke harness exposes the realistic
+            // worst case — a background indexer subprocess opens its own write
+            // connection right after `npm install` and walks the entire consumer
+            // tree (hundreds of guidance/skill files). The whole-tree first-pass
+            // can hold a RESERVED/EXCLUSIVE lock for 5–8s while the doctor
+            // foreground probe races against it. 5000ms was the original Phase 4
+            // value and ran the budget out under that exact load on Windows CI
+            // (#1098); 15000ms gives the indexer's full first-pass time to finish
+            // before doctor's probe gives up. The price of being wrong-high here
+            // is one slow probe per session, not lost data.
+            db.exec('PRAGMA busy_timeout = 15000');
+            db.exec('PRAGMA journal_mode = WAL');
+            db.exec('PRAGMA synchronous = NORMAL');
+            // The daemon is the process most exposed to network-FS edge cases
+            // (long-lived MCP server, ~30s of writes per indexer pass). NFS/SMB
+            // mounts silently fall back from WAL to a rollback journal — surface
+            // a one-line warning so the user knows to move the project off the
+            // network mount. Mirrors `bin/lib/get-backend.mjs:warnIfNotWal`.
+            warnIfNotWal(db, dbPath);
+        }
+        catch (err) {
+            try {
+                db.close();
+            }
+            catch {
+                /* handle already dead */
+            }
+            throw err;
+        }
+    }
+    // sql.js's `db.run(sql, params?)` and `prepare/run` share state; node:sqlite
+    // requires fresh statements per call. Cache prepared statements keyed by SQL
+    // text so the indexer-equivalent tight write loops don't churn the compiler.
+    const runStmtCache = new Map();
+    let changesStmt = null;
+    return {
+        kind: 'node-sqlite',
+        _raw: db,
+        prepare(sql) {
+            return wrapStatement(db.prepare(sql));
+        },
+        run(sql, params) {
+            const arr = toParamsArray(params);
+            if (arr.length > 0) {
+                let s = runStmtCache.get(sql);
+                if (!s) {
+                    s = db.prepare(sql);
+                    runStmtCache.set(sql, s);
+                }
+                s.run(...arr);
+            }
+            else {
+                db.exec(sql);
+            }
+            return undefined;
+        },
+        exec(sql, params) {
+            return execAsRowsNodeSqlite(db, sql, params);
+        },
+        getRowsModified() {
+            if (!changesStmt)
+                changesStmt = db.prepare('SELECT changes() AS c');
+            const row = changesStmt.get();
+            const c = row?.c ?? 0;
+            return typeof c === 'bigint' ? Number(c) : c;
+        },
+        save() {
+            // node:sqlite persists incrementally via WAL — no-op. The shape exists
+            // so bridge-core's persistBridgeDb can call it unconditionally during
+            // the Phase 4/5 transition window. Once everything routes through this
+            // adapter, the explicit persist call becomes dead code (Phase 5).
+        },
+        export() {
+            // Bridge-core's old persist path used `db.export()` + atomicWriteFileSync.
+            // node:sqlite ships a `serialize()` that returns the same shape so the
+            // few callers that still need a buffer (e.g. tests, backup tooling) work.
+            // The bridge-core persist call itself is being switched to `save()` so
+            // this exists only as a safety net during the migration.
+            const buf = db.prepare('SELECT 1').all();
+            void buf;
+            // Real serialize. node:sqlite added `DatabaseSync.prototype.serialize()`
+            // in Node 22; the TS type for it landed later, so cast through the
+            // engine's runtime API.
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const ser = db.serialize?.();
+            if (ser instanceof Uint8Array)
+                return ser;
+            if (ser && typeof ser === 'object' && 'buffer' in ser)
+                return new Uint8Array(ser);
+            // Last resort: read the file off disk. The caller knows the path; we
+            // don't, so this branch should never fire under normal flow.
+            return new Uint8Array();
+        },
+        close() {
+            runStmtCache.clear();
+            changesStmt = null;
+            db.close();
+        },
+    };
+}
+/**
+ * Seed an empty daemon DB from an existing file on disk. Equivalent to
+ * sql.js's `new SQL.Database(readFileSync(path))` round-trip — node:sqlite
+ * opens the path directly so this is just a wrapper that errors when the
+ * file doesn't exist (existing callers expected the sql.js behaviour).
+ */
+export function openDaemonDatabaseFromFile(dbPath) {
+    if (dbPath !== ':memory:' && !existsSync(dbPath)) {
+        throw new Error(`Database file not found: ${dbPath}`);
+    }
+    // Touch readFileSync so callers that previously expected eager I/O still
+    // observe the same failure shape (e.g. EACCES errors fire here, not
+    // lazily on first query). node:sqlite would lazy-error otherwise.
+    if (dbPath !== ':memory:')
+        readFileSync(dbPath, { flag: 'r' });
+    return openDaemonDatabase(dbPath);
+}
+//# sourceMappingURL=daemon-backend.js.map