claude-flow 3.21.0 → 3.22.0

package/v3/@claude-flow/cli/dist/src/memory/memory-bridge.js

@@ -365,11 +365,9 @@ async function getRegistry(dbPath) {
  * Replaces naive String.includes() with proper information retrieval scoring.
  * Parameters tuned for short memory entries (k1=1.2, b=0.75).
  */
-function bm25Score(queryTerms, docContent, avgDocLength, docCount, termDocFreqs) {
+function bm25Score(queryTerms, docWords, docLength, avgDocLength, docCount, termDocFreqs) {
     const k1 = 1.2;
     const b = 0.75;
-    const docWords = docContent.toLowerCase().split(/\s+/);
-    const docLength = docWords.length;
     let score = 0;
     for (const term of queryTerms) {
         const tf = docWords.filter(w => w === term || w.includes(term)).length;
@@ -382,23 +380,27 @@ function bm25Score(queryTerms, docContent, avgDocLength, docCount, termDocFreqs)
     }
     return score;
 }
+function tokenizeCorpus(rows) {
+    return rows.map(row => {
+        const contentLower = (row.content || '').toLowerCase();
+        return { contentLower, words: contentLower.split(/\s+/) };
+    });
+}
 /**
- * Compute BM25 term document frequencies for a set of rows.
+ * Compute BM25 term document frequencies over an already-tokenized corpus.
  */
-function computeTermDocFreqs(queryTerms, rows) {
+function computeTermDocFreqs(queryTerms, docs) {
     const termDocFreqs = new Map();
     let totalLength = 0;
-    for (const row of rows) {
-        const content = (row.content || '').toLowerCase();
-        const words = content.split(/\s+/);
-        totalLength += words.length;
+    for (const doc of docs) {
+        totalLength += doc.words.length;
         for (const term of queryTerms) {
-            if (content.includes(term)) {
+            if (doc.contentLower.includes(term)) {
                 termDocFreqs.set(term, (termDocFreqs.get(term) || 0) + 1);
             }
         }
     }
-    return { termDocFreqs, avgDocLength: rows.length > 0 ? totalLength / rows.length : 1 };
+    return { termDocFreqs, avgDocLength: docs.length > 0 ? totalLength / docs.length : 1 };
 }
 // ===== Phase 2: TieredCache helpers =====
 /**
@@ -484,6 +486,10 @@ async function logAttestation(registry, operation, entryId, metadata) {
         // Non-fatal — attestation is observability, not correctness
     }
 }
+// Tracks db handles whose schema DDL has already been ensured, so getDb()
+// runs the CREATE…IF NOT EXISTS block at most once per handle instead of on
+// every bridge call. WeakSet so handles GC without leaking.
+const _schemaEnsuredDbs = new WeakSet();
 /**
  * Get the AgentDB database handle and ensure memory_entries table exists.
  * Returns null if not available.
@@ -493,35 +499,42 @@ function getDb(registry) {
     if (!agentdb?.database)
         return null;
     const db = agentdb.database;
-    // Ensure memory_entries table exists (idempotent)
-    try {
-        db.exec(`CREATE TABLE IF NOT EXISTS memory_entries (
-      id TEXT PRIMARY KEY,
-      key TEXT NOT NULL,
-      namespace TEXT DEFAULT 'default',
-      content TEXT NOT NULL,
-      type TEXT DEFAULT 'semantic',
-      embedding TEXT,
-      embedding_model TEXT DEFAULT 'local',
-      embedding_dimensions INTEGER,
-      tags TEXT,
-      metadata TEXT,
-      owner_id TEXT,
-      created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
-      updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
-      expires_at INTEGER,
-      last_accessed_at INTEGER,
-      access_count INTEGER DEFAULT 0,
-      status TEXT DEFAULT 'active',
-      UNIQUE(namespace, key)
-    )`);
-        // Ensure indexes
-        db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_ns ON memory_entries(namespace)`);
-        db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_key ON memory_entries(key)`);
-        db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_status ON memory_entries(status)`);
-    }
-    catch {
-        // Table already exists or db is read-only — that's fine
+    // Ensure memory_entries table exists (idempotent). The DDL is run at most
+    // once per db handle — re-parsing 4× CREATE…IF NOT EXISTS on every bridge
+    // call (store/search/get) was pure per-op overhead. Keyed by handle via a
+    // WeakSet so a new db instance re-ensures without a stale global flag.
+    if (!_schemaEnsuredDbs.has(db)) {
+        try {
+            db.exec(`CREATE TABLE IF NOT EXISTS memory_entries (
+        id TEXT PRIMARY KEY,
+        key TEXT NOT NULL,
+        namespace TEXT DEFAULT 'default',
+        content TEXT NOT NULL,
+        type TEXT DEFAULT 'semantic',
+        embedding TEXT,
+        embedding_model TEXT DEFAULT 'local',
+        embedding_dimensions INTEGER,
+        tags TEXT,
+        metadata TEXT,
+        owner_id TEXT,
+        created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+        updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+        expires_at INTEGER,
+        last_accessed_at INTEGER,
+        access_count INTEGER DEFAULT 0,
+        status TEXT DEFAULT 'active',
+        UNIQUE(namespace, key)
+      )`);
+            // Ensure indexes
+            db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_ns ON memory_entries(namespace)`);
+            db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_key ON memory_entries(key)`);
+            db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_status ON memory_entries(status)`);
+            _schemaEnsuredDbs.add(db);
+        }
+        catch {
+            // Table already exists or db is read-only — that's fine. Don't mark
+            // ensured on failure so a later writable call can retry.
+        }
     }
     // ─── #2256-followup: rescue agentdb.embedder when its transformers.js
     // path fell through to mock embeddings.
@@ -637,6 +650,7 @@ export async function bridgeStoreEntry(options) {
         }
         // Generate embedding via AgentDB's embedder
         let embeddingJson = null;
+        let embeddingArr = null;
         let dimensions = 0;
         let model = 'local';
         if (options.generateEmbeddingFlag !== false && value.length > 0) {
@@ -645,7 +659,8 @@ export async function bridgeStoreEntry(options) {
                 if (embedder) {
                     const emb = await embedder.embed(value);
                     if (emb) {
-                        embeddingJson = JSON.stringify(Array.from(emb));
+                        embeddingArr = Array.from(emb);
+                        embeddingJson = JSON.stringify(embeddingArr);
                         dimensions = emb.length;
                         model = 'Xenova/all-MiniLM-L6-v2';
                     }
@@ -680,6 +695,31 @@ export async function bridgeStoreEntry(options) {
         catch { /* vector_indexes may not exist on legacy DBs — fall through */ }
         const stmt = ctx.db.prepare(insertSql);
         stmt.run(id, key, namespace, value, embeddingJson, dimensions || null, model, tags.length > 0 ? JSON.stringify(tags) : null, '{}', now, now, ttl ? now + (ttl * 1000) : null);
+        // #2558: keep `vector_indexes.total_vectors` accurate so status/tooling
+        // stop reporting "HNSW index: 0 vectors" while embedded entries exist.
+        try {
+            ctx.db
+                .prepare(`UPDATE vector_indexes SET
+             total_vectors = (SELECT COUNT(*) FROM memory_entries
+                              WHERE namespace = ? AND embedding IS NOT NULL),
+             updated_at = ?
+           WHERE name = ?`)
+                .run(namespace, now, namespace);
+        }
+        catch { /* vector_indexes may not exist on legacy DBs — non-fatal */ }
+        // #2558: better-sqlite3 opens the DB in WAL mode and, for the small write
+        // volumes typical of CLI usage, may never reach the auto-checkpoint
+        // threshold — leaving committed rows only in the -wal file. WAL-blind
+        // readers (the sql.js fallback search path; the statusline's read-only
+        // `sqlite3` vector count) then see a stale/empty main DB file and report
+        // "0 vectors" / empty search. A PASSIVE checkpoint flushes committed pages
+        // into the main file without blocking writers. Best-effort, never fatal.
+        try {
+            if (typeof ctx.db.pragma === 'function') {
+                ctx.db.pragma('wal_checkpoint(PASSIVE)');
+            }
+        }
+        catch { /* non-WAL, busy, or unsupported — non-fatal */ }
         // Phase 2: Write-through to TieredCache
         const safeNs = String(namespace).replace(/:/g, '_');
         const safeKey = String(key).replace(/:/g, '_');
@@ -691,7 +731,7 @@ export async function bridgeStoreEntry(options) {
             success: true,
             id,
             embedding: embeddingJson ? { dimensions, model } : undefined,
-            rawEmbedding: embeddingJson ? JSON.parse(embeddingJson) : undefined,
+            rawEmbedding: embeddingArr ?? undefined,
             guarded: true,
             cached: true,
             attested: true,
@@ -746,12 +786,17 @@ export async function bridgeSearchEntries(options) {
         catch {
             return null;
         }
-        // Phase 2: Compute BM25 term stats for the corpus
+        // Phase 2: Compute BM25 term stats for the corpus. Tokenize the corpus a
+        // single time and reuse the per-doc `{contentLower, words}` for term-freq,
+        // BM25 scoring, and the coverage floor below.
         const queryTerms = queryStr.toLowerCase().split(/\s+/).filter(t => t.length > 1);
-        const { termDocFreqs, avgDocLength } = computeTermDocFreqs(queryTerms, rows);
+        const docs = tokenizeCorpus(rows);
+        const { termDocFreqs, avgDocLength } = computeTermDocFreqs(queryTerms, docs);
         const docCount = rows.length;
         const results = [];
-        for (const row of rows) {
+        for (let i = 0; i < rows.length; i++) {
+            const row = rows[i];
+            const doc = docs[i];
             let semanticScore = 0;
             let bm25ScoreVal = 0;
             // Semantic scoring via cosine similarity
@@ -766,21 +811,37 @@ export async function bridgeSearchEntries(options) {
             }
             // Phase 2: BM25 keyword scoring (replaces String.includes fallback)
             if (queryTerms.length > 0 && row.content) {
-                bm25ScoreVal = bm25Score(queryTerms, row.content, avgDocLength, docCount, termDocFreqs);
+                bm25ScoreVal = bm25Score(queryTerms, doc.words, doc.words.length, avgDocLength, docCount, termDocFreqs);
                 // Normalize BM25 to 0-1 range (cap at 10 for normalization)
                 bm25ScoreVal = Math.min(bm25ScoreVal / 10, 1.0);
             }
-            // Reciprocal rank fusion: combine semantic and BM25
-            // Weight: 0.7 semantic + 0.3 BM25 when both embeddings present
-            // Fall back to BM25-only when either query or row lacks an embedding
-            const score = semanticScore > 0
-                ? (0.7 * semanticScore + 0.3 * bm25ScoreVal)
-                : bm25ScoreVal;
+            // #2558: keyword-coverage floor for the lexical signal.
+            // BM25's IDF collapses toward zero when a term appears in most/all
+            // documents (routine on small memory corpora), and the /10 normalization
+            // crushed exact-keyword hits well below the default 0.3 threshold — so
+            // `memory search` recalled NOTHING even when the content literally
+            // contained the query term (issue #2558: "keyword recall random"). The
+            // pre-BM25 fallback guaranteed keyword recall via matchCount/words*0.5;
+            // this restores that guarantee. `coverage` is the fraction of query
+            // terms present in the document — a full-coverage hit must always be
+            // recallable regardless of IDF.
+            const contentLower = doc.contentLower;
+            const matchedTerms = queryTerms.filter(t => contentLower.includes(t)).length;
+            const coverage = queryTerms.length > 0 ? matchedTerms / queryTerms.length : 0;
+            const lexicalScore = Math.max(bm25ScoreVal, coverage);
+            // Recall-friendly fusion: a strong semantic OR lexical signal alone must
+            // clear the threshold. `blended` (0.6 semantic + 0.4 lexical) drives
+            // ranking; taking max() with the raw semantic score means (a) a genuinely
+            // similar entry is never dropped just because it lacks the query's exact
+            // words, and (b) a full-coverage keyword hit (lexical=1 → blended≥0.4) is
+            // never dropped just because its embedding cosine is low or negative.
+            const blended = 0.6 * Math.max(0, semanticScore) + 0.4 * lexicalScore;
+            const score = Math.max(blended, semanticScore);
             if (score >= threshold) {
                 // Phase 4: ExplainableRecall provenance
                 const provenance = queryEmbedding
-                    ? `semantic:${semanticScore.toFixed(3)}+bm25:${bm25ScoreVal.toFixed(3)}`
-                    : `bm25:${bm25ScoreVal.toFixed(3)}`;
+                    ? `semantic:${semanticScore.toFixed(3)}+lexical:${lexicalScore.toFixed(3)}`
+                    : `lexical:${lexicalScore.toFixed(3)}`;
                 results.push({
                     id: String(row.id).substring(0, 12),
                     key: row.key || String(row.id).substring(0, 15),

package/v3/@claude-flow/cli/dist/src/memory/memory-initializer.d.ts

@@ -203,6 +203,77 @@ export declare function checkAndMigrateLegacy(options: {
     migrated?: boolean;
     migratedCount?: number;
 }>;
+/**
+ * Self-heal an EXISTING memory database that is missing the `vector_indexes`
+ * table or per-namespace rows.
+ *
+ * Why this exists: fresh installs create `vector_indexes` + seed rows, but a
+ * DB written by an older CLI or by agentdb directly may have thousands of
+ * embedded rows in `memory_entries` and NO `vector_indexes` table at all.
+ * Two things break as a result:
+ *   1. The statusline's vector count read collapsed to `0` (the count query
+ *      referenced the missing table and failed whole — now split, but the
+ *      HNSW flag still needs the table).
+ *   2. #1941 — `memory_search` routes per namespace via `vector_indexes`; a
+ *      namespace with no row returns 0 results even when entries exist.
+ *
+ * This is idempotent and conservative:
+ *   - Does NOTHING (no writes) when the table already exists and every embedded
+ *     namespace already has a row — the common already-healed path, hit on
+ *     every MCP start, must not write to the live DB unnecessarily.
+ *   - Before ANY write, runs `PRAGMA quick_check`; if the DB reports structural
+ *     corruption it SKIPS the repair entirely (returns `corrupt:true`) rather
+ *     than writing into a malformed btree and risking making it worse. The
+ *     caller/user should recover via `sqlite3 old.db .recover | sqlite3 new.db`.
+ *   - Does NOT checkpoint. mode=ro readers already see committed WAL frames, and
+ *     forcing a checkpoint on a DB with a torn WAL could persist latent damage.
+ *
+ * When a repair IS needed and the DB is healthy: creates the table if absent,
+ * seeds the fresh-install default rows, and backfills an accurate
+ * `total_vectors` per namespace. Runs on the existing-DB path of
+ * `initializeMemoryDatabase` (MCP start / `memory init`) and from `ruflo init`.
+ *
+ * Uses better-sqlite3 (WAL-safe, native). If the native module is unavailable
+ * it is a silent no-op — the split statusline query already prevents the count
+ * from zeroing; only the HNSW flag and namespace routing stay degraded.
+ */
+/**
+ * Auto-recover a structurally-corrupt memory DB into a clean one, universally
+ * (better-sqlite3 only — no dependency on the external `sqlite3` CLI, which is
+ * absent on many npx hosts). Safe by construction:
+ *   1. Confirms corruption (quick_check) — no-op on a healthy DB.
+ *   2. Acquires an EXCLUSIVE lock (BEGIN IMMEDIATE). If another process is
+ *      writing, it SKIPS (returns reason:'writer-active') rather than racing a
+ *      writer and losing its in-flight writes — the mistake that must not recur.
+ *   3. Rebuilds a fresh DB table-by-table (schema + rows), skipping any single
+ *      table whose pages won't scan so one bad table can't abort the whole
+ *      rebuild.
+ *   4. VERIFIES the rebuild (integrity_check == ok AND recovered
+ *      memory_entries count >= the readable source count) BEFORE touching the
+ *      original.
+ *   5. Backs up the corrupt DB to `<db>.corrupt-<ts>.bak`, then atomically
+ *      renames the verified rebuild into place and drops stale -wal/-shm.
+ * On any failure the original + backup are left intact — never destructive.
+ */
+export declare function recoverMemoryDatabase(dbPath: string, opts?: {
+    verbose?: boolean;
+}): Promise<{
+    recovered: boolean;
+    backupPath?: string;
+    rows?: number;
+    reason?: string;
+}>;
+export declare function repairVectorIndexes(dbPath: string, opts?: {
+    verbose?: boolean;
+    autoRecover?: boolean;
+}): Promise<{
+    repaired: boolean;
+    tableCreated: boolean;
+    namespaces: string[];
+    corrupt?: boolean;
+    recovered?: boolean;
+    backupPath?: string;
+}>;
 /**
  * Initialize the memory database properly using sql.js
  */