claude-flow 3.21.1 → 3.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/package.json +1 -1
  2. package/v3/@claude-flow/cli/dist/src/commands/daemon.js +23 -2
  3. package/v3/@claude-flow/cli/dist/src/commands/memory-distill.d.ts +27 -0
  4. package/v3/@claude-flow/cli/dist/src/commands/memory-distill.js +374 -0
  5. package/v3/@claude-flow/cli/dist/src/commands/memory.js +4 -2
  6. package/v3/@claude-flow/cli/dist/src/index.d.ts +1 -1
  7. package/v3/@claude-flow/cli/dist/src/index.js +22 -1
  8. package/v3/@claude-flow/cli/dist/src/init/executor.js +20 -0
  9. package/v3/@claude-flow/cli/dist/src/init/helper-refresh.d.ts +19 -0
  10. package/v3/@claude-flow/cli/dist/src/init/helper-refresh.js +175 -0
  11. package/v3/@claude-flow/cli/dist/src/init/helper-signing.d.ts +30 -0
  12. package/v3/@claude-flow/cli/dist/src/init/helper-signing.js +60 -0
  13. package/v3/@claude-flow/cli/dist/src/init/helpers-generator.d.ts +16 -0
  14. package/v3/@claude-flow/cli/dist/src/init/helpers-generator.js +63 -6
  15. package/v3/@claude-flow/cli/dist/src/init/statusline-generator.js +18 -7
  16. package/v3/@claude-flow/cli/dist/src/mcp-client.js +13 -0
  17. package/v3/@claude-flow/cli/dist/src/mcp-tools/agenticow-speculate-tools.js +9 -2
  18. package/v3/@claude-flow/cli/dist/src/mcp-tools/browser-intent-tools.d.ts +162 -0
  19. package/v3/@claude-flow/cli/dist/src/mcp-tools/browser-intent-tools.js +548 -0
  20. package/v3/@claude-flow/cli/dist/src/mcp-tools/browser-tools.d.ts +9 -1
  21. package/v3/@claude-flow/cli/dist/src/mcp-tools/browser-tools.js +4 -1
  22. package/v3/@claude-flow/cli/dist/src/memory/memory-bridge.js +67 -47
  23. package/v3/@claude-flow/cli/dist/src/memory/memory-initializer.d.ts +71 -0
  24. package/v3/@claude-flow/cli/dist/src/memory/memory-initializer.js +330 -2
  25. package/v3/@claude-flow/cli/dist/src/services/distill-tuning.d.ts +111 -0
  26. package/v3/@claude-flow/cli/dist/src/services/distill-tuning.js +510 -0
  27. package/v3/@claude-flow/cli/dist/src/services/memory-distillation.d.ts +41 -0
  28. package/v3/@claude-flow/cli/dist/src/services/memory-distillation.js +277 -0
  29. package/v3/@claude-flow/cli/dist/src/services/worker-daemon.d.ts +22 -1
  30. package/v3/@claude-flow/cli/dist/src/services/worker-daemon.js +117 -6
  31. package/v3/@claude-flow/cli/package.json +5 -2
@@ -365,11 +365,9 @@ async function getRegistry(dbPath) {
365
365
  * Replaces naive String.includes() with proper information retrieval scoring.
366
366
  * Parameters tuned for short memory entries (k1=1.2, b=0.75).
367
367
  */
368
- function bm25Score(queryTerms, docContent, avgDocLength, docCount, termDocFreqs) {
368
+ function bm25Score(queryTerms, docWords, docLength, avgDocLength, docCount, termDocFreqs) {
369
369
  const k1 = 1.2;
370
370
  const b = 0.75;
371
- const docWords = docContent.toLowerCase().split(/\s+/);
372
- const docLength = docWords.length;
373
371
  let score = 0;
374
372
  for (const term of queryTerms) {
375
373
  const tf = docWords.filter(w => w === term || w.includes(term)).length;
@@ -382,23 +380,27 @@ function bm25Score(queryTerms, docContent, avgDocLength, docCount, termDocFreqs)
382
380
  }
383
381
  return score;
384
382
  }
383
+ function tokenizeCorpus(rows) {
384
+ return rows.map(row => {
385
+ const contentLower = (row.content || '').toLowerCase();
386
+ return { contentLower, words: contentLower.split(/\s+/) };
387
+ });
388
+ }
385
389
  /**
386
- * Compute BM25 term document frequencies for a set of rows.
390
+ * Compute BM25 term document frequencies over an already-tokenized corpus.
387
391
  */
388
- function computeTermDocFreqs(queryTerms, rows) {
392
+ function computeTermDocFreqs(queryTerms, docs) {
389
393
  const termDocFreqs = new Map();
390
394
  let totalLength = 0;
391
- for (const row of rows) {
392
- const content = (row.content || '').toLowerCase();
393
- const words = content.split(/\s+/);
394
- totalLength += words.length;
395
+ for (const doc of docs) {
396
+ totalLength += doc.words.length;
395
397
  for (const term of queryTerms) {
396
- if (content.includes(term)) {
398
+ if (doc.contentLower.includes(term)) {
397
399
  termDocFreqs.set(term, (termDocFreqs.get(term) || 0) + 1);
398
400
  }
399
401
  }
400
402
  }
401
- return { termDocFreqs, avgDocLength: rows.length > 0 ? totalLength / rows.length : 1 };
403
+ return { termDocFreqs, avgDocLength: docs.length > 0 ? totalLength / docs.length : 1 };
402
404
  }
403
405
  // ===== Phase 2: TieredCache helpers =====
404
406
  /**
@@ -484,6 +486,10 @@ async function logAttestation(registry, operation, entryId, metadata) {
484
486
  // Non-fatal — attestation is observability, not correctness
485
487
  }
486
488
  }
489
+ // Tracks db handles whose schema DDL has already been ensured, so getDb()
490
+ // runs the CREATE…IF NOT EXISTS block at most once per handle instead of on
491
+ // every bridge call. WeakSet so handles GC without leaking.
492
+ const _schemaEnsuredDbs = new WeakSet();
487
493
  /**
488
494
  * Get the AgentDB database handle and ensure memory_entries table exists.
489
495
  * Returns null if not available.
@@ -493,35 +499,42 @@ function getDb(registry) {
493
499
  if (!agentdb?.database)
494
500
  return null;
495
501
  const db = agentdb.database;
496
- // Ensure memory_entries table exists (idempotent)
497
- try {
498
- db.exec(`CREATE TABLE IF NOT EXISTS memory_entries (
499
- id TEXT PRIMARY KEY,
500
- key TEXT NOT NULL,
501
- namespace TEXT DEFAULT 'default',
502
- content TEXT NOT NULL,
503
- type TEXT DEFAULT 'semantic',
504
- embedding TEXT,
505
- embedding_model TEXT DEFAULT 'local',
506
- embedding_dimensions INTEGER,
507
- tags TEXT,
508
- metadata TEXT,
509
- owner_id TEXT,
510
- created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
511
- updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
512
- expires_at INTEGER,
513
- last_accessed_at INTEGER,
514
- access_count INTEGER DEFAULT 0,
515
- status TEXT DEFAULT 'active',
516
- UNIQUE(namespace, key)
517
- )`);
518
- // Ensure indexes
519
- db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_ns ON memory_entries(namespace)`);
520
- db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_key ON memory_entries(key)`);
521
- db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_status ON memory_entries(status)`);
522
- }
523
- catch {
524
- // Table already exists or db is read-only — that's fine
502
+ // Ensure memory_entries table exists (idempotent). The DDL is run at most
503
+ // once per db handle — re-parsing 4× CREATE…IF NOT EXISTS on every bridge
504
+ // call (store/search/get) was pure per-op overhead. Keyed by handle via a
505
+ // WeakSet so a new db instance re-ensures without a stale global flag.
506
+ if (!_schemaEnsuredDbs.has(db)) {
507
+ try {
508
+ db.exec(`CREATE TABLE IF NOT EXISTS memory_entries (
509
+ id TEXT PRIMARY KEY,
510
+ key TEXT NOT NULL,
511
+ namespace TEXT DEFAULT 'default',
512
+ content TEXT NOT NULL,
513
+ type TEXT DEFAULT 'semantic',
514
+ embedding TEXT,
515
+ embedding_model TEXT DEFAULT 'local',
516
+ embedding_dimensions INTEGER,
517
+ tags TEXT,
518
+ metadata TEXT,
519
+ owner_id TEXT,
520
+ created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
521
+ updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
522
+ expires_at INTEGER,
523
+ last_accessed_at INTEGER,
524
+ access_count INTEGER DEFAULT 0,
525
+ status TEXT DEFAULT 'active',
526
+ UNIQUE(namespace, key)
527
+ )`);
528
+ // Ensure indexes
529
+ db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_ns ON memory_entries(namespace)`);
530
+ db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_key ON memory_entries(key)`);
531
+ db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_status ON memory_entries(status)`);
532
+ _schemaEnsuredDbs.add(db);
533
+ }
534
+ catch {
535
+ // Table already exists or db is read-only — that's fine. Don't mark
536
+ // ensured on failure so a later writable call can retry.
537
+ }
525
538
  }
526
539
  // ─── #2256-followup: rescue agentdb.embedder when its transformers.js
527
540
  // path fell through to mock embeddings.
@@ -637,6 +650,7 @@ export async function bridgeStoreEntry(options) {
637
650
  }
638
651
  // Generate embedding via AgentDB's embedder
639
652
  let embeddingJson = null;
653
+ let embeddingArr = null;
640
654
  let dimensions = 0;
641
655
  let model = 'local';
642
656
  if (options.generateEmbeddingFlag !== false && value.length > 0) {
@@ -645,7 +659,8 @@ export async function bridgeStoreEntry(options) {
645
659
  if (embedder) {
646
660
  const emb = await embedder.embed(value);
647
661
  if (emb) {
648
- embeddingJson = JSON.stringify(Array.from(emb));
662
+ embeddingArr = Array.from(emb);
663
+ embeddingJson = JSON.stringify(embeddingArr);
649
664
  dimensions = emb.length;
650
665
  model = 'Xenova/all-MiniLM-L6-v2';
651
666
  }
@@ -716,7 +731,7 @@ export async function bridgeStoreEntry(options) {
716
731
  success: true,
717
732
  id,
718
733
  embedding: embeddingJson ? { dimensions, model } : undefined,
719
- rawEmbedding: embeddingJson ? JSON.parse(embeddingJson) : undefined,
734
+ rawEmbedding: embeddingArr ?? undefined,
720
735
  guarded: true,
721
736
  cached: true,
722
737
  attested: true,
@@ -771,12 +786,17 @@ export async function bridgeSearchEntries(options) {
771
786
  catch {
772
787
  return null;
773
788
  }
774
- // Phase 2: Compute BM25 term stats for the corpus
789
+ // Phase 2: Compute BM25 term stats for the corpus. Tokenize the corpus a
790
+ // single time and reuse the per-doc `{contentLower, words}` for term-freq,
791
+ // BM25 scoring, and the coverage floor below.
775
792
  const queryTerms = queryStr.toLowerCase().split(/\s+/).filter(t => t.length > 1);
776
- const { termDocFreqs, avgDocLength } = computeTermDocFreqs(queryTerms, rows);
793
+ const docs = tokenizeCorpus(rows);
794
+ const { termDocFreqs, avgDocLength } = computeTermDocFreqs(queryTerms, docs);
777
795
  const docCount = rows.length;
778
796
  const results = [];
779
- for (const row of rows) {
797
+ for (let i = 0; i < rows.length; i++) {
798
+ const row = rows[i];
799
+ const doc = docs[i];
780
800
  let semanticScore = 0;
781
801
  let bm25ScoreVal = 0;
782
802
  // Semantic scoring via cosine similarity
@@ -791,7 +811,7 @@ export async function bridgeSearchEntries(options) {
791
811
  }
792
812
  // Phase 2: BM25 keyword scoring (replaces String.includes fallback)
793
813
  if (queryTerms.length > 0 && row.content) {
794
- bm25ScoreVal = bm25Score(queryTerms, row.content, avgDocLength, docCount, termDocFreqs);
814
+ bm25ScoreVal = bm25Score(queryTerms, doc.words, doc.words.length, avgDocLength, docCount, termDocFreqs);
795
815
  // Normalize BM25 to 0-1 range (cap at 10 for normalization)
796
816
  bm25ScoreVal = Math.min(bm25ScoreVal / 10, 1.0);
797
817
  }
@@ -805,7 +825,7 @@ export async function bridgeSearchEntries(options) {
805
825
  // this restores that guarantee. `coverage` is the fraction of query
806
826
  // terms present in the document — a full-coverage hit must always be
807
827
  // recallable regardless of IDF.
808
- const contentLower = String(row.content || '').toLowerCase();
828
+ const contentLower = doc.contentLower;
809
829
  const matchedTerms = queryTerms.filter(t => contentLower.includes(t)).length;
810
830
  const coverage = queryTerms.length > 0 ? matchedTerms / queryTerms.length : 0;
811
831
  const lexicalScore = Math.max(bm25ScoreVal, coverage);
@@ -203,6 +203,77 @@ export declare function checkAndMigrateLegacy(options: {
203
203
  migrated?: boolean;
204
204
  migratedCount?: number;
205
205
  }>;
206
+ /**
207
+ * Self-heal an EXISTING memory database that is missing the `vector_indexes`
208
+ * table or per-namespace rows.
209
+ *
210
+ * Why this exists: fresh installs create `vector_indexes` + seed rows, but a
211
+ * DB written by an older CLI or by agentdb directly may have thousands of
212
+ * embedded rows in `memory_entries` and NO `vector_indexes` table at all.
213
+ * Two things break as a result:
214
+ * 1. The statusline's vector count read collapsed to `0` (the count query
215
+ * referenced the missing table and failed whole — now split, but the
216
+ * HNSW flag still needs the table).
217
+ * 2. #1941 — `memory_search` routes per namespace via `vector_indexes`; a
218
+ * namespace with no row returns 0 results even when entries exist.
219
+ *
220
+ * This is idempotent and conservative:
221
+ * - Does NOTHING (no writes) when the table already exists and every embedded
222
+ * namespace already has a row — the common already-healed path, hit on
223
+ * every MCP start, must not write to the live DB unnecessarily.
224
+ * - Before ANY write, runs `PRAGMA quick_check`; if the DB reports structural
225
+ * corruption it SKIPS the repair entirely (returns `corrupt:true`) rather
226
+ * than writing into a malformed btree and risking making it worse. The
227
+ * caller/user should recover via `sqlite3 old.db .recover | sqlite3 new.db`.
228
+ * - Does NOT checkpoint. mode=ro readers already see committed WAL frames, and
229
+ * forcing a checkpoint on a DB with a torn WAL could persist latent damage.
230
+ *
231
+ * When a repair IS needed and the DB is healthy: creates the table if absent,
232
+ * seeds the fresh-install default rows, and backfills an accurate
233
+ * `total_vectors` per namespace. Runs on the existing-DB path of
234
+ * `initializeMemoryDatabase` (MCP start / `memory init`) and from `ruflo init`.
235
+ *
236
+ * Uses better-sqlite3 (WAL-safe, native). If the native module is unavailable
237
+ * it is a silent no-op — the split statusline query already prevents the count
238
+ * from zeroing; only the HNSW flag and namespace routing stay degraded.
239
+ */
240
+ /**
241
+ * Auto-recover a structurally-corrupt memory DB into a clean one, universally
242
+ * (better-sqlite3 only — no dependency on the external `sqlite3` CLI, which is
243
+ * absent on many npx hosts). Safe by construction:
244
+ * 1. Confirms corruption (quick_check) — no-op on a healthy DB.
245
+ * 2. Acquires an EXCLUSIVE lock (BEGIN IMMEDIATE). If another process is
246
+ * writing, it SKIPS (returns reason:'writer-active') rather than racing a
247
+ * writer and losing its in-flight writes — the mistake that must not recur.
248
+ * 3. Rebuilds a fresh DB table-by-table (schema + rows), skipping any single
249
+ * table whose pages won't scan so one bad table can't abort the whole
250
+ * rebuild.
251
+ * 4. VERIFIES the rebuild (integrity_check == ok AND recovered
252
+ * memory_entries count >= the readable source count) BEFORE touching the
253
+ * original.
254
+ * 5. Backs up the corrupt DB to `<db>.corrupt-<ts>.bak`, then atomically
255
+ * renames the verified rebuild into place and drops stale -wal/-shm.
256
+ * On any failure the original + backup are left intact — never destructive.
257
+ */
258
+ export declare function recoverMemoryDatabase(dbPath: string, opts?: {
259
+ verbose?: boolean;
260
+ }): Promise<{
261
+ recovered: boolean;
262
+ backupPath?: string;
263
+ rows?: number;
264
+ reason?: string;
265
+ }>;
266
+ export declare function repairVectorIndexes(dbPath: string, opts?: {
267
+ verbose?: boolean;
268
+ autoRecover?: boolean;
269
+ }): Promise<{
270
+ repaired: boolean;
271
+ tableCreated: boolean;
272
+ namespaces: string[];
273
+ corrupt?: boolean;
274
+ recovered?: boolean;
275
+ backupPath?: string;
276
+ }>;
206
277
  /**
207
278
  * Initialize the memory database properly using sql.js
208
279
  */
@@ -1127,6 +1127,329 @@ async function activateControllerRegistry(dbPath, verbose) {
1127
1127
  }
1128
1128
  return { activated, failed, initTimeMs: performance.now() - startTime };
1129
1129
  }
1130
+ /**
1131
+ * Self-heal an EXISTING memory database that is missing the `vector_indexes`
1132
+ * table or per-namespace rows.
1133
+ *
1134
+ * Why this exists: fresh installs create `vector_indexes` + seed rows, but a
1135
+ * DB written by an older CLI or by agentdb directly may have thousands of
1136
+ * embedded rows in `memory_entries` and NO `vector_indexes` table at all.
1137
+ * Two things break as a result:
1138
+ * 1. The statusline's vector count read collapsed to `0` (the count query
1139
+ * referenced the missing table and failed whole — now split, but the
1140
+ * HNSW flag still needs the table).
1141
+ * 2. #1941 — `memory_search` routes per namespace via `vector_indexes`; a
1142
+ * namespace with no row returns 0 results even when entries exist.
1143
+ *
1144
+ * This is idempotent and conservative:
1145
+ * - Does NOTHING (no writes) when the table already exists and every embedded
1146
+ * namespace already has a row — the common already-healed path, hit on
1147
+ * every MCP start, must not write to the live DB unnecessarily.
1148
+ * - Before ANY write, runs `PRAGMA quick_check`; if the DB reports structural
1149
+ * corruption it SKIPS the repair entirely (returns `corrupt:true`) rather
1150
+ * than writing into a malformed btree and risking making it worse. The
1151
+ * caller/user should recover via `sqlite3 old.db .recover | sqlite3 new.db`.
1152
+ * - Does NOT checkpoint. mode=ro readers already see committed WAL frames, and
1153
+ * forcing a checkpoint on a DB with a torn WAL could persist latent damage.
1154
+ *
1155
+ * When a repair IS needed and the DB is healthy: creates the table if absent,
1156
+ * seeds the fresh-install default rows, and backfills an accurate
1157
+ * `total_vectors` per namespace. Runs on the existing-DB path of
1158
+ * `initializeMemoryDatabase` (MCP start / `memory init`) and from `ruflo init`.
1159
+ *
1160
+ * Uses better-sqlite3 (WAL-safe, native). If the native module is unavailable
1161
+ * it is a silent no-op — the split statusline query already prevents the count
1162
+ * from zeroing; only the HNSW flag and namespace routing stay degraded.
1163
+ */
1164
+ /**
1165
+ * Auto-recover a structurally-corrupt memory DB into a clean one, universally
1166
+ * (better-sqlite3 only — no dependency on the external `sqlite3` CLI, which is
1167
+ * absent on many npx hosts). Safe by construction:
1168
+ * 1. Confirms corruption (quick_check) — no-op on a healthy DB.
1169
+ * 2. Acquires an EXCLUSIVE lock (BEGIN IMMEDIATE). If another process is
1170
+ * writing, it SKIPS (returns reason:'writer-active') rather than racing a
1171
+ * writer and losing its in-flight writes — the mistake that must not recur.
1172
+ * 3. Rebuilds a fresh DB table-by-table (schema + rows), skipping any single
1173
+ * table whose pages won't scan so one bad table can't abort the whole
1174
+ * rebuild.
1175
+ * 4. VERIFIES the rebuild (integrity_check == ok AND recovered
1176
+ * memory_entries count >= the readable source count) BEFORE touching the
1177
+ * original.
1178
+ * 5. Backs up the corrupt DB to `<db>.corrupt-<ts>.bak`, then atomically
1179
+ * renames the verified rebuild into place and drops stale -wal/-shm.
1180
+ * On any failure the original + backup are left intact — never destructive.
1181
+ */
1182
+ export async function recoverMemoryDatabase(dbPath, opts = {}) {
1183
+ if (!dbPath || !fs.existsSync(dbPath))
1184
+ return { recovered: false, reason: 'no-db' };
1185
+ let Database;
1186
+ try {
1187
+ // Module name behind a variable so TS does not statically resolve the
1188
+ // optional native dep's types at build time (CI may not install them).
1189
+ const mod = 'better-sqlite3';
1190
+ Database = (await import(mod)).default;
1191
+ }
1192
+ catch {
1193
+ return { recovered: false, reason: 'no-native' };
1194
+ }
1195
+ const ts = Date.now();
1196
+ const tmpPath = `${dbPath}.recovering-${ts}`;
1197
+ const bakPath = `${dbPath}.corrupt-${ts}.bak`;
1198
+ let src;
1199
+ let dst;
1200
+ try {
1201
+ src = new Database(dbPath, { timeout: 1500 });
1202
+ // Confirm corruption — never rewrite a healthy DB.
1203
+ const qc = src.prepare('PRAGMA quick_check(1)').get();
1204
+ const qcVal = qc ? String(Object.values(qc)[0] ?? '') : '';
1205
+ if (qcVal.toLowerCase() === 'ok') {
1206
+ src.close();
1207
+ return { recovered: false, reason: 'not-corrupt' };
1208
+ }
1209
+ // Exclusive-writer guard: acquire the write lock. If busy, another process
1210
+ // is writing — do NOT race it. Reading within this txn is still allowed.
1211
+ try {
1212
+ src.exec('BEGIN IMMEDIATE');
1213
+ }
1214
+ catch {
1215
+ src.close();
1216
+ return { recovered: false, reason: 'writer-active' };
1217
+ }
1218
+ let srcRows = 0;
1219
+ try {
1220
+ srcRows = src.prepare('SELECT COUNT(*) AS c FROM memory_entries').get()?.c ?? 0;
1221
+ }
1222
+ catch { /* unreadable */ }
1223
+ try {
1224
+ fs.rmSync(tmpPath, { force: true });
1225
+ }
1226
+ catch { /* fresh */ }
1227
+ dst = new Database(tmpPath);
1228
+ // Copy schema: tables first (so data can be inserted), then indexes/triggers.
1229
+ const objects = src
1230
+ .prepare("SELECT type, name, sql FROM sqlite_master WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%'")
1231
+ .all();
1232
+ const tables = objects.filter(o => o.type === 'table');
1233
+ const others = objects.filter(o => o.type !== 'table');
1234
+ for (const t of tables) {
1235
+ try {
1236
+ dst.exec(t.sql);
1237
+ }
1238
+ catch { /* skip an untranslatable table def */ }
1239
+ }
1240
+ // Copy rows table-by-table; a table whose pages won't scan is skipped whole.
1241
+ let copiedEntries = 0;
1242
+ for (const t of tables) {
1243
+ try {
1244
+ const cols = dst.prepare(`PRAGMA table_info("${t.name}")`).all().map(c => c.name);
1245
+ if (!cols.length)
1246
+ continue;
1247
+ const colList = cols.map(c => `"${c}"`).join(',');
1248
+ const placeholders = cols.map(() => '?').join(',');
1249
+ const insert = dst.prepare(`INSERT OR IGNORE INTO "${t.name}" (${colList}) VALUES (${placeholders})`);
1250
+ const rows = src.prepare(`SELECT ${colList} FROM "${t.name}"`).all();
1251
+ const runAll = dst.transaction((rs) => {
1252
+ for (const r of rs)
1253
+ insert.run(cols.map(c => r[c]));
1254
+ });
1255
+ runAll(rows);
1256
+ if (t.name === 'memory_entries')
1257
+ copiedEntries = rows.length;
1258
+ }
1259
+ catch { /* skip a table whose data pages are unreadable */ }
1260
+ }
1261
+ for (const o of others) {
1262
+ try {
1263
+ dst.exec(o.sql);
1264
+ }
1265
+ catch { /* an index over corrupt data — non-fatal */ }
1266
+ }
1267
+ dst.close();
1268
+ dst = null;
1269
+ try {
1270
+ src.exec('ROLLBACK');
1271
+ }
1272
+ catch { /* ignore */ }
1273
+ src.close();
1274
+ src = null;
1275
+ // Verify BEFORE touching the original.
1276
+ const check = new Database(tmpPath, { readonly: true });
1277
+ const integ = String(check.pragma('integrity_check', { simple: true }) ?? '');
1278
+ let dstRows = 0;
1279
+ try {
1280
+ dstRows = check.prepare('SELECT COUNT(*) AS c FROM memory_entries').get()?.c ?? 0;
1281
+ }
1282
+ catch { /* */ }
1283
+ check.close();
1284
+ if (integ.toLowerCase() !== 'ok' || dstRows < srcRows) {
1285
+ try {
1286
+ fs.rmSync(tmpPath, { force: true });
1287
+ }
1288
+ catch { /* */ }
1289
+ if (opts.verbose) {
1290
+ console.log(`memory DB auto-recovery aborted (integrity=${integ}, rows ${dstRows}/${srcRows}) — original untouched`);
1291
+ }
1292
+ return { recovered: false, reason: 'verify-failed' };
1293
+ }
1294
+ // Back up the corrupt DB, then atomically swap in the verified rebuild.
1295
+ fs.copyFileSync(dbPath, bakPath);
1296
+ fs.renameSync(tmpPath, dbPath);
1297
+ for (const s of ['-wal', '-shm']) {
1298
+ try {
1299
+ fs.rmSync(`${dbPath}${s}`, { force: true });
1300
+ }
1301
+ catch { /* */ }
1302
+ }
1303
+ if (opts.verbose) {
1304
+ console.log(`memory DB auto-recovered: ${dstRows} rows, integrity ok. Corrupt original saved to ${bakPath}`);
1305
+ }
1306
+ return { recovered: true, backupPath: bakPath, rows: dstRows };
1307
+ }
1308
+ catch (e) {
1309
+ try {
1310
+ dst?.close();
1311
+ }
1312
+ catch { /* */ }
1313
+ try {
1314
+ src?.exec('ROLLBACK');
1315
+ }
1316
+ catch { /* */ }
1317
+ try {
1318
+ src?.close();
1319
+ }
1320
+ catch { /* */ }
1321
+ try {
1322
+ fs.rmSync(tmpPath, { force: true });
1323
+ }
1324
+ catch { /* */ }
1325
+ if (opts.verbose)
1326
+ console.log(`memory DB auto-recovery error: ${e?.message ?? e}`);
1327
+ return { recovered: false, reason: 'error' };
1328
+ }
1329
+ }
1330
+ export async function repairVectorIndexes(dbPath, opts = {}) {
1331
+ const res = { repaired: false, tableCreated: false, namespaces: [], corrupt: false };
1332
+ if (!dbPath || !fs.existsSync(dbPath))
1333
+ return res;
1334
+ let Database;
1335
+ try {
1336
+ // Module name behind a variable so TS does not statically resolve the
1337
+ // optional native dep's types at build time (CI may not install them).
1338
+ const mod = 'better-sqlite3';
1339
+ Database = (await import(mod)).default;
1340
+ }
1341
+ catch {
1342
+ // Native module absent (e.g. WASM-only host). Statusline fix still covers
1343
+ // the display; nothing to repair here.
1344
+ return res;
1345
+ }
1346
+ let db;
1347
+ try {
1348
+ db = new Database(dbPath, { timeout: 3000 });
1349
+ const tableExists = (name) => (db.prepare("SELECT COUNT(*) AS c FROM sqlite_master WHERE type='table' AND name=?").get(name)?.c ?? 0) > 0;
1350
+ // Nothing to key off if there is no entries table.
1351
+ if (!tableExists('memory_entries')) {
1352
+ db.close();
1353
+ return res;
1354
+ }
1355
+ const hasVectorIndexes = tableExists('vector_indexes');
1356
+ // Namespaces that actually have embeddings.
1357
+ const nsRows = db
1358
+ .prepare("SELECT COALESCE(namespace, 'default') AS ns, COUNT(*) AS c " +
1359
+ 'FROM memory_entries WHERE embedding IS NOT NULL GROUP BY ns')
1360
+ .all();
1361
+ // Which of those are already present in vector_indexes? If the table exists
1362
+ // and every embedded namespace already has a row, the DB is already healed
1363
+ // — return WITHOUT writing anything (common path on every MCP start).
1364
+ let needsWrite = !hasVectorIndexes;
1365
+ if (hasVectorIndexes) {
1366
+ const present = new Set(db.prepare('SELECT name FROM vector_indexes').all().map(r => r.name));
1367
+ needsWrite = nsRows.some(r => !present.has(String(r.ns || 'default')));
1368
+ }
1369
+ if (!needsWrite) {
1370
+ db.close();
1371
+ return res;
1372
+ }
1373
+ // A write is needed. GUARD: never write into a structurally corrupt DB —
1374
+ // that risks worsening the damage. quick_check is cheaper than a full
1375
+ // integrity_check and only runs on the rare repair path, not every start.
1376
+ const qc = db.prepare('PRAGMA quick_check(1)').get();
1377
+ const qcVal = qc ? String(Object.values(qc)[0] ?? '') : '';
1378
+ if (qcVal.toLowerCase() !== 'ok') {
1379
+ res.corrupt = true;
1380
+ db.close(); // release our handle before recovery may swap the file
1381
+ if (opts.autoRecover) {
1382
+ // Auto-fix: rebuild the corrupt DB (backup + verify + atomic swap), then
1383
+ // provision vector_indexes on the clean rebuild. This is what makes any
1384
+ // npx-deployed ruflo self-repair a corrupt memory DB on init / MCP start.
1385
+ const rec = await recoverMemoryDatabase(dbPath, { verbose: opts.verbose });
1386
+ if (rec.recovered) {
1387
+ const healed = await repairVectorIndexes(dbPath, { verbose: opts.verbose });
1388
+ return { ...healed, corrupt: true, recovered: true, backupPath: rec.backupPath };
1389
+ }
1390
+ if (opts.verbose) {
1391
+ console.log(`vector_indexes repair skipped — corruption (${qcVal}); auto-recovery not run (${rec.reason ?? 'unknown'})`);
1392
+ }
1393
+ }
1394
+ else if (opts.verbose) {
1395
+ console.log('vector_indexes repair SKIPPED — memory DB reports corruption (' + qcVal + '). ' +
1396
+ 'Recover with: sqlite3 <db> .recover | sqlite3 <db>.recovered');
1397
+ }
1398
+ return res;
1399
+ }
1400
+ if (!hasVectorIndexes) {
1401
+ db.exec(`CREATE TABLE IF NOT EXISTS vector_indexes (
1402
+ id TEXT PRIMARY KEY,
1403
+ name TEXT NOT NULL UNIQUE,
1404
+ dimensions INTEGER NOT NULL,
1405
+ metric TEXT DEFAULT 'cosine' CHECK(metric IN ('cosine', 'euclidean', 'dot')),
1406
+ hnsw_m INTEGER DEFAULT 16,
1407
+ hnsw_ef_construction INTEGER DEFAULT 200,
1408
+ hnsw_ef_search INTEGER DEFAULT 100,
1409
+ quantization_type TEXT CHECK(quantization_type IN ('none', 'scalar', 'product')),
1410
+ quantization_bits INTEGER DEFAULT 8,
1411
+ total_vectors INTEGER DEFAULT 0,
1412
+ last_rebuild_at INTEGER,
1413
+ created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
1414
+ updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)
1415
+ )`);
1416
+ res.tableCreated = true;
1417
+ }
1418
+ // 384 = default ONNX model dim (Xenova/all-MiniLM-L6-v2); HNSW rejects
1419
+ // dim-mismatched inserts, so this must match the stored embeddings (#1947).
1420
+ const ensureRow = db.prepare('INSERT OR IGNORE INTO vector_indexes (id, name, dimensions) VALUES (?, ?, 384)');
1421
+ const setCount = db.prepare('UPDATE vector_indexes SET total_vectors = ?, updated_at = ? WHERE name = ?');
1422
+ const backfill = db.transaction(() => {
1423
+ // Parity with a fresh install's seed rows.
1424
+ ensureRow.run('default', 'default');
1425
+ ensureRow.run('patterns', 'patterns');
1426
+ const now = Date.now();
1427
+ for (const r of nsRows) {
1428
+ const ns = String(r.ns || 'default');
1429
+ ensureRow.run(ns, ns);
1430
+ setCount.run(r.c, now, ns);
1431
+ res.namespaces.push(ns);
1432
+ }
1433
+ });
1434
+ backfill();
1435
+ res.repaired = res.tableCreated || res.namespaces.length > 0;
1436
+ db.close();
1437
+ if (opts.verbose && res.repaired) {
1438
+ console.log(`vector_indexes ${res.tableCreated ? 'created' : 'refreshed'} — ` +
1439
+ `backfilled ${res.namespaces.length} namespace(s)`);
1440
+ }
1441
+ }
1442
+ catch (e) {
1443
+ try {
1444
+ db?.close();
1445
+ }
1446
+ catch { /* already closed */ }
1447
+ if (opts.verbose) {
1448
+ console.log(`vector_indexes repair skipped: ${e?.message ?? e}`);
1449
+ }
1450
+ }
1451
+ return res;
1452
+ }
1130
1453
  /**
1131
1454
  * Initialize the memory database properly using sql.js
1132
1455
  */
@@ -1155,19 +1478,24 @@ export async function initializeMemoryDatabase(options) {
1155
1478
  // surfaced an `[ERROR]` and a "Initialization failed" spinner even when
1156
1479
  // the existing DB was perfectly healthy.
1157
1480
  if (fs.existsSync(dbPath) && !force) {
1481
+ // #2568-followup: an existing DB may predate `vector_indexes` (or was
1482
+ // written by agentdb directly). Self-heal it here — this branch is hit on
1483
+ // every MCP-server start and `memory init`, so any ruflo repairs itself.
1484
+ // Idempotent + best-effort; never turns a healthy re-init into a failure.
1485
+ const heal = await repairVectorIndexes(dbPath, { verbose, autoRecover: true });
1158
1486
  return {
1159
1487
  success: true,
1160
1488
  alreadyExists: true,
1161
1489
  backend,
1162
1490
  dbPath,
1163
1491
  schemaVersion: '3.0.0',
1164
- tablesCreated: [],
1492
+ tablesCreated: heal.tableCreated ? ['vector_indexes'] : [],
1165
1493
  indexesCreated: [],
1166
1494
  features: {
1167
1495
  vectorEmbeddings: false,
1168
1496
  patternLearning: false,
1169
1497
  temporalDecay: false,
1170
- hnswIndexing: false,
1498
+ hnswIndexing: heal.repaired,
1171
1499
  migrationTracking: false
1172
1500
  }
1173
1501
  };