claude-flow 3.21.1 → 3.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/helpers/.helpers-version +1 -0
- package/.claude/helpers/auto-memory-hook.mjs +59 -274
- package/.claude/helpers/helpers.manifest.json +12 -0
- package/.claude/helpers/hook-handler.cjs +132 -112
- package/.claude/helpers/intelligence.cjs +992 -196
- package/package.json +1 -1
- package/v3/@claude-flow/cli/dist/src/commands/daemon.js +23 -2
- package/v3/@claude-flow/cli/dist/src/commands/memory-backup.d.ts +11 -0
- package/v3/@claude-flow/cli/dist/src/commands/memory-backup.js +46 -0
- package/v3/@claude-flow/cli/dist/src/commands/memory-distill.d.ts +27 -0
- package/v3/@claude-flow/cli/dist/src/commands/memory-distill.js +374 -0
- package/v3/@claude-flow/cli/dist/src/commands/memory.js +5 -2
- package/v3/@claude-flow/cli/dist/src/index.d.ts +1 -1
- package/v3/@claude-flow/cli/dist/src/index.js +22 -1
- package/v3/@claude-flow/cli/dist/src/init/executor.js +20 -0
- package/v3/@claude-flow/cli/dist/src/init/helper-refresh.d.ts +19 -0
- package/v3/@claude-flow/cli/dist/src/init/helper-refresh.js +175 -0
- package/v3/@claude-flow/cli/dist/src/init/helper-signing.d.ts +30 -0
- package/v3/@claude-flow/cli/dist/src/init/helper-signing.js +60 -0
- package/v3/@claude-flow/cli/dist/src/init/helpers-generator.d.ts +16 -0
- package/v3/@claude-flow/cli/dist/src/init/helpers-generator.js +63 -6
- package/v3/@claude-flow/cli/dist/src/init/statusline-generator.js +18 -7
- package/v3/@claude-flow/cli/dist/src/mcp-client.js +13 -0
- package/v3/@claude-flow/cli/dist/src/mcp-tools/agenticow-speculate-tools.js +9 -2
- package/v3/@claude-flow/cli/dist/src/mcp-tools/browser-intent-tools.d.ts +162 -0
- package/v3/@claude-flow/cli/dist/src/mcp-tools/browser-intent-tools.js +548 -0
- package/v3/@claude-flow/cli/dist/src/mcp-tools/browser-tools.d.ts +9 -1
- package/v3/@claude-flow/cli/dist/src/mcp-tools/browser-tools.js +4 -1
- package/v3/@claude-flow/cli/dist/src/memory/memory-bridge.js +67 -47
- package/v3/@claude-flow/cli/dist/src/memory/memory-initializer.d.ts +71 -0
- package/v3/@claude-flow/cli/dist/src/memory/memory-initializer.js +330 -2
- package/v3/@claude-flow/cli/dist/src/services/distill-tuning.d.ts +111 -0
- package/v3/@claude-flow/cli/dist/src/services/distill-tuning.js +510 -0
- package/v3/@claude-flow/cli/dist/src/services/memory-backup.d.ts +24 -0
- package/v3/@claude-flow/cli/dist/src/services/memory-backup.js +94 -0
- package/v3/@claude-flow/cli/dist/src/services/memory-distillation.d.ts +41 -0
- package/v3/@claude-flow/cli/dist/src/services/memory-distillation.js +277 -0
- package/v3/@claude-flow/cli/dist/src/services/worker-daemon.d.ts +31 -2
- package/v3/@claude-flow/cli/dist/src/services/worker-daemon.js +159 -6
- package/v3/@claude-flow/cli/package.json +5 -2
|
@@ -365,11 +365,9 @@ async function getRegistry(dbPath) {
|
|
|
365
365
|
* Replaces naive String.includes() with proper information retrieval scoring.
|
|
366
366
|
* Parameters tuned for short memory entries (k1=1.2, b=0.75).
|
|
367
367
|
*/
|
|
368
|
-
function bm25Score(queryTerms,
|
|
368
|
+
function bm25Score(queryTerms, docWords, docLength, avgDocLength, docCount, termDocFreqs) {
|
|
369
369
|
const k1 = 1.2;
|
|
370
370
|
const b = 0.75;
|
|
371
|
-
const docWords = docContent.toLowerCase().split(/\s+/);
|
|
372
|
-
const docLength = docWords.length;
|
|
373
371
|
let score = 0;
|
|
374
372
|
for (const term of queryTerms) {
|
|
375
373
|
const tf = docWords.filter(w => w === term || w.includes(term)).length;
|
|
@@ -382,23 +380,27 @@ function bm25Score(queryTerms, docContent, avgDocLength, docCount, termDocFreqs)
|
|
|
382
380
|
}
|
|
383
381
|
return score;
|
|
384
382
|
}
|
|
383
|
+
function tokenizeCorpus(rows) {
|
|
384
|
+
return rows.map(row => {
|
|
385
|
+
const contentLower = (row.content || '').toLowerCase();
|
|
386
|
+
return { contentLower, words: contentLower.split(/\s+/) };
|
|
387
|
+
});
|
|
388
|
+
}
|
|
385
389
|
/**
|
|
386
|
-
* Compute BM25 term document frequencies
|
|
390
|
+
* Compute BM25 term document frequencies over an already-tokenized corpus.
|
|
387
391
|
*/
|
|
388
|
-
function computeTermDocFreqs(queryTerms,
|
|
392
|
+
function computeTermDocFreqs(queryTerms, docs) {
|
|
389
393
|
const termDocFreqs = new Map();
|
|
390
394
|
let totalLength = 0;
|
|
391
|
-
for (const
|
|
392
|
-
|
|
393
|
-
const words = content.split(/\s+/);
|
|
394
|
-
totalLength += words.length;
|
|
395
|
+
for (const doc of docs) {
|
|
396
|
+
totalLength += doc.words.length;
|
|
395
397
|
for (const term of queryTerms) {
|
|
396
|
-
if (
|
|
398
|
+
if (doc.contentLower.includes(term)) {
|
|
397
399
|
termDocFreqs.set(term, (termDocFreqs.get(term) || 0) + 1);
|
|
398
400
|
}
|
|
399
401
|
}
|
|
400
402
|
}
|
|
401
|
-
return { termDocFreqs, avgDocLength:
|
|
403
|
+
return { termDocFreqs, avgDocLength: docs.length > 0 ? totalLength / docs.length : 1 };
|
|
402
404
|
}
|
|
403
405
|
// ===== Phase 2: TieredCache helpers =====
|
|
404
406
|
/**
|
|
@@ -484,6 +486,10 @@ async function logAttestation(registry, operation, entryId, metadata) {
|
|
|
484
486
|
// Non-fatal — attestation is observability, not correctness
|
|
485
487
|
}
|
|
486
488
|
}
|
|
489
|
+
// Tracks db handles whose schema DDL has already been ensured, so getDb()
|
|
490
|
+
// runs the CREATE…IF NOT EXISTS block at most once per handle instead of on
|
|
491
|
+
// every bridge call. WeakSet so handles GC without leaking.
|
|
492
|
+
const _schemaEnsuredDbs = new WeakSet();
|
|
487
493
|
/**
|
|
488
494
|
* Get the AgentDB database handle and ensure memory_entries table exists.
|
|
489
495
|
* Returns null if not available.
|
|
@@ -493,35 +499,42 @@ function getDb(registry) {
|
|
|
493
499
|
if (!agentdb?.database)
|
|
494
500
|
return null;
|
|
495
501
|
const db = agentdb.database;
|
|
496
|
-
// Ensure memory_entries table exists (idempotent)
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
502
|
+
// Ensure memory_entries table exists (idempotent). The DDL is run at most
|
|
503
|
+
// once per db handle — re-parsing 4× CREATE…IF NOT EXISTS on every bridge
|
|
504
|
+
// call (store/search/get) was pure per-op overhead. Keyed by handle via a
|
|
505
|
+
// WeakSet so a new db instance re-ensures without a stale global flag.
|
|
506
|
+
if (!_schemaEnsuredDbs.has(db)) {
|
|
507
|
+
try {
|
|
508
|
+
db.exec(`CREATE TABLE IF NOT EXISTS memory_entries (
|
|
509
|
+
id TEXT PRIMARY KEY,
|
|
510
|
+
key TEXT NOT NULL,
|
|
511
|
+
namespace TEXT DEFAULT 'default',
|
|
512
|
+
content TEXT NOT NULL,
|
|
513
|
+
type TEXT DEFAULT 'semantic',
|
|
514
|
+
embedding TEXT,
|
|
515
|
+
embedding_model TEXT DEFAULT 'local',
|
|
516
|
+
embedding_dimensions INTEGER,
|
|
517
|
+
tags TEXT,
|
|
518
|
+
metadata TEXT,
|
|
519
|
+
owner_id TEXT,
|
|
520
|
+
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
|
|
521
|
+
updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
|
|
522
|
+
expires_at INTEGER,
|
|
523
|
+
last_accessed_at INTEGER,
|
|
524
|
+
access_count INTEGER DEFAULT 0,
|
|
525
|
+
status TEXT DEFAULT 'active',
|
|
526
|
+
UNIQUE(namespace, key)
|
|
527
|
+
)`);
|
|
528
|
+
// Ensure indexes
|
|
529
|
+
db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_ns ON memory_entries(namespace)`);
|
|
530
|
+
db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_key ON memory_entries(key)`);
|
|
531
|
+
db.exec(`CREATE INDEX IF NOT EXISTS idx_bridge_status ON memory_entries(status)`);
|
|
532
|
+
_schemaEnsuredDbs.add(db);
|
|
533
|
+
}
|
|
534
|
+
catch {
|
|
535
|
+
// Table already exists or db is read-only — that's fine. Don't mark
|
|
536
|
+
// ensured on failure so a later writable call can retry.
|
|
537
|
+
}
|
|
525
538
|
}
|
|
526
539
|
// ─── #2256-followup: rescue agentdb.embedder when its transformers.js
|
|
527
540
|
// path fell through to mock embeddings.
|
|
@@ -637,6 +650,7 @@ export async function bridgeStoreEntry(options) {
|
|
|
637
650
|
}
|
|
638
651
|
// Generate embedding via AgentDB's embedder
|
|
639
652
|
let embeddingJson = null;
|
|
653
|
+
let embeddingArr = null;
|
|
640
654
|
let dimensions = 0;
|
|
641
655
|
let model = 'local';
|
|
642
656
|
if (options.generateEmbeddingFlag !== false && value.length > 0) {
|
|
@@ -645,7 +659,8 @@ export async function bridgeStoreEntry(options) {
|
|
|
645
659
|
if (embedder) {
|
|
646
660
|
const emb = await embedder.embed(value);
|
|
647
661
|
if (emb) {
|
|
648
|
-
|
|
662
|
+
embeddingArr = Array.from(emb);
|
|
663
|
+
embeddingJson = JSON.stringify(embeddingArr);
|
|
649
664
|
dimensions = emb.length;
|
|
650
665
|
model = 'Xenova/all-MiniLM-L6-v2';
|
|
651
666
|
}
|
|
@@ -716,7 +731,7 @@ export async function bridgeStoreEntry(options) {
|
|
|
716
731
|
success: true,
|
|
717
732
|
id,
|
|
718
733
|
embedding: embeddingJson ? { dimensions, model } : undefined,
|
|
719
|
-
rawEmbedding:
|
|
734
|
+
rawEmbedding: embeddingArr ?? undefined,
|
|
720
735
|
guarded: true,
|
|
721
736
|
cached: true,
|
|
722
737
|
attested: true,
|
|
@@ -771,12 +786,17 @@ export async function bridgeSearchEntries(options) {
|
|
|
771
786
|
catch {
|
|
772
787
|
return null;
|
|
773
788
|
}
|
|
774
|
-
// Phase 2: Compute BM25 term stats for the corpus
|
|
789
|
+
// Phase 2: Compute BM25 term stats for the corpus. Tokenize the corpus a
|
|
790
|
+
// single time and reuse the per-doc `{contentLower, words}` for term-freq,
|
|
791
|
+
// BM25 scoring, and the coverage floor below.
|
|
775
792
|
const queryTerms = queryStr.toLowerCase().split(/\s+/).filter(t => t.length > 1);
|
|
776
|
-
const
|
|
793
|
+
const docs = tokenizeCorpus(rows);
|
|
794
|
+
const { termDocFreqs, avgDocLength } = computeTermDocFreqs(queryTerms, docs);
|
|
777
795
|
const docCount = rows.length;
|
|
778
796
|
const results = [];
|
|
779
|
-
for (
|
|
797
|
+
for (let i = 0; i < rows.length; i++) {
|
|
798
|
+
const row = rows[i];
|
|
799
|
+
const doc = docs[i];
|
|
780
800
|
let semanticScore = 0;
|
|
781
801
|
let bm25ScoreVal = 0;
|
|
782
802
|
// Semantic scoring via cosine similarity
|
|
@@ -791,7 +811,7 @@ export async function bridgeSearchEntries(options) {
|
|
|
791
811
|
}
|
|
792
812
|
// Phase 2: BM25 keyword scoring (replaces String.includes fallback)
|
|
793
813
|
if (queryTerms.length > 0 && row.content) {
|
|
794
|
-
bm25ScoreVal = bm25Score(queryTerms,
|
|
814
|
+
bm25ScoreVal = bm25Score(queryTerms, doc.words, doc.words.length, avgDocLength, docCount, termDocFreqs);
|
|
795
815
|
// Normalize BM25 to 0-1 range (cap at 10 for normalization)
|
|
796
816
|
bm25ScoreVal = Math.min(bm25ScoreVal / 10, 1.0);
|
|
797
817
|
}
|
|
@@ -805,7 +825,7 @@ export async function bridgeSearchEntries(options) {
|
|
|
805
825
|
// this restores that guarantee. `coverage` is the fraction of query
|
|
806
826
|
// terms present in the document — a full-coverage hit must always be
|
|
807
827
|
// recallable regardless of IDF.
|
|
808
|
-
const contentLower =
|
|
828
|
+
const contentLower = doc.contentLower;
|
|
809
829
|
const matchedTerms = queryTerms.filter(t => contentLower.includes(t)).length;
|
|
810
830
|
const coverage = queryTerms.length > 0 ? matchedTerms / queryTerms.length : 0;
|
|
811
831
|
const lexicalScore = Math.max(bm25ScoreVal, coverage);
|
|
@@ -203,6 +203,77 @@ export declare function checkAndMigrateLegacy(options: {
|
|
|
203
203
|
migrated?: boolean;
|
|
204
204
|
migratedCount?: number;
|
|
205
205
|
}>;
|
|
206
|
+
/**
|
|
207
|
+
* Self-heal an EXISTING memory database that is missing the `vector_indexes`
|
|
208
|
+
* table or per-namespace rows.
|
|
209
|
+
*
|
|
210
|
+
* Why this exists: fresh installs create `vector_indexes` + seed rows, but a
|
|
211
|
+
* DB written by an older CLI or by agentdb directly may have thousands of
|
|
212
|
+
* embedded rows in `memory_entries` and NO `vector_indexes` table at all.
|
|
213
|
+
* Two things break as a result:
|
|
214
|
+
* 1. The statusline's vector count read collapsed to `0` (the count query
|
|
215
|
+
* referenced the missing table and failed whole — now split, but the
|
|
216
|
+
* HNSW flag still needs the table).
|
|
217
|
+
* 2. #1941 — `memory_search` routes per namespace via `vector_indexes`; a
|
|
218
|
+
* namespace with no row returns 0 results even when entries exist.
|
|
219
|
+
*
|
|
220
|
+
* This is idempotent and conservative:
|
|
221
|
+
* - Does NOTHING (no writes) when the table already exists and every embedded
|
|
222
|
+
* namespace already has a row — the common already-healed path, hit on
|
|
223
|
+
* every MCP start, must not write to the live DB unnecessarily.
|
|
224
|
+
* - Before ANY write, runs `PRAGMA quick_check`; if the DB reports structural
|
|
225
|
+
* corruption it SKIPS the repair entirely (returns `corrupt:true`) rather
|
|
226
|
+
* than writing into a malformed btree and risking making it worse. The
|
|
227
|
+
* caller/user should recover via `sqlite3 old.db .recover | sqlite3 new.db`.
|
|
228
|
+
* - Does NOT checkpoint. mode=ro readers already see committed WAL frames, and
|
|
229
|
+
* forcing a checkpoint on a DB with a torn WAL could persist latent damage.
|
|
230
|
+
*
|
|
231
|
+
* When a repair IS needed and the DB is healthy: creates the table if absent,
|
|
232
|
+
* seeds the fresh-install default rows, and backfills an accurate
|
|
233
|
+
* `total_vectors` per namespace. Runs on the existing-DB path of
|
|
234
|
+
* `initializeMemoryDatabase` (MCP start / `memory init`) and from `ruflo init`.
|
|
235
|
+
*
|
|
236
|
+
* Uses better-sqlite3 (WAL-safe, native). If the native module is unavailable
|
|
237
|
+
* it is a silent no-op — the split statusline query already prevents the count
|
|
238
|
+
* from zeroing; only the HNSW flag and namespace routing stay degraded.
|
|
239
|
+
*/
|
|
240
|
+
/**
|
|
241
|
+
* Auto-recover a structurally-corrupt memory DB into a clean one, universally
|
|
242
|
+
* (better-sqlite3 only — no dependency on the external `sqlite3` CLI, which is
|
|
243
|
+
* absent on many npx hosts). Safe by construction:
|
|
244
|
+
* 1. Confirms corruption (quick_check) — no-op on a healthy DB.
|
|
245
|
+
* 2. Acquires an EXCLUSIVE lock (BEGIN IMMEDIATE). If another process is
|
|
246
|
+
* writing, it SKIPS (returns reason:'writer-active') rather than racing a
|
|
247
|
+
* writer and losing its in-flight writes — the mistake that must not recur.
|
|
248
|
+
* 3. Rebuilds a fresh DB table-by-table (schema + rows), skipping any single
|
|
249
|
+
* table whose pages won't scan so one bad table can't abort the whole
|
|
250
|
+
* rebuild.
|
|
251
|
+
* 4. VERIFIES the rebuild (integrity_check == ok AND recovered
|
|
252
|
+
* memory_entries count >= the readable source count) BEFORE touching the
|
|
253
|
+
* original.
|
|
254
|
+
* 5. Backs up the corrupt DB to `<db>.corrupt-<ts>.bak`, then atomically
|
|
255
|
+
* renames the verified rebuild into place and drops stale -wal/-shm.
|
|
256
|
+
* On any failure the original + backup are left intact — never destructive.
|
|
257
|
+
*/
|
|
258
|
+
export declare function recoverMemoryDatabase(dbPath: string, opts?: {
|
|
259
|
+
verbose?: boolean;
|
|
260
|
+
}): Promise<{
|
|
261
|
+
recovered: boolean;
|
|
262
|
+
backupPath?: string;
|
|
263
|
+
rows?: number;
|
|
264
|
+
reason?: string;
|
|
265
|
+
}>;
|
|
266
|
+
export declare function repairVectorIndexes(dbPath: string, opts?: {
|
|
267
|
+
verbose?: boolean;
|
|
268
|
+
autoRecover?: boolean;
|
|
269
|
+
}): Promise<{
|
|
270
|
+
repaired: boolean;
|
|
271
|
+
tableCreated: boolean;
|
|
272
|
+
namespaces: string[];
|
|
273
|
+
corrupt?: boolean;
|
|
274
|
+
recovered?: boolean;
|
|
275
|
+
backupPath?: string;
|
|
276
|
+
}>;
|
|
206
277
|
/**
|
|
207
278
|
* Initialize the memory database properly using sql.js
|
|
208
279
|
*/
|
|
@@ -1127,6 +1127,329 @@ async function activateControllerRegistry(dbPath, verbose) {
|
|
|
1127
1127
|
}
|
|
1128
1128
|
return { activated, failed, initTimeMs: performance.now() - startTime };
|
|
1129
1129
|
}
|
|
1130
|
+
/**
|
|
1131
|
+
* Self-heal an EXISTING memory database that is missing the `vector_indexes`
|
|
1132
|
+
* table or per-namespace rows.
|
|
1133
|
+
*
|
|
1134
|
+
* Why this exists: fresh installs create `vector_indexes` + seed rows, but a
|
|
1135
|
+
* DB written by an older CLI or by agentdb directly may have thousands of
|
|
1136
|
+
* embedded rows in `memory_entries` and NO `vector_indexes` table at all.
|
|
1137
|
+
* Two things break as a result:
|
|
1138
|
+
* 1. The statusline's vector count read collapsed to `0` (the count query
|
|
1139
|
+
* referenced the missing table and failed whole — now split, but the
|
|
1140
|
+
* HNSW flag still needs the table).
|
|
1141
|
+
* 2. #1941 — `memory_search` routes per namespace via `vector_indexes`; a
|
|
1142
|
+
* namespace with no row returns 0 results even when entries exist.
|
|
1143
|
+
*
|
|
1144
|
+
* This is idempotent and conservative:
|
|
1145
|
+
* - Does NOTHING (no writes) when the table already exists and every embedded
|
|
1146
|
+
* namespace already has a row — the common already-healed path, hit on
|
|
1147
|
+
* every MCP start, must not write to the live DB unnecessarily.
|
|
1148
|
+
* - Before ANY write, runs `PRAGMA quick_check`; if the DB reports structural
|
|
1149
|
+
* corruption it SKIPS the repair entirely (returns `corrupt:true`) rather
|
|
1150
|
+
* than writing into a malformed btree and risking making it worse. The
|
|
1151
|
+
* caller/user should recover via `sqlite3 old.db .recover | sqlite3 new.db`.
|
|
1152
|
+
* - Does NOT checkpoint. mode=ro readers already see committed WAL frames, and
|
|
1153
|
+
* forcing a checkpoint on a DB with a torn WAL could persist latent damage.
|
|
1154
|
+
*
|
|
1155
|
+
* When a repair IS needed and the DB is healthy: creates the table if absent,
|
|
1156
|
+
* seeds the fresh-install default rows, and backfills an accurate
|
|
1157
|
+
* `total_vectors` per namespace. Runs on the existing-DB path of
|
|
1158
|
+
* `initializeMemoryDatabase` (MCP start / `memory init`) and from `ruflo init`.
|
|
1159
|
+
*
|
|
1160
|
+
* Uses better-sqlite3 (WAL-safe, native). If the native module is unavailable
|
|
1161
|
+
* it is a silent no-op — the split statusline query already prevents the count
|
|
1162
|
+
* from zeroing; only the HNSW flag and namespace routing stay degraded.
|
|
1163
|
+
*/
|
|
1164
|
+
/**
|
|
1165
|
+
* Auto-recover a structurally-corrupt memory DB into a clean one, universally
|
|
1166
|
+
* (better-sqlite3 only — no dependency on the external `sqlite3` CLI, which is
|
|
1167
|
+
* absent on many npx hosts). Safe by construction:
|
|
1168
|
+
* 1. Confirms corruption (quick_check) — no-op on a healthy DB.
|
|
1169
|
+
* 2. Acquires an EXCLUSIVE lock (BEGIN IMMEDIATE). If another process is
|
|
1170
|
+
* writing, it SKIPS (returns reason:'writer-active') rather than racing a
|
|
1171
|
+
* writer and losing its in-flight writes — the mistake that must not recur.
|
|
1172
|
+
* 3. Rebuilds a fresh DB table-by-table (schema + rows), skipping any single
|
|
1173
|
+
* table whose pages won't scan so one bad table can't abort the whole
|
|
1174
|
+
* rebuild.
|
|
1175
|
+
* 4. VERIFIES the rebuild (integrity_check == ok AND recovered
|
|
1176
|
+
* memory_entries count >= the readable source count) BEFORE touching the
|
|
1177
|
+
* original.
|
|
1178
|
+
* 5. Backs up the corrupt DB to `<db>.corrupt-<ts>.bak`, then atomically
|
|
1179
|
+
* renames the verified rebuild into place and drops stale -wal/-shm.
|
|
1180
|
+
* On any failure the original + backup are left intact — never destructive.
|
|
1181
|
+
*/
|
|
1182
|
+
export async function recoverMemoryDatabase(dbPath, opts = {}) {
|
|
1183
|
+
if (!dbPath || !fs.existsSync(dbPath))
|
|
1184
|
+
return { recovered: false, reason: 'no-db' };
|
|
1185
|
+
let Database;
|
|
1186
|
+
try {
|
|
1187
|
+
// Module name behind a variable so TS does not statically resolve the
|
|
1188
|
+
// optional native dep's types at build time (CI may not install them).
|
|
1189
|
+
const mod = 'better-sqlite3';
|
|
1190
|
+
Database = (await import(mod)).default;
|
|
1191
|
+
}
|
|
1192
|
+
catch {
|
|
1193
|
+
return { recovered: false, reason: 'no-native' };
|
|
1194
|
+
}
|
|
1195
|
+
const ts = Date.now();
|
|
1196
|
+
const tmpPath = `${dbPath}.recovering-${ts}`;
|
|
1197
|
+
const bakPath = `${dbPath}.corrupt-${ts}.bak`;
|
|
1198
|
+
let src;
|
|
1199
|
+
let dst;
|
|
1200
|
+
try {
|
|
1201
|
+
src = new Database(dbPath, { timeout: 1500 });
|
|
1202
|
+
// Confirm corruption — never rewrite a healthy DB.
|
|
1203
|
+
const qc = src.prepare('PRAGMA quick_check(1)').get();
|
|
1204
|
+
const qcVal = qc ? String(Object.values(qc)[0] ?? '') : '';
|
|
1205
|
+
if (qcVal.toLowerCase() === 'ok') {
|
|
1206
|
+
src.close();
|
|
1207
|
+
return { recovered: false, reason: 'not-corrupt' };
|
|
1208
|
+
}
|
|
1209
|
+
// Exclusive-writer guard: acquire the write lock. If busy, another process
|
|
1210
|
+
// is writing — do NOT race it. Reading within this txn is still allowed.
|
|
1211
|
+
try {
|
|
1212
|
+
src.exec('BEGIN IMMEDIATE');
|
|
1213
|
+
}
|
|
1214
|
+
catch {
|
|
1215
|
+
src.close();
|
|
1216
|
+
return { recovered: false, reason: 'writer-active' };
|
|
1217
|
+
}
|
|
1218
|
+
let srcRows = 0;
|
|
1219
|
+
try {
|
|
1220
|
+
srcRows = src.prepare('SELECT COUNT(*) AS c FROM memory_entries').get()?.c ?? 0;
|
|
1221
|
+
}
|
|
1222
|
+
catch { /* unreadable */ }
|
|
1223
|
+
try {
|
|
1224
|
+
fs.rmSync(tmpPath, { force: true });
|
|
1225
|
+
}
|
|
1226
|
+
catch { /* fresh */ }
|
|
1227
|
+
dst = new Database(tmpPath);
|
|
1228
|
+
// Copy schema: tables first (so data can be inserted), then indexes/triggers.
|
|
1229
|
+
const objects = src
|
|
1230
|
+
.prepare("SELECT type, name, sql FROM sqlite_master WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%'")
|
|
1231
|
+
.all();
|
|
1232
|
+
const tables = objects.filter(o => o.type === 'table');
|
|
1233
|
+
const others = objects.filter(o => o.type !== 'table');
|
|
1234
|
+
for (const t of tables) {
|
|
1235
|
+
try {
|
|
1236
|
+
dst.exec(t.sql);
|
|
1237
|
+
}
|
|
1238
|
+
catch { /* skip an untranslatable table def */ }
|
|
1239
|
+
}
|
|
1240
|
+
// Copy rows table-by-table; a table whose pages won't scan is skipped whole.
|
|
1241
|
+
let copiedEntries = 0;
|
|
1242
|
+
for (const t of tables) {
|
|
1243
|
+
try {
|
|
1244
|
+
const cols = dst.prepare(`PRAGMA table_info("${t.name}")`).all().map(c => c.name);
|
|
1245
|
+
if (!cols.length)
|
|
1246
|
+
continue;
|
|
1247
|
+
const colList = cols.map(c => `"${c}"`).join(',');
|
|
1248
|
+
const placeholders = cols.map(() => '?').join(',');
|
|
1249
|
+
const insert = dst.prepare(`INSERT OR IGNORE INTO "${t.name}" (${colList}) VALUES (${placeholders})`);
|
|
1250
|
+
const rows = src.prepare(`SELECT ${colList} FROM "${t.name}"`).all();
|
|
1251
|
+
const runAll = dst.transaction((rs) => {
|
|
1252
|
+
for (const r of rs)
|
|
1253
|
+
insert.run(cols.map(c => r[c]));
|
|
1254
|
+
});
|
|
1255
|
+
runAll(rows);
|
|
1256
|
+
if (t.name === 'memory_entries')
|
|
1257
|
+
copiedEntries = rows.length;
|
|
1258
|
+
}
|
|
1259
|
+
catch { /* skip a table whose data pages are unreadable */ }
|
|
1260
|
+
}
|
|
1261
|
+
for (const o of others) {
|
|
1262
|
+
try {
|
|
1263
|
+
dst.exec(o.sql);
|
|
1264
|
+
}
|
|
1265
|
+
catch { /* an index over corrupt data — non-fatal */ }
|
|
1266
|
+
}
|
|
1267
|
+
dst.close();
|
|
1268
|
+
dst = null;
|
|
1269
|
+
try {
|
|
1270
|
+
src.exec('ROLLBACK');
|
|
1271
|
+
}
|
|
1272
|
+
catch { /* ignore */ }
|
|
1273
|
+
src.close();
|
|
1274
|
+
src = null;
|
|
1275
|
+
// Verify BEFORE touching the original.
|
|
1276
|
+
const check = new Database(tmpPath, { readonly: true });
|
|
1277
|
+
const integ = String(check.pragma('integrity_check', { simple: true }) ?? '');
|
|
1278
|
+
let dstRows = 0;
|
|
1279
|
+
try {
|
|
1280
|
+
dstRows = check.prepare('SELECT COUNT(*) AS c FROM memory_entries').get()?.c ?? 0;
|
|
1281
|
+
}
|
|
1282
|
+
catch { /* */ }
|
|
1283
|
+
check.close();
|
|
1284
|
+
if (integ.toLowerCase() !== 'ok' || dstRows < srcRows) {
|
|
1285
|
+
try {
|
|
1286
|
+
fs.rmSync(tmpPath, { force: true });
|
|
1287
|
+
}
|
|
1288
|
+
catch { /* */ }
|
|
1289
|
+
if (opts.verbose) {
|
|
1290
|
+
console.log(`memory DB auto-recovery aborted (integrity=${integ}, rows ${dstRows}/${srcRows}) — original untouched`);
|
|
1291
|
+
}
|
|
1292
|
+
return { recovered: false, reason: 'verify-failed' };
|
|
1293
|
+
}
|
|
1294
|
+
// Back up the corrupt DB, then atomically swap in the verified rebuild.
|
|
1295
|
+
fs.copyFileSync(dbPath, bakPath);
|
|
1296
|
+
fs.renameSync(tmpPath, dbPath);
|
|
1297
|
+
for (const s of ['-wal', '-shm']) {
|
|
1298
|
+
try {
|
|
1299
|
+
fs.rmSync(`${dbPath}${s}`, { force: true });
|
|
1300
|
+
}
|
|
1301
|
+
catch { /* */ }
|
|
1302
|
+
}
|
|
1303
|
+
if (opts.verbose) {
|
|
1304
|
+
console.log(`memory DB auto-recovered: ${dstRows} rows, integrity ok. Corrupt original saved to ${bakPath}`);
|
|
1305
|
+
}
|
|
1306
|
+
return { recovered: true, backupPath: bakPath, rows: dstRows };
|
|
1307
|
+
}
|
|
1308
|
+
catch (e) {
|
|
1309
|
+
try {
|
|
1310
|
+
dst?.close();
|
|
1311
|
+
}
|
|
1312
|
+
catch { /* */ }
|
|
1313
|
+
try {
|
|
1314
|
+
src?.exec('ROLLBACK');
|
|
1315
|
+
}
|
|
1316
|
+
catch { /* */ }
|
|
1317
|
+
try {
|
|
1318
|
+
src?.close();
|
|
1319
|
+
}
|
|
1320
|
+
catch { /* */ }
|
|
1321
|
+
try {
|
|
1322
|
+
fs.rmSync(tmpPath, { force: true });
|
|
1323
|
+
}
|
|
1324
|
+
catch { /* */ }
|
|
1325
|
+
if (opts.verbose)
|
|
1326
|
+
console.log(`memory DB auto-recovery error: ${e?.message ?? e}`);
|
|
1327
|
+
return { recovered: false, reason: 'error' };
|
|
1328
|
+
}
|
|
1329
|
+
}
|
|
1330
|
+
export async function repairVectorIndexes(dbPath, opts = {}) {
|
|
1331
|
+
const res = { repaired: false, tableCreated: false, namespaces: [], corrupt: false };
|
|
1332
|
+
if (!dbPath || !fs.existsSync(dbPath))
|
|
1333
|
+
return res;
|
|
1334
|
+
let Database;
|
|
1335
|
+
try {
|
|
1336
|
+
// Module name behind a variable so TS does not statically resolve the
|
|
1337
|
+
// optional native dep's types at build time (CI may not install them).
|
|
1338
|
+
const mod = 'better-sqlite3';
|
|
1339
|
+
Database = (await import(mod)).default;
|
|
1340
|
+
}
|
|
1341
|
+
catch {
|
|
1342
|
+
// Native module absent (e.g. WASM-only host). Statusline fix still covers
|
|
1343
|
+
// the display; nothing to repair here.
|
|
1344
|
+
return res;
|
|
1345
|
+
}
|
|
1346
|
+
let db;
|
|
1347
|
+
try {
|
|
1348
|
+
db = new Database(dbPath, { timeout: 3000 });
|
|
1349
|
+
const tableExists = (name) => (db.prepare("SELECT COUNT(*) AS c FROM sqlite_master WHERE type='table' AND name=?").get(name)?.c ?? 0) > 0;
|
|
1350
|
+
// Nothing to key off if there is no entries table.
|
|
1351
|
+
if (!tableExists('memory_entries')) {
|
|
1352
|
+
db.close();
|
|
1353
|
+
return res;
|
|
1354
|
+
}
|
|
1355
|
+
const hasVectorIndexes = tableExists('vector_indexes');
|
|
1356
|
+
// Namespaces that actually have embeddings.
|
|
1357
|
+
const nsRows = db
|
|
1358
|
+
.prepare("SELECT COALESCE(namespace, 'default') AS ns, COUNT(*) AS c " +
|
|
1359
|
+
'FROM memory_entries WHERE embedding IS NOT NULL GROUP BY ns')
|
|
1360
|
+
.all();
|
|
1361
|
+
// Which of those are already present in vector_indexes? If the table exists
|
|
1362
|
+
// and every embedded namespace already has a row, the DB is already healed
|
|
1363
|
+
// — return WITHOUT writing anything (common path on every MCP start).
|
|
1364
|
+
let needsWrite = !hasVectorIndexes;
|
|
1365
|
+
if (hasVectorIndexes) {
|
|
1366
|
+
const present = new Set(db.prepare('SELECT name FROM vector_indexes').all().map(r => r.name));
|
|
1367
|
+
needsWrite = nsRows.some(r => !present.has(String(r.ns || 'default')));
|
|
1368
|
+
}
|
|
1369
|
+
if (!needsWrite) {
|
|
1370
|
+
db.close();
|
|
1371
|
+
return res;
|
|
1372
|
+
}
|
|
1373
|
+
// A write is needed. GUARD: never write into a structurally corrupt DB —
|
|
1374
|
+
// that risks worsening the damage. quick_check is cheaper than a full
|
|
1375
|
+
// integrity_check and only runs on the rare repair path, not every start.
|
|
1376
|
+
const qc = db.prepare('PRAGMA quick_check(1)').get();
|
|
1377
|
+
const qcVal = qc ? String(Object.values(qc)[0] ?? '') : '';
|
|
1378
|
+
if (qcVal.toLowerCase() !== 'ok') {
|
|
1379
|
+
res.corrupt = true;
|
|
1380
|
+
db.close(); // release our handle before recovery may swap the file
|
|
1381
|
+
if (opts.autoRecover) {
|
|
1382
|
+
// Auto-fix: rebuild the corrupt DB (backup + verify + atomic swap), then
|
|
1383
|
+
// provision vector_indexes on the clean rebuild. This is what makes any
|
|
1384
|
+
// npx-deployed ruflo self-repair a corrupt memory DB on init / MCP start.
|
|
1385
|
+
const rec = await recoverMemoryDatabase(dbPath, { verbose: opts.verbose });
|
|
1386
|
+
if (rec.recovered) {
|
|
1387
|
+
const healed = await repairVectorIndexes(dbPath, { verbose: opts.verbose });
|
|
1388
|
+
return { ...healed, corrupt: true, recovered: true, backupPath: rec.backupPath };
|
|
1389
|
+
}
|
|
1390
|
+
if (opts.verbose) {
|
|
1391
|
+
console.log(`vector_indexes repair skipped — corruption (${qcVal}); auto-recovery not run (${rec.reason ?? 'unknown'})`);
|
|
1392
|
+
}
|
|
1393
|
+
}
|
|
1394
|
+
else if (opts.verbose) {
|
|
1395
|
+
console.log('vector_indexes repair SKIPPED — memory DB reports corruption (' + qcVal + '). ' +
|
|
1396
|
+
'Recover with: sqlite3 <db> .recover | sqlite3 <db>.recovered');
|
|
1397
|
+
}
|
|
1398
|
+
return res;
|
|
1399
|
+
}
|
|
1400
|
+
if (!hasVectorIndexes) {
|
|
1401
|
+
db.exec(`CREATE TABLE IF NOT EXISTS vector_indexes (
|
|
1402
|
+
id TEXT PRIMARY KEY,
|
|
1403
|
+
name TEXT NOT NULL UNIQUE,
|
|
1404
|
+
dimensions INTEGER NOT NULL,
|
|
1405
|
+
metric TEXT DEFAULT 'cosine' CHECK(metric IN ('cosine', 'euclidean', 'dot')),
|
|
1406
|
+
hnsw_m INTEGER DEFAULT 16,
|
|
1407
|
+
hnsw_ef_construction INTEGER DEFAULT 200,
|
|
1408
|
+
hnsw_ef_search INTEGER DEFAULT 100,
|
|
1409
|
+
quantization_type TEXT CHECK(quantization_type IN ('none', 'scalar', 'product')),
|
|
1410
|
+
quantization_bits INTEGER DEFAULT 8,
|
|
1411
|
+
total_vectors INTEGER DEFAULT 0,
|
|
1412
|
+
last_rebuild_at INTEGER,
|
|
1413
|
+
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
|
|
1414
|
+
updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)
|
|
1415
|
+
)`);
|
|
1416
|
+
res.tableCreated = true;
|
|
1417
|
+
}
|
|
1418
|
+
// 384 = default ONNX model dim (Xenova/all-MiniLM-L6-v2); HNSW rejects
|
|
1419
|
+
// dim-mismatched inserts, so this must match the stored embeddings (#1947).
|
|
1420
|
+
const ensureRow = db.prepare('INSERT OR IGNORE INTO vector_indexes (id, name, dimensions) VALUES (?, ?, 384)');
|
|
1421
|
+
const setCount = db.prepare('UPDATE vector_indexes SET total_vectors = ?, updated_at = ? WHERE name = ?');
|
|
1422
|
+
const backfill = db.transaction(() => {
|
|
1423
|
+
// Parity with a fresh install's seed rows.
|
|
1424
|
+
ensureRow.run('default', 'default');
|
|
1425
|
+
ensureRow.run('patterns', 'patterns');
|
|
1426
|
+
const now = Date.now();
|
|
1427
|
+
for (const r of nsRows) {
|
|
1428
|
+
const ns = String(r.ns || 'default');
|
|
1429
|
+
ensureRow.run(ns, ns);
|
|
1430
|
+
setCount.run(r.c, now, ns);
|
|
1431
|
+
res.namespaces.push(ns);
|
|
1432
|
+
}
|
|
1433
|
+
});
|
|
1434
|
+
backfill();
|
|
1435
|
+
res.repaired = res.tableCreated || res.namespaces.length > 0;
|
|
1436
|
+
db.close();
|
|
1437
|
+
if (opts.verbose && res.repaired) {
|
|
1438
|
+
console.log(`vector_indexes ${res.tableCreated ? 'created' : 'refreshed'} — ` +
|
|
1439
|
+
`backfilled ${res.namespaces.length} namespace(s)`);
|
|
1440
|
+
}
|
|
1441
|
+
}
|
|
1442
|
+
catch (e) {
|
|
1443
|
+
try {
|
|
1444
|
+
db?.close();
|
|
1445
|
+
}
|
|
1446
|
+
catch { /* already closed */ }
|
|
1447
|
+
if (opts.verbose) {
|
|
1448
|
+
console.log(`vector_indexes repair skipped: ${e?.message ?? e}`);
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1451
|
+
return res;
|
|
1452
|
+
}
|
|
1130
1453
|
/**
|
|
1131
1454
|
* Initialize the memory database properly using sql.js
|
|
1132
1455
|
*/
|
|
@@ -1155,19 +1478,24 @@ export async function initializeMemoryDatabase(options) {
|
|
|
1155
1478
|
// surfaced an `[ERROR]` and a "Initialization failed" spinner even when
|
|
1156
1479
|
// the existing DB was perfectly healthy.
|
|
1157
1480
|
if (fs.existsSync(dbPath) && !force) {
|
|
1481
|
+
// #2568-followup: an existing DB may predate `vector_indexes` (or was
|
|
1482
|
+
// written by agentdb directly). Self-heal it here — this branch is hit on
|
|
1483
|
+
// every MCP-server start and `memory init`, so any ruflo repairs itself.
|
|
1484
|
+
// Idempotent + best-effort; never turns a healthy re-init into a failure.
|
|
1485
|
+
const heal = await repairVectorIndexes(dbPath, { verbose, autoRecover: true });
|
|
1158
1486
|
return {
|
|
1159
1487
|
success: true,
|
|
1160
1488
|
alreadyExists: true,
|
|
1161
1489
|
backend,
|
|
1162
1490
|
dbPath,
|
|
1163
1491
|
schemaVersion: '3.0.0',
|
|
1164
|
-
tablesCreated: [],
|
|
1492
|
+
tablesCreated: heal.tableCreated ? ['vector_indexes'] : [],
|
|
1165
1493
|
indexesCreated: [],
|
|
1166
1494
|
features: {
|
|
1167
1495
|
vectorEmbeddings: false,
|
|
1168
1496
|
patternLearning: false,
|
|
1169
1497
|
temporalDecay: false,
|
|
1170
|
-
hnswIndexing:
|
|
1498
|
+
hnswIndexing: heal.repaired,
|
|
1171
1499
|
migrationTracking: false
|
|
1172
1500
|
}
|
|
1173
1501
|
};
|