codesift-mcp 0.11.0 → 0.13.1

package/dist/storage/embedding-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"embedding-store.d.ts","sourceRoot":"","sources":["../../src/storage/embedding-store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAGjD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAE9D;AAOD;;;GAGG;AACH,wBAAsB,cAAc,CAClC,aAAa,EAAE,MAAM,EACrB,QAAQ,GAAE,MAAiC,GAC1C,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CA0FpC;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAClC,aAAa,EAAE,MAAM,EACrB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,GACpC,OAAO,CAAC,IAAI,CAAC,CAmCf;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,aAAa,GAClB,OAAO,CAAC,IAAI,CAAC,CAGf;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAgB/B;AAeD;;;;;;;;;;;GAWG;AACH,wBAAsB,UAAU,CAC9B,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,EACnC,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,EACjD,SAAS,EAAE,MAAM,EACjB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CA2CpC"}
+{"version":3,"file":"embedding-store.d.ts","sourceRoot":"","sources":["../../src/storage/embedding-store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAIjD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAE9D;AAkCD,4EAA4E;AAC5E,wBAAgB,oBAAoB,CAAC,aAAa,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAE/E;AAED,wBAAsB,cAAc,CAClC,aAAa,EAAE,MAAM,EACrB,QAAQ,GAAE,MAAiC,GAC1C,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAgGpC;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAClC,aAAa,EAAE,MAAM,EACrB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,EACrC,MAAM,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GACnC,OAAO,CAAC,IAAI,CAAC,CAsCf;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,aAAa,GAClB,OAAO,CAAC,IAAI,CAAC,CAGf;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAgB/B;AAeD;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAK9F;AAED,gFAAgF;AAChF,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAE9E;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,UAAU,CAC9B,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,EACnC,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,EACjD,SAAS,EAAE,MAAM,EACjB,QAAQ,CAAC,EAAE,MAAM;AACjB;;;;;;GAMG;AACH,WAAW,CAAC,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAClD,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAwEpC"}

package/dist/storage/embedding-store.js

@@ -1,4 +1,5 @@
 import { readFile } from "node:fs/promises";
+import { loadSharedCache, appendSharedCache, contentKey } from "./shared-embedding-cache.js";
 import { atomicWriteFile, cleanupOrphanTempFiles } from "./_shared.js";
 /**
  * Get the embedding file path from an index path.
@@ -18,8 +19,22 @@ export function getEmbeddingMetaPath(indexPath) {
  * Load all embeddings from an ndjson file.
  * Returns a Map of symbolId → Float32Array vector.
  */
+/**
+ * Content hashes read alongside the vectors on the last load of a given path.
+ *
+ * Keyed by path rather than repo name because that is what the loader knows.
+ * Populated as a side effect of `loadEmbeddings` — the data is already on the
+ * line, so collecting it costs nothing extra.
+ */
+const loadedContentHashes = new Map();
+/** Content hashes that came with the vectors at `embeddingPath`, if any. */
+export function contentHashesForPath(embeddingPath) {
+    return loadedContentHashes.get(embeddingPath) ?? new Map();
+}
 export async function loadEmbeddings(embeddingPath, maxBytes = Number.POSITIVE_INFINITY) {
     const embeddings = new Map();
+    const hashes = new Map();
+    loadedContentHashes.set(embeddingPath, hashes);
     // Stream line-by-line rather than readFile-ing the whole file into one string:
     // embedding files are GB-scale (e.g. 4.5GB), so a single slurp spikes the heap
     // by the full file size at load. readline keeps peak memory to one line plus
@@ -86,12 +101,17 @@ export async function loadEmbeddings(embeddingPath, maxBytes = Number.POSITIVE_I
                     if (residentBytes > maxBytes) {
                         overBudget = true;
                         embeddings.clear();
+                        // Hashes describe vectors we are dropping; keeping them would let
+                        // batchEmbed believe those symbols are still covered.
+                        hashes.clear();
                         rl.close();
                         stream.destroy();
                         resolve();
                         return;
                     }
                     embeddings.set(entry.id, vec);
+                    if (typeof entry.h === "number")
+                        hashes.set(entry.id, entry.h);
                 }
             }
             catch {
@@ -110,7 +130,7 @@ export async function loadEmbeddings(embeddingPath, maxBytes = Number.POSITIVE_I
  * Save all embeddings to an ndjson file using streaming writes.
  * Avoids building a single huge string (30K+ symbols × 1536 floats = >300MB).
  */
-export async function saveEmbeddings(embeddingPath, embeddings) {
+export async function saveEmbeddings(embeddingPath, embeddings, hashes) {
     // Sweep orphans from earlier runs that were killed mid-write. These files are
     // never overwritten (the name carries a timestamp), so without this they
     // accumulate indefinitely — see cleanupOrphanTempFiles.
@@ -125,7 +145,8 @@ export async function saveEmbeddings(embeddingPath, embeddings) {
         for (const [id, vec] of embeddings) {
             if (streamError)
                 throw streamError;
-            const line = JSON.stringify({ id, vec: Array.from(vec) }) + "\n";
+            const h = hashes?.get(id);
+            const line = JSON.stringify(h === undefined ? { id, vec: Array.from(vec) } : { id, vec: Array.from(vec), h }) + "\n";
             const canContinue = stream.write(line);
             if (!canContinue) {
                 await new Promise((resolve) => stream.once("drain", resolve));
@@ -187,6 +208,27 @@ function contentHash(text) {
 }
 /** Track content hashes so we re-embed when symbol content changes. */
 const embeddingContentHashes = new Map();
+/**
+ * Seed the in-process hash map for `cacheKey` from what was persisted.
+ *
+ * This map used to start empty on every process, so `hashes.get(id)` was always
+ * undefined, every symbol compared as changed, and the entire corpus was
+ * re-embedded on each MCP server start — 4.18M symbols across 1,587 repos, and
+ * servers restart once per session.
+ */
+export function primeContentHashes(cacheKey, hashes) {
+    if (hashes.size === 0)
+        return;
+    const target = embeddingContentHashes.get(cacheKey) ?? new Map();
+    for (const [id, h] of hashes)
+        if (!target.has(id))
+            target.set(id, h);
+    embeddingContentHashes.set(cacheKey, target);
+}
+/** Hashes currently known for `cacheKey`, for persisting beside the vectors. */
+export function contentHashesFor(cacheKey) {
+    return embeddingContentHashes.get(cacheKey) ?? new Map();
+}
 /**
  * Batch-embed symbols using the given provider, appending to existing embeddings.
  * Skips symbols whose ID exists AND content hash hasn't changed.
@@ -199,7 +241,15 @@ const embeddingContentHashes = new Map();
  * @param cacheKey - Optional key to track content hashes across calls
  * @returns Map of symbolId → Float32Array (existing + new)
  */
-export async function batchEmbed(symbolTexts, existing, embedFn, batchSize, cacheKey) {
+export async function batchEmbed(symbolTexts, existing, embedFn, batchSize, cacheKey, 
+/**
+ * Model identity for the cross-repo cache. A vector is a pure function of
+ * (model, dimensions, text), so the same text embedded for a second repo can
+ * be looked up instead of recomputed — which is what makes indexing a
+ * worktree cheap rather than a full duplicate pass. Omitted (CLI paths,
+ * tests) disables the shared lookup and behaves exactly as before.
+ */
+sharedModel) {
     const result = new Map(existing);
     const hashes = cacheKey ? (embeddingContentHashes.get(cacheKey) ?? new Map()) : new Map();
     // Find symbols that need embedding (new or content changed)
@@ -212,6 +262,29 @@ export async function batchEmbed(symbolTexts, existing, embedFn, batchSize, cach
         }
         hashes.set(id, hash);
     }
+    // Cross-repo lookup before calling the model. Half of the symbols still
+    // queued for embedding on this machine live in worktrees or temp copies whose
+    // text is byte-identical to a repo already embedded; those become a hash
+    // lookup here instead of a model call.
+    const shared = sharedModel ? await loadSharedCache() : null;
+    const freshlyEmbedded = [];
+    const stillToEmbed = [];
+    for (const item of toEmbed) {
+        if (!shared || !sharedModel) {
+            stillToEmbed.push(item);
+            continue;
+        }
+        const key = contentKey(sharedModel.model, sharedModel.dimensions, item.text);
+        const hit = shared.get(key);
+        if (hit) {
+            result.set(item.id, hit);
+        }
+        else {
+            stillToEmbed.push({ ...item, key });
+        }
+    }
+    toEmbed.length = 0;
+    toEmbed.push(...stillToEmbed);
     // Process in batches (only symbols that need embedding)
     for (let i = 0; i < toEmbed.length; i += batchSize) {
         const batch = toEmbed.slice(i, i + batchSize);
@@ -221,7 +294,11 @@ export async function batchEmbed(symbolTexts, existing, embedFn, batchSize, cach
             const entry = batch[j];
             const vec = vectors[j];
             if (entry && vec) {
-                result.set(entry.id, new Float32Array(vec));
+                const f32 = new Float32Array(vec);
+                result.set(entry.id, f32);
+                const key = entry.key;
+                if (key)
+                    freshlyEmbedded.push({ key, vec: f32 });
             }
         }
     }
@@ -234,6 +311,10 @@ export async function batchEmbed(symbolTexts, existing, embedFn, batchSize, cach
     if (cacheKey) {
         embeddingContentHashes.set(cacheKey, hashes);
     }
+    // Publish what was computed so the NEXT repo containing this text — a sibling
+    // worktree, a temp copy — gets it for free.
+    if (sharedModel)
+        appendSharedCache(freshlyEmbedded);
     return result;
 }
 //# sourceMappingURL=embedding-store.js.map

package/dist/storage/embedding-store.js.map

@@ -1 +1 @@
-{"version":3,"file":"embedding-store.js","sourceRoot":"","sources":["../../src/storage/embedding-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAC;AAEvE;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,SAAiB;IAChD,OAAO,SAAS,CAAC,OAAO,CAAC,gBAAgB,EAAE,oBAAoB,CAAC,CAAC;AACnE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAiB;IACpD,OAAO,SAAS,CAAC,OAAO,CAAC,gBAAgB,EAAE,uBAAuB,CAAC,CAAC;AACtE,CAAC;AAOD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,aAAqB,EACrB,QAAQ,GAAW,MAAM,CAAC,iBAAiB;IAE3C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwB,CAAC;IAEnD,+EAA+E;IAC/E,+EAA+E;IAC/E,6EAA6E;IAC7E,iCAAiC;IACjC,MAAM,EAAE,gBAAgB,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3E,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;IAE1D,gFAAgF;IAChF,mEAAmE;IACnE,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,UAAU,CAAC;IAElD,+EAA+E;IAC/E,gFAAgF;IAChF,gFAAgF;IAChF,iFAAiF;IACjF,uEAAuE;IACvE,EAAE;IACF,iFAAiF;IACjF,iFAAiF;IACjF,+EAA+E;IAC/E,8EAA8E;IAC9E,iFAAiF;IACjF,+EAA+E;IAC/E,6EAA6E;IAC7E,MAAM,sBAAsB,GAAG,CAAC,CAAC;IACjC,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC;QAC1C,IAAI,IAAI,GAAG,QAAQ,GAAG,sBAAsB,EAAE,CAAC;YAC7C,OAAO,CAAC,KAAK,CACX,kCAAkC,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ;gBAC/D,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,6CAA6C;gBAC3E,iDAAiD,CACpD,CAAC;YACF,OAAO,UAAU,CAAC;QACpB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,yEAAyE,CAAC,CAAC;IAErF,+EAA+E;IAC/E,0EAA0E;IAC1E,gFAAgF;IAChF,yEAAyE;IACzE,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,IAAI,MAAoC,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,GAAG,gBAAgB,CAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAClE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;YACV,OAAO;QACT,CAAC;QACD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,8CAA8C;QACnF,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QACnE,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QAChC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACrB,IAAI,UAAU;gBAAE,OAAO;YACvB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO;gBAAE,OAAO;YACrB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAkB,CAAC;gBACnD,IAAI,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzC,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBACxC,aAAa,IAAI,GAAG,CAAC,UAAU,GAAG,KAAK,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,8BAA8B;oBAC1F,IAAI,aAAa,GAAG,QAAQ,EAAE,CAAC;wBAC7B,UAAU,GAAG,IAAI,CAAC;wBAClB,UAAU,CAAC,KAAK,EAAE,CAAC;wBACnB,EAAE,CAAC,KAAK,EAAE,CAAC;wBACX,MAAM,CAAC,OAAO,EAAE,CAAC;wBACjB,OAAO,EAAE,CAAC;wBACV,OAAO;oBACT,CAAC;oBACD,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;gBAChC,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,uBAAuB;YACzB,CAAC;QACH,CAAC,CAAC,CAAC;QACH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CACX,2CAA2C,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;YAC1E,iDAAiD,CACpD,CAAC;IACJ,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,aAAqB,EACrB,UAAqC;IAErC,8EAA8E;IAC9E,yEAAyE;IACzE,wDAAwD;IACxD,MAAM,sBAAsB,CAAC,aAAa,CAAC,CAAC;IAE5C,MAAM,OAAO,GAAG,GAAG,aAAa,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IACrD,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IAEjE,uEAAuE;IACvE,IAAI,WAAW,GAAiB,IAAI,CAAC;IACrC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,GAAG,WAAW,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAEpD,IAAI,CAAC;QACH,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,WAAW;gBAAE,MAAM,WAAW,CAAC;YACnC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC;YACjE,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YACtE,CAAC;QACH,CAAC;QACD,IAAI,WAAW;YAAE,MAAM,WAAW,CAAC;QACnC,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QACH,gBAAgB;QAChB,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;QACpD,MAAM,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,8BAA8B;QAC9B,IAAI,CAAC;YAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;YAAC,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QAC1G,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAgB,EAChB,IAAmB;IAEnB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;AACxC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAExC,IACE,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI;YAC7C,OAAQ,MAAkC,CAAC,OAAO,CAAC,KAAK,QAAQ;YAChE,OAAQ,MAAkC,CAAC,YAAY,CAAC,KAAK,QAAQ,EACrE,CAAC;YACD,OAAO,MAAuB,CAAC;QACjC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,gEAAgE;AAChE,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,CAAC,GAAG,UAAU,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,CAAC;AACjB,CAAC;AAED,uEAAuE;AACvE,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAA+B,CAAC;AAEtE;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,WAAgC,EAChC,QAAmC,EACnC,OAAiD,EACjD,SAAiB,EACjB,QAAiB;IAEjB,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,EAAkB,CAAC;IAE1H,4DAA4D;IAC5D,MAAM,OAAO,GAAwC,EAAE,CAAC;IACxD,KAAK,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,UAAU,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC;QAC5F,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7B,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IACvB,CAAC;IAED,wDAAwD;IACxD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEvC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;QAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IAED,mEAAmE;IACnE,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACtE,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACpB,CAAC;IAED,IAAI,QAAQ,EAAE,CAAC;QACb,sBAAsB,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"embedding-store.js","sourceRoot":"","sources":["../../src/storage/embedding-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAC7F,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAC;AAEvE;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,SAAiB;IAChD,OAAO,SAAS,CAAC,OAAO,CAAC,gBAAgB,EAAE,oBAAoB,CAAC,CAAC;AACnE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAiB;IACpD,OAAO,SAAS,CAAC,OAAO,CAAC,gBAAgB,EAAE,uBAAuB,CAAC,CAAC;AACtE,CAAC;AAqBD;;;GAGG;AACH;;;;;;GAMG;AACH,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAA+B,CAAC;AAEnE,4EAA4E;AAC5E,MAAM,UAAU,oBAAoB,CAAC,aAAqB;IACxD,OAAO,mBAAmB,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC;AAC7E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,aAAqB,EACrB,QAAQ,GAAW,MAAM,CAAC,iBAAiB;IAE3C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwB,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,mBAAmB,CAAC,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;IAE/C,+EAA+E;IAC/E,+EAA+E;IAC/E,6EAA6E;IAC7E,iCAAiC;IACjC,MAAM,EAAE,gBAAgB,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3E,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;IAE1D,gFAAgF;IAChF,mEAAmE;IACnE,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;QAAE,OAAO,UAAU,CAAC;IAElD,+EAA+E;IAC/E,gFAAgF;IAChF,gFAAgF;IAChF,iFAAiF;IACjF,uEAAuE;IACvE,EAAE;IACF,iFAAiF;IACjF,iFAAiF;IACjF,+EAA+E;IAC/E,8EAA8E;IAC9E,iFAAiF;IACjF,+EAA+E;IAC/E,6EAA6E;IAC7E,MAAM,sBAAsB,GAAG,CAAC,CAAC;IACjC,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC;QAC1C,IAAI,IAAI,GAAG,QAAQ,GAAG,sBAAsB,EAAE,CAAC;YAC7C,OAAO,CAAC,KAAK,CACX,kCAAkC,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ;gBAC/D,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,6CAA6C;gBAC3E,iDAAiD,CACpD,CAAC;YACF,OAAO,UAAU,CAAC;QACpB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,yEAAyE,CAAC,CAAC;IAErF,+EAA+E;IAC/E,0EAA0E;IAC1E,gFAAgF;IAChF,yEAAyE;IACzE,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,IAAI,MAAoC,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,GAAG,gBAAgB,CAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAClE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;YACV,OAAO;QACT,CAAC;QACD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,8CAA8C;QACnF,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QACnE,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QAChC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACrB,IAAI,UAAU;gBAAE,OAAO;YACvB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO;gBAAE,OAAO;YACrB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAkB,CAAC;gBACnD,IAAI,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzC,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBACxC,aAAa,IAAI,GAAG,CAAC,UAAU,GAAG,KAAK,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,8BAA8B;oBAC1F,IAAI,aAAa,GAAG,QAAQ,EAAE,CAAC;wBAC7B,UAAU,GAAG,IAAI,CAAC;wBAClB,UAAU,CAAC,KAAK,EAAE,CAAC;wBACnB,kEAAkE;wBAClE,sDAAsD;wBACtD,MAAM,CAAC,KAAK,EAAE,CAAC;wBACf,EAAE,CAAC,KAAK,EAAE,CAAC;wBACX,MAAM,CAAC,OAAO,EAAE,CAAC;wBACjB,OAAO,EAAE,CAAC;wBACV,OAAO;oBACT,CAAC;oBACD,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;oBAC9B,IAAI,OAAO,KAAK,CAAC,CAAC,KAAK,QAAQ;wBAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;gBACjE,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,uBAAuB;YACzB,CAAC;QACH,CAAC,CAAC,CAAC;QACH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CACX,2CAA2C,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;YAC1E,iDAAiD,CACpD,CAAC;IACJ,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,aAAqB,EACrB,UAAqC,EACrC,MAAoC;IAEpC,8EAA8E;IAC9E,yEAAyE;IACzE,wDAAwD;IACxD,MAAM,sBAAsB,CAAC,aAAa,CAAC,CAAC;IAE5C,MAAM,OAAO,GAAG,GAAG,aAAa,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IACrD,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IAEjE,uEAAuE;IACvE,IAAI,WAAW,GAAiB,IAAI,CAAC;IACrC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,GAAG,WAAW,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAEpD,IAAI,CAAC;QACH,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,WAAW;gBAAE,MAAM,WAAW,CAAC;YACnC,MAAM,CAAC,GAAG,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CACzB,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CACjF,GAAG,IAAI,CAAC;YACT,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YACtE,CAAC;QACH,CAAC;QACD,IAAI,WAAW;YAAE,MAAM,WAAW,CAAC;QACnC,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QACH,gBAAgB;QAChB,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;QACpD,MAAM,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,8BAA8B;QAC9B,IAAI,CAAC;YAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;YAAC,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QAC1G,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAgB,EAChB,IAAmB;IAEnB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;AACxC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAExC,IACE,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI;YAC7C,OAAQ,MAAkC,CAAC,OAAO,CAAC,KAAK,QAAQ;YAChE,OAAQ,MAAkC,CAAC,YAAY,CAAC,KAAK,QAAQ,EACrE,CAAC;YACD,OAAO,MAAuB,CAAC;QACjC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,gEAAgE;AAChE,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,CAAC,GAAG,UAAU,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,CAAC;AACjB,CAAC;AAED,uEAAuE;AACvE,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAA+B,CAAC;AAEtE;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAgB,EAAE,MAAmC;IACtF,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO;IAC9B,MAAM,MAAM,GAAG,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC;IACjF,KAAK,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,MAAM;QAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACrE,sBAAsB,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;AAC/C,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,OAAO,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC;AAC3E,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,WAAgC,EAChC,QAAmC,EACnC,OAAiD,EACjD,SAAiB,EACjB,QAAiB;AACjB;;;;;;GAMG;AACH,WAAmD;IAEnD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,EAAkB,CAAC;IAE1H,4DAA4D;IAC5D,MAAM,OAAO,GAAwC,EAAE,CAAC;IACxD,KAAK,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,UAAU,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC;QAC5F,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7B,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IACvB,CAAC;IAED,wEAAwE;IACxE,8EAA8E;IAC9E,yEAAyE;IACzE,uCAAuC;IACvC,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5D,MAAM,eAAe,GAA8C,EAAE,CAAC;IACtE,MAAM,YAAY,GAAsD,EAAE,CAAC;IAC3E,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5B,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,SAAS;QACX,CAAC;QACD,MAAM,GAAG,GAAG,UAAU,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7E,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IACD,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IACnB,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;IAE9B,wDAAwD;IACxD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEvC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;QAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;gBAClC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;gBAC1B,MAAM,GAAG,GAAI,KAA0B,CAAC,GAAG,CAAC;gBAC5C,IAAI,GAAG;oBAAE,eAAe,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;IACH,CAAC;IAED,mEAAmE;IACnE,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACtE,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACpB,CAAC;IAED,IAAI,QAAQ,EAAE,CAAC;QACb,sBAAsB,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IACD,8EAA8E;IAC9E,4CAA4C;IAC5C,IAAI,WAAW;QAAE,iBAAiB,CAAC,eAAe,CAAC,CAAC;IAEpD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/storage/index-store.d.ts

@@ -1,4 +1,11 @@
 import type { CodeIndex, CodeSymbol, FileEntry } from "../types.js";
+export type IndexBackend = "json" | "sqlite";
+export declare function resolveIndexBackend(): Promise<IndexBackend>;
+export declare function resetIndexBackendForTesting(): void;
+/** `<hash>.index.json` -> `<hash>.index.db`, so both formats sit side by side and the JSON
+ *  stays available as the rollback artifact. */
+export declare function sqlitePathFor(indexPath: string): string;
+export declare function resetMigrationCacheForTesting(): void;
 /**
  * Save a code index atomically.
  * Writes to a temp file first, then renames to prevent partial reads.
@@ -15,6 +22,21 @@ export declare function saveIndex(indexPath: string, index: CodeIndex): Promise<
  * where version enforcement would cause spurious reindexes.
  */
 export declare function loadIndex(indexPath: string, currentVersions?: Record<string, string>): Promise<CodeIndex | null>;
+/**
+ * One file's `files[]` entry.
+ *
+ * This is the accessor ADR-003 exists for. `file-indexer` needs a single file's `mtime_ms`
+ * and `symbol_count` to decide whether an edit changed anything, and used to obtain them by
+ * loading the entire index — a 262 MB parse on tgm-survey-platform, paid on the first touch
+ * of every file, in a hook process that exits immediately afterwards and so can never reuse
+ * the result. Under SQLite it is one indexed row.
+ *
+ * The JSON backend keeps the old cost; there is no way to read one record out of a blob.
+ */
+export declare function getFileEntry(indexPath: string, filePath: string): Promise<FileEntry | undefined>;
+export declare function resetIndexCacheForTesting(): void;
+export declare function getIndexCacheSizeForTesting(): number;
+export declare function resetStaleRollbackWarningForTesting(): void;
 /** Discriminated union returned by loadIndexOrStale: distinguishes a healthy
  *  load from a version-mismatch stale-index case. Tools route through this
  *  helper instead of calling loadIndex directly so stale indexes surface as
@@ -23,6 +45,15 @@ export declare function loadIndex(indexPath: string, currentVersions?: Record<st
 export type IndexOrStaleResult = {
     status: "ok";
     index: CodeIndex;
+} | {
+    /** The store exists but could not be read — locked, corrupt, unreadable. Distinct from a
+     *  null return, which means "nothing is indexed here". A caller that treats this as an
+     *  empty index reports a confident, wrong "no results". */
+    status: "unreadable";
+    reason: "storage_error";
+    /** SQLITE_* / errno code, for operators and for deciding whether a retry is sane. */
+    code: string;
+    message: string;
 } | {
     status: "stale";
     reason: "extractor_version_mismatch";

package/dist/storage/index-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index-store.d.ts","sourceRoot":"","sources":["../../src/storage/index-store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAMpE;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC,CAGf;AAED;;;;;;;;;GASG;AACH,wBAAsB,SAAS,CAC7B,SAAS,EAAE,MAAM,EACjB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACvC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAiB3B;AAED;;;;4BAI4B;AAC5B,MAAM,MAAM,kBAAkB,GAC1B;IAAE,MAAM,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,SAAS,CAAA;CAAE,GAClC;IACE,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,4BAA4B,CAAC;IACrC,+EAA+E;IAC/E,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB;sFACkF;IAClF,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEN,MAAM,MAAM,2BAA2B,GAAG;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF;;qEAEqE;AACrE,wBAAgB,iCAAiC,CAC/C,KAAK,EAAE,SAAS,EAChB,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACtC,2BAA2B,EAAE,CAmC/B;AAED;;;;;;;;;;;;;;;;wCAgBwC;AACxC,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACtC,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC,CA8BpC;AAED;;;;;;;;;GASG;AACH,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,SAAS,EAChB,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACtC,OAAO,CAGT;AAyBD,wBAAgB,4BAA4B,IAAI,MAAM,CAErD;AACD,wBAAgB,8BAA8B,IAAI,IAAI,CAErD;AAoFD;;;;GAIG;AACH,wBAAsB,eAAe,CACnC,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,UAAU,EAAE,EACxB,SAAS,CAAC,EAAE,SAAS,GACpB,OAAO,CAAC,IAAI,CAAC,CAiBf;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CACvC,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAOtE"}
+{"version":3,"file":"index-store.d.ts","sourceRoot":"","sources":["../../src/storage/index-store.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAsBpE,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,QAAQ,CAAC;AAgB7C,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,YAAY,CAAC,CAGjE;AAgBD,wBAAgB,2BAA2B,IAAI,IAAI,CAElD;AAED;gDACgD;AAChD,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAIvD;AAsCD,wBAAgB,6BAA6B,IAAI,IAAI,CAEpD;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC,CASf;AAED;;;;;;;;;GASG;AACH,wBAAsB,SAAS,CAC7B,SAAS,EAAE,MAAM,EACjB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACvC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAK3B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,YAAY,CAChC,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CAQhC;AAsED,wBAAgB,yBAAyB,IAAI,IAAI,CAEhD;AAED,wBAAgB,2BAA2B,IAAI,MAAM,CAEpD;AAgED,wBAAgB,mCAAmC,IAAI,IAAI,CAE1D;AAwDD;;;;4BAI4B;AAC5B,MAAM,MAAM,kBAAkB,GAC1B;IAAE,MAAM,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,SAAS,CAAA;CAAE,GAClC;IACE;;+DAE2D;IAC3D,MAAM,EAAE,YAAY,CAAC;IACrB,MAAM,EAAE,eAAe,CAAC;IACxB,qFAAqF;IACrF,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB,GACD;IACE,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,4BAA4B,CAAC;IACrC,+EAA+E;IAC/E,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB;sFACkF;IAClF,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEN,MAAM,MAAM,2BAA2B,GAAG;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF;;qEAEqE;AACrE,wBAAgB,iCAAiC,CAC/C,KAAK,EAAE,SAAS,EAChB,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACtC,2BAA2B,EAAE,CAmC/B;AAED;;;;;;;;;;;;;;;;wCAgBwC;AACxC,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACtC,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC,CA0CpC;AAED;;;;;;;;;GASG;AACH,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,SAAS,EAChB,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACtC,OAAO,CAGT;AAyBD,wBAAgB,4BAA4B,IAAI,MAAM,CAErD;AACD,wBAAgB,8BAA8B,IAAI,IAAI,CAErD;AAoFD;;;;GAIG;AACH,wBAAsB,eAAe,CACnC,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,UAAU,EAAE,EACxB,SAAS,CAAC,EAAE,SAAS,GACpB,OAAO,CAAC,IAAI,CAAC,CA0Bf;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CACvC,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CAmBf;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAOtE"}

package/dist/storage/index-store.js

@@ -1,14 +1,100 @@
 import { readFile } from "node:fs/promises";
+import { statSync } from "node:fs";
 import { join } from "node:path";
 import { createHash } from "node:crypto";
 import { atomicWriteFile } from "./_shared.js";
+import { isSqliteAvailable, loadIndexSqlite, saveIndexSqlite, saveIncrementalSqlite, removeFileFromIndexSqlite, getFileEntrySqlite, getDataVersion, importLegacyIndexIfEmpty, classifyStorageError, IndexStorageError, } from "./sqlite-index-store.js";
 /** Serialize concurrent writes to the same index path. */
 const writeLocks = new Map();
+/**
+ * Which on-disk format to use.
+ *
+ * `CODESIFT_INDEX_BACKEND=json` pins the legacy path (the documented rollback);
+ * `=sqlite` demands the new one and fails loudly if `node:sqlite` is missing, so a CI run
+ * cannot quietly exercise the wrong backend. Unset auto-detects, which is what keeps the
+ * `engines: >=20` floor honest: Node 20 has no `node:sqlite` and simply stays on JSON.
+ *
+ * Read from the environment here rather than `config.ts` because the choice must be
+ * resolvable from the short-lived `codesift postindex-file` hook process too, which never
+ * builds a full config.
+ */
+let backendPromise;
+export async function resolveIndexBackend() {
+    backendPromise ??= computeIndexBackend();
+    return backendPromise;
+}
+async function computeIndexBackend() {
+    const explicit = process.env["CODESIFT_INDEX_BACKEND"];
+    if (explicit === "json")
+        return "json";
+    if (explicit === "sqlite") {
+        if (!(await isSqliteAvailable())) {
+            throw new Error("CODESIFT_INDEX_BACKEND=sqlite but node:sqlite is unavailable (requires Node >= 22.5)");
+        }
+        return "sqlite";
+    }
+    return (await isSqliteAvailable()) ? "sqlite" : "json";
+}
+export function resetIndexBackendForTesting() {
+    backendPromise = undefined;
+}
+/** `<hash>.index.json` -> `<hash>.index.db`, so both formats sit side by side and the JSON
+ *  stays available as the rollback artifact. */
+export function sqlitePathFor(indexPath) {
+    return indexPath.endsWith(".json")
+        ? `${indexPath.slice(0, -".json".length)}.db`
+        : `${indexPath}.db`;
+}
+/** Guards one-time JSON->SQLite migration per index path. */
+const migrations = new Map();
+/**
+ * Bring an existing JSON index across on first touch.
+ *
+ * Deliberately non-destructive: the source `.json` is left in place. Deleting it would make
+ * `CODESIFT_INDEX_BACKEND=json` a one-way door, and the whole point of shipping a rollback
+ * switch is that it still has something to roll back to.
+ */
+async function ensureSqliteMigrated(indexPath, dbPath) {
+    const inFlight = migrations.get(dbPath);
+    if (inFlight)
+        return inFlight;
+    const run = (async () => {
+        if ((await loadIndexSqlite(dbPath)) !== null)
+            return; // already migrated
+        const legacy = await loadJsonIndex(indexPath);
+        // importLegacyIndexIfEmpty re-checks emptiness under the write lock, so a second
+        // process that raced us here imports nothing instead of overwriting our rows.
+        if (legacy)
+            await importLegacyIndexIfEmpty(dbPath, legacy);
+    })();
+    migrations.set(dbPath, run);
+    try {
+        await run;
+    }
+    catch (err) {
+        // A failed migration is retried, not remembered as done — and we do NOT re-read the same
+        // failing database to decide that. Probing it again inside a `finally` would hit an
+        // already-struggling store a second time AND, if that probe throws too, JS try/finally
+        // semantics would replace the original error with the second one.
+        migrations.delete(dbPath);
+        throw err;
+    }
+    migrations.set(dbPath, Promise.resolve());
+}
+export function resetMigrationCacheForTesting() {
+    migrations.clear();
+}
 /**
  * Save a code index atomically.
  * Writes to a temp file first, then renames to prevent partial reads.
  */
 export async function saveIndex(indexPath, index) {
+    if ((await resolveIndexBackend()) === "sqlite") {
+        const dbPath = sqlitePathFor(indexPath);
+        await saveIndexSqlite(dbPath, index);
+        invalidateIndexCache(dbPath);
+        return;
+    }
     const data = JSON.stringify(index);
     await atomicWriteFile(indexPath, data);
 }
@@ -23,19 +109,187 @@ export async function saveIndex(indexPath, index) {
  * where version enforcement would cause spurious reindexes.
  */
 export async function loadIndex(indexPath, currentVersions) {
-    try {
-        const raw = await readFile(indexPath, "utf-8");
-        const parsed = JSON.parse(raw);
-        if (!isValidIndex(parsed)) {
-            return null;
+    const index = await readIndex(indexPath);
+    if (!index)
+        return null;
+    if (currentVersions && !isExtractorVersionCurrent(index, currentVersions))
+        return null;
+    return index;
+}
+/**
+ * One file's `files[]` entry.
+ *
+ * This is the accessor ADR-003 exists for. `file-indexer` needs a single file's `mtime_ms`
+ * and `symbol_count` to decide whether an edit changed anything, and used to obtain them by
+ * loading the entire index — a 262 MB parse on tgm-survey-platform, paid on the first touch
+ * of every file, in a hook process that exits immediately afterwards and so can never reuse
+ * the result. Under SQLite it is one indexed row.
+ *
+ * The JSON backend keeps the old cost; there is no way to read one record out of a blob.
+ */
+export async function getFileEntry(indexPath, filePath) {
+    if ((await resolveIndexBackend()) === "sqlite") {
+        const dbPath = sqlitePathFor(indexPath);
+        await ensureSqliteMigrated(indexPath, dbPath);
+        return getFileEntrySqlite(dbPath, filePath);
+    }
+    const index = await loadJsonIndex(indexPath);
+    return index?.files.find((f) => f.path === filePath);
+}
+/**
+ * Bounded, LRU by insertion order. A materialised index is the largest object this process
+ * holds — the JSON equivalent is 262 MB on tgm-survey-platform — and `codesift serve` is a
+ * long-lived daemon that touches many repos. An unbounded map here would have been the one
+ * cache in this codebase without a ceiling, next to an embedding cache that is explicitly
+ * RAM-budgeted (`CODESIFT_MAX_EMBEDDING_MEM_MB`); that asymmetry is how the OOM reports
+ * documented in CLAUDE.md started.
+ */
+// `|| 3` would be wrong here: an operator setting 0 to minimise memory would get 3, because
+// 0 is falsy. Only a non-numeric value should fall back.
+const MAX_CACHED_INDEXES = (() => {
+    const raw = process.env["CODESIFT_MAX_CACHED_INDEXES"];
+    const parsed = raw === undefined ? 3 : Number(raw);
+    return Math.max(1, Number.isNaN(parsed) ? 3 : parsed);
+})();
+const indexCache = new Map();
+function cacheIndex(dbPath, entry) {
+    // Re-insert to move to the most-recent end of Map iteration order.
+    indexCache.delete(dbPath);
+    indexCache.set(dbPath, entry);
+    while (indexCache.size > MAX_CACHED_INDEXES) {
+        const oldest = indexCache.keys().next();
+        if (oldest.done)
+            break;
+        indexCache.delete(oldest.value);
+    }
+}
+/**
+ * Detach the two arrays callers actually touch.
+ *
+ * A bare `{...index}` only stops a caller REASSIGNING `files`/`symbols`; an in-place
+ * `push`/`sort`/`splice` would still hit the array the cache holds — and `data_version`
+ * cannot see an in-memory mutation, so the corruption would survive every later hit. The
+ * elements stay shared (symbols are treated as immutable records); this copies the two
+ * array containers, which is the boundary that was actually being crossed.
+ */
+function copyIndex(index) {
+    return { ...index, files: [...index.files], symbols: [...index.symbols] };
+}
+function invalidateIndexCache(dbPath) {
+    indexCache.delete(dbPath);
+}
+export function resetIndexCacheForTesting() {
+    indexCache.clear();
+}
+export function getIndexCacheSizeForTesting() {
+    return indexCache.size;
+}
+/** Backend-agnostic read, without version enforcement. */
+async function readIndex(indexPath) {
+    if ((await resolveIndexBackend()) === "sqlite") {
+        const dbPath = sqlitePathFor(indexPath);
+        await ensureSqliteMigrated(indexPath, dbPath);
+        const dataVersion = await getDataVersion(dbPath);
+        const cached = indexCache.get(dbPath);
+        if (cached && cached.dataVersion === dataVersion) {
+            // Re-insert on HIT too, or the eviction order is FIFO-by-first-load rather than LRU:
+            // the hottest repo would be evicted the moment an (N+1)th repo is touched, while a
+            // repo read once and never again outlives it. That thrashes exactly the working set
+            // the cache exists to keep warm.
+            cacheIndex(dbPath, cached);
+            return copyIndex(cached.index);
         }
-        if (currentVersions && !isExtractorVersionCurrent(parsed, currentVersions)) {
+        const index = await loadIndexSqlite(dbPath);
+        if (!index)
             return null;
+        cacheIndex(dbPath, { index, dataVersion });
+        // Copy on the miss path too: returning the freshly-cached object itself would let the
+        // very first caller mutate the entry every later reader is about to be handed.
+        return copyIndex(index);
+    }
+    warnIfRollbackIsStale(indexPath);
+    return loadJsonIndex(indexPath);
+}
+/**
+ * Warn when the JSON backend is serving a snapshot that SQLite has since moved past.
+ *
+ * The rollback switch keeps the `.json` untouched, which is what makes rolling back possible
+ * — and also means that after a repo has run on SQLite for a while, the JSON is a frozen
+ * snapshot from migration day. Falling back to it then answers every query from a stale index
+ * with no error and no empty result: the shape is valid, the extractor versions still match,
+ * it is simply old. That is the worst failure mode we have (confidently wrong), so it gets a
+ * loud one-time line rather than silence.
+ */
+const staleRollbackWarned = new Set();
+function warnIfRollbackIsStale(indexPath) {
+    if (staleRollbackWarned.has(indexPath))
+        return;
+    try {
+        // Mark only once a real comparison happened. Marking up front would permanently silence
+        // the warning for a long-lived JSON-pinned process that read this repo before any `.db`
+        // existed — precisely the process that most needs telling when one appears later.
+        const dbStat = statSync(sqlitePathFor(indexPath));
+        const jsonStat = statSync(indexPath);
+        staleRollbackWarned.add(indexPath);
+        if (dbStat.mtimeMs <= jsonStat.mtimeMs)
+            return;
+        const ageHours = Math.round((dbStat.mtimeMs - jsonStat.mtimeMs) / 3_600_000);
+        console.error(`[codesift] WARNING: reading the legacy JSON index at ${indexPath}, but a SQLite index ` +
+            `updated ~${ageHours}h more recently exists alongside it. Every answer from this repo ` +
+            `is from the pre-rollback snapshot. Run index_folder to rebuild the JSON index, or ` +
+            `unset CODESIFT_INDEX_BACKEND to go back to SQLite.`);
+    }
+    catch {
+        /* no .db, or unreadable — nothing to compare against, so nothing to warn about */
+    }
+}
+export function resetStaleRollbackWarningForTesting() {
+    staleRollbackWarned.clear();
+}
+/**
+ * Codes that genuinely mean "there is no index file here". Everything else that `readFile` can
+ * fail with — EISDIR, ENOTDIR, ENAMETOOLONG, ELOOP, platform-specific I/O errors — describes a
+ * path that is wrong or unreadable, not a repo that was never indexed. Reporting those as
+ * absence is the same misdiagnosis this change exists to remove, one layer down.
+ */
+const ABSENCE_CODES = new Set(["ENOENT", "ENOTDIR_PARENT"]);
+function nonAbsenceReadCode(err) {
+    const code = typeof err === "object" && err !== null ? err.code : undefined;
+    // No recognisable code at all: this is NOT evidence of absence. Only an explicit ENOENT is.
+    // Defaulting the unknown case to "nothing indexed here" is how a real fault becomes a
+    // confident empty answer — the whole failure mode being removed.
+    if (typeof code !== "string")
+        return "UNKNOWN_READ_ERROR";
+    return ABSENCE_CODES.has(code) ? null : code;
+}
+/**
+ * The legacy path, kept intact: it is both the Node 20 backend and the rollback target.
+ *
+ * ENOENT is absence and stays `null`; a malformed document is also `null` (an invalid index is
+ * rebuildable, which is the historical contract). But EACCES/EIO/EBUSY mean the file exists and
+ * we simply could not read it — reporting that as "no index" is how a permissions problem gets
+ * misdiagnosed as an unindexed repo.
+ */
+async function loadJsonIndex(indexPath) {
+    let raw;
+    try {
+        raw = await readFile(indexPath, "utf-8");
+    }
+    catch (err) {
+        const code = classifyStorageError(err) ?? nonAbsenceReadCode(err);
+        if (code !== null) {
+            throw new IndexStorageError(`index storage at ${indexPath} is unreadable (${code}): ${err instanceof Error ? err.message : String(err)}`, code, indexPath, { cause: err });
         }
+        return null; // ENOENT and friends: nothing indexed here
+    }
+    try {
+        const parsed = JSON.parse(raw);
+        if (!isValidIndex(parsed))
+            return null;
         return parsed;
     }
     catch {
-        return null;
+        return null; // malformed JSON is a rebuildable index, not a storage fault
     }
 }
 /** All languages whose stored `extractor_version` entry does not match
@@ -96,10 +350,23 @@ export function collectExtractorVersionMismatches(index, currentVersions) {
  * to `loadIndex(...)` returning null. Callers must still handle null (no
  * structured error) for those cases. */
 export async function loadIndexOrStale(indexPath, currentVersions) {
+    let parsed;
     try {
-        const raw = await readFile(indexPath, "utf-8");
-        const parsed = JSON.parse(raw);
-        if (!isValidIndex(parsed))
+        parsed = await readIndex(indexPath);
+    }
+    catch (err) {
+        const code = classifyStorageError(err);
+        if (code === null)
+            throw err;
+        return {
+            status: "unreadable",
+            reason: "storage_error",
+            code,
+            message: err instanceof Error ? err.message : String(err),
+        };
+    }
+    try {
+        if (!parsed)
             return null;
         const mismatches = collectExtractorVersionMismatches(parsed, currentVersions);
         if (mismatches.length > 0) {
@@ -241,6 +508,15 @@ function enqueueIndexMutation(indexPath, missing, apply) {
  * Serialized per indexPath to prevent read-modify-write races.
  */
 export async function saveIncremental(indexPath, updatedFile, newSymbols, fileEntry) {
+    if ((await resolveIndexBackend()) === "sqlite") {
+        const dbPath = sqlitePathFor(indexPath);
+        await ensureSqliteMigrated(indexPath, dbPath);
+        // No mutation batching here on purpose: batching exists to amortise a whole-blob
+        // rewrite, and this backend writes one file's rows inside a transaction instead.
+        await saveIncrementalSqlite(dbPath, updatedFile, newSymbols, fileEntry);
+        invalidateIndexCache(dbPath);
+        return;
+    }
     return enqueueIndexMutation(indexPath, "throw", (existing) => {
         const filtered = existing.symbols.filter((symbol) => symbol.file !== updatedFile);
         const merged = [...filtered, ...newSymbols];
@@ -261,6 +537,13 @@ export async function saveIncremental(indexPath, updatedFile, newSymbols, fileEn
  * Serialized per indexPath to prevent read-modify-write races.
  */
 export async function removeFileFromIndex(indexPath, deletedFile) {
+    if ((await resolveIndexBackend()) === "sqlite") {
+        const dbPath = sqlitePathFor(indexPath);
+        await ensureSqliteMigrated(indexPath, dbPath);
+        await removeFileFromIndexSqlite(dbPath, deletedFile);
+        invalidateIndexCache(dbPath);
+        return;
+    }
     return enqueueIndexMutation(indexPath, "skip", (existing) => {
         const hadSymbols = existing.symbols.some((s) => s.file === deletedFile);
         const hadFile = existing.files.some((f) => f.path === deletedFile);