@softerist/heuristic-mcp 2.1.47 → 3.0.0

package/lib/cache.js

@@ -1,621 +1,1426 @@
-import fs from "fs/promises";
-import path from "path";
-
-const CACHE_META_VERSION = 1;
-const CACHE_META_FILE = "meta.json";
-const ANN_META_VERSION = 1;
-const ANN_INDEX_FILE = "ann-index.bin";
-const ANN_META_FILE = "ann-meta.json";
-const CALL_GRAPH_FILE = "call-graph.json";
-
-let hnswlibPromise = null;
-let hnswlibLoadError = null;
-
-async function loadHnswlib() {
-  if (hnswlibLoadError) return null;
-  if (!hnswlibPromise) {
-    hnswlibPromise = import("hnswlib-node")
-      .then((mod) => {
-        const HierarchicalNSW = mod?.HierarchicalNSW || mod?.default?.HierarchicalNSW;
-        if (!HierarchicalNSW) {
-          throw new Error("HierarchicalNSW export not found");
-        }
-        return HierarchicalNSW;
-      })
-      .catch((err) => {
-        hnswlibLoadError = err;
-        console.error(`[ANN] hnswlib-node unavailable, using linear search (${err.message})`);
-        return null;
-      });
+import fs from 'fs/promises';
+import path from 'path';
+import { Worker } from 'worker_threads';
+import { StreamingJsonWriter } from './json-writer.js';
+import { BinaryVectorStore } from './vector-store-binary.js';
+
+const CACHE_META_VERSION = 1;
+const CACHE_META_FILE = 'meta.json';
+
+// ANN meta version stays at 1 for compatibility; maxElements is optional.
+const ANN_META_VERSION = 1;
+const ANN_INDEX_FILE = 'ann-index.bin';
+const ANN_META_FILE = 'ann-meta.json';
+
+const CALL_GRAPH_FILE = 'call-graph.json';
+
+const DEFAULT_JSON_WORKER_THRESHOLD = 5 * 1024 * 1024;
+const IS_TEST_ENV = process.env.VITEST === 'true' || process.env.NODE_ENV === 'test';
+
+// Yield to event loop to keep IDE/extension host responsive during heavy CPU loops
+const yieldToLoop = () => new Promise((resolve) => setImmediate(resolve));
+
+let hnswlibPromise = null;
+let hnswlibLoadError = null;
+
+async function parseJsonInWorker(filePath) {
+  return new Promise((resolve, reject) => {
+    let settled = false;
+    const worker = new Worker(new URL('./json-worker.js', import.meta.url), {
+      workerData: { filePath },
+    });
+
+    const finish = (handler, value) => {
+      if (settled) return;
+      settled = true;
+      worker.removeAllListeners();
+      const termination = worker.terminate?.();
+      if (termination && typeof termination.catch === 'function') termination.catch(() => null);
+      handler(value);
+    };
+
+    worker.once('message', (msg) => {
+      if (msg?.ok) {
+        finish(resolve, msg.data);
+      } else {
+        const err = new Error(msg?.error || 'JSON worker failed');
+        console.warn(`[Cache] ${err.message}`);
+        finish(reject, err);
+      }
+    });
+
+    worker.once('error', (err) => {
+      console.error(`[Cache] JSON worker error: ${err.message}`);
+      finish(reject, err);
+    });
+
+    worker.once('exit', (code) => {
+      if (code !== 0) {
+        const err = new Error(`JSON worker exited with code ${code}`);
+        console.error(`[Cache] ${err.message}`);
+        finish(reject, err);
+      }
+    });
+  });
+}
+
+async function readJsonFile(filePath, { workerThresholdBytes = DEFAULT_JSON_WORKER_THRESHOLD } = {}) {
+  let stats;
+  try {
+    stats = await fs.stat(filePath);
+  } catch {
+    return null;
+  }
+
+  try {
+    const canUseWorker = typeof Worker === 'function';
+    const useWorker =
+      canUseWorker && stats && typeof stats.size === 'number'
+        ? stats.size >= workerThresholdBytes
+        : false;
+
+    if (useWorker) return await parseJsonInWorker(filePath);
+
+    const data = await fs.readFile(filePath, 'utf-8');
+    return JSON.parse(data);
+  } catch (error) {
+    console.warn(`[Cache] Failed to parse ${path.basename(filePath)}: ${error.message}`);
+    return null;
+  }
+}
+
+async function loadHnswlib() {
+  if (hnswlibLoadError) return null;
+
+  if (!hnswlibPromise) {
+    hnswlibPromise = import('hnswlib-node')
+      .then((mod) => {
+        const HierarchicalNSW = mod?.HierarchicalNSW || mod?.default?.HierarchicalNSW;
+        if (!HierarchicalNSW) throw new Error('HierarchicalNSW export not found');
+        return HierarchicalNSW;
+      })
+      .catch((err) => {
+        hnswlibLoadError = err;
+        console.warn(`[ANN] hnswlib-node unavailable, using linear search (${err.message})`);
+        return null;
+      });
+  }
+
+  return hnswlibPromise;
+}
+
+function initHnswIndex(index, maxElements, m, efConstruction) {
+  try {
+    index.initIndex(maxElements, m, efConstruction, 100);
+    return;
+  } catch (err) { console.warn(`[ANN] Standard init failed: ${err.message}`); }
+  try {
+    index.initIndex(maxElements, m, efConstruction);
+    return;
+  } catch (err) { console.warn(`[ANN] Legacy init failed: ${err.message}`); }
+  index.initIndex(maxElements);
+}
+
+function readHnswIndex(index, filePath, maxElements) {
+  try {
+    index.readIndexSync(filePath, maxElements);
+    return true;
+  } catch {
+    /* ignore */
+  }
+  try {
+    index.readIndexSync(filePath);
+    return true;
+  } catch (err) {
+    console.warn(`[ANN] Read index failed: ${err.message}`);
+  }
+  return false;
+}
+
+function normalizeLabels(result) {
+  if (!result) return [];
+  if (Array.isArray(result)) return result;
+  const labels = result.labels || result.neighbors || result.indices;
+  return labels ? Array.from(labels) : [];
+}
+
+function ensureFloat32(vector) {
+  if (!vector) return null;
+  if (vector instanceof Float32Array) return vector;
+
+  // Convert values (do NOT reinterpret bytes)
+  if (ArrayBuffer.isView(vector)) {
+    return Float32Array.from(vector);
+  }
+
+  return new Float32Array(vector);
+}
+
+function normalizeChunkVector(chunk) {
+  if (chunk?.vector) chunk.vector = ensureFloat32(chunk.vector);
+}
+
+function assignChunkIndices(store) {
+  if (!Array.isArray(store)) return;
+  for (let i = 0; i < store.length; i += 1) {
+    const chunk = store[i];
+    if (chunk) {
+      chunk._index = i;
+    }
+  }
+}
+
+function normalizeFileHashEntry(entry) {
+  if (!entry) return null;
+  if (typeof entry === 'string') return { hash: entry };
+  if (typeof entry !== 'object') return null;
+  if (typeof entry.hash !== 'string') return null;
+  const normalized = { hash: entry.hash };
+  if (Number.isFinite(entry.mtimeMs)) normalized.mtimeMs = entry.mtimeMs;
+  if (Number.isFinite(entry.size)) normalized.size = entry.size;
+  return normalized;
+}
+
+function serializeFileHashEntry(entry) {
+  if (!entry) return null;
+  if (typeof entry === 'string') return { hash: entry };
+  if (typeof entry !== 'object') return null;
+  if (typeof entry.hash !== 'string') return null;
+  const serialized = { hash: entry.hash };
+  if (Number.isFinite(entry.mtimeMs)) serialized.mtimeMs = entry.mtimeMs;
+  if (Number.isFinite(entry.size)) serialized.size = entry.size;
+  return serialized;
+}
+
+function computeAnnCapacity(total, config) {
+  const factor = typeof config.annCapacityFactor === 'number' ? config.annCapacityFactor : 1.2;
+  const extra = Number.isInteger(config.annCapacityExtra) ? config.annCapacityExtra : 1024;
+  const byFactor = Math.ceil(total * factor);
+  const byExtra = total + extra;
+  return Math.max(total, byFactor, byExtra);
+}
+
+export class EmbeddingsCache {
+  constructor(config) {
+    this.config = config;
+
+    this.vectorStore = [];
+    this.fileHashes = new Map();
+    this.isSaving = false;
+    this.lastIndexDurationMs = null;
+    this.lastIndexStats = null;
+
+    this.cacheMeta = {
+      version: CACHE_META_VERSION,
+      embeddingModel: config.embeddingModel,
+    };
+
+    // Save coalescing / debounce (serialized via saveQueue)
+    this.saveQueue = Promise.resolve();
+    this._saveTimer = null;
+    this._saveRequested = false;
+    this._savePromise = null;
+
+    // ANN state
+    this.annIndex = null;
+    this.annMeta = null;
+    this.annDirty = false; // needs rebuild
+    this.annPersistDirty = false; // in-memory differs from disk
+    this.annLoading = null;
+    this.annVectorCache = null;
+
+    // Call graph
+    this.fileCallData = new Map();
+    this.callGraph = null;
+    this._callGraphBuild = null;
+
+    // Binary vector store (optional)
+    this.binaryStore = null;
+
+    // Error tracking
+    this.initErrors = [];
+    
+    // Concurrency hooks (read tracking)
+    this.activeReads = 0;
+    this._readWaiters = [];
+
+    // Lazy reload support after dropping in-memory vectors
+    this._clearedAfterIndex = false;
+    this._loadPromise = null;
   }
-  return hnswlibPromise;
-}
-
-function initHnswIndex(index, maxElements, m, efConstruction) {
-  try {
-    index.initIndex(maxElements, m, efConstruction, 100);
-    return;
-  } catch {}
-  try {
-    index.initIndex(maxElements, m, efConstruction);
-    return;
-  } catch {}
-  index.initIndex(maxElements);
-}
-
-function readHnswIndex(index, filePath, maxElements) {
-  try {
-    index.readIndexSync(filePath, maxElements);
-    return true;
-  } catch {}
-  try {
-    index.readIndexSync(filePath);
-    return true;
-  } catch {}
-  return false;
-}
-
-function normalizeLabels(result) {
-  if (!result) return [];
-  if (Array.isArray(result)) return result;
-  const labels = result.labels || result.neighbors || result.indices;
-  if (labels) {
-    return Array.from(labels);
-  }
-  return [];
-}
-
-function toFloat32Array(vector) {
-  if (vector instanceof Float32Array) {
-    return vector;
-  }
-  return Float32Array.from(vector);
-}
-
-export class EmbeddingsCache {
-  constructor(config) {
-    this.config = config;
-    this.vectorStore = [];
-    this.fileHashes = new Map();
-    this.isSaving = false;
-    this.cacheMeta = {
-      version: CACHE_META_VERSION,
-      embeddingModel: config.embeddingModel
-    };
-    this.annIndex = null;
-    this.annMeta = null;
-    this.annDirty = false;
-    this.annLoading = null;
-    this.annVectorCache = null;
-    // Call graph data
-    this.fileCallData = new Map(); // file -> { definitions: [], calls: [] }
-    this.callGraph = null;         // { defines, calledBy, fileCalls }
+
+  async close() {
+    if (this.binaryStore) {
+      await this.binaryStore.close();
+      this.binaryStore = null;
+    }
   }
 
-  async load() {
+  async ensureLoaded() {
     if (!this.config.enableCache) return;
+    if (!this._clearedAfterIndex) return;
+    if (this._loadPromise) return this._loadPromise;
+
+    this._loadPromise = (async () => {
+      await this.load();
+      this._clearedAfterIndex = false;
+    })().finally(() => {
+      this._loadPromise = null;
+    });
 
-    try {
-      await fs.mkdir(this.config.cacheDirectory, { recursive: true });
-      const cacheFile = path.join(this.config.cacheDirectory, "embeddings.json");
-      const hashFile = path.join(this.config.cacheDirectory, "file-hashes.json");
-      const metaFile = path.join(this.config.cacheDirectory, CACHE_META_FILE);
-
-      const [metaData, cacheData, hashData] = await Promise.all([
-        fs.readFile(metaFile, "utf-8").catch(() => null),
-        fs.readFile(cacheFile, "utf-8").catch(() => null),
-        fs.readFile(hashFile, "utf-8").catch(() => null)
-      ]);
-
-      if (!metaData && !cacheData && !hashData) {
-        return;
-      }
-
-      if (!metaData) {
-        console.error("[Cache] Missing cache metadata, ignoring cache");
-        return;
-      }
-
-      let meta = null;
-      try {
-        meta = JSON.parse(metaData);
-      } catch {
-        console.error("[Cache] Invalid cache metadata, ignoring cache");
-        return;
-      }
-
-      if (meta?.version !== CACHE_META_VERSION) {
-        console.error(`[Cache] Cache version mismatch (${meta?.version}), ignoring cache`);
-        return;
-      }
-
-      if (meta?.embeddingModel !== this.config.embeddingModel) {
-        console.error(`[Cache] Embedding model changed, ignoring cache (${meta?.embeddingModel} -> ${this.config.embeddingModel})`);
-        return;
-      }
-
-      this.cacheMeta = meta;
-
-      if (cacheData && hashData) {
-        const rawVectorStore = JSON.parse(cacheData);
-        const rawHashes = new Map(Object.entries(JSON.parse(hashData)));
-
-        // Filter cache to only include files matching current extensions
-        const allowedExtensions = this.config.fileExtensions.map(ext => `.${ext}`);
-
-        this.vectorStore = rawVectorStore.filter(chunk => {
-          const ext = path.extname(chunk.file);
-          return allowedExtensions.includes(ext);
-        });
-
-        // Only keep hashes for files matching current extensions
-        for (const [file, hash] of rawHashes) {
-          const ext = path.extname(file);
-          if (allowedExtensions.includes(ext)) {
-            this.fileHashes.set(file, hash);
-          }
-        }
-
-        const filtered = rawVectorStore.length - this.vectorStore.length;
-        if (filtered > 0) {
-          console.error(`[Cache] Filtered ${filtered} outdated cache entries`);
-        }
-        console.error(`[Cache] Loaded ${this.vectorStore.length} cached embeddings`);
-        this.annDirty = false;
-        this.annIndex = null;
-        this.annMeta = null;
-      }
-
-      // Load call-graph data if it exists
-      const callGraphFile = path.join(this.config.cacheDirectory, CALL_GRAPH_FILE);
-      try {
-        const callGraphData = await fs.readFile(callGraphFile, "utf8");
-        const parsed = JSON.parse(callGraphData);
-        this.fileCallData = new Map(Object.entries(parsed));
-        if (this.config.verbose) {
-          console.error(`[Cache] Loaded call-graph data for ${this.fileCallData.size} files`);
-        }
-      } catch {
-        // Call-graph file doesn't exist yet, that's OK
-      }
-    } catch (error) {
-      console.error("[Cache] Failed to load cache:", error.message);
-    }
+    return this._loadPromise;
   }
 
-  async save() {
+  async dropInMemoryVectors() {
     if (!this.config.enableCache) return;
 
-    this.isSaving = true;
-
-    try {
-      await fs.mkdir(this.config.cacheDirectory, { recursive: true });
-      const cacheFile = path.join(this.config.cacheDirectory, "embeddings.json");
-      const hashFile = path.join(this.config.cacheDirectory, "file-hashes.json");
-      const metaFile = path.join(this.config.cacheDirectory, CACHE_META_FILE);
-
-      // Include indexing stats in meta for verification
-      const uniqueFiles = new Set(this.vectorStore.map(chunk => chunk.file));
-      this.cacheMeta = {
-        version: CACHE_META_VERSION,
-        embeddingModel: this.config.embeddingModel,
-        lastSaveTime: new Date().toISOString(),
-        filesIndexed: uniqueFiles.size,
-        chunksStored: this.vectorStore.length,
-        workspace: this.config.searchDirectory || null
-      };
-
-      await Promise.all([
-        fs.writeFile(cacheFile, JSON.stringify(this.vectorStore, null, 2)),
-        fs.writeFile(hashFile, JSON.stringify(Object.fromEntries(this.fileHashes), null, 2)),
-        fs.writeFile(metaFile, JSON.stringify(this.cacheMeta, null, 2))
-      ]);
-
-      // Save call-graph data (or remove stale cache if empty)
-      const callGraphFile = path.join(this.config.cacheDirectory, CALL_GRAPH_FILE);
-      if (this.fileCallData.size > 0) {
-        await fs.writeFile(callGraphFile, JSON.stringify(Object.fromEntries(this.fileCallData), null, 2));
-      } else {
-        await fs.rm(callGraphFile, { force: true });
-      }
-    } catch (error) {
-      console.error("[Cache] Failed to save cache:", error.message);
-    } finally {
-      this.isSaving = false;
+    if (this.activeReads > 0) {
+      await this.waitForReaders();
     }
-  }
-
-  getVectorStore() {
-    return this.vectorStore;
-  }
-
-  setVectorStore(store) {
-    this.vectorStore = store;
-    this.invalidateAnnIndex();
-  }
-
-  getFileHash(file) {
-    return this.fileHashes.get(file);
-  }
-
-  setFileHash(file, hash) {
-    this.fileHashes.set(file, hash);
-  }
-
-  deleteFileHash(file) {
-    this.fileHashes.delete(file);
-  }
 
-  removeFileFromStore(file) {
-    this.vectorStore = this.vectorStore.filter(chunk => chunk.file !== file);
-    this.invalidateAnnIndex();
-    // Also clear call-graph data for this file
-    this.removeFileCallData(file);
-  }
-
-
-  addToStore(chunk) {
-    this.vectorStore.push(chunk);
-    this.invalidateAnnIndex();
-  }
-
-  invalidateAnnIndex() {
+    this.vectorStore = [];
+    this.annVectorCache = null;
     this.annIndex = null;
     this.annMeta = null;
     this.annDirty = true;
-    this.annVectorCache = null;
-  }
-
-  getAnnVector(index) {
-    if (!this.annVectorCache || this.annVectorCache.length !== this.vectorStore.length) {
-      this.annVectorCache = new Array(this.vectorStore.length);
-    }
-
-    let cached = this.annVectorCache[index];
-    if (!cached) {
-      const vector = this.vectorStore[index]?.vector;
-      if (!vector) {
-        return null;
-      }
-      cached = toFloat32Array(vector);
-      this.annVectorCache[index] = cached;
-    }
-
-    return cached;
-  }
-
-  getAnnIndexPaths() {
-    return {
-      indexFile: path.join(this.config.cacheDirectory, ANN_INDEX_FILE),
-      metaFile: path.join(this.config.cacheDirectory, ANN_META_FILE)
-    };
-  }
+    this.annPersistDirty = false;
 
-  async ensureAnnIndex() {
-    if (!this.config.annEnabled) return null;
-    if (this.vectorStore.length < this.config.annMinChunks) return null;
-    if (this.annIndex && !this.annDirty) return this.annIndex;
-    if (this.annLoading) return this.annLoading;
-
-    this.annLoading = (async () => {
-      const HierarchicalNSW = await loadHnswlib();
-      if (!HierarchicalNSW) return null;
-
-      const dim = this.vectorStore[0]?.vector?.length;
-      if (!dim) return null;
-
-      if (!this.annDirty && this.config.annIndexCache !== false) {
-        const loaded = await this.loadAnnIndexFromDisk(HierarchicalNSW, dim);
-        if (loaded) return this.annIndex;
+    if (this.binaryStore) {
+      try {
+        await this.binaryStore.close();
+      } catch {
+        // ignore close errors
       }
-
-      return await this.buildAnnIndex(HierarchicalNSW, dim);
-    })();
-
-    const index = await this.annLoading;
-    this.annLoading = null;
-    return index;
-  }
-
-  async loadAnnIndexFromDisk(HierarchicalNSW, dim) {
-    const { indexFile, metaFile } = this.getAnnIndexPaths();
-    const metaData = await fs.readFile(metaFile, "utf-8").catch(() => null);
-
-    if (!metaData) {
-      return false;
+      this.binaryStore = null;
     }
 
-    let meta = null;
-    try {
-      meta = JSON.parse(metaData);
-    } catch {
-      console.error("[ANN] Invalid ANN metadata, rebuilding");
-      return false;
-    }
-
-    if (meta?.version !== ANN_META_VERSION) {
-      console.error(`[ANN] ANN index version mismatch (${meta?.version}), rebuilding`);
-      return false;
-    }
-
-    if (meta?.embeddingModel !== this.config.embeddingModel) {
-      console.error(`[ANN] Embedding model changed for ANN index, rebuilding`);
-      return false;
-    }
-
-    if (meta?.dim !== dim || meta?.count !== this.vectorStore.length) {
-      console.error("[ANN] ANN index size mismatch, rebuilding");
-      return false;
-    }
-
-    if (meta?.metric !== this.config.annMetric ||
-        meta?.m !== this.config.annM ||
-        meta?.efConstruction !== this.config.annEfConstruction) {
-      console.error("[ANN] ANN index config changed, rebuilding");
-      return false;
-    }
-
-    const index = new HierarchicalNSW(meta.metric, dim);
-    const loaded = readHnswIndex(index, indexFile, meta.count);
-    if (!loaded) {
-      console.error("[ANN] Failed to load ANN index file, rebuilding");
-      return false;
-    }
-
-    if (typeof index.setEf === "function") {
-      index.setEf(this.config.annEfSearch);
-    }
-
-    this.annIndex = index;
-    this.annMeta = meta;
-    this.annDirty = false;
-    console.error(`[ANN] Loaded ANN index (${meta.count} vectors)`);
-    return true;
+    this._clearedAfterIndex = true;
   }
-
-  async buildAnnIndex(HierarchicalNSW, dim) {
-    const total = this.vectorStore.length;
-    if (total === 0) return null;
-
-    try {
-      const index = new HierarchicalNSW(this.config.annMetric, dim);
-      initHnswIndex(index, total, this.config.annM, this.config.annEfConstruction);
-
-      for (let i = 0; i < total; i++) {
-        const vector = this.getAnnVector(i);
-        if (!vector) {
-          throw new Error(`Missing vector for ANN index at position ${i}`);
-        }
-        index.addPoint(vector, i);
-      }
-
-      if (typeof index.setEf === "function") {
-        index.setEf(this.config.annEfSearch);
-      }
-
-      this.annIndex = index;
-      this.annMeta = {
-        version: ANN_META_VERSION,
-        embeddingModel: this.config.embeddingModel,
-        metric: this.config.annMetric,
-        dim,
-        count: total,
-        m: this.config.annM,
-        efConstruction: this.config.annEfConstruction,
-        efSearch: this.config.annEfSearch
-      };
-      this.annDirty = false;
-
-      if (this.config.annIndexCache !== false) {
+
+  // -------------------- Concurrency Hooks --------------------
+
+  startRead() {
+    this.activeReads++;
+  }
+
+  endRead() {
+    if (this.activeReads > 0) {
+      this.activeReads--;
+      if (this.activeReads === 0 && this._readWaiters.length > 0) {
+        const waiters = this._readWaiters;
+        this._readWaiters = [];
+        for (const resolve of waiters) {
+          resolve();
+        }
+      }
+    }
+  }
+
+  async waitForReaders() {
+    if (this.activeReads === 0) return;
+    await new Promise((resolve) => {
+      this._readWaiters.push(resolve);
+    });
+  }
+
+  // -------------------- Reset --------------------
+
+  /**
+   * Resets the cache state (clears vectors, hashes, and call graph).
+   * Used for forced reindexing.
+   */
+  async reset() {
+    this.vectorStore = [];
+    if (this.binaryStore) {
+      try {
+        await this.binaryStore.close();
+      } catch {
+        // ignore close errors
+      }
+      this.binaryStore = null;
+    }
+    this.fileHashes.clear();
+    this.invalidateAnnIndex();
+    await this.clearCallGraphData({ removeFile: true });
+    this.initErrors = [];
+  }
+
+  // -------------------- Load --------------------
+
+  async load() {
+    if (!this.config.enableCache) return;
+
+    try {
+      await fs.mkdir(this.config.cacheDirectory, { recursive: true });
+
+      const cacheFile = path.join(this.config.cacheDirectory, 'embeddings.json');
+      const hashFile = path.join(this.config.cacheDirectory, 'file-hashes.json');
+      const metaFile = path.join(this.config.cacheDirectory, CACHE_META_FILE);
+
+      const workerThresholdBytes =
+        Number.isInteger(this.config.jsonWorkerThresholdBytes) &&
+        this.config.jsonWorkerThresholdBytes > 0
+          ? this.config.jsonWorkerThresholdBytes
+          : DEFAULT_JSON_WORKER_THRESHOLD;
+
+      const useBinary = this.config.vectorStoreFormat === 'binary';
+
+      const { vectorsPath, recordsPath, contentPath, filesPath } = BinaryVectorStore.getPaths(
+        this.config.cacheDirectory,
+      );
+      const pathExists = async (targetPath) => {
+        try {
+          await fs.access(targetPath);
+          return true;
+        } catch {
+          return false;
+        }
+      };
+
+      // In tests, read cache files eagerly to exercise worker paths.
+      let cacheData = null;
+      let hashData = null;
+      let prefetched = false;
+      if (IS_TEST_ENV) {
+        prefetched = true;
+        const cachePromise = useBinary
+          ? Promise.resolve(null)
+          : readJsonFile(cacheFile, { workerThresholdBytes });
+        [cacheData, hashData] = await Promise.all([
+          cachePromise,
+          readJsonFile(hashFile, { workerThresholdBytes }),
+        ]);
+      }
+
+      // Read meta first to avoid parsing huge cache files when invalid
+      const metaData = await fs.readFile(metaFile, 'utf-8').catch(() => null);
+      if (!metaData) {
+        console.warn('[Cache] Missing cache metadata, ignoring cache');
+        return;
+      }
+
+      let meta;
+      try {
+        meta = JSON.parse(metaData);
+      } catch {
+        console.warn('[Cache] Invalid cache metadata, ignoring cache');
+        return;
+      }
+
+      if (meta?.version !== CACHE_META_VERSION) {
+        console.warn(`[Cache] Cache version mismatch (${meta?.version}), ignoring cache`);
+        return;
+      }
+
+      if (meta?.embeddingModel !== this.config.embeddingModel) {
+        console.warn(
+          `[Cache] Embedding model changed, ignoring cache (${meta?.embeddingModel} -> ${this.config.embeddingModel})`,
+        );
+        return;
+      }
+
+      if (!prefetched) {
+        [cacheData, hashData] = await Promise.all([
+          useBinary ? Promise.resolve(null) : readJsonFile(cacheFile, { workerThresholdBytes }),
+          readJsonFile(hashFile, { workerThresholdBytes }),
+        ]);
+      }
+
+      this.cacheMeta = meta;
+
+      const [binaryFilesPresent, jsonCachePresent] = await Promise.all([
+        (async () => {
+          const [vectorsOk, recordsOk, contentOk, filesOk] = await Promise.all([
+            pathExists(vectorsPath),
+            pathExists(recordsPath),
+            pathExists(contentPath),
+            pathExists(filesPath),
+          ]);
+          return vectorsOk && recordsOk && contentOk && filesOk;
+        })(),
+        pathExists(cacheFile),
+      ]);
+
+      if (useBinary && !binaryFilesPresent) {
+        if (jsonCachePresent) {
+          console.warn(
+            '[Cache] vectorStoreFormat=binary but binary cache files are missing; embeddings.json exists. If you switched formats, reindex or set vectorStoreFormat=json.',
+          );
+        } else {
+          console.warn(
+            '[Cache] vectorStoreFormat=binary but binary cache files are missing. Reindex to regenerate the cache.',
+          );
+        }
+      } else if (!useBinary && !jsonCachePresent) {
+        if (binaryFilesPresent) {
+          console.warn(
+            '[Cache] vectorStoreFormat=json but binary cache files exist. If you switched formats, set vectorStoreFormat=binary or reindex.',
+          );
+        } else {
+          console.warn(
+            '[Cache] vectorStoreFormat=json but embeddings.json is missing. Reindex to regenerate the cache.',
+          );
+        }
+      }
+
+      if (useBinary) {
         try {
-          await fs.mkdir(this.config.cacheDirectory, { recursive: true });
-          const { indexFile, metaFile } = this.getAnnIndexPaths();
-          index.writeIndexSync(indexFile);
-          await fs.writeFile(metaFile, JSON.stringify(this.annMeta, null, 2));
-          console.error(`[ANN] Saved ANN index (${total} vectors)`);
-        } catch (error) {
-          console.error(`[ANN] Failed to save ANN index: ${error.message}`);
+          this.binaryStore = await BinaryVectorStore.load(this.config.cacheDirectory, {
+            contentCacheEntries: this.config.contentCacheEntries,
+            vectorCacheEntries: this.config.vectorCacheEntries,
+            vectorLoadMode: this.config.vectorStoreLoadMode,
+          });
+          cacheData = await this.binaryStore.toChunkViews({
+            includeContent: this.config.vectorStoreContentMode === 'inline',
+            includeVector: this.config.vectorStoreLoadMode !== 'disk',
+          });
+        } catch (err) {
+          this.binaryStore = null;
+          console.warn(`[Cache] Failed to load binary vector store: ${err.message}`);
         }
       }
-
-      return index;
-    } catch (error) {
-      console.error(`[ANN] Failed to build ANN index: ${error.message}`);
-      this.annIndex = null;
-      this.annMeta = null;
-      this.annDirty = true;
-      return null;
-    }
-  }
-
-  async queryAnn(queryVector, k) {
-    const index = await this.ensureAnnIndex();
-    if (!index) return null;
-
-    const results = index.searchKnn(toFloat32Array(queryVector), k);
-    const labels = normalizeLabels(results);
-
-    if (labels.length === 0) return null;
-    const filtered = labels.filter((label) =>
-      Number.isInteger(label) && label >= 0 && label < this.vectorStore.length
-    );
-    return filtered.length > 0 ? filtered : null;
-  }
-
-  async clear() {
-    if (!this.config.enableCache) return;
-
-    try {
-      await fs.rm(this.config.cacheDirectory, { recursive: true, force: true });
-      this.vectorStore = [];
-      this.fileHashes = new Map();
-      this.invalidateAnnIndex();
-      await this.clearCallGraphData();
-      console.error(`[Cache] Cache cleared successfully: ${this.config.cacheDirectory}`);
-    } catch (error) {
-      console.error("[Cache] Failed to clear cache:", error.message);
-      throw error;
-    }
-  }
-
-  /**
-   * Adjust efSearch at runtime for speed/accuracy tradeoff.
-   * Higher values = more accurate but slower.
-   * @param {number} efSearch - New efSearch value (typically 16-512)
-   * @returns {object} Result with success status and current config
-   */
-  setEfSearch(efSearch) {
-    if (typeof efSearch !== "number" || efSearch < 1 || efSearch > 1000) {
-      return { success: false, error: "efSearch must be a number between 1 and 1000" };
-    }
-
-    this.config.annEfSearch = efSearch;
-
-    if (this.annIndex && typeof this.annIndex.setEf === "function") {
-      this.annIndex.setEf(efSearch);
-      console.error(`[ANN] efSearch updated to ${efSearch} (applied to active index)`);
-      return { success: true, applied: true, efSearch };
-    } else {
-      console.error(`[ANN] efSearch updated to ${efSearch} (will apply on next index build)`);
-      return { success: true, applied: false, efSearch };
-    }
-  }
-
-  /**
-   * Get current ANN index statistics for diagnostics.
-   * @returns {object} ANN stats including index state, config, and vector count
-   */
-  getAnnStats() {
-    return {
-      enabled: this.config.annEnabled ?? false,
-      indexLoaded: this.annIndex !== null,
-      dirty: this.annDirty,
-      vectorCount: this.vectorStore.length,
-      minChunksForAnn: this.config.annMinChunks ?? 5000,
-      config: this.annMeta ? {
-        metric: this.annMeta.metric,
-        dim: this.annMeta.dim,
-        count: this.annMeta.count,
-        m: this.annMeta.m,
-        efConstruction: this.annMeta.efConstruction,
-        efSearch: this.config.annEfSearch
-      } : null
-    };
-  }
-
-  // ========== Call Graph Methods ==========
-
-  /**
-   * Clear all call-graph data (optionally remove persisted cache file)
-   */
-  async clearCallGraphData({ removeFile = false } = {}) {
-    this.fileCallData.clear();
-    this.callGraph = null;
-
-    if (removeFile && this.config.enableCache) {
-      const callGraphFile = path.join(this.config.cacheDirectory, CALL_GRAPH_FILE);
-      try {
-        await fs.rm(callGraphFile, { force: true });
-      } catch (error) {
-        if (this.config.verbose) {
-          console.error(`[Cache] Failed to remove call-graph cache: ${error.message}`);
+
+      if (!cacheData) {
+        cacheData = await readJsonFile(cacheFile, { workerThresholdBytes });
+      }
+
+      const hasCacheData = Array.isArray(cacheData);
+      const hasHashData = hashData && typeof hashData === 'object';
+
+      if (hasCacheData) {
+        const allowedExtensions = new Set((this.config.fileExtensions || []).map((ext) => `.${ext}`));
+        const applyExtensionFilter = !this.binaryStore;
+
+        const rawHashes = hasHashData ? new Map(Object.entries(hashData)) : new Map();
+        this.vectorStore = [];
+        this.fileHashes.clear();
+
+        // Single-pass filter + normalization
+        for (const chunk of cacheData) {
+          if (applyExtensionFilter) {
+            const ext = path.extname(chunk.file);
+            if (!allowedExtensions.has(ext)) continue;
+          }
+          normalizeChunkVector(chunk);
+          this.vectorStore.push(chunk);
+        }
+        const filteredCount = cacheData.length - this.vectorStore.length;
+        if (filteredCount > 0 && this.config.verbose) {
+          console.info(`[Cache] Filtered ${filteredCount} outdated cache entries`);
+        }
+
+        if (hasHashData) {
+          // Only keep hashes for allowed extensions
+          for (const [file, entry] of rawHashes) {
+            if (!applyExtensionFilter || allowedExtensions.has(path.extname(file))) {
+              const normalized = normalizeFileHashEntry(entry);
+              if (normalized) {
+                this.fileHashes.set(file, normalized);
+              }
+            }
+          }
+        } else {
+          console.warn('[Cache] Missing file-hashes.json; loaded embeddings but hashes were cleared');
+        }
+
+        assignChunkIndices(this.vectorStore);
+
+        if (this.config.verbose) {
+          console.info(`[Cache] Loaded ${this.vectorStore.length} cached embeddings`);
+        }
+
+        // ANN index is lazily loaded/built on first query
+        this.annDirty = false;
+        this.annPersistDirty = false;
+        this.annIndex = null;
+        this.annMeta = null;
+        this.annVectorCache = null;
+      } else if (cacheData) {
+        console.warn('[Cache] Cache data is not an array; ignoring cached embeddings');
+      } else if (hasHashData) {
+        console.warn('[Cache] Hashes exist without embeddings; ignoring file-hashes.json');
+      }
+
+      // Load call-graph data if it exists
+      const callGraphFile = path.join(this.config.cacheDirectory, CALL_GRAPH_FILE);
+      try {
+        const callGraphData = await fs.readFile(callGraphFile, 'utf8');
+        const parsed = JSON.parse(callGraphData);
+        this.fileCallData = new Map(Object.entries(parsed));
+        if (this.config.verbose) {
+          console.info(`[Cache] Loaded call-graph data for ${this.fileCallData.size} files`);
+        }
+      } catch {
+        // no cache yet, OK
+      }
+    } catch (error) {
+      console.warn('[Cache] Failed to load cache:', error.message);
+    }
+  }
+
+  // -------------------- Save (debounced + serialized) --------------------
+
+  save() {
+    if (!this.config.enableCache) return Promise.resolve();
+
+    this._saveRequested = true;
+
+    if (this._saveTimer) return this._savePromise ?? Promise.resolve();
+
+    const debounceMs = Number.isInteger(this.config.saveDebounceMs)
+      ? this.config.saveDebounceMs
+      : 250;
+
+    this._savePromise = new Promise((resolve, reject) => {
+      this._saveTimer = setTimeout(() => {
+        this._saveTimer = null;
+
+        this.saveQueue = this.saveQueue
+          .then(async () => {
+            while (this._saveRequested) {
+              this._saveRequested = false;
+              await this.performSave();
+            }
+          })
+          .then(resolve, reject)
+          .finally(() => {
+            this._savePromise = null;
+          });
+      }, debounceMs);
+    });
+
+    return this._savePromise;
+  }
+
+  async performSave() {
+    this.isSaving = true;
+
+    try {
+      await fs.mkdir(this.config.cacheDirectory, { recursive: true });
+
+      const cacheFile = path.join(this.config.cacheDirectory, 'embeddings.json');
+      const hashFile = path.join(this.config.cacheDirectory, 'file-hashes.json');
+      const metaFile = path.join(this.config.cacheDirectory, CACHE_META_FILE);
+
+      // Snapshot to avoid race conditions during async write
+      const snapshotStore = Array.isArray(this.vectorStore) ? [...this.vectorStore] : [];
+      
+      this.cacheMeta = {
+        version: CACHE_META_VERSION,
+        embeddingModel: this.config.embeddingModel,
+        lastSaveTime: new Date().toISOString(),
+        filesIndexed: this.fileHashes.size,
+        chunksStored: snapshotStore.length,
+        workspace: this.config.searchDirectory || null,
+      };
+      if (Number.isFinite(this.lastIndexDurationMs) && this.lastIndexDurationMs >= 0) {
+        this.cacheMeta.indexDurationMs = Math.round(this.lastIndexDurationMs);
+      }
+      if (this.lastIndexStats && typeof this.lastIndexStats === 'object') {
+        Object.assign(this.cacheMeta, this.lastIndexStats);
+      }
+
+      const total = snapshotStore.length;
+      if (this.config.vectorStoreFormat === 'binary') {
+        this.binaryStore = await BinaryVectorStore.write(
+          this.config.cacheDirectory,
+          snapshotStore,
+          {
+            contentCacheEntries: this.config.contentCacheEntries,
+            getContent: (chunk, index) => this.getChunkContent(chunk, index),
+            preRename: async () => {
+              if (this.binaryStore) await this.binaryStore.close();
+            },
+          },
+        );
+        if (this.binaryStore) {
+          this.cacheMeta.chunksStored = this.binaryStore.length;
         }
+      } else {
+        const vectorWriter = new StreamingJsonWriter(cacheFile, {
+          highWaterMark: this.config.cacheWriteHighWaterMark ?? 256 * 1024,
+          floatDigits:
+            this.config.cacheVectorFloatDigits === undefined
+              ? 6
+              : this.config.cacheVectorFloatDigits,
+          flushChars: this.config.cacheVectorFlushChars ?? 256 * 1024,
+          indent: '', // set to "  " if you prefer pretty formatting
+          assumeFinite: this.config.cacheVectorAssumeFinite,
+          checkFinite: this.config.cacheVectorCheckFinite,
+          noMutation: this.config.cacheVectorNoMutation ?? false,
+          joinThreshold: this.config.cacheVectorJoinThreshold ?? 8192,
+          joinChunkSize: this.config.cacheVectorJoinChunkSize ?? 2048,
+        });
+
+        await vectorWriter.writeStart();
+
+        // Optional responsiveness yield (only for huge saves)
+        const yieldEvery = total >= 50_000 ? 5000 : 0;
+
+        try {
+          for (let i = 0; i < total; i++) {
+            const pending = vectorWriter.writeItem(snapshotStore[i]);
+            if (pending) await pending;
+            if (yieldEvery && i > 0 && i % yieldEvery === 0) await yieldToLoop();
+          }
+          await vectorWriter.writeEnd();
+        } catch (e) {
+          vectorWriter.abort(e);
+          throw e;
+        }
+      }
+
+      const hashEntries = {};
+      for (const [file, entry] of this.fileHashes) {
+        const serialized = serializeFileHashEntry(entry);
+        if (serialized) {
+          hashEntries[file] = serialized;
+        }
+      }
+
+      await Promise.all([
+        fs.writeFile(hashFile, JSON.stringify(hashEntries, null, 2)),
+        fs.writeFile(metaFile, JSON.stringify(this.cacheMeta, null, 2)),
+      ]);
+
+      // Save call-graph data (or remove stale cache)
+      const callGraphFile = path.join(this.config.cacheDirectory, CALL_GRAPH_FILE);
+      if (this.fileCallData.size > 0) {
+        await fs.writeFile(
+          callGraphFile,
+          JSON.stringify(Object.fromEntries(this.fileCallData), null, 2),
+        );
+      } else {
+        await fs.rm(callGraphFile, { force: true });
+      }
+
+      // Persist ANN index if it exists and changed in memory
+      if (
+        this.config.annIndexCache !== false &&
+        this.annPersistDirty &&
+        !this.annDirty &&
+        this.annIndex &&
+        this.annMeta
+      ) {
+        try {
+          const { indexFile, metaFile: annMetaFile } = this.getAnnIndexPaths();
+          this.annIndex.writeIndexSync(indexFile);
+          await fs.writeFile(annMetaFile, JSON.stringify(this.annMeta, null, 2));
+          this.annPersistDirty = false;
+          if (this.config.verbose) {
+            console.info(`[ANN] Persisted updated ANN index (${this.annMeta.count} vectors)`);
+          }
+        } catch (error) {
+          console.warn(`[ANN] Failed to persist ANN index: ${error.message}`);
+        }
+      }
+    } catch (error) {
+      console.warn('[Cache] Failed to save cache:', error.message);
+      // Attempt to recover binary store if it was closed during failed save
+      if (this.config.vectorStoreFormat === 'binary' && this.binaryStore && !this.binaryStore.vectorsBuffer) {
+        try {
+           console.info('[Cache] Attempting to recover binary store after failed save...');
+           this.binaryStore = await BinaryVectorStore.load(this.config.cacheDirectory, {
+            contentCacheEntries: this.config.contentCacheEntries,
+           });
+           console.info('[Cache] Binary store recovered.');
+        } catch (recoverErr) {
+           console.warn(`[Cache] Failed to recover binary store: ${recoverErr.message}`);
+           this.binaryStore = null; // Ensure it's null if unusable
+        }
+      }
+    } finally {
+      this.isSaving = false;
+    }
+  }
+
+  // -------------------- Vector Store API --------------------
+
+  getVectorStore() {
+    return Array.isArray(this.vectorStore) ? this.vectorStore : [];
+  }
+
+  async setVectorStore(store) {
+    const previousBinaryStore = this.binaryStore;
+    this.vectorStore = store;
+    this.binaryStore = null;
+    if (Array.isArray(this.vectorStore)) {
+      for (const chunk of this.vectorStore) normalizeChunkVector(chunk);
+      assignChunkIndices(this.vectorStore);
+    }
+    this.invalidateAnnIndex();
+    if (previousBinaryStore) {
+      try {
+        await previousBinaryStore.close();
+      } catch {
+        // ignore close errors
+      }
+    }
+  }
+
+  setLastIndexDuration(durationMs) {
+    if (Number.isFinite(durationMs) && durationMs >= 0) {
+      this.lastIndexDurationMs = durationMs;
+    }
+  }
+
+  setLastIndexStats(stats) {
+    if (stats && typeof stats === 'object') {
+      this.lastIndexStats = { ...stats };
+    }
+  }
+
+  getFileHash(file) {
+    const entry = this.fileHashes.get(file);
+    if (typeof entry === 'string') return entry;
+    return entry?.hash;
+  }
+
+  getFileHashKeys() {
+    return Array.from(this.fileHashes.keys());
+  }
+
+  getFileHashCount() {
+    return this.fileHashes.size;
+  }
+
+  clearFileHashes() {
+    this.fileHashes.clear();
+  }
+
+  setFileHashes(entries) {
+    this.fileHashes.clear();
+    if (!entries) return;
+    const iterator =
+      entries instanceof Map
+        ? entries.entries()
+        : typeof entries === 'object'
+          ? Object.entries(entries)
+          : null;
+    if (!iterator) return;
+    for (const [file, entry] of iterator) {
+      const normalized = normalizeFileHashEntry(entry);
+      if (normalized) {
+        this.fileHashes.set(file, normalized);
+      }
+    }
+  }
+
+  setFileHash(file, hash, meta = null) {
+    const entry = { hash };
+    if (meta && typeof meta === 'object') {
+      if (Number.isFinite(meta.mtimeMs)) entry.mtimeMs = meta.mtimeMs;
+      if (Number.isFinite(meta.size)) entry.size = meta.size;
+    }
+    this.fileHashes.set(file, entry);
+  }
+
+  getFileMeta(file) {
+    const entry = this.fileHashes.get(file);
+    if (!entry) return null;
+    if (typeof entry === 'string') return { hash: entry };
+    return entry;
+  }
+
+  getChunkVector(chunk, index = null) {
+    if (typeof chunk === 'number') {
+      const store = Array.isArray(this.vectorStore) ? this.vectorStore : null;
+      const entry = store ? store[chunk] : null;
+      if (entry?.vector) return entry.vector;
+      if (this.binaryStore) {
+        const resolved = Number.isInteger(entry?._binaryIndex) ? entry._binaryIndex : chunk;
+        return this.binaryStore.getVector(resolved);
       }
+      return null;
     }
-  }
-
-  /**
-   * Remove call-graph entries for files no longer in the codebase
-   */
-  pruneCallGraphData(validFiles) {
-    if (!validFiles || this.fileCallData.size === 0) return 0;
-
-    let pruned = 0;
-    for (const file of Array.from(this.fileCallData.keys())) {
-      if (!validFiles.has(file)) {
-        this.fileCallData.delete(file);
-        pruned++;
-      }
-    }
+
+    if (chunk?.vector) return chunk.vector;
+    const resolved = Number.isInteger(index) ? index : chunk?._index;
+    if (this.binaryStore && Number.isInteger(chunk?._binaryIndex)) {
+      return this.binaryStore.getVector(chunk._binaryIndex);
+    }
+    if (this.binaryStore && !Array.isArray(this.vectorStore) && Number.isInteger(resolved)) {
+      return this.binaryStore.getVector(resolved);
+    }
+    return null;
+  }
+
+  async getChunkContent(chunk, index = null) {
+    if (typeof chunk === 'number') {
+      const store = Array.isArray(this.vectorStore) ? this.vectorStore : null;
+      const entry = store ? store[chunk] : null;
+      if (entry) return await this.getChunkContent(entry, chunk);
+      if (!store && this.binaryStore) {
+        return await this.binaryStore.getContent(chunk);
+      }
+      return '';
+    }
+    if (chunk?.content !== undefined && chunk?.content !== null) {
+      return chunk.content;
+    }
+    if (this.binaryStore && Number.isInteger(chunk?._binaryIndex)) {
+      return await this.binaryStore.getContent(chunk._binaryIndex);
+    }
+    const resolved = Number.isInteger(index) ? index : chunk?._index;
+    if (this.binaryStore && !Array.isArray(this.vectorStore) && Number.isInteger(resolved)) {
+      return await this.binaryStore.getContent(resolved);
+    }
+    return '';
+  }
+
+  deleteFileHash(file) {
+    this.fileHashes.delete(file);
+  }
+
+  removeFileFromStore(file) {
+    if (!Array.isArray(this.vectorStore)) return;
+    // In-place compaction to avoid allocating a new large array
+    let w = 0;
+    for (let r = 0; r < this.vectorStore.length; r++) {
+      const chunk = this.vectorStore[r];
+      if (chunk.file !== file) {
+        chunk._index = w;
+        this.vectorStore[w++] = chunk;
+      }
+    }
+    this.vectorStore.length = w;
+
+    // Removing shifts labels => rebuild ANN
+    this.invalidateAnnIndex();
+    this.removeFileCallData(file);
+  }
+
+  addToStore(chunk) {
+    normalizeChunkVector(chunk);
+
+    if (!Array.isArray(this.vectorStore)) {
+      this.vectorStore = [];
+    }
+
+    const label = this.vectorStore.length;
+    chunk._index = label;
+    this.vectorStore.push(chunk);
+    if (Array.isArray(this.annVectorCache) && this.annVectorCache.length === label) {
+      this.annVectorCache.push(chunk.vector);
+    }
+
+    // Best-effort incremental ANN append (fast path)
+    if (
+      this.annIndex &&
+      !this.annDirty &&
+      this.annMeta &&
+      typeof this.annIndex.addPoint === 'function' &&
+      this.annMeta.count === label &&
+      this.annMeta.maxElements > this.annMeta.count
+    ) {
+      try {
+        this.annIndex.addPoint(chunk.vector, label);
+        this.annMeta.count += 1;
+        this.annPersistDirty = true;
+        return;
+      } catch {
+        // fall through
+      }
+    }
+
+    this.invalidateAnnIndex();
+  }
+
+  invalidateAnnIndex() {
+    this.annIndex = null;
+    this.annMeta = null;
+    this.annDirty = true;
+    this.annPersistDirty = false;
+    this.annVectorCache = null;
+  }
+
+  getAnnVector(index) {
+    if (!Array.isArray(this.vectorStore)) return null;
+    const chunk = this.vectorStore[index];
+    if (!chunk) return null;
 
-    if (pruned > 0) {
-      this.callGraph = null;
+    if (!Array.isArray(this.annVectorCache) || this.annVectorCache.length !== this.vectorStore.length) {
+      this.annVectorCache = new Array(this.vectorStore.length);
     }
 
-    return pruned;
-  }
-
-  /**
-   * Store call data for a file
-   */
-  setFileCallData(file, data) {
-    this.fileCallData.set(file, data);
-    this.callGraph = null; // Invalidate cached graph
-  }
-
-  /**
-   * Get call data for a file
-   */
-  getFileCallData(file) {
-    return this.fileCallData.get(file);
-  }
-
-  /**
-   * Remove call data for a file
-   */
-  removeFileCallData(file) {
-    this.fileCallData.delete(file);
-    this.callGraph = null; // Invalidate cached graph
-  }
-
-  /**
-   * Rebuild the call graph from file data
-   */
-  rebuildCallGraph() {
-    // Lazy import to avoid circular dependencies
-    import("./call-graph.js").then(({ buildCallGraph }) => {
-      this.callGraph = buildCallGraph(this.fileCallData);
-      if (this.config.verbose) {
-        console.error(`[CallGraph] Built graph: ${this.callGraph.defines.size} definitions, ${this.callGraph.calledBy.size} call targets`);
-      }
-    }).catch(err => {
-      console.error(`[CallGraph] Failed to build: ${err.message}`);
-      this.callGraph = null;
-    });
-  }
+    const cached = this.annVectorCache[index];
+    if (cached) return cached;
 
-  /**
-   * Get files related to symbols via call graph
-   */
-  async getRelatedFiles(symbols) {
-    if (!this.config.callGraphEnabled || symbols.length === 0) {
-      return new Map();
+    let vec = null;
+    if (chunk.vector) {
+      vec = ensureFloat32(chunk.vector);
+    } else if (this.binaryStore && Number.isInteger(chunk._binaryIndex)) {
+      vec = this.binaryStore.getVector(chunk._binaryIndex);
     }
 
-    // Rebuild graph if needed
-    if (!this.callGraph && this.fileCallData.size > 0) {
-      const { buildCallGraph } = await import("./call-graph.js");
-      this.callGraph = buildCallGraph(this.fileCallData);
-    }
+    if (!vec) return null;
 
-    if (!this.callGraph) {
-      return new Map();
+    if (this.config.vectorStoreLoadMode !== 'disk') {
+      chunk.vector = vec;
     }
-
-    const { getRelatedFiles } = await import("./call-graph.js");
-    return getRelatedFiles(this.callGraph, symbols, this.config.callGraphMaxHops);
-  }
-
-  /**
-   * Get call graph statistics
-   */
-  getCallGraphStats() {
-    return {
-      enabled: this.config.callGraphEnabled ?? false,
-      filesWithData: this.fileCallData.size,
-      graphBuilt: this.callGraph !== null,
-      definitions: this.callGraph?.defines.size ?? 0,
-      callTargets: this.callGraph?.calledBy.size ?? 0
-    };
+    this.annVectorCache[index] = vec;
+    return vec;
   }
-}
+
+  getAnnIndexPaths() {
+    return {
+      indexFile: path.join(this.config.cacheDirectory, ANN_INDEX_FILE),
+      metaFile: path.join(this.config.cacheDirectory, ANN_META_FILE),
+    };
+  }
+
+  // -------------------- ANN --------------------
+
+  async ensureAnnIndex() {
+    if (!this.config.annEnabled) return null;
+    if (!Array.isArray(this.vectorStore)) return null;
+    if (this.vectorStore.length < (this.config.annMinChunks ?? 5000)) return null;
+    if (this.annIndex && !this.annDirty) return this.annIndex;
+    if (this.annLoading) return this.annLoading;
+
+    this.annLoading = (async () => {
+      const HierarchicalNSW = await loadHnswlib();
+      if (!HierarchicalNSW) {
+        if (hnswlibLoadError) {
+          this.initErrors.push({
+            stage: 'loadHnswlib',
+            message: hnswlibLoadError.message,
+            stack: hnswlibLoadError.stack
+          });
+        }
+        return null;
+      }
+
+      const dim = this.vectorStore[0]?.vector?.length;
+      if (!dim) return null;
+
+      if (!this.annDirty && this.config.annIndexCache !== false) {
+        const loaded = await this.loadAnnIndexFromDisk(HierarchicalNSW, dim);
+        if (loaded) return this.annIndex;
+      }
+
+      return await this.buildAnnIndex(HierarchicalNSW, dim);
+    })();
+
+    const index = await this.annLoading;
+    this.annLoading = null;
+    return index;
+  }
+
+  async loadAnnIndexFromDisk(HierarchicalNSW, dim) {
+    const { indexFile, metaFile } = this.getAnnIndexPaths();
+    const metaData = await fs.readFile(metaFile, 'utf-8').catch(() => null);
+    if (!metaData) return false;
+
+    let meta;
+    try {
+      meta = JSON.parse(metaData);
+    } catch {
+      console.warn('[ANN] Invalid ANN metadata, rebuilding');
+      return false;
+    }
+
+    if (meta?.version !== ANN_META_VERSION) {
+      console.warn(`[ANN] ANN index version mismatch (${meta?.version}), rebuilding`);
+      return false;
+    }
+
+    if (meta?.embeddingModel !== this.config.embeddingModel) {
+      console.warn('[ANN] Embedding model changed for ANN index, rebuilding');
+      return false;
+    }
+
+    if (meta?.dim !== dim || meta?.count !== this.vectorStore.length) {
+      console.warn('[ANN] ANN index size mismatch, rebuilding');
+      return false;
+    }
+
+    if (
+      meta?.metric !== this.config.annMetric ||
+      meta?.m !== this.config.annM ||
+      meta?.efConstruction !== this.config.annEfConstruction
+    ) {
+      console.warn('[ANN] ANN index config changed, rebuilding');
+      return false;
+    }
+
+    let maxElements = meta?.maxElements;
+    if (!Number.isInteger(maxElements)) {
+      maxElements = meta.count;
+    } else if (maxElements < meta.count) {
+      console.warn('[ANN] ANN capacity invalid, rebuilding');
+      return false;
+    }
+
+    const index = new HierarchicalNSW(meta.metric, dim);
+    const loaded = readHnswIndex(index, indexFile, maxElements);
+    if (!loaded) {
+      console.warn('[ANN] Failed to load ANN index file, rebuilding');
+      return false;
+    }
+
+    if (typeof index.setEf === 'function') {
+      index.setEf(this.config.annEfSearch);
+    }
+
+    this.annIndex = index;
+    this.annMeta = { ...meta, maxElements };
+    this.annDirty = false;
+    this.annPersistDirty = false;
+
+    if (this.config.verbose) {
+      console.info(`[ANN] Loaded ANN index (${meta.count} vectors, cap=${maxElements})`);
+    }
+    return true;
+  }
+
+  async buildAnnIndex(HierarchicalNSW, dim) {
+    if (!Array.isArray(this.vectorStore)) return null;
+    const total = this.vectorStore.length;
+    if (total === 0) return null;
+
+    try {
+      const index = new HierarchicalNSW(this.config.annMetric, dim);
+
+      const maxElements = computeAnnCapacity(total, this.config);
+      initHnswIndex(index, maxElements, this.config.annM, this.config.annEfConstruction);
+
+      const yieldEvery = Number.isInteger(this.config.annBuildYieldEvery)
+        ? this.config.annBuildYieldEvery
+        : 1000;
+
+      for (let i = 0; i < total; i++) {
+        const vector = this.getAnnVector(i);
+        if (!vector) throw new Error(`Missing vector for ANN index at position ${i}`);
+        index.addPoint(vector, i);
+
+        if (yieldEvery > 0 && i > 0 && i % yieldEvery === 0) {
+          await yieldToLoop();
+        }
+      }
+
+      if (typeof index.setEf === 'function') {
+        index.setEf(this.config.annEfSearch);
+      }
+
+      this.annIndex = index;
+      this.annMeta = {
+        version: ANN_META_VERSION,
+        embeddingModel: this.config.embeddingModel,
+        metric: this.config.annMetric,
+        dim,
+        count: total,
+        maxElements,
+        m: this.config.annM,
+        efConstruction: this.config.annEfConstruction,
+        efSearch: this.config.annEfSearch,
+      };
+      this.annDirty = false;
+      this.annPersistDirty = true;
+
+      if (this.config.annIndexCache !== false) {
+        try {
+          await fs.mkdir(this.config.cacheDirectory, { recursive: true });
+          const { indexFile, metaFile } = this.getAnnIndexPaths();
+          index.writeIndexSync(indexFile);
+          await fs.writeFile(metaFile, JSON.stringify(this.annMeta, null, 2));
+          this.annPersistDirty = false;
+          if (this.config.verbose) {
+            console.info(`[ANN] Saved ANN index (${total} vectors, cap=${maxElements})`);
+          }
+        } catch (error) {
+          console.warn(`[ANN] Failed to save ANN index: ${error.message}`);
+        }
+      }
+
+      return index;
+    } catch (error) {
+      console.warn(`[ANN] Failed to build ANN index: ${error.message}`);
+      this.initErrors.push({
+        stage: 'buildAnnIndex',
+        message: error.message,
+        stack: error.stack
+      });
+      this.annIndex = null;
+      this.annMeta = null;
+      this.annDirty = true;
+      this.annPersistDirty = false;
+      return null;
+    }
+  }
+
+  async queryAnn(queryVector, k) {
+    if (!Array.isArray(this.vectorStore) || this.vectorStore.length === 0) return [];
+    const index = await this.ensureAnnIndex();
+    if (!index) return [];
+
+    const qVec = queryVector instanceof Float32Array ? queryVector : new Float32Array(queryVector);
+    const results = index.searchKnn(qVec, k);
+    const labels = normalizeLabels(results);
+
+    if (labels.length === 0) return [];
+
+    const filtered = labels.filter(
+      (label) => Number.isInteger(label) && label >= 0 && label < this.vectorStore.length,
+    );
+
+    return filtered;
+  }
+
+  async clear() {
+    if (!this.config.enableCache) return;
+
+    try {
+      await fs.rm(this.config.cacheDirectory, { recursive: true, force: true });
+      this.vectorStore = [];
+      if (this.binaryStore) {
+        try {
+          await this.binaryStore.close();
+        } catch {
+          // ignore close errors
+        }
+      }
+      this.binaryStore = null;
+      this.fileHashes = new Map();
+      this.invalidateAnnIndex();
+      await this.clearCallGraphData();
+      if (this.config.verbose) {
+        console.info(`[Cache] Cache cleared successfully: ${this.config.cacheDirectory}`);
+      }
+    } catch (error) {
+      console.error('[Cache] Failed to clear cache:', error.message);
+      throw error;
+    }
+  }
+
+  /**
+   * Adjust efSearch at runtime for speed/accuracy tradeoff.
+   * Higher values = more accurate but slower.
+   * @param {number} efSearch - New efSearch value (typically 16-512)
+   * @returns {object} Result with success status and current config
+   */
+  setEfSearch(efSearch) {
+    if (typeof efSearch !== 'number' || efSearch < 1 || efSearch > 1000) {
+      return {
+        success: false,
+        error: 'efSearch must be a number between 1 and 1000',
+      };
+    }
+
+    this.config.annEfSearch = efSearch;
+
+    if (this.annIndex && typeof this.annIndex.setEf === 'function') {
+      this.annIndex.setEf(efSearch);
+      if (this.annMeta) this.annMeta.efSearch = efSearch;
+      this.annPersistDirty = true;
+      if (this.config.verbose) {
+        console.info(`[ANN] efSearch updated to ${efSearch} (applied to active index)`);
+      }
+      return { success: true, applied: true, efSearch };
+    }
+
+    if (this.config.verbose) {
+      console.info(`[ANN] efSearch updated to ${efSearch} (will apply on next index build)`);
+    }
+    return { success: true, applied: false, efSearch };
+  }
+
+  /**
+   * Get current ANN index statistics for diagnostics.
+   * @returns {object} ANN stats including index state, config, and vector count
+   */
+  getAnnStats() {
+    return {
+      enabled: this.config.annEnabled ?? false,
+      indexLoaded: this.annIndex !== null,
+      dirty: this.annDirty,
+      vectorCount: Array.isArray(this.vectorStore) ? this.vectorStore.length : 0,
+      minChunksForAnn: this.config.annMinChunks ?? 5000,
+      config: this.annMeta
+        ? {
+            metric: this.annMeta.metric,
+            dim: this.annMeta.dim,
+            count: this.annMeta.count,
+            m: this.annMeta.m,
+            efConstruction: this.annMeta.efConstruction,
+            efSearch: this.config.annEfSearch,
+          }
+        : null,
+    };
+  }
+
+  // -------------------- Call Graph --------------------
+
+  async clearCallGraphData({ removeFile = false } = {}) {
+    this.fileCallData.clear();
+    this.callGraph = null;
+
+    if (removeFile && this.config.enableCache) {
+      const callGraphFile = path.join(this.config.cacheDirectory, CALL_GRAPH_FILE);
+      try {
+        await fs.rm(callGraphFile, { force: true });
+      } catch (error) {
+        if (this.config.verbose) {
+          console.warn(`[Cache] Failed to remove call-graph cache: ${error.message}`);
+        }
+      }
+    }
+  }
+
+  pruneCallGraphData(validFiles) {
+    if (!validFiles || this.fileCallData.size === 0) return 0;
+
+    let pruned = 0;
+    for (const file of Array.from(this.fileCallData.keys())) {
+      if (!validFiles.has(file)) {
+        this.fileCallData.delete(file);
+        pruned++;
+      }
+    }
+
+    if (pruned > 0) this.callGraph = null;
+    return pruned;
+  }
+
+  getFileCallData(file) {
+    return this.fileCallData.get(file);
+  }
+
+  hasFileCallData(file) {
+    return this.fileCallData.has(file);
+  }
+
+  getFileCallDataKeys() {
+    return Array.from(this.fileCallData.keys());
+  }
+
+  getFileCallDataCount() {
+    return this.fileCallData.size;
+  }
+
+  /**
+   * Sets call data for a specific file.
+   * @param {string} file
+   * @param {object} data
+   */
+  setFileCallData(file, data) {
+    this.fileCallData.set(file, data);
+    this.callGraph = null;
+  }
+
+  /**
+   * Sets the entire file call data map.
+   * @param {Map<string, object>|object} entries
+   */
+  setFileCallDataEntries(entries) {
+    if (entries instanceof Map) {
+      this.fileCallData = entries;
+    } else {
+      this.fileCallData.clear();
+      if (entries && typeof entries === 'object') {
+        for (const [file, data] of Object.entries(entries)) {
+          this.fileCallData.set(file, data);
+        }
+      }
+    }
+    this.callGraph = null;
+  }
+
+  clearFileCallData() {
+    this.fileCallData.clear();
+    this.callGraph = null;
+  }
+
+  removeFileCallData(file) {
+    this.fileCallData.delete(file);
+    this.callGraph = null;
+  }
+
+  async rebuildCallGraph() {
+    if (this._callGraphBuild) return this._callGraphBuild;
+
+    this._callGraphBuild = (async () => {
+      try {
+        const { buildCallGraph } = await import('./call-graph.js');
+        this.callGraph = buildCallGraph(this.fileCallData);
+        if (this.config.verbose && this.callGraph) {
+          console.info(
+            `[CallGraph] Built graph: ${this.callGraph.defines.size} definitions, ${this.callGraph.calledBy.size} call targets`,
+          );
+        }
+      } catch (err) {
+        console.error(`[CallGraph] Failed to build: ${err.message}`);
+        this.callGraph = null;
+      } finally {
+        this._callGraphBuild = null;
+      }
+    })();
+
+    return this._callGraphBuild;
+  }
+
+  async getRelatedFiles(symbols) {
+    if (!this.config.callGraphEnabled || symbols.length === 0) return new Map();
+    if (!this.callGraph && this.fileCallData.size > 0) await this.rebuildCallGraph();
+    if (!this.callGraph) return new Map();
+
+    const { getRelatedFiles } = await import('./call-graph.js');
+    return getRelatedFiles(this.callGraph, symbols, this.config.callGraphMaxHops);
+  }
+
+  getCallGraphStats() {
+    return {
+      enabled: this.config.callGraphEnabled ?? false,
+      filesWithData: this.fileCallData.size,
+      graphBuilt: this.callGraph !== null,
+      definitions: this.callGraph?.defines.size ?? 0,
+      callTargets: this.callGraph?.calledBy.size ?? 0,
+    };
+  }
+
+  // -------------------- Abstraction Layer --------------------
+
+  /**
+   * Returns the total number of chunks in the store.
+   * @returns {number}
+   */
+  getStoreSize() {
+    if (Array.isArray(this.vectorStore)) return this.vectorStore.length;
+    if (this.binaryStore) return this.binaryStore.length;
+    return 0;
+  }
+
+  /**
+   * Retrieves a vector by its store index.
+   * @param {number} index
+   * @returns {Float32Array|null}
+   */
+  getVector(index) {
+    return this.getChunkVector(index);
+  }
+
+  /**
+   * Retrieves a chunk object by its store index.
+   * @param {number} index
+   * @returns {object|null}
+   */
+  getChunk(index) {
+    if (Array.isArray(this.vectorStore) && index >= 0 && index < this.vectorStore.length) {
+      return this.vectorStore[index];
+    }
+    if (this.binaryStore) {
+      const record = this.binaryStore.getRecord(index);
+      if (record) {
+        return {
+          file: record.file,
+          startLine: record.startLine,
+          endLine: record.endLine,
+          vector: this.binaryStore.getVector(index),
+          _index: index,
+          _binaryIndex: index,
+        };
+      }
+    }
+    return null;
+  }
+}