graph-indexer 1.1.0 → 2.0.0

package/{core-engine.mjs → engine/memory.mjs}

@@ -1,81 +1,20 @@
 /**
- * @file core-engine.mjs
+ * @file engine/memory.mjs
  * @description In-Memory Graph Indexer Core Engine.
  * @author MaquinaTech <https://github.com/MaquinaTech>
  * @copyright (c) 2026 MaquinaTech. All rights reserved.
  * @license MIT
  */
 import fs from 'fs';
-
-export const EMBEDDING_CONTEXT_LIMIT = 8000;
-
-// Thresholds for adaptive vector loading strategy
-const HNSW_THRESHOLD = 5000;      // Build HNSW index above this (eager mode only)
-const LAZY_VEC_THRESHOLD = 10000; // Switch to lazy (disk-backed) vector access above this
-const TFIDF_PREFILTER_K = 2000;   // TF-IDF candidates fed into lazy vector scoring
-
-// Optional HNSW accelerator for medium corpora (eager mode only)
-let HierarchicalNSW = null;
-try {
-    const mod = await import('hnswlib-node');
-    HierarchicalNSW = mod.HierarchicalNSW ?? mod.default?.HierarchicalNSW ?? null;
-} catch { /* not installed — flat scan used */ }
-
-export function truncateForEmbedding(text) {
-    return text.length > EMBEDDING_CONTEXT_LIMIT ? text.slice(0, EMBEDDING_CONTEXT_LIMIT) : text;
-}
-
-export function cosineSimilarity(vecA, vecB) {
-    let dot = 0, nA = 0, nB = 0;
-    for (let i = 0; i < vecA.length; i++) {
-        dot += vecA[i] * vecB[i];
-        nA  += vecA[i] * vecA[i];
-        nB  += vecB[i] * vecB[i];
-    }
-    return nA === 0 || nB === 0 ? 0 : dot / (Math.sqrt(nA) * Math.sqrt(nB));
-}
-
-function tokenize(text) {
-    if (!text) return [];
-    const rawTokens = text.split(/[\s\W_]+/);
-    const tokens = [];
-    for (const word of rawTokens) {
-        if (word.length < 2) continue;
-        tokens.push(word.toLowerCase());
-        const camelParts = word.replace(/([a-z])([A-Z])/g, '$1 $2').split(' ');
-        if (camelParts.length > 1) {
-            for (const part of camelParts) {
-                if (part.length >= 2) tokens.push(part.toLowerCase());
-            }
-        }
-    }
-    return tokens;
-}
-
-/**
- * Serializes embeddingCache to compact binary.
- * Format per entry: [uint32 hashLen][utf8 hash][uint32 dim][float32 × dim]
- */
-export function writeEmbeddingBinary(embeddingCache) {
-    const entries = embeddingCache instanceof Map
-        ? Array.from(embeddingCache.entries())
-        : Object.entries(embeddingCache);
-    let size = 4;
-    for (const [hash, vec] of entries) {
-        size += 4 + Buffer.byteLength(hash, 'utf8') + 4 + vec.length * 4;
-    }
-    const buf = Buffer.allocUnsafe(size);
-    let off = 0;
-    buf.writeUInt32LE(entries.length, off); off += 4;
-    for (const [hash, vec] of entries) {
-        const hashBytes = Buffer.from(hash, 'utf8');
-        buf.writeUInt32LE(hashBytes.length, off); off += 4;
-        hashBytes.copy(buf, off); off += hashBytes.length;
-        buf.writeUInt32LE(vec.length, off); off += 4;
-        for (let d = 0; d < vec.length; d++) { buf.writeFloatLE(vec[d], off); off += 4; }
-    }
-    return buf;
-}
+import {
+    tokenize, okapiIdf, bm25Score, fuseAndRank, buildLexicalDocument, embeddingKeyFor,
+    SUMMARY_VEC_SUFFIX, LEXICAL_FUSION_CAP, VECTOR_SCAN_RAW_N, finalizeVectorCandidates,
+    isNaturalLanguageQuery, WINDOW_VEC_SUFFIX, EMBEDDING_MAX_WINDOWS, baseEmbeddingKey,
+} from '../search-core.mjs';
+import {
+    HierarchicalNSW, HNSW_THRESHOLD, LAZY_VEC_THRESHOLD, SKETCH_THRESHOLD,
+    writeEmbeddingBinary, scanEmbeddingBinary, updateVectorSketch, searchVectorSketch,
+} from './binary.mjs';
 
 export class MemoryGraphIndex {
     /**
@@ -126,6 +65,8 @@ export class MemoryGraphIndex {
         this._embeddingBuf = null;         // Buffer kept for small-lazy corpora (<50k)
         this._vecFd       = -1;            // open fd for disk-backed access (≥50k)
         this._lazyMode    = false;
+        this._keyToIds    = null;          // embedding key → chunkId[] (built lazily for scans)
+        this._sketch      = null;          // binary-quantized sketch (lazy mode, large corpora)
 
         this.rrfK = rrfK;
         this._saveTimer = null;
@@ -137,10 +78,13 @@ export class MemoryGraphIndex {
         if (!fs.existsSync(this.indexPath)) return;
         const data = JSON.parse(fs.readFileSync(this.indexPath, 'utf-8'));
         this.graph = data.graph || { dependencies: {}, importedBy: {} };
+        // Does this corpus carry LLM enrichment? Drives the NL vector-channel weight
+        // in fuseAndRank (strong summary vectors earn full weight; plain code vectors
+        // stay a low-weight rescue). Set during the chunk load loop below.
+        this._corpusEnriched = false;
 
         const chunkCount = (data.chunks || []).length;
 
-        // Decide loading strategy based on corpus size and cacheEmbeddings flag
         this._lazyMode = (!this._cacheEmbeddings) && chunkCount >= LAZY_VEC_THRESHOLD;
 
         if (fs.existsSync(this._embeddingPath)) {
@@ -150,13 +94,10 @@ export class MemoryGraphIndex {
                 // For very large corpora: open persistent fd, release buffer from heap
                 if (chunkCount >= 50000) {
                     try { this._vecFd = fs.openSync(this._embeddingPath, 'r'); } catch { this._vecFd = -1; }
-                    // binBuf goes out of scope → GC can collect it
                 } else {
-                    // Medium-large: keep buffer for zero-copy slice access
                     this._embeddingBuf = binBuf;
                 }
             } else {
-                // Eager: fill embeddingCache (needed by indexer + small-corpus MCP server)
                 this._loadEmbeddingBinary(binBuf);
             }
         } else if (data.embeddingCache) {
@@ -167,10 +108,28 @@ export class MemoryGraphIndex {
 
         for (const chunk of (data.chunks || [])) {
             this.chunks.set(chunk.id, chunk);
+            if (!this._corpusEnriched && (chunk.summary || (chunk.concepts && chunk.concepts.length))) {
+                this._corpusEnriched = true;
+            }
 
-            // Eager vector population
+            // Eager vector population. Vectors are keyed by embeddingKeyFor(chunk)
+            // (content_hash + enrichment digest); plain content_hash is the
+            // backward-compatible fallback for bins written before enrichment keys.
             if (!this._lazyMode) {
-                if (chunk.content_hash && this.embeddingCache.has(chunk.content_hash)) {
+                const vecKey = embeddingKeyFor(chunk);
+                if (chunk.content_hash && this.embeddingCache.has(vecKey)) {
+                    this.vectors.set(chunk.id, this.embeddingCache.get(vecKey));
+                    // Summary-only and per-window vectors (enriched / oversized chunks):
+                    // stored under pseudo row ids; searchHybrid folds hits back onto the
+                    // chunk id (max-sim across all of the chunk's vectors).
+                    const sVec = this.embeddingCache.get(vecKey + SUMMARY_VEC_SUFFIX);
+                    if (sVec) this.vectors.set(chunk.id + SUMMARY_VEC_SUFFIX, sVec);
+                    for (let i = 1; i < EMBEDDING_MAX_WINDOWS; i++) {
+                        const wVec = this.embeddingCache.get(vecKey + WINDOW_VEC_SUFFIX + i);
+                        if (!wVec) break;
+                        this.vectors.set(chunk.id + WINDOW_VEC_SUFFIX + i, wVec);
+                    }
+                } else if (chunk.content_hash && this.embeddingCache.has(chunk.content_hash)) {
                     this.vectors.set(chunk.id, this.embeddingCache.get(chunk.content_hash));
                 } else if (chunk.embedding) {
                     const vec = new Float32Array(chunk.embedding);
@@ -179,23 +138,9 @@ export class MemoryGraphIndex {
                 }
             }
 
-            // Build inverted lexical index
             const deps = this.graph.dependencies[chunk.file_path] || [];
-            const cleanDeps = deps.map(d => d.split('/').pop().split('.')[0]);
-            const enrichedContext = [
-                chunk.name,
-                chunk.docstring || '',
-                cleanDeps.join(' '),
-                (chunk.calls || []).join(' '),
-                (chunk.params || []).join(' '),
-                chunk.return_type || '',
-                chunk.class_context ? `${chunk.class_context}.${chunk.name}` : '',
-                (chunk.type_refs || []).join(' '),
-                chunk.code_snippet
-            ].join(' ');
-            this._indexLexical(chunk.id, enrichedContext, chunk.file_path);
-
-            // Build symbol table (Frontier 2)
+            this._indexLexical(chunk.id, buildLexicalDocument(chunk, deps), chunk.file_path);
+
             if (chunk.name && chunk.name !== 'anonymous') {
                 const n = chunk.name.toLowerCase();
                 if (!this.symbolTable.has(n)) this.symbolTable.set(n, new Set());
@@ -237,7 +182,8 @@ export class MemoryGraphIndex {
     _getVecForChunk(chunkId) {
         const chunk = this.chunks.get(chunkId);
         if (!chunk?.content_hash) return null;
-        const entry = this._vecOffsets.get(chunk.content_hash);
+        const entry = this._vecOffsets.get(embeddingKeyFor(chunk))
+            ?? this._vecOffsets.get(chunk.content_hash); // pre-enrichment-key bins
         if (!entry) return null;
 
         if (this._embeddingBuf) {
@@ -275,7 +221,7 @@ export class MemoryGraphIndex {
     // ─── Lexical index (TRUE inverted index) ──────────────────────────────────
 
     _indexLexical(chunkId, text, filePath = '') {
-        const tokens = tokenize(text);
+        const tokens = tokenize(text);               // raw + additive Porter stems
         if (tokens.length === 0) return;
 
         const termCounts = new Map();
@@ -283,16 +229,16 @@ export class MemoryGraphIndex {
             termCounts.set(token, (termCounts.get(token) || 0) + 1);
         }
 
-        // Track document length for BM25 length normalization.
-        // Only content tokens are counted — path tokens are handled separately
-        // in searchHybrid to avoid BM25 length normalisation amplifying short
-        // export stubs that share a path with a larger implementation.
-        this.docLens.set(chunkId, tokens.length);
-        this.totalDocLen += tokens.length;
+        // Additive stem tokens earn their own postings but must NOT inflate docLen —
+        // inflated length would perturb BM25 normalisation for exact symbolic matches.
+        // raw-based docLen keeps raw-term BM25 byte-identical to the pre-stem index.
+        // Path tokens are tallied separately in searchHybrid, not here.
+        const rawLen = tokenize(text, false).length;
+        this.docLens.set(chunkId, rawLen);
+        this.totalDocLen += rawLen;
 
         const chunkTokenSet = new Set();
         for (const [term, count] of termCounts) {
-            // Store raw term count — BM25 applies saturation at search time
             this.df.set(term, (this.df.get(term) || 0) + 1);
             let posting = this.invertedIndex.get(term);
             if (!posting) { posting = new Map(); this.invertedIndex.set(term, posting); }
@@ -307,7 +253,7 @@ export class MemoryGraphIndex {
         // Path matching is done multiplicatively in searchHybrid, which avoids
         // amplifying short stubs that happen to share a path with a long function.
         if (filePath) {
-            const pathTokenSet = new Set(tokenize(filePath.replace(/[/\-_.]/g, ' ')));
+            const pathTokenSet = new Set(tokenize(filePath.replace(/[/\-_.]/g, ' '), false));
             if (!this.pathTokens) this.pathTokens = new Map();
             this.pathTokens.set(chunkId, pathTokenSet);
         }
@@ -328,7 +274,6 @@ export class MemoryGraphIndex {
                 else this.df.set(term, freq - 1);
             }
         }
-        // Update BM25 length accounting
         const dl = this.docLens.get(chunkId);
         if (dl !== undefined) {
             this.totalDocLen = Math.max(0, this.totalDocLen - dl);
@@ -340,32 +285,27 @@ export class MemoryGraphIndex {
     }
 
     _searchLexical(queryText) {
-        const queryTokens = tokenize(queryText);
+        // Asymmetric stemming: the index always carries stems, but only stem the
+        // QUERY for natural-language/behavioural queries — symbolic name lookups
+        // stay exact so stem collisions never dilute a precise symbol match.
+        const queryTokens = tokenize(queryText, isNaturalLanguageQuery(queryText));
         const scores = new Map();
-
-        // BM25 parameters tuned for code (b lowered from 0.75 to 0.3 — code chunks
-        // have purposeful length variation unlike prose, so heavy length normalization
-        // incorrectly penalises long implementations vs short export stubs).
-        const k1 = 1.5;
-        const b  = 0.3;
         const avgdl = this.docCount > 0 ? this.totalDocLen / this.docCount : 1;
 
         for (const token of queryTokens) {
             const docFreq = this.df.get(token);
             if (!docFreq) continue;
-            // Okapi IDF — always positive, avoids negative IDF for very common terms
-            const idf = Math.log((this.docCount - docFreq + 0.5) / (docFreq + 0.5) + 1);
+            const idf = okapiIdf(this.docCount, docFreq);
             const posting = this.invertedIndex.get(token);
             if (!posting) continue;
             for (const [chunkId, tf] of posting) {
                 const dl = this.docLens.get(chunkId) ?? avgdl;
-                // BM25: diminishing returns for high TF + document-length normalisation
-                const bm25 = idf * (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * dl / avgdl));
-                scores.set(chunkId, (scores.get(chunkId) || 0) + bm25);
+                scores.set(chunkId, (scores.get(chunkId) || 0) + bm25Score(idf, tf, dl, avgdl));
             }
         }
         return Array.from(scores.entries())
-            .sort((a, b) => b[1] - a[1])
+            .sort((a, b) => (b[1] - a[1]) || (a[0] < b[0] ? -1 : 1)) // id tie-break: backend parity
+            .slice(0, LEXICAL_FUSION_CAP)                            // same cap as SQLite: parity
             .map(([id, score], rank) => ({ id, score, rank: rank + 1 }));
     }
 
@@ -397,7 +337,10 @@ export class MemoryGraphIndex {
         const dim = this.vectors.get(ids[0]).length;
         this._dim = dim;
         const matrix = new Float32Array(n * dim);
-        const norms  = new Float32Array(n);
+        // Float64: norms must carry full precision — f32-rounded norms produced
+        // ~1e-7 score skew vs the disk-backed store's double-precision rescore,
+        // which was enough to flip near-tied ranks between backends.
+        const norms  = new Float64Array(n);
         for (let i = 0; i < n; i++) {
             const v = this.vectors.get(ids[i]);
             let normSq = 0;
@@ -413,7 +356,6 @@ export class MemoryGraphIndex {
         this._vecNorms  = norms;
         this._matrixDirty = false;
 
-        // HNSW for large eager corpora
         if (HierarchicalNSW && n >= HNSW_THRESHOLD) {
             try {
                 const hnsw = new HierarchicalNSW('cosine', dim);
@@ -427,7 +369,7 @@ export class MemoryGraphIndex {
                 }
                 this._hnsw = hnsw;
             } catch (e) {
-                process.stderr.write(`[core-engine] HNSW build failed: ${e.message}\n`);
+                process.stderr.write(`[engine/memory] HNSW build failed: ${e.message}\n`);
                 this._hnsw = null;
             }
         }
@@ -447,7 +389,6 @@ export class MemoryGraphIndex {
         qNorm = Math.sqrt(qNorm);
         if (qNorm === 0) return [];
 
-        // HNSW fast path
         if (this._hnsw) {
             const topK = Math.min(200, n);
             try {
@@ -462,7 +403,6 @@ export class MemoryGraphIndex {
             } catch { /* fall through to flat scan */ }
         }
 
-        // Exact flat scan
         const results = [];
         for (let i = 0; i < n; i++) {
             const base = i * dim;
@@ -477,34 +417,53 @@ export class MemoryGraphIndex {
     }
 
     /**
-     * Two-phase vector search (lazy mode):
-     * Only compute cosine for candidateIds (top-K TF-IDF pre-filter).
-     * Reads vectors on-demand from buffer or disk — no full matrix in RAM.
+     * Full vector search in lazy mode: stream the entire embeddings bin (bounded
+     * buffers, no full matrix in RAM) and map the top entry keys back to chunks.
+     * Replaces the old lexical-prefiltered scoring, which could never surface a
+     * chunk for a conceptual query that shared no tokens with the code.
      */
-    _searchVectorCandidates(queryVector, candidateIds, minScore = 0.3) {
-        const qDim = queryVector.length;
-        let qNorm = 0;
-        for (let d = 0; d < qDim; d++) qNorm += queryVector[d] * queryVector[d];
-        qNorm = Math.sqrt(qNorm);
-        if (qNorm === 0) return [];
-
-        const results = [];
-        for (const chunkId of candidateIds) {
-            const vec = this._getVecForChunk(chunkId);
-            if (!vec || vec.length !== qDim) continue;
-            let dp = 0, normVec = 0;
-            for (let d = 0; d < qDim; d++) {
-                dp      += queryVector[d] * vec[d];
-                normVec += vec[d] * vec[d];
+    _scanVectorsLazy(queryVector, minScore = 0.3) {
+        const source = this._embeddingBuf ? { buffer: this._embeddingBuf } : { fd: this._vecFd };
+        if (!this._embeddingBuf && this._vecFd < 0) return [];
+        // Large corpora: binary sketch (built once on first vector query) replaces
+        // the exact O(corpus) scan with a Hamming prefilter + bounded rescore.
+        // SKETCH_THRESHOLD is shared with the SQLite store so both switch to the
+        // sketch at the same vector count. NOTE: this lazy path only runs when the
+        // store is in lazy mode (cacheEmbeddings off AND chunkCount ≥ LAZY_VEC_THRESHOLD);
+        // a corpus with < LAZY_VEC_THRESHOLD chunks but ≥ SKETCH_THRESHOLD vectors
+        // (heavy enrichment/oversized windows) still scans exactly here while SQLite
+        // sketches — a narrow band worth a dedicated scale parity test before relying
+        // on byte-identical top-5 above the threshold.
+        if (!this._sketch && this._vecOffsets.size >= SKETCH_THRESHOLD) {
+            try { this._sketch = updateVectorSketch(null, source); } catch { this._sketch = null; }
+        }
+        const hits = this._sketch
+            ? searchVectorSketch(this._sketch, source, queryVector, { topN: VECTOR_SCAN_RAW_N, minScore })
+            : scanEmbeddingBinary(source, queryVector, { topN: VECTOR_SCAN_RAW_N, minScore });
+        if (hits.length === 0) return [];
+
+        if (!this._keyToIds) {
+            const map = new Map();
+            const put = (key, id) => {
+                if (!map.has(key)) map.set(key, []);
+                map.get(key).push(id);
+            };
+            for (const c of this.chunks.values()) {
+                if (!c.content_hash) continue;
+                const key = embeddingKeyFor(c);
+                put(key, c.id);
+                put(key + SUMMARY_VEC_SUFFIX, c.id);     // summary-only vector → same chunk
+                for (let i = 1; i < EMBEDDING_MAX_WINDOWS; i++) put(key + WINDOW_VEC_SUFFIX + i, c.id); // window vectors → same chunk
+                if (key !== c.content_hash) put(c.content_hash, c.id); // legacy bins
             }
-            const normV = Math.sqrt(normVec);
-            if (normV === 0) continue;
-            const score = dp / (qNorm * normV);
-            if (score > minScore) results.push({ id: chunkId, score });
+            this._keyToIds = map;
         }
-        results.sort((a, b) => b.score - a.score);
-        for (let i = 0; i < results.length; i++) results[i].rank = i + 1;
-        return results;
+
+        const entries = [];
+        for (const { key, score } of hits) {
+            for (const id of (this._keyToIds.get(key) || [])) entries.push({ id, score });
+        }
+        return finalizeVectorCandidates(entries);
     }
 
     // ─── Hybrid search ─────────────────────────────────────────────────────────
@@ -515,111 +474,39 @@ export class MemoryGraphIndex {
         let vectorResults;
         if (queryVector) {
             if (this._lazyMode && this._vecOffsets.size > 0) {
-                // Two-phase: TF-IDF pre-filter → lazy vector load
-                const candidateIds = lexicalResults.slice(0, TFIDF_PREFILTER_K).map(r => r.id);
-                vectorResults = this._searchVectorCandidates(queryVector, candidateIds, minScore);
+                // Full streaming scan of the bin — covers conceptual queries with
+                // zero lexical overlap, which the old TF-IDF prefilter never could.
+                vectorResults = this._scanVectorsLazy(queryVector, minScore);
             } else {
-                vectorResults = this._searchVector(queryVector, minScore);
+                // Eager rows include summary-only and per-window pseudo ids
+                // (`<id>|s`, `<id>|wN`) — fold each back onto its chunk via the shared
+                // finalizer (best/max-sim score per chunk, deterministic order, same
+                // cap as the disk-backed paths).
+                const entries = this._searchVector(queryVector, minScore).map(r => ({
+                    id: baseEmbeddingKey(r.id),
+                    score: r.score,
+                }));
+                vectorResults = finalizeVectorCandidates(entries);
             }
         } else {
             vectorResults = [];
         }
 
-        const rrfScores  = new Map();
-        const K          = this.rrfK;
-        const queryLower = queryText.toLowerCase();
-
-        const LEXICAL_WEIGHT = 1.5;
-        const VECTOR_WEIGHT  = 1.0;
-
-        // Pre-compute query file tokens once for path boosting
-        const _queryPathTokens = queryLower.split(/[\s\W_]+/).filter(t => t.length >= 3);
-
-        const allResults = [
-            ...vectorResults.map(r => ({ ...r, _w: VECTOR_WEIGHT })),
-            ...lexicalResults.map(r => ({ ...r, _w: LEXICAL_WEIGHT })),
-        ];
-
-        for (const { id, rank, _w } of allResults) {
-            let baseScore = (_w ?? 1.0) / (K + rank);
-            const chunk = this.chunks.get(id);
-            if (!chunk) continue;
-
-            // Demotion: test / spec files
-            if (/\.(test|spec)\.|[/\\]__tests__[/\\]|_test\.|^tests?[/\\]|[/\\]tests?[/\\]|[/\\]spec[/\\]/.test(chunk.file_path)) {
-                if (!queryLower.includes('test') && !queryLower.includes('spec')) baseScore *= 0.25;
-            }
-            // Demotion: example / docs dirs (tutorial snippets over-rank due to short length
-            // and high keyword density relative to the actual framework implementation).
-            if (/^examples?[/\\]|[/\\]examples?[/\\]|^samples?[/\\]|[/\\]samples?[/\\]|^demos?[/\\]|[/\\]demos?[/\\]|[/\\]tutorials?[/\\]|^docs_src[/\\]|[/\\]docs_src[/\\]/.test(chunk.file_path)) {
-                baseScore *= 0.5;
-            }
-            // Demotion: pure expression sites
-            if (chunk.node_type === 'expression_statement' || chunk.node_type === 'call_expression') {
-                baseScore *= 0.8;
-            }
-            // Demotion: TypeScript barrel re-exports (e.g. "controller.decorator_export_statement"
-            // in .ts files). These are synthetic chunks created for `export { X } from 'y'`
-            // declarations — no implementation, just re-exporting from another file.
-            // JavaScript .js exports (e.g. "http_export_statement") are intentionally excluded
-            // since those often ARE the primary module export (the file IS the implementation).
-            if (chunk.name && chunk.name.endsWith('_export_statement') && chunk.file_path?.endsWith('.ts')) {
-                baseScore *= 0.7;
-            }
-
-            // File-path boost: applied to ALL queries (not just when vectors are absent).
-            // Uses the separate pathTokens index (not the BM25 inverted index) so that
-            // length normalisation does not penalise long implementations sharing a path
-            // with short stubs.
-            if (_queryPathTokens.length > 0) {
-                const pathToks = this.pathTokens.get(id);
-                if (pathToks) {
-                    const hasExact = _queryPathTokens.some(t => pathToks.has(t));
-                    const hasPrefix = !hasExact && _queryPathTokens.some(t =>
-                        t.length >= 4 && Array.from(pathToks).some(pt => pt.startsWith(t.slice(0, 5)))
-                    );
-                    if (hasExact)       baseScore *= 1.4;
-                    else if (hasPrefix) baseScore *= 1.2;
-                }
-            }
-
-            // Name boost:
-            //   2.0x exact match — a query token IS the full chunk name.
-            //   1.4x snake_case suffix — the last underscore/dot component of the name
-            //     matches a query token.  Intentionally uses only [._] splitting so
-            //     camelCase names stay atomic: "dispatchRequest" is one token, not
-            //     ["dispatch","request"], preventing false boosts from generic suffixes.
-            if (chunk.name && chunk.name !== 'anonymous') {
-                const nameLower      = chunk.name.toLowerCase();
-                const queryTokensAll = queryLower.split(/[\s\W_]+/).filter(t => t.length >= 5);
-                if (queryTokensAll.some(t => nameLower === t || nameLower.endsWith('.' + t))) {
-                    baseScore *= 2.0;
-                } else {
-                    const snakeParts = nameLower.split(/[._]+/);
-                    const lastSnake  = snakeParts[snakeParts.length - 1] ?? '';
-                    if (snakeParts.length >= 2 && lastSnake.length >= 5 && queryTokensAll.includes(lastSnake)) {
-                        baseScore *= 1.4;
-                    }
-                }
-            }
-
-            rrfScores.set(id, (rrfScores.get(id) || 0) + baseScore);
-        }
-
-        if (exactBoostName) {
-            const boostTerm = String(exactBoostName).toLowerCase().trim();
-            for (const [id, chunk] of this.chunks.entries()) {
-                if (chunk.name && String(chunk.name).toLowerCase() === boostTerm) {
-                    rrfScores.set(id, (rrfScores.get(id) || 0) + 1 / (K + 1));
-                }
-            }
-        }
-
-        return Array.from(rrfScores.entries())
-            .sort((a, b) => b[1] - a[1])
-            .slice(0, topK)
-            .map(([id, rrfScore]) => ({ score: rrfScore, chunk: this.chunks.get(id) }))
-            .filter(r => r.chunk !== undefined);
+        return fuseAndRank({
+            lexicalResults,
+            vectorResults,
+            getChunk:      (id) => this.chunks.get(id),
+            getPathTokens: (id) => this.pathTokens.get(id),
+            getDf:         (t)  => this.df.get(t) || 0,
+            docCount:      this.docCount,
+            rrfK:          this.rrfK,
+            topK,
+            queryText,
+            exactBoostName,
+            corpusEnriched: this._corpusEnriched,
+            // O(1) vs the old full-scan; same contract implemented by the SQLite backend.
+            resolveExact:  (term) => this.symbolTable.get(term) || [],
+        });
     }
 
     // ─── Persistence ───────────────────────────────────────────────────────────
@@ -629,9 +516,11 @@ export class MemoryGraphIndex {
             id: c.id, file_path: c.file_path, node_type: c.node_type,
             name: c.name, docstring: c.docstring || '', code_snippet: c.code_snippet,
             content_hash: c.content_hash, start_line: c.start_line, end_line: c.end_line,
-            calls: c.calls || [], params: c.params || [],
+            calls: c.calls || [], call_sites: c.call_sites || [], params: c.params || [],
             return_type: c.return_type || '', class_context: c.class_context || '',
-            type_refs: c.type_refs || [],
+            type_refs: c.type_refs || [], decorators: c.decorators || [],
+            extends: c.extends || [],
+            hyde: c.hyde || '', summary: c.summary || '', concepts: c.concepts || [],
         }));
         const payload  = JSON.stringify({ chunks: chunksData, graph: this.graph });
         const tmpPath    = `${this.indexPath}.tmp`;
@@ -651,10 +540,107 @@ export class MemoryGraphIndex {
         this._saveTimer = setTimeout(async () => {
             this._saveTimer = null;
             try { await this.save(); }
-            catch (err) { process.stderr.write(`[core-engine] ❌ Async save failed: ${err.message}\n`); }
+            catch (err) { process.stderr.write(`[engine/memory] ❌ Async save failed: ${err.message}\n`); }
         }, delayMs);
     }
 
+    _removeSymbol(chunk) {
+        if (!chunk.name || chunk.name === 'anonymous') return;
+        const n = chunk.name.toLowerCase();
+        const set = this.symbolTable.get(n);
+        if (set) {
+            set.delete(chunk.id);
+            if (set.size === 0) this.symbolTable.delete(n);
+        }
+    }
+
+    /** Whether a vector for this embedding key is already cached (skip re-embedding). */
+    hasEmbedding(key) { return this.embeddingCache.has(key); }
+
+    /**
+     * Atomically replace every chunk of one file — the incremental-update entry
+     * point used by the watch daemon. Removes the file's old chunks from the
+     * lexical, vector AND symbol indexes (the old daemon path leaked symbol-table
+     * entries), stores newly computed embeddings, indexes the new chunks, and
+     * schedules a debounced save. Pass empty chunks/imports for a deleted file.
+     *
+     * @param {string} filePath
+     * @param {object} p
+     * @param {object[]} [p.chunks]
+     * @param {string[]} [p.imports]
+     * @param {Map<string, Float32Array|number[]>} [p.embeddings] New vectors keyed by embeddingKeyFor(chunk).
+     */
+    applyFileUpdate(filePath, { chunks = [], imports = [], embeddings = null } = {}) {
+        this.updateFileGraph(filePath, imports);
+
+        for (const [id, chunk] of Array.from(this.chunks.entries())) {
+            if (chunk.file_path !== filePath) continue;
+            this._removeLexical(id);
+            this.removeVector(id);
+            this.removeVector(id + SUMMARY_VEC_SUFFIX);
+            for (let i = 1; i < EMBEDDING_MAX_WINDOWS; i++) this.removeVector(id + WINDOW_VEC_SUFFIX + i);
+            this._removeSymbol(chunk);
+            this.chunks.delete(id);
+        }
+
+        if (embeddings) {
+            for (const [key, vec] of embeddings) {
+                this.embeddingCache.set(key, vec instanceof Float32Array ? vec : new Float32Array(vec));
+            }
+        }
+
+        for (const chunk of chunks) {
+            const vecKey = embeddingKeyFor(chunk);
+            const vec = this.embeddingCache.get(vecKey) ?? this.embeddingCache.get(chunk.content_hash);
+            if (vec) this.addVector(chunk.id, vec);
+            const sVec = this.embeddingCache.get(vecKey + SUMMARY_VEC_SUFFIX);
+            if (sVec) this.addVector(chunk.id + SUMMARY_VEC_SUFFIX, sVec);
+            for (let i = 1; i < EMBEDDING_MAX_WINDOWS; i++) {
+                const wVec = this.embeddingCache.get(vecKey + WINDOW_VEC_SUFFIX + i);
+                if (wVec) this.addVector(chunk.id + WINDOW_VEC_SUFFIX + i, wVec);
+            }
+            this._indexLexical(chunk.id, buildLexicalDocument(chunk, imports), chunk.file_path);
+            this.chunks.set(chunk.id, chunk);
+            if (chunk.name && chunk.name !== 'anonymous') {
+                const n = chunk.name.toLowerCase();
+                if (!this.symbolTable.has(n)) this.symbolTable.set(n, new Set());
+                this.symbolTable.get(n).add(chunk.id);
+            }
+        }
+
+        this._keyToIds = null;
+        this.saveDebounced();
+    }
+
+    /**
+     * Drop every in-memory structure and re-read the index artifacts from disk.
+     * Used by the MCP server when the watch daemon (a separate process) rewrites
+     * code-index.json — without this the server would serve a stale snapshot
+     * until restart.
+     */
+    reload() {
+        if (this._vecFd >= 0) { try { fs.closeSync(this._vecFd); } catch {} this._vecFd = -1; }
+        this.chunks = new Map();
+        this.graph = { dependencies: {}, importedBy: {} };
+        this.embeddingCache = new Map();
+        this.symbolTable = new Map();
+        this.invertedIndex = new Map();
+        this.chunkTerms = new Map();
+        this.docLens = new Map();
+        this.totalDocLen = 0;
+        this.pathTokens = new Map();
+        this.docCount = 0;
+        this.df = new Map();
+        this.vectors = new Map();
+        this._invalidateMatrix();
+        this._vecOffsets = new Map();
+        this._embeddingBuf = null;
+        this._keyToIds = null;
+        this._sketch = null;
+        this._lazyMode = false;
+        this.load();
+    }
+
     updateFileGraph(filePath, imports) {
         const oldDeps = this.graph.dependencies[filePath] || [];
         for (const oldDep of oldDeps) {
@@ -669,6 +655,130 @@ export class MemoryGraphIndex {
         }
     }
 
+    // ─── Store contract ────────────────────────────────────────────────────────
+    // The read surface the MCP tools consume. The SQLite store implements the same
+    // methods over disk so tools never reach into a backend's internals.
+
+    get backend() { return 'memory'; }
+
+    /** @returns {object|null} */
+    getChunk(id) { return this.chunks.get(id) ?? null; }
+
+    /** All chunks defined in a given file. @returns {object[]} */
+    getChunksByFile(filePath) {
+        const out = [];
+        for (const c of this.chunks.values()) if (c.file_path === filePath) out.push(c);
+        return out;
+    }
+
+    /** Chunks whose symbol name matches exactly (case-insensitive). @returns {object[]} */
+    resolveSymbol(name) {
+        const ids = this.symbolTable.get(String(name).toLowerCase().trim());
+        if (!ids) return [];
+        const out = [];
+        for (const id of ids) { const c = this.chunks.get(id); if (c) out.push(c); }
+        return out;
+    }
+
+    /** Chunks that call the given function name. @returns {object[]} */
+    findCallers(funcName) {
+        const out = [];
+        for (const c of this.chunks.values()) if (c.calls?.includes(funcName)) out.push(c);
+        return out;
+    }
+
+    /**
+     * Chunks that reference a symbol *by name* as a type (`type_refs`) or as a
+     * base class/interface (`extends`) — the non-call half of symbol-level
+     * references. Calls are served by findCallers; mcp/topology.mjs findReferences fuses
+     * the two and classifies each by confidence. Case-insensitive exact match.
+     * @returns {object[]}
+     */
+    findReferers(symbol) {
+        const key = String(symbol).toLowerCase().trim();
+        if (!key) return [];
+        const out = [];
+        for (const c of this.chunks.values()) {
+            // A class/type lists its own name in type_refs (the name node is a
+            // type_identifier) — that self-mention is a definition, not a reference.
+            if (c.name && c.name.toLowerCase() === key) continue;
+            const hit = (c.type_refs && c.type_refs.some(t => t.toLowerCase() === key))
+                || (c.extends && c.extends.some(t => t.toLowerCase() === key));
+            if (hit) out.push(c);
+        }
+        return out;
+    }
+
+    /**
+     * HTTP routes filtered by method and/or path prefix, each augmented with its
+     * resolved handler chunk (`chunk`, looked up by handler_chunk_id, or null).
+     * `graph.routes` round-trips verbatim through load(); a route-less index yields
+     * []. Method is matched case-insensitively (HTTP verbs are upper-cased); the
+     * path prefix match is case-insensitive. Mirrors SqliteGraphStore.findRoutes so
+     * both backends return identical records.
+     * @returns {object[]}
+     */
+    findRoutes({ method = null, pathPrefix = null } = {}) {
+        const routes = (this.graph && this.graph.routes) || [];
+        const m = method ? String(method).toUpperCase().trim() : null;
+        const p = pathPrefix ? String(pathPrefix).toLowerCase() : null;
+        const out = [];
+        for (const r of routes) {
+            if (m && String(r.method || '').toUpperCase() !== m) continue;
+            if (p && !String(r.path || '').toLowerCase().startsWith(p)) continue;
+            const chunk = r.handler_chunk_id ? (this.chunks.get(r.handler_chunk_id) || null) : null;
+            out.push({ ...r, chunk });
+        }
+        return out;
+    }
+
+    /** Lazily iterate every chunk (cursor-friendly parity with the SQLite store). */
+    *iterateChunks() { yield* this.chunks.values(); }
+
+    chunkCount()  { return this.chunks.size; }
+    symbolCount() { return this.symbolTable.size; }
+    fileCount() {
+        const files = new Set();
+        for (const c of this.chunks.values()) files.add(c.file_path);
+        return files.size;
+    }
+    vectorCount() {
+        if (this._lazyMode) {
+            let n = 0;
+            for (const k of this._vecOffsets.keys()) if (baseEmbeddingKey(k) === k) n++;
+            return n;
+        }
+        let n = 0;
+        for (const k of this.vectors.keys()) if (baseEmbeddingKey(k) === k) n++;
+        return n;
+    }
+
+    getDependencies(filePath) { return this.graph.dependencies[filePath] || []; }
+    getImportedBy(filePath)   { return this.graph.importedBy[filePath] || []; }
+
+    /** Engine-level health facts for list_index_stats (daemon/age added by the tool). */
+    stats() {
+        const extCounts = new Map();
+        for (const c of this.chunks.values()) {
+            const ext = c.file_path.split('.').pop() || 'unknown';
+            extCounts.set(ext, (extCounts.get(ext) || 0) + 1);
+        }
+        const vectorSource = this._lazyMode
+            ? (this._vecFd >= 0 ? 'disk-backed fd' : 'buffer-lazy')
+            : 'eager (in-memory matrix)';
+        return {
+            backend: 'memory',
+            chunks: this.chunks.size,
+            files: this.fileCount(),
+            symbols: this.symbolTable.size,
+            vectors: this.vectorCount(),
+            hasVectors: this.vectors.size > 0 || this._vecOffsets.size > 0,
+            lazyMode: this._lazyMode,
+            vectorSource,
+            extCounts,
+        };
+    }
+
     // ─── Cleanup ───────────────────────────────────────────────────────────────
 
     close() {