graph-indexer 1.2.0 → 2.0.0

package/{core-engine.mjs → engine/memory.mjs}

@@ -1,5 +1,5 @@
 /**
- * @file core-engine.mjs
+ * @file engine/memory.mjs
  * @description In-Memory Graph Indexer Core Engine.
  * @author MaquinaTech <https://github.com/MaquinaTech>
  * @copyright (c) 2026 MaquinaTech. All rights reserved.
@@ -7,405 +7,14 @@
  */
 import fs from 'fs';
 import {
-    EMBEDDING_CONTEXT_LIMIT, truncateForEmbedding, cosineSimilarity, tokenize,
-    okapiIdf, bm25Score, fuseAndRank, buildLexicalDocument, embeddingKeyFor,
+    tokenize, okapiIdf, bm25Score, fuseAndRank, buildLexicalDocument, embeddingKeyFor,
     SUMMARY_VEC_SUFFIX, LEXICAL_FUSION_CAP, VECTOR_SCAN_RAW_N, finalizeVectorCandidates,
-} from './search-core.mjs';
-
-// Re-exported for backward compatibility — callers historically imported these
-// retrieval primitives from core-engine; they now live in search-core.mjs.
-export { EMBEDDING_CONTEXT_LIMIT, truncateForEmbedding, cosineSimilarity };
-
-// Thresholds for adaptive vector loading strategy
-const HNSW_THRESHOLD = 5000;      // Build HNSW index above this (eager mode only)
-const LAZY_VEC_THRESHOLD = 10000; // Switch to lazy (disk-backed) vector access above this
-
-// Optional HNSW accelerator for medium corpora (eager mode only)
-let HierarchicalNSW = null;
-try {
-    const mod = await import('hnswlib-node');
-    HierarchicalNSW = mod.HierarchicalNSW ?? mod.default?.HierarchicalNSW ?? null;
-} catch { /* not installed — flat scan used */ }
-
-/**
- * Serializes embeddingCache to compact binary.
- * Format per entry: [uint32 hashLen][utf8 hash][uint32 dim][float32 × dim]
- */
-export function writeEmbeddingBinary(embeddingCache) {
-    const entries = embeddingCache instanceof Map
-        ? Array.from(embeddingCache.entries())
-        : Object.entries(embeddingCache);
-    let size = 4;
-    for (const [hash, vec] of entries) {
-        size += 4 + Buffer.byteLength(hash, 'utf8') + 4 + vec.length * 4;
-    }
-    const buf = Buffer.allocUnsafe(size);
-    let off = 0;
-    buf.writeUInt32LE(entries.length, off); off += 4;
-    for (const [hash, vec] of entries) {
-        const hashBytes = Buffer.from(hash, 'utf8');
-        buf.writeUInt32LE(hashBytes.length, off); off += 4;
-        hashBytes.copy(buf, off); off += hashBytes.length;
-        buf.writeUInt32LE(vec.length, off); off += 4;
-        for (let d = 0; d < vec.length; d++) { buf.writeFloatLE(vec[d], off); off += 4; }
-    }
-    return buf;
-}
-
-/**
- * Decode an `.embeddings.bin` file into a Map<content_hash, Float32Array>.
- * Standalone (mirrors the instance loader) so the indexer can reuse cached
- * vectors across runs regardless of the active storage backend.
- */
-export function readEmbeddingBinary(filePath) {
-    const cache = new Map();
-    if (!fs.existsSync(filePath)) return cache;
-    const buf = fs.readFileSync(filePath);
-    let off = 0;
-    const count = buf.readUInt32LE(off); off += 4;
-    for (let i = 0; i < count; i++) {
-        const hashLen = buf.readUInt32LE(off); off += 4;
-        const hash = buf.subarray(off, off + hashLen).toString('utf8'); off += hashLen;
-        const dim = buf.readUInt32LE(off); off += 4;
-        const vec = new Float32Array(dim);
-        for (let d = 0; d < dim; d++) { vec[d] = buf.readFloatLE(off); off += 4; }
-        cache.set(hash, vec);
-    }
-    return cache;
-}
-
-/**
- * Append embedding entries to an `.embeddings.bin` file in place (creating it if
- * absent) and return the absolute float-data offset of each appended entry.
- * The leading uint32 entry count is updated so readEmbeddingBinary stays valid.
- *
- * Used by the watch daemon for incremental updates: a full bin rewrite on every
- * file change would be O(corpus) — appending is O(changed chunks). Dead entries
- * from replaced chunks are compacted away on the next full index run.
- *
- * @param {string} filePath
- * @param {Map<string, Float32Array|number[]>|object} entries  key → vector
- * @returns {Map<string, {offset:number, dim:number}>}
- */
-export function appendEmbeddingBinary(filePath, entries) {
-    const list = entries instanceof Map ? Array.from(entries.entries()) : Object.entries(entries || {});
-    const offsets = new Map();
-    if (list.length === 0) return offsets;
-
-    if (!fs.existsSync(filePath)) {
-        const hdr = Buffer.alloc(4);
-        hdr.writeUInt32LE(0, 0);
-        fs.writeFileSync(filePath, hdr);
-    }
-    const fd = fs.openSync(filePath, 'r+');
-    try {
-        const pos = fs.fstatSync(fd).size;
-        let size = 0;
-        for (const [key, vec] of list) size += 4 + Buffer.byteLength(key, 'utf8') + 4 + vec.length * 4;
-        const buf = Buffer.allocUnsafe(size);
-        let off = 0;
-        for (const [key, vec] of list) {
-            const kb = Buffer.from(key, 'utf8');
-            buf.writeUInt32LE(kb.length, off); off += 4;
-            kb.copy(buf, off); off += kb.length;
-            buf.writeUInt32LE(vec.length, off); off += 4;
-            offsets.set(key, { offset: pos + off, dim: vec.length });
-            for (let d = 0; d < vec.length; d++) { buf.writeFloatLE(vec[d], off); off += 4; }
-        }
-        fs.writeSync(fd, buf, 0, buf.length, pos);
-        const hdr = Buffer.allocUnsafe(4);
-        fs.readSync(fd, hdr, 0, 4, 0);
-        hdr.writeUInt32LE(hdr.readUInt32LE(0) + list.length, 0);
-        fs.writeSync(fd, hdr, 0, 4, 0);
-    } finally {
-        fs.closeSync(fd);
-    }
-    return offsets;
-}
-
-/**
- * Cosine-score EVERY vector in an `.embeddings.bin` against a query vector,
- * streaming from disk in bounded buffers, and return the top-N entry keys.
- *
- * This is the semantic channel for the disk-backed stores. The previous design
- * scored vectors only for chunks that already matched lexically, which silently
- * disabled semantic search exactly where it matters: conceptual queries that
- * share no tokens with the code. A sequential scan of the bin is fast (the OS
- * page cache holds it after the first query) and keeps resident RAM flat.
- *
- * @param {{fd?: number, buffer?: Buffer}} source  Open fd or in-memory buffer of the bin.
- * @param {Float32Array|number[]} queryVector
- * @param {{topN?: number, minScore?: number}} [opts]
- * @returns {Array<{key: string, score: number}>}  Sorted by score, descending.
- */
-export function scanEmbeddingBinary({ fd = -1, buffer = null }, queryVector, { topN = 200, minScore = 0 } = {}) {
-    const qDim = queryVector.length;
-    let qNorm = 0;
-    for (let d = 0; d < qDim; d++) qNorm += queryVector[d] * queryVector[d];
-    qNorm = Math.sqrt(qNorm);
-    if (qNorm === 0) return [];
-
-    const top = []; // sorted desc, capped at topN
-    let worst = -Infinity;
-    const push = (key, score) => {
-        if (score <= minScore) return;
-        if (top.length >= topN && score <= worst) return;
-        let lo = 0, hi = top.length;
-        while (lo < hi) { const mid = (lo + hi) >> 1; if (top[mid].score >= score) lo = mid + 1; else hi = mid; }
-        top.splice(lo, 0, { key, score });
-        if (top.length > topN) top.pop();
-        worst = top[top.length - 1].score;
-    };
-
-    // Parses complete entries in buf starting at startOff; returns the offset of
-    // the first incomplete entry (callers carry the remainder into the next read).
-    const processEntries = (buf, startOff) => {
-        let off = startOff;
-        for (;;) {
-            if (off + 4 > buf.length) return off;
-            const keyLen = buf.readUInt32LE(off);
-            if (off + 4 + keyLen + 4 > buf.length) return off;
-            const dim = buf.readUInt32LE(off + 4 + keyLen);
-            const entryEnd = off + 4 + keyLen + 4 + dim * 4;
-            if (entryEnd > buf.length) return off;
-            if (dim === qDim) {
-                const key = buf.toString('utf8', off + 4, off + 4 + keyLen);
-                let dp = 0, nv = 0;
-                let p = off + 4 + keyLen + 4;
-                for (let d = 0; d < dim; d++, p += 4) {
-                    const v = buf.readFloatLE(p);
-                    dp += queryVector[d] * v;
-                    nv += v * v;
-                }
-                const n = Math.sqrt(nv);
-                if (n > 0) push(key, dp / (qNorm * n));
-            }
-            off = entryEnd;
-        }
-    };
-
-    if (buffer) { processEntries(buffer, 4); return top; }
-    if (fd < 0) return [];
-
-    const CHUNK = 4 * 1024 * 1024;
-    let filePos = 0;
-    let carry = null;
-    let first = true;
-    for (;;) {
-        const buf = Buffer.allocUnsafe(CHUNK);
-        const read = fs.readSync(fd, buf, 0, CHUNK, filePos);
-        if (read <= 0) break;
-        filePos += read;
-        const work = carry && carry.length
-            ? Buffer.concat([carry, buf.subarray(0, read)])
-            : buf.subarray(0, read);
-        const consumed = processEntries(work, first ? 4 : 0);
-        first = false;
-        carry = work.subarray(consumed);
-        if (read < CHUNK) break;
-    }
-    return top;
-}
-
-// ─── Binary-quantized vector sketch ─────────────────────────────────────────────
-// The streaming full scan above is exact but O(corpus × dim) per query: ~104 ms
-// at 50k chunks and ~519 ms at 200k (measured warm). The sketch removes that
-// ceiling while keeping resident RAM bounded: each vector is reduced to its SIGN
-// BITS (768 dims → 96 bytes, 0.1% of the float32 data). A query is answered by
-// (1) a Hamming-distance pass over the packed bits (XOR + popcount over Uint32
-// words — a few ms even at 200k rows), keeping the best `oversample × topN`
-// candidates, then (2) an exact cosine rescore of only those candidates, pread
-// from the bin. Sign quantization of normalized embeddings preserves cosine
-// ordering well enough that a 4× oversampled rescore recovers the exact top-N
-// (validated in test/unit.mjs against the exhaustive scan).
-//
-// The sketch is APPEND-AWARE: the bin only ever grows between full rebuilds
-// (the daemon appends), so `updateVectorSketch` re-reads just the tail beyond
-// what it has already consumed, keeping refresh O(changed chunks).
-
-function popcnt32(x) {
-    x -= (x >> 1) & 0x55555555;
-    x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
-    x = (x + (x >> 4)) & 0x0f0f0f0f;
-    return (x * 0x01010101) >> 24;
-}
-
-/** Pack a float vector's sign bits into `words` Uint32 words. */
-function quantizeToBits(vec, out, base, words) {
-    for (let w = 0; w < words; w++) {
-        let bits = 0;
-        const d0 = w * 32;
-        const dMax = Math.min(d0 + 32, vec.length);
-        for (let d = d0; d < dMax; d++) {
-            if (vec[d] > 0) bits |= (1 << (d - d0));
-        }
-        out[base + w] = bits;
-    }
-}
-
-/**
- * Build or incrementally extend a binary sketch of an `.embeddings.bin`.
- *
- * @param {object|null} sketch  Existing sketch to extend, or null to create.
- * @param {{fd?: number, buffer?: Buffer}} source
- * @returns {object} sketch { dim, words, n, keys[], offsets[], bits: Uint32Array,
- *                            consumed, headerCount, firstKey }
- */
-export function updateVectorSketch(sketch, { fd = -1, buffer = null }) {
-    const readAt = (buf, pos, len) => {
-        if (buffer) return buffer.subarray(pos, pos + len);
-        const b = Buffer.allocUnsafe(len);
-        const r = fs.readSync(fd, b, 0, len, pos);
-        return r < len ? b.subarray(0, r) : b;
-    };
-    const fileSize = buffer ? buffer.length : (fd >= 0 ? fs.fstatSync(fd).size : 0);
-    if (fileSize < 4) return sketch;
-
-    const headerCount = readAt(null, 0, 4).readUInt32LE(0);
-    if (!sketch) {
-        sketch = {
-            dim: 0, words: 0, n: 0, keys: [], offsets: [],
-            bits: new Uint32Array(0), consumed: 4, headerCount: 0, firstKey: null,
-        };
-    }
-    if (fileSize <= sketch.consumed) return sketch;
-
-    // Read the unseen tail in large blocks. No carry buffers / Buffer.concat —
-    // the next block simply re-reads from the last complete entry boundary
-    // (at most one partial entry of overlap), so the build leaves no transient
-    // garbage behind. Sign bits are quantized straight off the block buffer.
-    const CHUNK = 8 * 1024 * 1024;
-    const reusable = buffer ? null : Buffer.allocUnsafe(CHUNK);
-    let pos = sketch.consumed;
-    while (pos < fileSize) {
-        const len = Math.min(CHUNK, fileSize - pos);
-        let work;
-        if (buffer) {
-            work = buffer.subarray(pos, pos + len);
-        } else {
-            const r = fs.readSync(fd, reusable, 0, len, pos);
-            work = r < len ? reusable.subarray(0, r) : (len < CHUNK ? reusable.subarray(0, len) : reusable);
-        }
-        let off = 0;
-        let parsedAny = false;
-        for (;;) {
-            if (off + 4 > work.length) break;
-            const keyLen = work.readUInt32LE(off);
-            if (off + 4 + keyLen + 4 > work.length) break;
-            const dim = work.readUInt32LE(off + 4 + keyLen);
-            const entryEnd = off + 4 + keyLen + 4 + dim * 4;
-            if (entryEnd > work.length) break;
-
-            const key = work.toString('utf8', off + 4, off + 4 + keyLen);
-            const floatBase = off + 4 + keyLen + 4;
-            if (sketch.dim === 0) {
-                sketch.dim = dim;
-                sketch.words = Math.ceil(dim / 32);
-            }
-            if (dim === sketch.dim) {
-                if ((sketch.n + 1) * sketch.words > sketch.bits.length) {
-                    const grown = new Uint32Array(Math.max(1024 * sketch.words, sketch.bits.length * 2));
-                    grown.set(sketch.bits);
-                    sketch.bits = grown;
-                }
-                // Quantize sign bits directly from the block buffer.
-                const bitBase = sketch.n * sketch.words;
-                for (let w = 0; w < sketch.words; w++) {
-                    let bits = 0;
-                    const d0 = w * 32;
-                    const dMax = Math.min(d0 + 32, dim);
-                    for (let d = d0; d < dMax; d++) {
-                        if (work.readFloatLE(floatBase + d * 4) > 0) bits |= (1 << (d - d0));
-                    }
-                    sketch.bits[bitBase + w] = bits;
-                }
-                sketch.keys.push(key);
-                sketch.offsets.push(pos + floatBase);
-                if (sketch.firstKey === null) sketch.firstKey = key;
-                sketch.n++;
-            }
-            off = entryEnd;
-            parsedAny = true;
-        }
-        if (!parsedAny) break; // partial entry larger than the block — corrupt tail
-        pos += off;            // resume exactly at the last complete entry boundary
-        sketch.consumed = pos;
-    }
-    sketch.headerCount = headerCount;
-    return sketch;
-}
-
-/**
- * Approximate top-N search via the sketch + exact cosine rescore from the bin.
- * Drop-in replacement for scanEmbeddingBinary (same return shape), ~20–50×
- * faster at scale.
- *
- * @returns {Array<{key: string, score: number}>} sorted by exact cosine, desc.
- */
-export function searchVectorSketch(sketch, { fd = -1, buffer = null }, queryVector, {
-    topN = 200, minScore = 0, oversample = 4,
-} = {}) {
-    if (!sketch || sketch.n === 0 || queryVector.length !== sketch.dim) return [];
-    const words = sketch.words;
-
-    // 1. Quantize the query and rank rows by Hamming distance (bounded top-M).
-    const qbits = new Uint32Array(words);
-    quantizeToBits(queryVector, qbits, 0, words);
-
-    const M = Math.min(sketch.n, Math.max(topN * oversample, 64));
-    const candIdx = new Int32Array(M);
-    const candHam = new Int32Array(M);
-    let count = 0, worst = -1;
-    const bits = sketch.bits;
-    for (let i = 0; i < sketch.n; i++) {
-        const base = i * words;
-        let ham = 0;
-        for (let w = 0; w < words; w++) ham += popcnt32((bits[base + w] ^ qbits[w]) >>> 0);
-        if (count >= M && ham >= worst) continue;
-        // bounded insertion sort (ascending hamming)
-        let lo = 0, hi = count;
-        while (lo < hi) { const mid = (lo + hi) >> 1; if (candHam[mid] <= ham) lo = mid + 1; else hi = mid; }
-        const end = Math.min(count, M - 1);
-        for (let j = end; j > lo; j--) { candHam[j] = candHam[j - 1]; candIdx[j] = candIdx[j - 1]; }
-        candHam[lo] = ham; candIdx[lo] = i;
-        if (count < M) count++;
-        worst = candHam[count - 1];
-    }
-
-    // 2. Exact cosine rescore of the candidates only (pread per candidate).
-    let qNorm = 0;
-    for (let d = 0; d < queryVector.length; d++) qNorm += queryVector[d] * queryVector[d];
-    qNorm = Math.sqrt(qNorm);
-    if (qNorm === 0) return [];
-
-    const byteLen = sketch.dim * 4;
-    const raw = Buffer.allocUnsafe(byteLen);
-    const scored = [];
-    for (let c = 0; c < count; c++) {
-        const i = candIdx[c];
-        let view;
-        if (buffer) {
-            view = buffer.subarray(sketch.offsets[i], sketch.offsets[i] + byteLen);
-            if (view.length < byteLen) continue;
-        } else {
-            const read = fs.readSync(fd, raw, 0, byteLen, sketch.offsets[i]);
-            if (read < byteLen) continue;
-            view = raw;
-        }
-        let dp = 0, nv = 0;
-        for (let d = 0; d < sketch.dim; d++) {
-            const v = view.readFloatLE(d * 4);
-            dp += queryVector[d] * v;
-            nv += v * v;
-        }
-        const n = Math.sqrt(nv);
-        if (n === 0) continue;
-        const score = dp / (qNorm * n);
-        if (score > minScore) scored.push({ key: sketch.keys[i], score });
-    }
-    scored.sort((a, b) => b.score - a.score);
-    return scored.slice(0, topN);
-}
+    isNaturalLanguageQuery, WINDOW_VEC_SUFFIX, EMBEDDING_MAX_WINDOWS, baseEmbeddingKey,
+} from '../search-core.mjs';
+import {
+    HierarchicalNSW, HNSW_THRESHOLD, LAZY_VEC_THRESHOLD, SKETCH_THRESHOLD,
+    writeEmbeddingBinary, scanEmbeddingBinary, updateVectorSketch, searchVectorSketch,
+} from './binary.mjs';
 
 export class MemoryGraphIndex {
     /**
@@ -469,10 +78,13 @@ export class MemoryGraphIndex {
         if (!fs.existsSync(this.indexPath)) return;
         const data = JSON.parse(fs.readFileSync(this.indexPath, 'utf-8'));
         this.graph = data.graph || { dependencies: {}, importedBy: {} };
+        // Does this corpus carry LLM enrichment? Drives the NL vector-channel weight
+        // in fuseAndRank (strong summary vectors earn full weight; plain code vectors
+        // stay a low-weight rescue). Set during the chunk load loop below.
+        this._corpusEnriched = false;
 
         const chunkCount = (data.chunks || []).length;
 
-        // Decide loading strategy based on corpus size and cacheEmbeddings flag
         this._lazyMode = (!this._cacheEmbeddings) && chunkCount >= LAZY_VEC_THRESHOLD;
 
         if (fs.existsSync(this._embeddingPath)) {
@@ -482,13 +94,10 @@ export class MemoryGraphIndex {
                 // For very large corpora: open persistent fd, release buffer from heap
                 if (chunkCount >= 50000) {
                     try { this._vecFd = fs.openSync(this._embeddingPath, 'r'); } catch { this._vecFd = -1; }
-                    // binBuf goes out of scope → GC can collect it
                 } else {
-                    // Medium-large: keep buffer for zero-copy slice access
                     this._embeddingBuf = binBuf;
                 }
             } else {
-                // Eager: fill embeddingCache (needed by indexer + small-corpus MCP server)
                 this._loadEmbeddingBinary(binBuf);
             }
         } else if (data.embeddingCache) {
@@ -499,6 +108,9 @@ export class MemoryGraphIndex {
 
         for (const chunk of (data.chunks || [])) {
             this.chunks.set(chunk.id, chunk);
+            if (!this._corpusEnriched && (chunk.summary || (chunk.concepts && chunk.concepts.length))) {
+                this._corpusEnriched = true;
+            }
 
             // Eager vector population. Vectors are keyed by embeddingKeyFor(chunk)
             // (content_hash + enrichment digest); plain content_hash is the
@@ -507,10 +119,16 @@ export class MemoryGraphIndex {
                 const vecKey = embeddingKeyFor(chunk);
                 if (chunk.content_hash && this.embeddingCache.has(vecKey)) {
                     this.vectors.set(chunk.id, this.embeddingCache.get(vecKey));
-                    // Summary-only second vector (enriched chunks): stored under a
-                    // pseudo row id; searchHybrid folds hits back onto the chunk id.
+                    // Summary-only and per-window vectors (enriched / oversized chunks):
+                    // stored under pseudo row ids; searchHybrid folds hits back onto the
+                    // chunk id (max-sim across all of the chunk's vectors).
                     const sVec = this.embeddingCache.get(vecKey + SUMMARY_VEC_SUFFIX);
                     if (sVec) this.vectors.set(chunk.id + SUMMARY_VEC_SUFFIX, sVec);
+                    for (let i = 1; i < EMBEDDING_MAX_WINDOWS; i++) {
+                        const wVec = this.embeddingCache.get(vecKey + WINDOW_VEC_SUFFIX + i);
+                        if (!wVec) break;
+                        this.vectors.set(chunk.id + WINDOW_VEC_SUFFIX + i, wVec);
+                    }
                 } else if (chunk.content_hash && this.embeddingCache.has(chunk.content_hash)) {
                     this.vectors.set(chunk.id, this.embeddingCache.get(chunk.content_hash));
                 } else if (chunk.embedding) {
@@ -520,12 +138,9 @@ export class MemoryGraphIndex {
                 }
             }
 
-            // Build inverted lexical index from the shared document builder
-            // (search-core.buildLexicalDocument) — identical text across backends.
             const deps = this.graph.dependencies[chunk.file_path] || [];
             this._indexLexical(chunk.id, buildLexicalDocument(chunk, deps), chunk.file_path);
 
-            // Build symbol table (Frontier 2)
             if (chunk.name && chunk.name !== 'anonymous') {
                 const n = chunk.name.toLowerCase();
                 if (!this.symbolTable.has(n)) this.symbolTable.set(n, new Set());
@@ -606,7 +221,7 @@ export class MemoryGraphIndex {
     // ─── Lexical index (TRUE inverted index) ──────────────────────────────────
 
     _indexLexical(chunkId, text, filePath = '') {
-        const tokens = tokenize(text);
+        const tokens = tokenize(text);               // raw + additive Porter stems
         if (tokens.length === 0) return;
 
         const termCounts = new Map();
@@ -614,16 +229,16 @@ export class MemoryGraphIndex {
             termCounts.set(token, (termCounts.get(token) || 0) + 1);
         }
 
-        // Track document length for BM25 length normalization.
-        // Only content tokens are counted — path tokens are handled separately
-        // in searchHybrid to avoid BM25 length normalisation amplifying short
-        // export stubs that share a path with a larger implementation.
-        this.docLens.set(chunkId, tokens.length);
-        this.totalDocLen += tokens.length;
+        // Additive stem tokens earn their own postings but must NOT inflate docLen —
+        // inflated length would perturb BM25 normalisation for exact symbolic matches.
+        // raw-based docLen keeps raw-term BM25 byte-identical to the pre-stem index.
+        // Path tokens are tallied separately in searchHybrid, not here.
+        const rawLen = tokenize(text, false).length;
+        this.docLens.set(chunkId, rawLen);
+        this.totalDocLen += rawLen;
 
         const chunkTokenSet = new Set();
         for (const [term, count] of termCounts) {
-            // Store raw term count — BM25 applies saturation at search time
             this.df.set(term, (this.df.get(term) || 0) + 1);
             let posting = this.invertedIndex.get(term);
             if (!posting) { posting = new Map(); this.invertedIndex.set(term, posting); }
@@ -638,7 +253,7 @@ export class MemoryGraphIndex {
         // Path matching is done multiplicatively in searchHybrid, which avoids
         // amplifying short stubs that happen to share a path with a long function.
         if (filePath) {
-            const pathTokenSet = new Set(tokenize(filePath.replace(/[/\-_.]/g, ' ')));
+            const pathTokenSet = new Set(tokenize(filePath.replace(/[/\-_.]/g, ' '), false));
             if (!this.pathTokens) this.pathTokens = new Map();
             this.pathTokens.set(chunkId, pathTokenSet);
         }
@@ -659,7 +274,6 @@ export class MemoryGraphIndex {
                 else this.df.set(term, freq - 1);
             }
         }
-        // Update BM25 length accounting
         const dl = this.docLens.get(chunkId);
         if (dl !== undefined) {
             this.totalDocLen = Math.max(0, this.totalDocLen - dl);
@@ -671,7 +285,10 @@ export class MemoryGraphIndex {
     }
 
     _searchLexical(queryText) {
-        const queryTokens = tokenize(queryText);
+        // Asymmetric stemming: the index always carries stems, but only stem the
+        // QUERY for natural-language/behavioural queries — symbolic name lookups
+        // stay exact so stem collisions never dilute a precise symbol match.
+        const queryTokens = tokenize(queryText, isNaturalLanguageQuery(queryText));
         const scores = new Map();
         const avgdl = this.docCount > 0 ? this.totalDocLen / this.docCount : 1;
 
@@ -739,7 +356,6 @@ export class MemoryGraphIndex {
         this._vecNorms  = norms;
         this._matrixDirty = false;
 
-        // HNSW for large eager corpora
         if (HierarchicalNSW && n >= HNSW_THRESHOLD) {
             try {
                 const hnsw = new HierarchicalNSW('cosine', dim);
@@ -753,7 +369,7 @@ export class MemoryGraphIndex {
                 }
                 this._hnsw = hnsw;
             } catch (e) {
-                process.stderr.write(`[core-engine] HNSW build failed: ${e.message}\n`);
+                process.stderr.write(`[engine/memory] HNSW build failed: ${e.message}\n`);
                 this._hnsw = null;
             }
         }
@@ -773,7 +389,6 @@ export class MemoryGraphIndex {
         qNorm = Math.sqrt(qNorm);
         if (qNorm === 0) return [];
 
-        // HNSW fast path
         if (this._hnsw) {
             const topK = Math.min(200, n);
             try {
@@ -788,7 +403,6 @@ export class MemoryGraphIndex {
             } catch { /* fall through to flat scan */ }
         }
 
-        // Exact flat scan
         const results = [];
         for (let i = 0; i < n; i++) {
             const base = i * dim;
@@ -813,9 +427,14 @@ export class MemoryGraphIndex {
         if (!this._embeddingBuf && this._vecFd < 0) return [];
         // Large corpora: binary sketch (built once on first vector query) replaces
         // the exact O(corpus) scan with a Hamming prefilter + bounded rescore.
-        // 10k threshold matches the SQLite store so backends stay rank-identical
-        // wherever the exact scan is still fast.
-        if (!this._sketch && this._vecOffsets.size >= 10000) {
+        // SKETCH_THRESHOLD is shared with the SQLite store so both switch to the
+        // sketch at the same vector count. NOTE: this lazy path only runs when the
+        // store is in lazy mode (cacheEmbeddings off AND chunkCount ≥ LAZY_VEC_THRESHOLD);
+        // a corpus with < LAZY_VEC_THRESHOLD chunks but ≥ SKETCH_THRESHOLD vectors
+        // (heavy enrichment/oversized windows) still scans exactly here while SQLite
+        // sketches — a narrow band worth a dedicated scale parity test before relying
+        // on byte-identical top-5 above the threshold.
+        if (!this._sketch && this._vecOffsets.size >= SKETCH_THRESHOLD) {
             try { this._sketch = updateVectorSketch(null, source); } catch { this._sketch = null; }
         }
         const hits = this._sketch
@@ -834,6 +453,7 @@ export class MemoryGraphIndex {
                 const key = embeddingKeyFor(c);
                 put(key, c.id);
                 put(key + SUMMARY_VEC_SUFFIX, c.id);     // summary-only vector → same chunk
+                for (let i = 1; i < EMBEDDING_MAX_WINDOWS; i++) put(key + WINDOW_VEC_SUFFIX + i, c.id); // window vectors → same chunk
                 if (key !== c.content_hash) put(c.content_hash, c.id); // legacy bins
             }
             this._keyToIds = map;
@@ -858,12 +478,12 @@ export class MemoryGraphIndex {
                 // zero lexical overlap, which the old TF-IDF prefilter never could.
                 vectorResults = this._scanVectorsLazy(queryVector, minScore);
             } else {
-                // Eager rows include summary-only pseudo ids (`<id>|s`) — fold each
-                // back onto its chunk via the shared finalizer (best score per
-                // chunk, deterministic order, same cap as the disk-backed paths).
+                // Eager rows include summary-only and per-window pseudo ids
+                // (`<id>|s`, `<id>|wN`) — fold each back onto its chunk via the shared
+                // finalizer (best/max-sim score per chunk, deterministic order, same
+                // cap as the disk-backed paths).
                 const entries = this._searchVector(queryVector, minScore).map(r => ({
-                    id: r.id.endsWith(SUMMARY_VEC_SUFFIX)
-                        ? r.id.slice(0, -SUMMARY_VEC_SUFFIX.length) : r.id,
+                    id: baseEmbeddingKey(r.id),
                     score: r.score,
                 }));
                 vectorResults = finalizeVectorCandidates(entries);
@@ -872,9 +492,6 @@ export class MemoryGraphIndex {
             vectorResults = [];
         }
 
-        // Fusion + boost ladder lives in search-core.mjs so the in-memory engine
-        // and the SQLite store rank identically. Backend state is reached through
-        // accessors; the math is measured once and shared.
         return fuseAndRank({
             lexicalResults,
             vectorResults,
@@ -886,8 +503,8 @@ export class MemoryGraphIndex {
             topK,
             queryText,
             exactBoostName,
-            // Equivalent to the former full-scan (symbolTable is keyed by name.toLowerCase()),
-            // but O(1) and reusable by the SQLite backend.
+            corpusEnriched: this._corpusEnriched,
+            // O(1) vs the old full-scan; same contract implemented by the SQLite backend.
             resolveExact:  (term) => this.symbolTable.get(term) || [],
         });
     }
@@ -899,7 +516,7 @@ export class MemoryGraphIndex {
             id: c.id, file_path: c.file_path, node_type: c.node_type,
             name: c.name, docstring: c.docstring || '', code_snippet: c.code_snippet,
             content_hash: c.content_hash, start_line: c.start_line, end_line: c.end_line,
-            calls: c.calls || [], params: c.params || [],
+            calls: c.calls || [], call_sites: c.call_sites || [], params: c.params || [],
             return_type: c.return_type || '', class_context: c.class_context || '',
             type_refs: c.type_refs || [], decorators: c.decorators || [],
             extends: c.extends || [],
@@ -923,7 +540,7 @@ export class MemoryGraphIndex {
         this._saveTimer = setTimeout(async () => {
             this._saveTimer = null;
             try { await this.save(); }
-            catch (err) { process.stderr.write(`[core-engine] ❌ Async save failed: ${err.message}\n`); }
+            catch (err) { process.stderr.write(`[engine/memory] ❌ Async save failed: ${err.message}\n`); }
         }, delayMs);
     }
 
@@ -961,6 +578,7 @@ export class MemoryGraphIndex {
             this._removeLexical(id);
             this.removeVector(id);
             this.removeVector(id + SUMMARY_VEC_SUFFIX);
+            for (let i = 1; i < EMBEDDING_MAX_WINDOWS; i++) this.removeVector(id + WINDOW_VEC_SUFFIX + i);
             this._removeSymbol(chunk);
             this.chunks.delete(id);
         }
@@ -977,6 +595,10 @@ export class MemoryGraphIndex {
             if (vec) this.addVector(chunk.id, vec);
             const sVec = this.embeddingCache.get(vecKey + SUMMARY_VEC_SUFFIX);
             if (sVec) this.addVector(chunk.id + SUMMARY_VEC_SUFFIX, sVec);
+            for (let i = 1; i < EMBEDDING_MAX_WINDOWS; i++) {
+                const wVec = this.embeddingCache.get(vecKey + WINDOW_VEC_SUFFIX + i);
+                if (wVec) this.addVector(chunk.id + WINDOW_VEC_SUFFIX + i, wVec);
+            }
             this._indexLexical(chunk.id, buildLexicalDocument(chunk, imports), chunk.file_path);
             this.chunks.set(chunk.id, chunk);
             if (chunk.name && chunk.name !== 'anonymous') {
@@ -1065,6 +687,51 @@ export class MemoryGraphIndex {
         return out;
     }
 
+    /**
+     * Chunks that reference a symbol *by name* as a type (`type_refs`) or as a
+     * base class/interface (`extends`) — the non-call half of symbol-level
+     * references. Calls are served by findCallers; mcp/topology.mjs findReferences fuses
+     * the two and classifies each by confidence. Case-insensitive exact match.
+     * @returns {object[]}
+     */
+    findReferers(symbol) {
+        const key = String(symbol).toLowerCase().trim();
+        if (!key) return [];
+        const out = [];
+        for (const c of this.chunks.values()) {
+            // A class/type lists its own name in type_refs (the name node is a
+            // type_identifier) — that self-mention is a definition, not a reference.
+            if (c.name && c.name.toLowerCase() === key) continue;
+            const hit = (c.type_refs && c.type_refs.some(t => t.toLowerCase() === key))
+                || (c.extends && c.extends.some(t => t.toLowerCase() === key));
+            if (hit) out.push(c);
+        }
+        return out;
+    }
+
+    /**
+     * HTTP routes filtered by method and/or path prefix, each augmented with its
+     * resolved handler chunk (`chunk`, looked up by handler_chunk_id, or null).
+     * `graph.routes` round-trips verbatim through load(); a route-less index yields
+     * []. Method is matched case-insensitively (HTTP verbs are upper-cased); the
+     * path prefix match is case-insensitive. Mirrors SqliteGraphStore.findRoutes so
+     * both backends return identical records.
+     * @returns {object[]}
+     */
+    findRoutes({ method = null, pathPrefix = null } = {}) {
+        const routes = (this.graph && this.graph.routes) || [];
+        const m = method ? String(method).toUpperCase().trim() : null;
+        const p = pathPrefix ? String(pathPrefix).toLowerCase() : null;
+        const out = [];
+        for (const r of routes) {
+            if (m && String(r.method || '').toUpperCase() !== m) continue;
+            if (p && !String(r.path || '').toLowerCase().startsWith(p)) continue;
+            const chunk = r.handler_chunk_id ? (this.chunks.get(r.handler_chunk_id) || null) : null;
+            out.push({ ...r, chunk });
+        }
+        return out;
+    }
+
     /** Lazily iterate every chunk (cursor-friendly parity with the SQLite store). */
     *iterateChunks() { yield* this.chunks.values(); }
 
@@ -1078,11 +745,11 @@ export class MemoryGraphIndex {
     vectorCount() {
         if (this._lazyMode) {
             let n = 0;
-            for (const k of this._vecOffsets.keys()) if (!k.endsWith(SUMMARY_VEC_SUFFIX)) n++;
+            for (const k of this._vecOffsets.keys()) if (baseEmbeddingKey(k) === k) n++;
             return n;
         }
         let n = 0;
-        for (const k of this.vectors.keys()) if (!k.endsWith(SUMMARY_VEC_SUFFIX)) n++;
+        for (const k of this.vectors.keys()) if (baseEmbeddingKey(k) === k) n++;
         return n;
     }