graph-indexer 1.2.0 → 2.1.0

package/{core-engine.mjs → engine/memory.mjs}

@@ -1,5 +1,5 @@
 /**
- * @file core-engine.mjs
+ * @file engine/memory.mjs
  * @description In-Memory Graph Indexer Core Engine.
  * @author MaquinaTech <https://github.com/MaquinaTech>
  * @copyright (c) 2026 MaquinaTech. All rights reserved.
@@ -7,405 +7,16 @@
  */
 import fs from 'fs';
 import {
-    EMBEDDING_CONTEXT_LIMIT, truncateForEmbedding, cosineSimilarity, tokenize,
-    okapiIdf, bm25Score, fuseAndRank, buildLexicalDocument, embeddingKeyFor,
+    tokenize, okapiIdf, bm25Score, fuseAndRank, buildLexicalDocument, embeddingKeyFor,
     SUMMARY_VEC_SUFFIX, LEXICAL_FUSION_CAP, VECTOR_SCAN_RAW_N, finalizeVectorCandidates,
-} from './search-core.mjs';
-
-// Re-exported for backward compatibility — callers historically imported these
-// retrieval primitives from core-engine; they now live in search-core.mjs.
-export { EMBEDDING_CONTEXT_LIMIT, truncateForEmbedding, cosineSimilarity };
-
-// Thresholds for adaptive vector loading strategy
-const HNSW_THRESHOLD = 5000;      // Build HNSW index above this (eager mode only)
-const LAZY_VEC_THRESHOLD = 10000; // Switch to lazy (disk-backed) vector access above this
-
-// Optional HNSW accelerator for medium corpora (eager mode only)
-let HierarchicalNSW = null;
-try {
-    const mod = await import('hnswlib-node');
-    HierarchicalNSW = mod.HierarchicalNSW ?? mod.default?.HierarchicalNSW ?? null;
-} catch { /* not installed — flat scan used */ }
-
-/**
- * Serializes embeddingCache to compact binary.
- * Format per entry: [uint32 hashLen][utf8 hash][uint32 dim][float32 × dim]
- */
-export function writeEmbeddingBinary(embeddingCache) {
-    const entries = embeddingCache instanceof Map
-        ? Array.from(embeddingCache.entries())
-        : Object.entries(embeddingCache);
-    let size = 4;
-    for (const [hash, vec] of entries) {
-        size += 4 + Buffer.byteLength(hash, 'utf8') + 4 + vec.length * 4;
-    }
-    const buf = Buffer.allocUnsafe(size);
-    let off = 0;
-    buf.writeUInt32LE(entries.length, off); off += 4;
-    for (const [hash, vec] of entries) {
-        const hashBytes = Buffer.from(hash, 'utf8');
-        buf.writeUInt32LE(hashBytes.length, off); off += 4;
-        hashBytes.copy(buf, off); off += hashBytes.length;
-        buf.writeUInt32LE(vec.length, off); off += 4;
-        for (let d = 0; d < vec.length; d++) { buf.writeFloatLE(vec[d], off); off += 4; }
-    }
-    return buf;
-}
-
-/**
- * Decode an `.embeddings.bin` file into a Map<content_hash, Float32Array>.
- * Standalone (mirrors the instance loader) so the indexer can reuse cached
- * vectors across runs regardless of the active storage backend.
- */
-export function readEmbeddingBinary(filePath) {
-    const cache = new Map();
-    if (!fs.existsSync(filePath)) return cache;
-    const buf = fs.readFileSync(filePath);
-    let off = 0;
-    const count = buf.readUInt32LE(off); off += 4;
-    for (let i = 0; i < count; i++) {
-        const hashLen = buf.readUInt32LE(off); off += 4;
-        const hash = buf.subarray(off, off + hashLen).toString('utf8'); off += hashLen;
-        const dim = buf.readUInt32LE(off); off += 4;
-        const vec = new Float32Array(dim);
-        for (let d = 0; d < dim; d++) { vec[d] = buf.readFloatLE(off); off += 4; }
-        cache.set(hash, vec);
-    }
-    return cache;
-}
-
-/**
- * Append embedding entries to an `.embeddings.bin` file in place (creating it if
- * absent) and return the absolute float-data offset of each appended entry.
- * The leading uint32 entry count is updated so readEmbeddingBinary stays valid.
- *
- * Used by the watch daemon for incremental updates: a full bin rewrite on every
- * file change would be O(corpus) — appending is O(changed chunks). Dead entries
- * from replaced chunks are compacted away on the next full index run.
- *
- * @param {string} filePath
- * @param {Map<string, Float32Array|number[]>|object} entries  key → vector
- * @returns {Map<string, {offset:number, dim:number}>}
- */
-export function appendEmbeddingBinary(filePath, entries) {
-    const list = entries instanceof Map ? Array.from(entries.entries()) : Object.entries(entries || {});
-    const offsets = new Map();
-    if (list.length === 0) return offsets;
-
-    if (!fs.existsSync(filePath)) {
-        const hdr = Buffer.alloc(4);
-        hdr.writeUInt32LE(0, 0);
-        fs.writeFileSync(filePath, hdr);
-    }
-    const fd = fs.openSync(filePath, 'r+');
-    try {
-        const pos = fs.fstatSync(fd).size;
-        let size = 0;
-        for (const [key, vec] of list) size += 4 + Buffer.byteLength(key, 'utf8') + 4 + vec.length * 4;
-        const buf = Buffer.allocUnsafe(size);
-        let off = 0;
-        for (const [key, vec] of list) {
-            const kb = Buffer.from(key, 'utf8');
-            buf.writeUInt32LE(kb.length, off); off += 4;
-            kb.copy(buf, off); off += kb.length;
-            buf.writeUInt32LE(vec.length, off); off += 4;
-            offsets.set(key, { offset: pos + off, dim: vec.length });
-            for (let d = 0; d < vec.length; d++) { buf.writeFloatLE(vec[d], off); off += 4; }
-        }
-        fs.writeSync(fd, buf, 0, buf.length, pos);
-        const hdr = Buffer.allocUnsafe(4);
-        fs.readSync(fd, hdr, 0, 4, 0);
-        hdr.writeUInt32LE(hdr.readUInt32LE(0) + list.length, 0);
-        fs.writeSync(fd, hdr, 0, 4, 0);
-    } finally {
-        fs.closeSync(fd);
-    }
-    return offsets;
-}
-
-/**
- * Cosine-score EVERY vector in an `.embeddings.bin` against a query vector,
- * streaming from disk in bounded buffers, and return the top-N entry keys.
- *
- * This is the semantic channel for the disk-backed stores. The previous design
- * scored vectors only for chunks that already matched lexically, which silently
- * disabled semantic search exactly where it matters: conceptual queries that
- * share no tokens with the code. A sequential scan of the bin is fast (the OS
- * page cache holds it after the first query) and keeps resident RAM flat.
- *
- * @param {{fd?: number, buffer?: Buffer}} source  Open fd or in-memory buffer of the bin.
- * @param {Float32Array|number[]} queryVector
- * @param {{topN?: number, minScore?: number}} [opts]
- * @returns {Array<{key: string, score: number}>}  Sorted by score, descending.
- */
-export function scanEmbeddingBinary({ fd = -1, buffer = null }, queryVector, { topN = 200, minScore = 0 } = {}) {
-    const qDim = queryVector.length;
-    let qNorm = 0;
-    for (let d = 0; d < qDim; d++) qNorm += queryVector[d] * queryVector[d];
-    qNorm = Math.sqrt(qNorm);
-    if (qNorm === 0) return [];
-
-    const top = []; // sorted desc, capped at topN
-    let worst = -Infinity;
-    const push = (key, score) => {
-        if (score <= minScore) return;
-        if (top.length >= topN && score <= worst) return;
-        let lo = 0, hi = top.length;
-        while (lo < hi) { const mid = (lo + hi) >> 1; if (top[mid].score >= score) lo = mid + 1; else hi = mid; }
-        top.splice(lo, 0, { key, score });
-        if (top.length > topN) top.pop();
-        worst = top[top.length - 1].score;
-    };
-
-    // Parses complete entries in buf starting at startOff; returns the offset of
-    // the first incomplete entry (callers carry the remainder into the next read).
-    const processEntries = (buf, startOff) => {
-        let off = startOff;
-        for (;;) {
-            if (off + 4 > buf.length) return off;
-            const keyLen = buf.readUInt32LE(off);
-            if (off + 4 + keyLen + 4 > buf.length) return off;
-            const dim = buf.readUInt32LE(off + 4 + keyLen);
-            const entryEnd = off + 4 + keyLen + 4 + dim * 4;
-            if (entryEnd > buf.length) return off;
-            if (dim === qDim) {
-                const key = buf.toString('utf8', off + 4, off + 4 + keyLen);
-                let dp = 0, nv = 0;
-                let p = off + 4 + keyLen + 4;
-                for (let d = 0; d < dim; d++, p += 4) {
-                    const v = buf.readFloatLE(p);
-                    dp += queryVector[d] * v;
-                    nv += v * v;
-                }
-                const n = Math.sqrt(nv);
-                if (n > 0) push(key, dp / (qNorm * n));
-            }
-            off = entryEnd;
-        }
-    };
-
-    if (buffer) { processEntries(buffer, 4); return top; }
-    if (fd < 0) return [];
-
-    const CHUNK = 4 * 1024 * 1024;
-    let filePos = 0;
-    let carry = null;
-    let first = true;
-    for (;;) {
-        const buf = Buffer.allocUnsafe(CHUNK);
-        const read = fs.readSync(fd, buf, 0, CHUNK, filePos);
-        if (read <= 0) break;
-        filePos += read;
-        const work = carry && carry.length
-            ? Buffer.concat([carry, buf.subarray(0, read)])
-            : buf.subarray(0, read);
-        const consumed = processEntries(work, first ? 4 : 0);
-        first = false;
-        carry = work.subarray(consumed);
-        if (read < CHUNK) break;
-    }
-    return top;
-}
-
-// ─── Binary-quantized vector sketch ─────────────────────────────────────────────
-// The streaming full scan above is exact but O(corpus × dim) per query: ~104 ms
-// at 50k chunks and ~519 ms at 200k (measured warm). The sketch removes that
-// ceiling while keeping resident RAM bounded: each vector is reduced to its SIGN
-// BITS (768 dims → 96 bytes, 0.1% of the float32 data). A query is answered by
-// (1) a Hamming-distance pass over the packed bits (XOR + popcount over Uint32
-// words — a few ms even at 200k rows), keeping the best `oversample × topN`
-// candidates, then (2) an exact cosine rescore of only those candidates, pread
-// from the bin. Sign quantization of normalized embeddings preserves cosine
-// ordering well enough that a 4× oversampled rescore recovers the exact top-N
-// (validated in test/unit.mjs against the exhaustive scan).
-//
-// The sketch is APPEND-AWARE: the bin only ever grows between full rebuilds
-// (the daemon appends), so `updateVectorSketch` re-reads just the tail beyond
-// what it has already consumed, keeping refresh O(changed chunks).
-
-function popcnt32(x) {
-    x -= (x >> 1) & 0x55555555;
-    x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
-    x = (x + (x >> 4)) & 0x0f0f0f0f;
-    return (x * 0x01010101) >> 24;
-}
-
-/** Pack a float vector's sign bits into `words` Uint32 words. */
-function quantizeToBits(vec, out, base, words) {
-    for (let w = 0; w < words; w++) {
-        let bits = 0;
-        const d0 = w * 32;
-        const dMax = Math.min(d0 + 32, vec.length);
-        for (let d = d0; d < dMax; d++) {
-            if (vec[d] > 0) bits |= (1 << (d - d0));
-        }
-        out[base + w] = bits;
-    }
-}
-
-/**
- * Build or incrementally extend a binary sketch of an `.embeddings.bin`.
- *
- * @param {object|null} sketch  Existing sketch to extend, or null to create.
- * @param {{fd?: number, buffer?: Buffer}} source
- * @returns {object} sketch { dim, words, n, keys[], offsets[], bits: Uint32Array,
- *                            consumed, headerCount, firstKey }
- */
-export function updateVectorSketch(sketch, { fd = -1, buffer = null }) {
-    const readAt = (buf, pos, len) => {
-        if (buffer) return buffer.subarray(pos, pos + len);
-        const b = Buffer.allocUnsafe(len);
-        const r = fs.readSync(fd, b, 0, len, pos);
-        return r < len ? b.subarray(0, r) : b;
-    };
-    const fileSize = buffer ? buffer.length : (fd >= 0 ? fs.fstatSync(fd).size : 0);
-    if (fileSize < 4) return sketch;
-
-    const headerCount = readAt(null, 0, 4).readUInt32LE(0);
-    if (!sketch) {
-        sketch = {
-            dim: 0, words: 0, n: 0, keys: [], offsets: [],
-            bits: new Uint32Array(0), consumed: 4, headerCount: 0, firstKey: null,
-        };
-    }
-    if (fileSize <= sketch.consumed) return sketch;
-
-    // Read the unseen tail in large blocks. No carry buffers / Buffer.concat —
-    // the next block simply re-reads from the last complete entry boundary
-    // (at most one partial entry of overlap), so the build leaves no transient
-    // garbage behind. Sign bits are quantized straight off the block buffer.
-    const CHUNK = 8 * 1024 * 1024;
-    const reusable = buffer ? null : Buffer.allocUnsafe(CHUNK);
-    let pos = sketch.consumed;
-    while (pos < fileSize) {
-        const len = Math.min(CHUNK, fileSize - pos);
-        let work;
-        if (buffer) {
-            work = buffer.subarray(pos, pos + len);
-        } else {
-            const r = fs.readSync(fd, reusable, 0, len, pos);
-            work = r < len ? reusable.subarray(0, r) : (len < CHUNK ? reusable.subarray(0, len) : reusable);
-        }
-        let off = 0;
-        let parsedAny = false;
-        for (;;) {
-            if (off + 4 > work.length) break;
-            const keyLen = work.readUInt32LE(off);
-            if (off + 4 + keyLen + 4 > work.length) break;
-            const dim = work.readUInt32LE(off + 4 + keyLen);
-            const entryEnd = off + 4 + keyLen + 4 + dim * 4;
-            if (entryEnd > work.length) break;
-
-            const key = work.toString('utf8', off + 4, off + 4 + keyLen);
-            const floatBase = off + 4 + keyLen + 4;
-            if (sketch.dim === 0) {
-                sketch.dim = dim;
-                sketch.words = Math.ceil(dim / 32);
-            }
-            if (dim === sketch.dim) {
-                if ((sketch.n + 1) * sketch.words > sketch.bits.length) {
-                    const grown = new Uint32Array(Math.max(1024 * sketch.words, sketch.bits.length * 2));
-                    grown.set(sketch.bits);
-                    sketch.bits = grown;
-                }
-                // Quantize sign bits directly from the block buffer.
-                const bitBase = sketch.n * sketch.words;
-                for (let w = 0; w < sketch.words; w++) {
-                    let bits = 0;
-                    const d0 = w * 32;
-                    const dMax = Math.min(d0 + 32, dim);
-                    for (let d = d0; d < dMax; d++) {
-                        if (work.readFloatLE(floatBase + d * 4) > 0) bits |= (1 << (d - d0));
-                    }
-                    sketch.bits[bitBase + w] = bits;
-                }
-                sketch.keys.push(key);
-                sketch.offsets.push(pos + floatBase);
-                if (sketch.firstKey === null) sketch.firstKey = key;
-                sketch.n++;
-            }
-            off = entryEnd;
-            parsedAny = true;
-        }
-        if (!parsedAny) break; // partial entry larger than the block — corrupt tail
-        pos += off;            // resume exactly at the last complete entry boundary
-        sketch.consumed = pos;
-    }
-    sketch.headerCount = headerCount;
-    return sketch;
-}
-
-/**
- * Approximate top-N search via the sketch + exact cosine rescore from the bin.
- * Drop-in replacement for scanEmbeddingBinary (same return shape), ~20–50×
- * faster at scale.
- *
- * @returns {Array<{key: string, score: number}>} sorted by exact cosine, desc.
- */
-export function searchVectorSketch(sketch, { fd = -1, buffer = null }, queryVector, {
-    topN = 200, minScore = 0, oversample = 4,
-} = {}) {
-    if (!sketch || sketch.n === 0 || queryVector.length !== sketch.dim) return [];
-    const words = sketch.words;
-
-    // 1. Quantize the query and rank rows by Hamming distance (bounded top-M).
-    const qbits = new Uint32Array(words);
-    quantizeToBits(queryVector, qbits, 0, words);
-
-    const M = Math.min(sketch.n, Math.max(topN * oversample, 64));
-    const candIdx = new Int32Array(M);
-    const candHam = new Int32Array(M);
-    let count = 0, worst = -1;
-    const bits = sketch.bits;
-    for (let i = 0; i < sketch.n; i++) {
-        const base = i * words;
-        let ham = 0;
-        for (let w = 0; w < words; w++) ham += popcnt32((bits[base + w] ^ qbits[w]) >>> 0);
-        if (count >= M && ham >= worst) continue;
-        // bounded insertion sort (ascending hamming)
-        let lo = 0, hi = count;
-        while (lo < hi) { const mid = (lo + hi) >> 1; if (candHam[mid] <= ham) lo = mid + 1; else hi = mid; }
-        const end = Math.min(count, M - 1);
-        for (let j = end; j > lo; j--) { candHam[j] = candHam[j - 1]; candIdx[j] = candIdx[j - 1]; }
-        candHam[lo] = ham; candIdx[lo] = i;
-        if (count < M) count++;
-        worst = candHam[count - 1];
-    }
-
-    // 2. Exact cosine rescore of the candidates only (pread per candidate).
-    let qNorm = 0;
-    for (let d = 0; d < queryVector.length; d++) qNorm += queryVector[d] * queryVector[d];
-    qNorm = Math.sqrt(qNorm);
-    if (qNorm === 0) return [];
-
-    const byteLen = sketch.dim * 4;
-    const raw = Buffer.allocUnsafe(byteLen);
-    const scored = [];
-    for (let c = 0; c < count; c++) {
-        const i = candIdx[c];
-        let view;
-        if (buffer) {
-            view = buffer.subarray(sketch.offsets[i], sketch.offsets[i] + byteLen);
-            if (view.length < byteLen) continue;
-        } else {
-            const read = fs.readSync(fd, raw, 0, byteLen, sketch.offsets[i]);
-            if (read < byteLen) continue;
-            view = raw;
-        }
-        let dp = 0, nv = 0;
-        for (let d = 0; d < sketch.dim; d++) {
-            const v = view.readFloatLE(d * 4);
-            dp += queryVector[d] * v;
-            nv += v * v;
-        }
-        const n = Math.sqrt(nv);
-        if (n === 0) continue;
-        const score = dp / (qNorm * n);
-        if (score > minScore) scored.push({ key: sketch.keys[i], score });
-    }
-    scored.sort((a, b) => b.score - a.score);
-    return scored.slice(0, topN);
-}
+    isNaturalLanguageQuery, WINDOW_VEC_SUFFIX, EMBEDDING_MAX_WINDOWS, baseEmbeddingKey,
+    scoreSparseExpanded,
+} from '../search-core.mjs';
+import { expandSparseQuery } from '../search-sparse.mjs';
+import {
+    HierarchicalNSW, HNSW_THRESHOLD, LAZY_VEC_THRESHOLD, SKETCH_THRESHOLD,
+    writeEmbeddingBinary, scanEmbeddingBinary, updateVectorSketch, searchVectorSketch,
+} from './binary.mjs';
 
 export class MemoryGraphIndex {
     /**
@@ -461,6 +72,72 @@ export class MemoryGraphIndex {
 
         this.rrfK = rrfK;
         this._saveTimer = null;
+
+        // ── A4: persistent resolved symbol graph (opt-in; null when not built) ──
+        // _edges is the serialized edge list; the two maps index it by endpoint for
+        // O(1) getEdges and edge-backed findCallers/findReferers. Cleared on any
+        // incremental update (daemon) so a stale graph never serves wrong callers.
+        this._edges       = null;
+        this._edgesByTo   = new Map();   // to_chunk_id   → edge[]
+        this._edgesByFrom = new Map();   // from_chunk_id → edge[]
+
+        // ── A5: symbol centrality (PageRank over the edges; built with the graph) ──
+        // _centrality maps chunk id → { score, rank }; precomputed at index time and
+        // serialized, so both backends serve identical numbers. Cleared with the edges.
+        this._centrality      = null;
+        this._centralityTotal = 0;
+        this._centralRanked   = null;    // lazy id-by-rank cache for topCentral
+
+        // ── C2: serialized taint flows ({ flows, meta }); precomputed at index time with --taint,
+        //    so the taint tools serve instantly and both backends serve identical flows. Cleared
+        //    on any incremental file update (a code change invalidates the flow set).
+        this._taint = null;
+
+        // ── A2 v2: SCIP precise references — `{ defChunkId: [refererChunkId, …] }`, built at index
+        //    time from a locally-generated SCIP index (--resolver scip) and serialized as an
+        //    isolated artifact so it cannot perturb A4 edges / A5 centrality. find_references reads
+        //    it for a binding-precise, cross-file `resolved` reference tier. Cleared with the graph.
+        this._scipRefs = null;
+
+        // ── B3: learned-sparse association model — `{ assoc: { term: [[assocTerm, weight], …] } }`,
+        //    learned from corpus co-occurrence at index time with --learned-sparse and serialized as
+        //    an isolated artifact. searchHybrid uses it to add an NL-only vocabulary-EXPANSION
+        //    channel; absent on default indexes → channel inert → byte-identical. Whole-program →
+        //    cleared on any incremental file update.
+        this._sparseModel = null;
+    }
+
+    /** Index the (already deterministically-ordered) edge list by endpoint. */
+    _indexEdges(edges) {
+        this._edges = edges;
+        this._edgesByTo = new Map();
+        this._edgesByFrom = new Map();
+        for (const e of edges) {
+            if (!this._edgesByTo.has(e.to_chunk_id)) this._edgesByTo.set(e.to_chunk_id, []);
+            this._edgesByTo.get(e.to_chunk_id).push(e);
+            if (!this._edgesByFrom.has(e.from_chunk_id)) this._edgesByFrom.set(e.from_chunk_id, []);
+            this._edgesByFrom.get(e.from_chunk_id).push(e);
+        }
+    }
+
+    /** Load the precomputed symbol-centrality map (A5). */
+    _indexCentrality(centrality) {
+        this._centrality = new Map(Object.entries(centrality));
+        this._centralityTotal = this._centrality.size;
+        this._centralRanked = null;
+    }
+
+    /** Drop the symbol graph + its centrality + serialized taint + SCIP references (an incremental update makes them stale). */
+    _clearEdges() {
+        this._edges = null;
+        this._edgesByTo = new Map();
+        this._edgesByFrom = new Map();
+        this._centrality = null;
+        this._centralityTotal = 0;
+        this._centralRanked = null;
+        this._taint = null;
+        this._scipRefs = null;
+        this._sparseModel = null;
     }
 
     // ─── Load ─────────────────────────────────────────────────────────────────
@@ -469,10 +146,23 @@ export class MemoryGraphIndex {
         if (!fs.existsSync(this.indexPath)) return;
         const data = JSON.parse(fs.readFileSync(this.indexPath, 'utf-8'));
         this.graph = data.graph || { dependencies: {}, importedBy: {} };
+        // A4 resolved symbol graph (opt-in; absent on default indexes → no-op).
+        if (Array.isArray(data.edges)) this._indexEdges(data.edges);
+        // A5 symbol centrality (built with the graph; absent on default indexes → no-op).
+        if (data.centrality && typeof data.centrality === 'object') this._indexCentrality(data.centrality);
+        // C2 serialized taint flows (opt-in --taint; absent on default indexes → no-op).
+        if (data.taint && typeof data.taint === 'object') this._taint = data.taint;
+        // A2 v2 SCIP precise references (opt-in --resolver scip; absent on default indexes → no-op).
+        if (data.scip_refs && typeof data.scip_refs === 'object') this._scipRefs = data.scip_refs;
+        // B3 learned-sparse model (opt-in --learned-sparse; absent on default indexes → no-op).
+        if (data.sparse_model && typeof data.sparse_model === 'object') this._sparseModel = data.sparse_model;
+        // Does this corpus carry LLM enrichment? Drives the NL vector-channel weight
+        // in fuseAndRank (strong summary vectors earn full weight; plain code vectors
+        // stay a low-weight rescue). Set during the chunk load loop below.
+        this._corpusEnriched = false;
 
         const chunkCount = (data.chunks || []).length;
 
-        // Decide loading strategy based on corpus size and cacheEmbeddings flag
         this._lazyMode = (!this._cacheEmbeddings) && chunkCount >= LAZY_VEC_THRESHOLD;
 
         if (fs.existsSync(this._embeddingPath)) {
@@ -482,13 +172,10 @@ export class MemoryGraphIndex {
                 // For very large corpora: open persistent fd, release buffer from heap
                 if (chunkCount >= 50000) {
                     try { this._vecFd = fs.openSync(this._embeddingPath, 'r'); } catch { this._vecFd = -1; }
-                    // binBuf goes out of scope → GC can collect it
                 } else {
-                    // Medium-large: keep buffer for zero-copy slice access
                     this._embeddingBuf = binBuf;
                 }
             } else {
-                // Eager: fill embeddingCache (needed by indexer + small-corpus MCP server)
                 this._loadEmbeddingBinary(binBuf);
             }
         } else if (data.embeddingCache) {
@@ -499,6 +186,9 @@ export class MemoryGraphIndex {
 
         for (const chunk of (data.chunks || [])) {
             this.chunks.set(chunk.id, chunk);
+            if (!this._corpusEnriched && (chunk.summary || (chunk.concepts && chunk.concepts.length))) {
+                this._corpusEnriched = true;
+            }
 
             // Eager vector population. Vectors are keyed by embeddingKeyFor(chunk)
             // (content_hash + enrichment digest); plain content_hash is the
@@ -507,10 +197,16 @@ export class MemoryGraphIndex {
                 const vecKey = embeddingKeyFor(chunk);
                 if (chunk.content_hash && this.embeddingCache.has(vecKey)) {
                     this.vectors.set(chunk.id, this.embeddingCache.get(vecKey));
-                    // Summary-only second vector (enriched chunks): stored under a
-                    // pseudo row id; searchHybrid folds hits back onto the chunk id.
+                    // Summary-only and per-window vectors (enriched / oversized chunks):
+                    // stored under pseudo row ids; searchHybrid folds hits back onto the
+                    // chunk id (max-sim across all of the chunk's vectors).
                     const sVec = this.embeddingCache.get(vecKey + SUMMARY_VEC_SUFFIX);
                     if (sVec) this.vectors.set(chunk.id + SUMMARY_VEC_SUFFIX, sVec);
+                    for (let i = 1; i < EMBEDDING_MAX_WINDOWS; i++) {
+                        const wVec = this.embeddingCache.get(vecKey + WINDOW_VEC_SUFFIX + i);
+                        if (!wVec) break;
+                        this.vectors.set(chunk.id + WINDOW_VEC_SUFFIX + i, wVec);
+                    }
                 } else if (chunk.content_hash && this.embeddingCache.has(chunk.content_hash)) {
                     this.vectors.set(chunk.id, this.embeddingCache.get(chunk.content_hash));
                 } else if (chunk.embedding) {
@@ -520,12 +216,9 @@ export class MemoryGraphIndex {
                 }
             }
 
-            // Build inverted lexical index from the shared document builder
-            // (search-core.buildLexicalDocument) — identical text across backends.
             const deps = this.graph.dependencies[chunk.file_path] || [];
             this._indexLexical(chunk.id, buildLexicalDocument(chunk, deps), chunk.file_path);
 
-            // Build symbol table (Frontier 2)
             if (chunk.name && chunk.name !== 'anonymous') {
                 const n = chunk.name.toLowerCase();
                 if (!this.symbolTable.has(n)) this.symbolTable.set(n, new Set());
@@ -606,7 +299,7 @@ export class MemoryGraphIndex {
     // ─── Lexical index (TRUE inverted index) ──────────────────────────────────
 
     _indexLexical(chunkId, text, filePath = '') {
-        const tokens = tokenize(text);
+        const tokens = tokenize(text);               // raw + additive Porter stems
         if (tokens.length === 0) return;
 
         const termCounts = new Map();
@@ -614,16 +307,16 @@ export class MemoryGraphIndex {
             termCounts.set(token, (termCounts.get(token) || 0) + 1);
         }
 
-        // Track document length for BM25 length normalization.
-        // Only content tokens are counted — path tokens are handled separately
-        // in searchHybrid to avoid BM25 length normalisation amplifying short
-        // export stubs that share a path with a larger implementation.
-        this.docLens.set(chunkId, tokens.length);
-        this.totalDocLen += tokens.length;
+        // Additive stem tokens earn their own postings but must NOT inflate docLen —
+        // inflated length would perturb BM25 normalisation for exact symbolic matches.
+        // raw-based docLen keeps raw-term BM25 byte-identical to the pre-stem index.
+        // Path tokens are tallied separately in searchHybrid, not here.
+        const rawLen = tokenize(text, false).length;
+        this.docLens.set(chunkId, rawLen);
+        this.totalDocLen += rawLen;
 
         const chunkTokenSet = new Set();
         for (const [term, count] of termCounts) {
-            // Store raw term count — BM25 applies saturation at search time
             this.df.set(term, (this.df.get(term) || 0) + 1);
             let posting = this.invertedIndex.get(term);
             if (!posting) { posting = new Map(); this.invertedIndex.set(term, posting); }
@@ -638,7 +331,7 @@ export class MemoryGraphIndex {
         // Path matching is done multiplicatively in searchHybrid, which avoids
         // amplifying short stubs that happen to share a path with a long function.
         if (filePath) {
-            const pathTokenSet = new Set(tokenize(filePath.replace(/[/\-_.]/g, ' ')));
+            const pathTokenSet = new Set(tokenize(filePath.replace(/[/\-_.]/g, ' '), false));
             if (!this.pathTokens) this.pathTokens = new Map();
             this.pathTokens.set(chunkId, pathTokenSet);
         }
@@ -659,7 +352,6 @@ export class MemoryGraphIndex {
                 else this.df.set(term, freq - 1);
             }
         }
-        // Update BM25 length accounting
         const dl = this.docLens.get(chunkId);
         if (dl !== undefined) {
             this.totalDocLen = Math.max(0, this.totalDocLen - dl);
@@ -671,7 +363,10 @@ export class MemoryGraphIndex {
     }
 
     _searchLexical(queryText) {
-        const queryTokens = tokenize(queryText);
+        // Asymmetric stemming: the index always carries stems, but only stem the
+        // QUERY for natural-language/behavioural queries — symbolic name lookups
+        // stay exact so stem collisions never dilute a precise symbol match.
+        const queryTokens = tokenize(queryText, isNaturalLanguageQuery(queryText));
         const scores = new Map();
         const avgdl = this.docCount > 0 ? this.totalDocLen / this.docCount : 1;
 
@@ -739,7 +434,6 @@ export class MemoryGraphIndex {
         this._vecNorms  = norms;
         this._matrixDirty = false;
 
-        // HNSW for large eager corpora
         if (HierarchicalNSW && n >= HNSW_THRESHOLD) {
             try {
                 const hnsw = new HierarchicalNSW('cosine', dim);
@@ -753,7 +447,7 @@ export class MemoryGraphIndex {
                 }
                 this._hnsw = hnsw;
             } catch (e) {
-                process.stderr.write(`[core-engine] HNSW build failed: ${e.message}\n`);
+                process.stderr.write(`[engine/memory] HNSW build failed: ${e.message}\n`);
                 this._hnsw = null;
             }
         }
@@ -773,7 +467,6 @@ export class MemoryGraphIndex {
         qNorm = Math.sqrt(qNorm);
         if (qNorm === 0) return [];
 
-        // HNSW fast path
         if (this._hnsw) {
             const topK = Math.min(200, n);
             try {
@@ -788,7 +481,6 @@ export class MemoryGraphIndex {
             } catch { /* fall through to flat scan */ }
         }
 
-        // Exact flat scan
         const results = [];
         for (let i = 0; i < n; i++) {
             const base = i * dim;
@@ -813,9 +505,14 @@ export class MemoryGraphIndex {
         if (!this._embeddingBuf && this._vecFd < 0) return [];
         // Large corpora: binary sketch (built once on first vector query) replaces
         // the exact O(corpus) scan with a Hamming prefilter + bounded rescore.
-        // 10k threshold matches the SQLite store so backends stay rank-identical
-        // wherever the exact scan is still fast.
-        if (!this._sketch && this._vecOffsets.size >= 10000) {
+        // SKETCH_THRESHOLD is shared with the SQLite store so both switch to the
+        // sketch at the same vector count. NOTE: this lazy path only runs when the
+        // store is in lazy mode (cacheEmbeddings off AND chunkCount ≥ LAZY_VEC_THRESHOLD);
+        // a corpus with < LAZY_VEC_THRESHOLD chunks but ≥ SKETCH_THRESHOLD vectors
+        // (heavy enrichment/oversized windows) still scans exactly here while SQLite
+        // sketches — a narrow band worth a dedicated scale parity test before relying
+        // on byte-identical top-5 above the threshold.
+        if (!this._sketch && this._vecOffsets.size >= SKETCH_THRESHOLD) {
             try { this._sketch = updateVectorSketch(null, source); } catch { this._sketch = null; }
         }
         const hits = this._sketch
@@ -834,6 +531,7 @@ export class MemoryGraphIndex {
                 const key = embeddingKeyFor(c);
                 put(key, c.id);
                 put(key + SUMMARY_VEC_SUFFIX, c.id);     // summary-only vector → same chunk
+                for (let i = 1; i < EMBEDDING_MAX_WINDOWS; i++) put(key + WINDOW_VEC_SUFFIX + i, c.id); // window vectors → same chunk
                 if (key !== c.content_hash) put(c.content_hash, c.id); // legacy bins
             }
             this._keyToIds = map;
@@ -858,12 +556,12 @@ export class MemoryGraphIndex {
                 // zero lexical overlap, which the old TF-IDF prefilter never could.
                 vectorResults = this._scanVectorsLazy(queryVector, minScore);
             } else {
-                // Eager rows include summary-only pseudo ids (`<id>|s`) — fold each
-                // back onto its chunk via the shared finalizer (best score per
-                // chunk, deterministic order, same cap as the disk-backed paths).
+                // Eager rows include summary-only and per-window pseudo ids
+                // (`<id>|s`, `<id>|wN`) — fold each back onto its chunk via the shared
+                // finalizer (best/max-sim score per chunk, deterministic order, same
+                // cap as the disk-backed paths).
                 const entries = this._searchVector(queryVector, minScore).map(r => ({
-                    id: r.id.endsWith(SUMMARY_VEC_SUFFIX)
-                        ? r.id.slice(0, -SUMMARY_VEC_SUFFIX.length) : r.id,
+                    id: baseEmbeddingKey(r.id),
                     score: r.score,
                 }));
                 vectorResults = finalizeVectorCandidates(entries);
@@ -872,12 +570,14 @@ export class MemoryGraphIndex {
             vectorResults = [];
         }
 
-        // Fusion + boost ladder lives in search-core.mjs so the in-memory engine
-        // and the SQLite store rank identically. Backend state is reached through
-        // accessors; the math is measured once and shared.
+        // B3 learned-sparse channel: opt-in (model present) + NL-asymmetric (symbolic queries skip
+        // it → byte-identical). expandSparseQuery returns null when nothing fires → no channel.
+        const sparseResults = this._searchSparseExpanded(queryText);
+
         return fuseAndRank({
             lexicalResults,
             vectorResults,
+            sparseResults,
             getChunk:      (id) => this.chunks.get(id),
             getPathTokens: (id) => this.pathTokens.get(id),
             getDf:         (t)  => this.df.get(t) || 0,
@@ -886,12 +586,37 @@ export class MemoryGraphIndex {
             topK,
             queryText,
             exactBoostName,
-            // Equivalent to the former full-scan (symbolTable is keyed by name.toLowerCase()),
-            // but O(1) and reusable by the SQLite backend.
+            corpusEnriched: this._corpusEnriched,
+            // O(1) vs the old full-scan; same contract implemented by the SQLite backend.
             resolveExact:  (term) => this.symbolTable.get(term) || [],
         });
     }
 
+    /**
+     * B3 — the learned-sparse expansion channel: NL queries only, model present only. Returns the
+     * ranked sparse candidates, or [] when the channel is inert (no model / symbolic query / no
+     * association fired) so fusion is byte-identical to the default path.
+     */
+    _searchSparseExpanded(queryText) {
+        if (!this._sparseModel || !isNaturalLanguageQuery(queryText)) return [];
+        const termWeights = expandSparseQuery(queryText, this._sparseModel);
+        if (!termWeights) return [];
+        const avgdl = this.docCount > 0 ? this.totalDocLen / this.docCount : 1;
+        return scoreSparseExpanded(termWeights, {
+            docCount:   this.docCount,
+            avgdl,
+            getDf:      (t) => this.df.get(t) || 0,
+            getPostings:(t) => this.invertedIndex.get(t) || [],
+            getDocLen:  (id) => this.docLens.get(id),
+        });
+    }
+
+    /** Whether opt-in learned-sparse associations (B3 --learned-sparse) are loaded. */
+    hasSparseModel() { return this._sparseModel != null; }
+
+    /** Path to the shared `.embeddings.bin` (B4 late-interaction reranker reads candidate vectors from it). */
+    embeddingBinPath() { return this._embeddingPath; }
+
     // ─── Persistence ───────────────────────────────────────────────────────────
 
     async save() {
@@ -899,13 +624,20 @@ export class MemoryGraphIndex {
             id: c.id, file_path: c.file_path, node_type: c.node_type,
             name: c.name, docstring: c.docstring || '', code_snippet: c.code_snippet,
             content_hash: c.content_hash, start_line: c.start_line, end_line: c.end_line,
-            calls: c.calls || [], params: c.params || [],
+            calls: c.calls || [], call_sites: c.call_sites || [], params: c.params || [],
             return_type: c.return_type || '', class_context: c.class_context || '',
             type_refs: c.type_refs || [], decorators: c.decorators || [],
             extends: c.extends || [],
             hyde: c.hyde || '', summary: c.summary || '', concepts: c.concepts || [],
         }));
-        const payload  = JSON.stringify({ chunks: chunksData, graph: this.graph });
+        const payload  = JSON.stringify({
+            chunks: chunksData, graph: this.graph,
+            ...(this._edges ? { edges: this._edges } : {}),
+            ...(this._centrality ? { centrality: Object.fromEntries(this._centrality) } : {}),
+            ...(this._taint ? { taint: this._taint } : {}),
+            ...(this._scipRefs ? { scip_refs: this._scipRefs } : {}),
+            ...(this._sparseModel ? { sparse_model: this._sparseModel } : {}),
+        });
         const tmpPath    = `${this.indexPath}.tmp`;
         const tmpBinPath = `${this._embeddingPath}.tmp`;
         await Promise.all([
@@ -923,7 +655,7 @@ export class MemoryGraphIndex {
         this._saveTimer = setTimeout(async () => {
             this._saveTimer = null;
             try { await this.save(); }
-            catch (err) { process.stderr.write(`[core-engine] ❌ Async save failed: ${err.message}\n`); }
+            catch (err) { process.stderr.write(`[engine/memory] ❌ Async save failed: ${err.message}\n`); }
         }, delayMs);
     }
 
@@ -954,6 +686,10 @@ export class MemoryGraphIndex {
      * @param {Map<string, Float32Array|number[]>} [p.embeddings] New vectors keyed by embeddingKeyFor(chunk).
      */
     applyFileUpdate(filePath, { chunks = [], imports = [], embeddings = null } = {}) {
+        // A per-file edit makes the whole-program symbol graph + its centrality stale; drop
+        // both so findCallers/findReferers fall back to the always-correct scan until the next
+        // full `idx-index` rebuilds them (the daemon never rebuilds the graph).
+        if (this._edges || this._centrality || this._taint || this._scipRefs || this._sparseModel) this._clearEdges();
         this.updateFileGraph(filePath, imports);
 
         for (const [id, chunk] of Array.from(this.chunks.entries())) {
@@ -961,6 +697,7 @@ export class MemoryGraphIndex {
             this._removeLexical(id);
             this.removeVector(id);
             this.removeVector(id + SUMMARY_VEC_SUFFIX);
+            for (let i = 1; i < EMBEDDING_MAX_WINDOWS; i++) this.removeVector(id + WINDOW_VEC_SUFFIX + i);
             this._removeSymbol(chunk);
             this.chunks.delete(id);
         }
@@ -977,6 +714,10 @@ export class MemoryGraphIndex {
             if (vec) this.addVector(chunk.id, vec);
             const sVec = this.embeddingCache.get(vecKey + SUMMARY_VEC_SUFFIX);
             if (sVec) this.addVector(chunk.id + SUMMARY_VEC_SUFFIX, sVec);
+            for (let i = 1; i < EMBEDDING_MAX_WINDOWS; i++) {
+                const wVec = this.embeddingCache.get(vecKey + WINDOW_VEC_SUFFIX + i);
+                if (wVec) this.addVector(chunk.id + WINDOW_VEC_SUFFIX + i, wVec);
+            }
             this._indexLexical(chunk.id, buildLexicalDocument(chunk, imports), chunk.file_path);
             this.chunks.set(chunk.id, chunk);
             if (chunk.name && chunk.name !== 'anonymous') {
@@ -1058,13 +799,152 @@ export class MemoryGraphIndex {
         return out;
     }
 
+    /**
+     * Resolved symbol-graph edges incident to a chunk (A4). Returns [] unless the
+     * opt-in graph was built. Deterministically ordered (the edge list is pre-sorted).
+     * @param {string} chunkId
+     * @param {object} [opts]
+     * @param {string|null} [opts.kind]      Filter to 'calls' | 'extends' | 'type'.
+     * @param {'in'|'out'} [opts.direction]  'in' = referrers of chunkId (default), 'out' = its referents.
+     * @returns {Array<{from_chunk_id:string,to_chunk_id:string,kind:string,confidence:string,chunk:object|null}>}
+     */
+    /** Whether the opt-in resolved symbol graph (A4) is loaded. */
+    hasSymbolGraph() { return this._edges != null; }
+
+    getEdges(chunkId, { kind = null, direction = 'in' } = {}) {
+        if (!this._edges) return [];
+        const src = direction === 'out' ? this._edgesByFrom : this._edgesByTo;
+        const list = src.get(chunkId) || [];
+        const out = [];
+        for (const e of list) {
+            if (kind && e.kind !== kind) continue;
+            const otherId = direction === 'out' ? e.to_chunk_id : e.from_chunk_id;
+            out.push({ ...e, chunk: this.chunks.get(otherId) ?? null });
+        }
+        return out;
+    }
+
+    /** Whether opt-in symbol centrality (A5) is loaded. */
+    hasCentrality() { return this._centrality != null; }
+
+    /** Whether opt-in serialized taint flows (C2 --taint) are loaded. */
+    hasTaint() { return this._taint != null; }
+
+    /** The serialized taint payload `{ flows, meta }` (C2), or null. */
+    getTaintFlows() { return this._taint; }
+
+    /** Whether opt-in SCIP precise references (A2 v2 --resolver scip) are loaded. */
+    hasScipRefs() { return this._scipRefs != null; }
+
+    /** SCIP-confirmed referer chunk ids for a definition chunk (A2 v2), or [] when none / not loaded. */
+    getScipReferers(defChunkId) {
+        const refs = this._scipRefs && this._scipRefs[defChunkId];
+        return Array.isArray(refs) ? refs : [];
+    }
+
+    /** How many definitions carry SCIP precise references (A2 v2). 0 when not loaded. */
+    scipRefCount() { return this._scipRefs ? Object.keys(this._scipRefs).length : 0; }
+
+    /**
+     * Symbol centrality (A5) for one chunk: its PageRank score, dense rank (1 = most central),
+     * and the total number of ranked (connected) chunks. Null when the chunk is unranked
+     * (isolated) or centrality was not built. @returns {{score:number,rank:number,total:number}|null}
+     */
+    getCentrality(chunkId) {
+        if (!this._centrality) return null;
+        const e = this._centrality.get(chunkId);
+        return e ? { score: e.score, rank: e.rank, total: this._centralityTotal } : null;
+    }
+
+    /** The most central symbols (A5), rank-ascending. @returns {Array<{chunk,score,rank}>} */
+    topCentral(limit = 20) {
+        if (!this._centrality) return [];
+        if (!this._centralRanked) {
+            this._centralRanked = [...this._centrality.entries()]
+                .map(([id, v]) => ({ id, score: v.score, rank: v.rank }))
+                .sort((a, b) => a.rank - b.rank || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+        }
+        const out = [];
+        for (const { id, score, rank } of this._centralRanked) {
+            const chunk = this.chunks.get(id);
+            if (!chunk) continue;
+            out.push({ chunk, score, rank });
+            if (out.length >= limit) break;
+        }
+        return out;
+    }
+
     /** Chunks that call the given function name. @returns {object[]} */
     findCallers(funcName) {
+        // Edge-backed (A4) when the resolved graph is present and the name is defined —
+        // returns the SAME set as the scan below (every name-match caller → each def).
+        if (this._edges) {
+            const defs = this.resolveSymbol(funcName);
+            if (defs.length > 0) return this._referrersByEdge(defs, e => e.kind === 'calls');
+        }
         const out = [];
         for (const c of this.chunks.values()) if (c.calls?.includes(funcName)) out.push(c);
         return out;
     }
 
+    /** From-chunks of edges of a given kind that point at any of `defs`, id-sorted. */
+    _referrersByEdge(defs, keep) {
+        const ids = new Set();
+        for (const def of defs) for (const e of (this._edgesByTo.get(def.id) || [])) if (keep(e)) ids.add(e.from_chunk_id);
+        const out = [];
+        for (const id of [...ids].sort()) { const c = this.chunks.get(id); if (c) out.push(c); }
+        return out;
+    }
+
+    /**
+     * Chunks that reference a symbol *by name* as a type (`type_refs`) or as a
+     * base class/interface (`extends`) — the non-call half of symbol-level
+     * references. Calls are served by findCallers; mcp/topology.mjs findReferences fuses
+     * the two and classifies each by confidence. Case-insensitive exact match.
+     * @returns {object[]}
+     */
+    findReferers(symbol) {
+        const key = String(symbol).toLowerCase().trim();
+        if (!key) return [];
+        if (this._edges) {
+            const defs = this.resolveSymbol(symbol);
+            if (defs.length > 0) return this._referrersByEdge(defs, e => e.kind === 'extends' || e.kind === 'type');
+        }
+        const out = [];
+        for (const c of this.chunks.values()) {
+            // A class/type lists its own name in type_refs (the name node is a
+            // type_identifier) — that self-mention is a definition, not a reference.
+            if (c.name && c.name.toLowerCase() === key) continue;
+            const hit = (c.type_refs && c.type_refs.some(t => t.toLowerCase() === key))
+                || (c.extends && c.extends.some(t => t.toLowerCase() === key));
+            if (hit) out.push(c);
+        }
+        return out;
+    }
+
+    /**
+     * HTTP routes filtered by method and/or path prefix, each augmented with its
+     * resolved handler chunk (`chunk`, looked up by handler_chunk_id, or null).
+     * `graph.routes` round-trips verbatim through load(); a route-less index yields
+     * []. Method is matched case-insensitively (HTTP verbs are upper-cased); the
+     * path prefix match is case-insensitive. Mirrors SqliteGraphStore.findRoutes so
+     * both backends return identical records.
+     * @returns {object[]}
+     */
+    findRoutes({ method = null, pathPrefix = null } = {}) {
+        const routes = (this.graph && this.graph.routes) || [];
+        const m = method ? String(method).toUpperCase().trim() : null;
+        const p = pathPrefix ? String(pathPrefix).toLowerCase() : null;
+        const out = [];
+        for (const r of routes) {
+            if (m && String(r.method || '').toUpperCase() !== m) continue;
+            if (p && !String(r.path || '').toLowerCase().startsWith(p)) continue;
+            const chunk = r.handler_chunk_id ? (this.chunks.get(r.handler_chunk_id) || null) : null;
+            out.push({ ...r, chunk });
+        }
+        return out;
+    }
+
     /** Lazily iterate every chunk (cursor-friendly parity with the SQLite store). */
     *iterateChunks() { yield* this.chunks.values(); }
 
@@ -1078,11 +958,11 @@ export class MemoryGraphIndex {
     vectorCount() {
         if (this._lazyMode) {
             let n = 0;
-            for (const k of this._vecOffsets.keys()) if (!k.endsWith(SUMMARY_VEC_SUFFIX)) n++;
+            for (const k of this._vecOffsets.keys()) if (baseEmbeddingKey(k) === k) n++;
             return n;
         }
         let n = 0;
-        for (const k of this.vectors.keys()) if (!k.endsWith(SUMMARY_VEC_SUFFIX)) n++;
+        for (const k of this.vectors.keys()) if (baseEmbeddingKey(k) === k) n++;
         return n;
     }