graph-indexer 1.2.0 → 2.1.0

package/search-core.mjs

@@ -13,6 +13,10 @@
 
 import { createHash } from 'crypto';
 
+// Max embedding-input length in chars. ~8000 chars ≈ the usable context of the
+// default embedders (nomic-embed-text / MiniLM) once a query/document prefix is
+// added; longer inputs are silently truncated by the backend, so this is also the
+// per-window size for oversized definitions (see embeddingWindows).
 export const EMBEDDING_CONTEXT_LIMIT = 8000;
 
 /**
@@ -43,31 +47,173 @@ export function truncateForEmbedding(text) {
     return text.length > EMBEDDING_CONTEXT_LIMIT ? text.slice(0, EMBEDDING_CONTEXT_LIMIT) : text;
 }
 
+// Window-and-pool sub-chunking for the dense channel. A large definition (e.g. a
+// ~600-line free function) used to be embedded as a single vector over only its
+// first EMBEDDING_CONTEXT_LIMIT chars — its tail was invisible to semantic search
+// (BM25 still indexed the whole text, so this gap is dense-only). Instead we embed
+// a few overlapping windows and retrieve the chunk by the MAX cosine over its
+// windows (max-sim), which preserves tail recall where mean-pooling would dilute it.
+//
+// 12% overlap: a definition that straddles a boundary still lands wholly inside an
+// adjacent window (EMBEDDING_CONTEXT_LIMIT*0.12 ≈ 960 chars of shared context).
+export const EMBEDDING_WINDOW_OVERLAP = 0.12;
+// Hard cap on windows per chunk. Bounds embed cost and .bin growth at ~MAX× a
+// single vector. 4 windows at the 7040-char stride cover ~29k chars (~700 lines);
+// beyond that the god-class split already shards classes into method chunks, and a
+// non-class definition that large is rare enough to accept head-only coverage.
+export const EMBEDDING_MAX_WINDOWS = 4;
+// Suffix for a chunk's window vectors past the first: <key>|w1, <key>|w2, … Window 0
+// is the base key itself (identical to the single-vector path), so small chunks and
+// the head of large chunks are byte-identical to before windowing existed.
+export const WINDOW_VEC_SUFFIX = '|w';
+
+const _VEC_SUFFIX_RE = /\|(?:s|w\d+)$/;
+/** Strip a vector-key suffix (summary `|s` or window `|wN`) back to the base key, so
+ *  every per-chunk vector folds onto one fusion candidate. Both backends use this. */
+export function baseEmbeddingKey(key) {
+    return key.replace(_VEC_SUFFIX_RE, '');
+}
+
+/**
+ * Split an oversized embedding payload into overlapping windows of
+ * ≤EMBEDDING_CONTEXT_LIMIT chars, capped at EMBEDDING_MAX_WINDOWS. Returns [] when
+ * the payload fits in a single window (the default path — the caller embeds it under
+ * the base key unchanged). Window 0 is the same first-EMBEDDING_CONTEXT_LIMIT slice
+ * the single-vector path produced, so the base vector is unchanged; windows 1..N
+ * cover the tail and are stored under WINDOW_VEC_SUFFIX keys.
+ *
+ * @param {string} payload
+ * @returns {string[]}  [] or [window0, window1, …] (length ≥ 2 when it splits).
+ */
+export function embeddingWindows(payload) {
+    if (!payload || payload.length <= EMBEDDING_CONTEXT_LIMIT) return [];
+    const stride = Math.max(1, Math.round(EMBEDDING_CONTEXT_LIMIT * (1 - EMBEDDING_WINDOW_OVERLAP)));
+    const windows = [];
+    for (let i = 0; i < EMBEDDING_MAX_WINDOWS; i++) {
+        const start = i * stride;
+        if (start >= payload.length) break;
+        windows.push(payload.slice(start, start + EMBEDDING_CONTEXT_LIMIT));
+        if (start + EMBEDDING_CONTEXT_LIMIT >= payload.length) break; // tail covered
+    }
+    return windows;
+}
+
 export function cosineSimilarity(vecA, vecB) {
     let dot = 0, nA = 0, nB = 0;
     for (let i = 0; i < vecA.length; i++) {
         dot += vecA[i] * vecB[i];
-        nA  += vecA[i] * vecA[i];
-        nB  += vecB[i] * vecB[i];
+        nA += vecA[i] * vecA[i];
+        nB += vecB[i] * vecB[i];
     }
     return nA === 0 || nB === 0 ? 0 : dot / (Math.sqrt(nA) * Math.sqrt(nB));
 }
 
+// ─── Light Porter stemmer (additive recall bridge) ─────────────────────────────
+// Collapses English morphological variants to a shared root so behavioural queries
+// reach code identifiers across the inflection gap that lexical match otherwise
+// misses: "intercepting"↔"Interceptor", "injection"↔"Injectable",
+// "bootstrapping"↔"bootstrap", "managing"↔"Manager". Classic Porter (1980) steps
+// 1–5 plus an agent-noun "-or" rule for code (Interceptor→intercept,
+// Constructor→construct). Used ADDITIVELY in tokenize(): the raw token is always
+// emitted and the stem only when it differs — so exact matches, df statistics and
+// the name/path boosts are byte-for-byte unchanged; only recall is added.
+// Deterministic and pure → both backends produce identical postings (parity-safe).
+const _isVowel = (s, i) => {
+    const c = s.charCodeAt(i);
+    // a e i o u
+    if (c === 97 || c === 101 || c === 105 || c === 111 || c === 117) return true;
+    if (s[i] === 'y') return i === 0 ? true : !_isVowel(s, i - 1);
+    return false;
+};
+const _measure = (s) => {
+    let n = 0, prevV = false;
+    for (let i = 0; i < s.length; i++) { const v = _isVowel(s, i); if (prevV && !v) n++; prevV = v; }
+    return n;
+};
+const _hasVowel = (s) => { for (let i = 0; i < s.length; i++) if (_isVowel(s, i)) return true; return false; };
+const _endsDoubleCons = (s) => s.length >= 2 && s[s.length - 1] === s[s.length - 2] && !_isVowel(s, s.length - 1);
+const _cvc = (s) => {
+    const n = s.length; if (n < 3) return false;
+    if (_isVowel(s, n - 1) || !_isVowel(s, n - 2) || _isVowel(s, n - 3)) return false;
+    const c = s[n - 1]; return c !== 'w' && c !== 'x' && c !== 'y';
+};
+const _STEP2 = [['ational', 'ate'], ['tional', 'tion'], ['enci', 'ence'], ['anci', 'ance'], ['izer', 'ize'], ['abli', 'able'], ['alli', 'al'], ['entli', 'ent'], ['eli', 'e'], ['ousli', 'ous'], ['ization', 'ize'], ['ation', 'ate'], ['ator', 'ate'], ['alism', 'al'], ['iveness', 'ive'], ['fulness', 'ful'], ['ousness', 'ous'], ['aliti', 'al'], ['iviti', 'ive'], ['biliti', 'ble'], ['logi', 'log']];
+const _STEP3 = [['icate', 'ic'], ['ative', ''], ['alize', 'al'], ['iciti', 'ic'], ['ical', 'ic'], ['ful', ''], ['ness', '']];
+// Longest-first; 'or' added to Porter's set so agent nouns reduce like '-er'.
+const _STEP4 = ['ement', 'ance', 'ence', 'able', 'ible', 'ment', 'ant', 'ent', 'ion', 'ism', 'ate', 'iti', 'ous', 'ive', 'ize', 'al', 'er', 'or', 'ic', 'ou'];
+
+/** Porter stem of a single lowercase word. Returns the input unchanged for short
+ *  tokens (≤3 chars: acronyms, short API names) so precise symbols never blur. */
+export function stemToken(word) {
+    let w = word;
+    if (w.length <= 3) return w;
+
+    if (w.endsWith('sses')) w = w.slice(0, -2);
+    else if (w.endsWith('ies')) w = w.slice(0, -2);
+    else if (!w.endsWith('ss') && w.endsWith('s')) w = w.slice(0, -1);
+
+    let fix = false;
+    if (w.endsWith('eed')) { if (_measure(w.slice(0, -3)) > 0) w = w.slice(0, -1); }
+    else if (w.endsWith('ed') && _hasVowel(w.slice(0, -2))) { w = w.slice(0, -2); fix = true; }
+    else if (w.endsWith('ing') && _hasVowel(w.slice(0, -3))) { w = w.slice(0, -3); fix = true; }
+    if (fix) {
+        if (w.endsWith('at') || w.endsWith('bl') || w.endsWith('iz')) w += 'e';
+        else if (_endsDoubleCons(w) && !/[lsz]$/.test(w)) w = w.slice(0, -1);
+        else if (_measure(w) === 1 && _cvc(w)) w += 'e';
+    }
+
+    if (w.length > 2 && w.endsWith('y') && _hasVowel(w.slice(0, -1))) w = w.slice(0, -1) + 'i';
+
+    for (const [suf, rep] of _STEP2) { if (w.endsWith(suf)) { const st = w.slice(0, -suf.length); if (_measure(st) > 0) w = st + rep; break; } }
+    for (const [suf, rep] of _STEP3) { if (w.endsWith(suf)) { const st = w.slice(0, -suf.length); if (_measure(st) > 0) w = st + rep; break; } }
+
+    for (const suf of _STEP4) {
+        if (w.endsWith(suf)) {
+            const st = w.slice(0, -suf.length);
+            if (suf === 'ion') { if (_measure(st) > 1 && /[st]$/.test(st)) w = st; }
+            else if (_measure(st) > 1) w = st;
+            break;
+        }
+    }
+
+    if (w.endsWith('e')) { const st = w.slice(0, -1); const m = _measure(st); if (m > 1 || (m === 1 && !_cvc(st))) w = st; }
+    if (_measure(w) > 1 && w.endsWith('l') && _endsDoubleCons(w)) w = w.slice(0, -1);
+
+    return w;
+}
+
 /**
  * Tokenise code/identifiers: lowercase words plus camelCase sub-parts, so
- * `dispatchRequest` indexes as `dispatchrequest`, `dispatch`, `request`.
+ * `dispatchRequest` indexes as `dispatchrequest`, `dispatch`, `request`. Each
+ * emitted token also contributes its Porter stem (when different) so behavioural
+ * queries bridge the inflection gap to code names — see stemToken. Additive: the
+ * raw token is never dropped, so exact match and the boost ladder are unaffected.
  */
-export function tokenize(text) {
+// Stem postings live in a separate term namespace (sentinel-prefixed) so a raw
+// query token can NEVER match a stem posting: symbolic/exact lookups stay precisely
+// what they were before stemming existed, and only a query that opts in (emits the
+// prefixed stem) reaches the morphological bridges. Index always emits both; the
+// query emits the prefixed stem only for natural-language queries.
+export const STEM_PREFIX = '~stem~'; // sentinel namespace: raw tokens are [A-Za-z0-9]+, so '~' can never collide
+
+export function tokenize(text, stem = true) {
     if (!text) return [];
     const rawTokens = text.split(/[\s\W_]+/);
     const tokens = [];
+    const emit = (t) => {
+        const lo = t.toLowerCase();
+        tokens.push(lo);
+        if (!stem) return;
+        const s = stemToken(lo);
+        if (s !== lo && s.length >= 3) tokens.push(STEM_PREFIX + s);
+    };
     for (const word of rawTokens) {
         if (word.length < 2) continue;
-        tokens.push(word.toLowerCase());
+        emit(word);
         const camelParts = word.replace(/([a-z])([A-Z])/g, '$1 $2').split(' ');
         if (camelParts.length > 1) {
             for (const part of camelParts) {
-                if (part.length >= 2) tokens.push(part.toLowerCase());
+                if (part.length >= 2) emit(part);
             }
         }
     }
@@ -84,10 +230,14 @@ export function tokenize(text) {
  */
 export const SUMMARY_VEC_SUFFIX = '|s';
 
-/** The text embedded as the summary-only vector, or null when not enriched. */
+/** The text embedded as the summary-only vector, or null when not enriched.
+ * The leading LLM summary carries the behavioural signal; the concept tags add
+ * domain vocabulary; the qualified name (class_context.name when present) anchors
+ * the vector to the symbol so a method is disambiguated from same-named peers. */
 export function summaryEmbeddingText(chunk) {
     if (!chunk.summary && !(chunk.concepts?.length > 0)) return null;
-    return [chunk.summary, (chunk.concepts || []).join(', '), chunk.name]
+    const qualifiedName = chunk.class_context ? `${chunk.class_context}.${chunk.name}` : chunk.name;
+    return [chunk.summary, (chunk.concepts || []).join(', '), qualifiedName]
         .filter(Boolean).join('. ');
 }
 
@@ -98,7 +248,7 @@ export function summaryEmbeddingText(chunk) {
  *
  * Single source of truth shared by the in-memory engine, the watch daemon and the
  * SQLite writer so every backend indexes identical text. Decorators are
- * deliberately excluded — measured to regress framework repos (see core-engine).
+ * deliberately excluded — measured to regress framework repos (see engine/memory.mjs).
  * When enrichment ran, chunk.concepts contains domain keyword strings that bridge
  * lexical gaps (e.g. "authentication JWT middleware"); chunk.hyde is concepts.join(' ')
  * and serves as the backward-compatible field for existing serialized indexes.
@@ -133,7 +283,7 @@ export function buildLexicalDocument(chunk, depRelPaths = []) {
 // prose, so heavy length normalisation wrongly penalises long implementations vs
 // short export stubs.
 export const BM25_K1 = 1.5;
-export const BM25_B  = 0.3;
+export const BM25_B = 0.3;
 
 /** Okapi IDF — always positive, avoids negative IDF for very common terms. */
 export function okapiIdf(docCount, docFreq) {
@@ -145,6 +295,43 @@ export function bm25Score(idf, tf, docLen, avgdl, k1 = BM25_K1, b = BM25_B) {
     return idf * (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * docLen / avgdl));
 }
 
+/**
+ * B3 — score the learned-sparse expansion channel: a weighted-term retrieval over the EXISTING BM25
+ * postings, used identically by both backends so the channel is parity-by-construction.
+ *
+ * PARITY INVARIANT (do not break): each chunk receives EXACTLY ONE addend per associate term, so a
+ * chunk's final float sum is the cross-term accumulation in `termWeights`' iteration order — which is
+ * deterministic and identical across backends (expandSparseQuery rebuilds the Map sorted: weight desc,
+ * term asc). The INNER getPostings(term) order legitimately differs between backends (memory Map vs
+ * SQL rows) and is parity-IRRELEVANT precisely because the keys within one term are distinct (one
+ * scores.set per chunk per term). Never accumulate within a term, or emit a chunk twice for one term.
+ *
+ * The backends differ only in HOW postings are reached (in-memory Map vs SQL), injected as accessors:
+ *   getDf(term)            → document frequency (0 ⇒ skip the term)
+ *   getPostings(term)      → iterable of [chunkId, tf]
+ *   getDocLen(chunkId)     → that chunk's BM25 doc length (falls back to avgdl)
+ *
+ * @param {Iterable<[string, number]>} termWeights  associate term → fusion weight (ordered)
+ * @param {{docCount:number, avgdl:number, getDf:Function, getPostings:Function, getDocLen:Function}} acc
+ * @returns {Array<{id:string, score:number, rank:number}>}  ranked, capped, id-tie-broken
+ */
+export function scoreSparseExpanded(termWeights, { docCount, avgdl, getDf, getPostings, getDocLen }) {
+    const scores = new Map();
+    for (const [term, weight] of termWeights) {
+        const dfreq = getDf(term);
+        if (!dfreq) continue;
+        const idf = okapiIdf(docCount, dfreq);
+        for (const [chunkId, tf] of getPostings(term)) {
+            const dl = getDocLen(chunkId) ?? avgdl;
+            scores.set(chunkId, (scores.get(chunkId) || 0) + weight * bm25Score(idf, tf, dl, avgdl));
+        }
+    }
+    return Array.from(scores.entries())
+        .sort((a, b) => (b[1] - a[1]) || (a[0] < b[0] ? -1 : 1))   // id tie-break: backend parity
+        .slice(0, LEXICAL_FUSION_CAP)
+        .map(([id, score], rank) => ({ id, score, rank: rank + 1 }));
+}
+
 // ─── Hybrid fusion (RRF + boost ladder) ────────────────────────────────────────
 
 // Lexical candidates handed to fusion — shared by BOTH backends. An asymmetric
@@ -180,14 +367,57 @@ export function finalizeVectorCandidates(entries, cap = VECTOR_FUSION_CAP) {
         .map(([id, score], i) => ({ id, score, rank: i + 1 }));
 }
 
+// Non-natural-language channel weights (keyword / symbol-lookup queries). The
+// vector weight was lowered 1.0 → 0.7 after the 15-fixture search-eval was fixed
+// to actually exercise embeddings (its "semantic" pass had been silently lexical,
+// because the test bridge forced embeddings off — so the prior 1.0 was tuned
+// against a vector-less harness). With honest measurement, a full-strength vector
+// channel slightly OVER-contributes on keyword queries (BM25's home turf): it
+// displaces a confident exact-name lexical rank-1 with a same-prefixed sibling
+// (`Map`→`key`, `GardenPlantingRepository`→`…Dao`). 0.7 is the top of the winning
+// plateau — it restores those rank-1s (broad kw MRR 0.569 → 0.588, +1 hit@1) while
+// keeping the vector channel's recall lift, and leaves the 5-suite repo-eval
+// byte-identical (symbolic 0.78/0.83). Below 0.7 the repo-eval symbolic starts to
+// erode; the NL path keeps its own (higher) vector weight below.
 const LEXICAL_WEIGHT = 1.5;
-const VECTOR_WEIGHT  = 1.0;
-
-// Natural-language queries flip the channel weights: behavioural descriptions
-// ("how does the app parse incoming JSON payloads?") carry their signal in the
-// embedding, while their common English words only add BM25 noise.
-const NL_LEXICAL_WEIGHT = 1.0;
-const NL_VECTOR_WEIGHT  = 1.6;
+const VECTOR_WEIGHT = 0.7;
+
+// Natural-language queries keep the lexical channel at full strength (exact-name and
+// keyword matches are the most reliable signal); the vector channel's weight depends on
+// whether the corpus carries LLM enrichment:
+//
+//   • PLAIN corpus (no enrichment) — the only vectors are raw-code embeddings, a weak
+//     conceptual signal that DISPLACES correct lexical hits at any real weight. It acts
+//     as a low-weight rescue (0.4). This was the historical "joint optimum" finding.
+//   • ENRICHED corpus — every central chunk also owns a summary vector that speaks the
+//     vocabulary of behavioural queries, so the vector channel becomes a strong semantic
+//     signal and earns more weight. The value is the JOINT optimum of two independent
+//     suites that DISAGREE: the repo-eval behavioural channel keeps climbing to ~1.0,
+//     while the broader 15-fixture search-eval (mixed nl/kw/xc) peaks at 0.4 and decays
+//     above it (a high weight lets a confident-but-wrong summary vector displace a good
+//     lexical hit on cross-cutting queries). 0.6 improves BOTH over baseline and avoids
+//     overfitting either; the opt-in LLM reranker recovers the top-rank precision that a
+//     lower weight gives up. (Plain stays 0.4 — raw-code vectors are a weak rescue.)
+//
+// The engine selects the regime per index via `corpusEnriched` (see engine/memory.mjs /
+// engine/sqlite.mjs), so a repo indexed without --enrich is never hurt by the strong weight.
+const NL_LEXICAL_WEIGHT = 1.5;
+const NL_VECTOR_WEIGHT_PLAIN = 0.4;
+const NL_VECTOR_WEIGHT_ENRICHED = 0.6;
+
+// B3 learned-sparse channel weight. The sparse channel is the corpus-learned vocabulary EXPANSION
+// view (associate terms the literal query never mentions); it only fires for NL queries on a
+// --learned-sparse index, so it never touches symbolic lookups or the default path.
+//
+// HONEST MEASUREMENT (strict test:eval, 5 core suites, in-process A/B — see the weight sweep in
+// IMPROVEMENT_LEARNED_SPARSE.md): the channel does NOT beat the lexical baseline at any weight.
+// Higher weights (0.5/0.3) REGRESS semantic s@5 (the co-occurrence associates add noise that buries
+// correct hits); the best case is ≈neutral at 0.2 (semantic + symbolic rank-1 both unchanged,
+// overall s@5 −0.5). Like BM25F / AST / the code-embedding NL weight before it, this measured
+// net-neutral-to-negative, so the DEFAULT stays lexical (B3 ships opt-in via --learned-sparse only).
+// 0.2 is the measured least-harmful active setting for users who opt in / whose corpus has the
+// vocabulary mismatch the benchmark fixtures lack.
+const NL_SPARSE_WEIGHT = 0.2;
 
 const QUERY_STOPWORDS = new Set([
     'the', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'for', 'on', 'with', 'at',
@@ -209,10 +439,16 @@ export function isNaturalLanguageQuery(queryText) {
     return stops >= 2 && stops / words.length >= 0.2;
 }
 
-export const TEST_FILE_RE = /\.(test|spec)\.|[/\\]__tests__[/\\]|_test\.|^tests?[/\\]|[/\\]tests?[/\\]|[/\\]spec[/\\]/;
-// `^integration/` is root-anchored ONLY: a repo-root integration/ tree is an e2e
-// test-app convention (NestJS), while nested src/integrations/ is real code.
-export const EXAMPLE_DIR_RE = /^examples?[/\\]|[/\\]examples?[/\\]|^samples?[/\\]|[/\\]samples?[/\\]|^demos?[/\\]|[/\\]demos?[/\\]|[/\\]tutorials?[/\\]|^docs_src[/\\]|[/\\]docs_src[/\\]|^sandbox[/\\]|[/\\]sandbox[/\\]|^benchmarks?[/\\]|[/\\]benchmarks?[/\\]|^scripts?[/\\]|[/\\]scripts?[/\\]/;
+export const TEST_FILE_RE = /\.(test|spec)\.|[/\\]__tests__[/\\]|_test\.|^tests?[/\\]|[/\\]tests?[/\\]|[/\\]spec[/\\]/i;
+// Example/sample/demo/sandbox/etc. trees are a top-level project convention
+// (examples/, samples/, demo/, docs_src/, …). The keyword is matched ONLY as the
+// FIRST or SECOND path segment. Matching it at ANY depth (the previous behaviour)
+// false-positived on PACKAGE PATHS that merely contain the word — e.g. Spring's
+// `src/main/java/org/springframework/samples/petclinic/…` and Google's
+// `com/google/samples/apps/…` — silently excluding real application code from LLM
+// enrichment and demoting it in search. The fixtures' legitimate example/sample
+// dirs all sit at depth ≤1, so this only stops the deep package-path false hits.
+export const EXAMPLE_DIR_RE = /^(?:[^/\\]+[/\\])?(?:examples?|samples?|demos?|tutorials?|docs_src|sandbox|benchmarks?|scripts?)[/\\]/;
 
 /**
  * Reciprocal-Rank-Fusion of lexical + vector candidate lists, with the measured
@@ -239,13 +475,31 @@ export const EXAMPLE_DIR_RE = /^examples?[/\\]|[/\\]examples?[/\\]|^samples?[/\\
 export function fuseAndRank({
     lexicalResults, vectorResults, getChunk, getPathTokens, getDf,
     docCount, rrfK, topK, queryText, exactBoostName = null, resolveExact = null,
+    corpusEnriched = false, sparseResults = [],
 }) {
-    const rrfScores  = new Map();
-    const K          = rrfK;
+    const rrfScores = new Map();
+    const K = rrfK;
     const queryLower = queryText.toLowerCase();
 
     const _queryPathTokens = queryLower.split(/[\s\W_]+/).filter(t => t.length >= 3);
 
+    // File-path boost discriminativeness gate (natural-language queries only).
+    // The file-path boost (further down) fires when a query token matches a
+    // filename segment — but a common low-IDF word ("path", "url", "config",
+    // "index") that merely appears in a filename then over-promotes EVERY chunk in
+    // that file (an NL query mentioning "path" boosts all of path.go, burying the
+    // real answer that lives elsewhere). Restrict the boost to query terms
+    // discriminative enough to be a genuine filename signal: IDF ≥ ln(docCount/2),
+    // a per-corpus structural threshold (not tuned to any query). Symbolic/keyword
+    // queries are excluded by the NL gate and keep the original token set
+    // byte-for-byte. Measured: lifts agent-style recall@5 (overall s@5 0.77→0.81;
+    // tuning-semantic s@5 0.52→0.61) with symbolic rankings unchanged, no regression.
+    const _pathBoostIsNL = isNaturalLanguageQuery(queryText);
+    const _pathBoostMinIdf = Math.log(Math.max(docCount, 2) / 2);
+    const _queryPathTokensBoost = _pathBoostIsNL
+        ? _queryPathTokens.filter(t => okapiIdf(docCount, getDf(t)) >= _pathBoostMinIdf)
+        : _queryPathTokens;
+
     // Name-boost eligibility: long tokens, or short-but-discriminative ones whose
     // document frequency is ≤15% of the corpus (self-tunes per repo/language,
     // surfacing exact matches on short API names without re-introducing stopwords).
@@ -258,8 +512,9 @@ export function fuseAndRank({
     // vector-led for natural-language behavioural queries (only when a vector
     // channel actually produced candidates — lexical-only mode is unaffected).
     const nlQuery = vectorResults.length > 0 && isNaturalLanguageQuery(queryText);
+    const nlVecWeight = corpusEnriched ? NL_VECTOR_WEIGHT_ENRICHED : NL_VECTOR_WEIGHT_PLAIN;
     const wLex = nlQuery ? NL_LEXICAL_WEIGHT : LEXICAL_WEIGHT;
-    const wVec = nlQuery ? NL_VECTOR_WEIGHT  : VECTOR_WEIGHT;
+    const wVec = nlQuery ? nlVecWeight : VECTOR_WEIGHT;
 
     // Re-rank the vector channel with test/example demotion applied to the cosine
     // scores BEFORE ranks are assigned: a test helper that out-scores the real
@@ -280,15 +535,29 @@ export function fuseAndRank({
         vectorResults = adjusted.map((r, i) => ({ ...r, rank: i + 1 }));
     }
 
-    // On NL queries the name/path boosts misfire: identifiers that happen to share
-    // a generic English word with the query ("_global" for "global configuration")
-    // get 1.4–2.0× promotions over the semantically right answer. Gate the boosts
-    // on semantic agreement — a chunk must also be a vector candidate to earn them.
-    const boostEligible = nlQuery ? new Set(vectorResults.map(r => r.id)) : null;
-
+    // Boosts are NOT gated on vector-candidate membership. Gating was intended to
+    // stop generic-word name collisions on NL queries, but because the code-embedding
+    // channel is weak it mostly suppressed the EXACT-name boost on the correct hit
+    // (which then sank below a noisy vector candidate). Removing the gate measured
+    // strictly better on both retrieval suites; the exact-name boost is the single
+    // most reliable signal and must always apply.
+    const boostEligible = null;
+
+    // Style-intent detection: a stylesheet bundle vendored into a source tree
+    // (e.g. a bundled bootstrap.css) produces hundreds of synthetic-named
+    // `*_rule_set` chunks that pollute CODE searches in mixed repos. Demote those
+    // synthetic rule_sets UNLESS the query is actually about styling (a selector
+    // char, an exact_tokens pin, or a style keyword). Named SCSS @mixin/@function
+    // chunks are unaffected — only nameless rule_sets are noise for code queries.
+    const queryIsStyle = exactBoostName != null ||
+        /[.#]|css|scss|stylesheet|selector|\bstyle\b|\bclass\b|color|margin|padding|font|border|background|hover|keyframe|animation|mixin|breakpoint|responsive|\bwidth\b|\bheight\b/.test(queryLower);
+
+    // B3 learned-sparse channel (opt-in, presence + NL gated by the caller). When the caller passes
+    // no sparse candidates the spread contributes nothing → byte-identical to the default path.
     const allResults = [
         ...vectorResults.map(r => ({ ...r, _w: wVec })),
         ...lexicalResults.map(r => ({ ...r, _w: wLex })),
+        ...sparseResults.map(r => ({ ...r, _w: NL_SPARSE_WEIGHT })),
     ];
 
     for (const { id, rank, _w } of allResults) {
@@ -296,14 +565,13 @@ export function fuseAndRank({
         const chunk = getChunk(id);
         if (!chunk) continue;
 
-        // Demotion: test / spec files (unless the query is itself about tests).
         if (TEST_FILE_RE.test(chunk.file_path)) {
             if (!queryLower.includes('test') && !queryLower.includes('spec')) baseScore *= 0.25;
         }
-        // Demotion: example / docs dirs (tutorial snippets over-rank on short length
-        // + high keyword density vs the real implementation).
+        // Demotion: synthetic-named stylesheet rule_sets on non-style queries
+        // (vendored CSS bundle noise in mixed repos). Named mixins/functions exempt.
+        if (!queryIsStyle && chunk.name && chunk.name.endsWith('_rule_set')) baseScore *= 0.2;
         if (EXAMPLE_DIR_RE.test(chunk.file_path)) baseScore *= 0.5;
-        // Demotion: pure expression sites.
         if (chunk.node_type === 'expression_statement' || chunk.node_type === 'call_expression') {
             baseScore *= 0.8;
         }
@@ -327,14 +595,14 @@ export function fuseAndRank({
         // File-path boost via the separate path-token set (not the BM25 index), so
         // length normalisation never penalises long implementations sharing a path.
         const canBoost = !boostEligible || boostEligible.has(id);
-        if (canBoost && _queryPathTokens.length > 0) {
+        if (canBoost && _queryPathTokensBoost.length > 0) {
             const pathToks = getPathTokens(id);
             if (pathToks) {
-                const hasExact = _queryPathTokens.some(t => pathToks.has(t));
-                const hasPrefix = !hasExact && _queryPathTokens.some(t =>
+                const hasExact = _queryPathTokensBoost.some(t => pathToks.has(t));
+                const hasPrefix = !hasExact && _queryPathTokensBoost.some(t =>
                     t.length >= 4 && Array.from(pathToks).some(pt => pt.startsWith(t.slice(0, 5)))
                 );
-                if (hasExact)       baseScore *= 1.4;
+                if (hasExact) baseScore *= 1.4;
                 else if (hasPrefix) baseScore *= 1.2;
             }
         }
@@ -344,8 +612,8 @@ export function fuseAndRank({
         // Plural equivalence (`BackgroundTask` ↔ `BackgroundTasks`) is included —
         // API names pluralize and a strict equality check missed them.
         if (canBoost && chunk.name && chunk.name !== 'anonymous') {
-            const nameLower      = chunk.name.toLowerCase();
-            const lastDotted     = nameLower.split('.').pop() ?? nameLower;
+            const nameLower = chunk.name.toLowerCase();
+            const lastDotted = nameLower.split('.').pop() ?? nameLower;
             const queryTokensAll = _queryNameTokens;
             const eq = (a, b) => a === b || a === b + 's' || b === a + 's';
             if (queryTokensAll.some(t => eq(nameLower, t) || eq(lastDotted, t))) {
@@ -355,7 +623,7 @@ export function fuseAndRank({
                 baseScore *= 2.0;
             } else {
                 const snakeParts = nameLower.split(/[._]+/);
-                const lastSnake  = snakeParts[snakeParts.length - 1] ?? '';
+                const lastSnake = snakeParts[snakeParts.length - 1] ?? '';
                 if (snakeParts.length >= 2 && lastSnake.length >= 3 && queryTokensAll.includes(lastSnake)) {
                     baseScore *= 1.4;
                 }
@@ -365,7 +633,6 @@ export function fuseAndRank({
         rrfScores.set(id, (rrfScores.get(id) || 0) + baseScore);
     }
 
-    // Optional guaranteed boost for an exactly-named symbol (search_code exact_tokens).
     if (exactBoostName && resolveExact) {
         const boostTerm = String(exactBoostName).toLowerCase().trim();
         for (const id of resolveExact(boostTerm)) {
@@ -373,8 +640,6 @@ export function fuseAndRank({
         }
     }
 
-    // Deterministic tie-break on id: equal fused scores must order identically
-    // on every backend (Map iteration vs SQL row order would otherwise differ).
     return Array.from(rrfScores.entries())
         .sort((a, b) => (b[1] - a[1]) || (a[0] < b[0] ? -1 : 1))
         .slice(0, topK)
@@ -382,6 +647,102 @@ export function fuseAndRank({
         .filter(r => r.chunk !== undefined);
 }
 
+// ─── D3: learned re-ranker (opt-in, post-fusion) ─────────────────────────────────
+//
+// A learned linear model that RE-ORDERS the fused top-N using features the engine
+// already computes — the RRF fused score plus PROGRAM-STRUCTURE signals A4/A5/A1 added
+// (symbol centrality, resolved-edge in-degree, SCIP-resolved in-degree) and git recency.
+// It is a *post-fusion* re-rank (like the LLM reranker / git boost — tool layer, never
+// inside fuseAndRank), so the default RRF ordering is byte-identical: this runs ONLY when
+// the caller opts in (config.ranker === 'learned'). Inference is a dot product — zero
+// runtime dependency, no model server, deterministic (ties broken on id) → parity-safe.
+//
+// Honest scope: the discriminating features (centrality / in-degree / resolved-in) are
+// ZERO without --symbol-graph, so on a default index the model degrades to the RRF score
+// + git + node-type and is ≈neutral; its lift is on symbol-graph indexes. Trained offline
+// (bench/train-ranker.mjs) on the benchmark's labelled query→chunk pairs; RRF stays the
+// default. See docs/internals/IMPROVEMENT_LEARNED_RANKER.md for the measured deltas.
+
+/** Feature order — the model's weight vector is keyed by these names. */
+export const RANKER_FEATURES = ['rrf', 'rank', 'centrality', 'in_degree', 'resolved_in', 'git', 'is_def', 'is_test'];
+
+// Node types that are NOT a definition (a re-export / bare expression / import is a weaker hit).
+const NON_DEF_NODE_RE = /expression|call_expression|re_export|_export_statement|import|comment/;
+
+/**
+ * Default shipped model — CONSERVATIVE and RRF-DOMINANT by design. The offline trainer
+ * (bench/train-ranker.mjs) measured that the program-structure features (centrality / resolved
+ * in-degree) carry ~no signal for symbol retrieval on the benchmark — the correct symbol is usually
+ * NOT the most central one (the fit assigns them near-zero weight) — and a fully data-fit model only
+ * TIES RRF on the held-out split (rank-1 0.579 vs 0.579). So the shipped default keeps RRF in charge
+ * (it already
+ * encodes lexical+vector+boosts), adds only a whisper of structure as a tie-breaker, and an is_test
+ * penalty that aligns with the existing demotion. RRF remains the engine default (config.ranker);
+ * this model only runs under the opt-in `--ranker learned`, where it is ≈neutral on the generic
+ * benchmark and exists for users to RETRAIN on their own symbol-graph-rich repo. See
+ * docs/internals/IMPROVEMENT_LEARNED_RANKER.md for the full measured story.
+ */
+export const DEFAULT_RANKER_MODEL = Object.freeze({
+    bias: 0,
+    weights: Object.freeze({
+        rrf: 1.0, rank: 0.30, centrality: 0.04, in_degree: 0.04,
+        resolved_in: 0.04, git: 0.03, is_def: 0.0, is_test: -0.20,
+    }),
+});
+
+/**
+ * Extract the feature vector for one fused candidate. Pure; all backend data arrives through
+ * accessors so memory and SQLite produce identical features (parity).
+ *
+ * @param {{score:number, chunk:object}} result
+ * @param {number} idx  0-based position in the fused pool.
+ * @param {object} ctx  { maxScore, getCentrality?, getInEdges?, gitScoreFor? }
+ */
+export function extractRankerFeatures(result, idx, ctx) {
+    const c = result.chunk || {};
+    const cen = ctx.getCentrality ? ctx.getCentrality(c.id) : null;
+    const inEdges = ctx.getInEdges ? (ctx.getInEdges(c.id) || []) : [];
+    let resolvedIn = 0;
+    for (const e of inEdges) if (e.confidence === 'resolved') resolvedIn++;
+    const nodeType = String(c.node_type || '');
+    return {
+        rrf: ctx.maxScore > 0 ? (result.score || 0) / ctx.maxScore : 0,
+        rank: 1 / (1 + idx),
+        centrality: cen ? cen.score : 0,
+        in_degree: Math.min(inEdges.length / 10, 1),     // squashed into [0,1]
+        resolved_in: Math.min(resolvedIn / 5, 1),
+        git: ctx.gitScoreFor ? (ctx.gitScoreFor(c.file_path) || 0) : 0,
+        is_def: nodeType && !NON_DEF_NODE_RE.test(nodeType) ? 1 : 0,
+        is_test: c.file_path && TEST_FILE_RE.test(c.file_path) ? 1 : 0,
+    };
+}
+
+/** Linear score of a feature vector under a model. */
+export function scoreLearned(features, model = DEFAULT_RANKER_MODEL) {
+    let s = model.bias || 0;
+    const w = model.weights || {};
+    for (const f of RANKER_FEATURES) s += (w[f] || 0) * (features[f] || 0);
+    return s;
+}
+
+/**
+ * Re-rank fused results with the learned model. Returns a NEW array (score replaced by the learned
+ * score), ordered by learned score desc with id tie-break. ≤1 result is returned unchanged.
+ * Deterministic → both backends re-rank identically (parity-safe).
+ *
+ * @param {Array<{score:number, chunk:object}>} results  The fused (RRF) pool, over-fetched.
+ * @param {object} ctx  Accessors: { getCentrality?, getInEdges?, gitScoreFor? }.
+ * @param {object} [model]
+ */
+export function learnedRerank(results, ctx = {}, model = DEFAULT_RANKER_MODEL) {
+    if (!Array.isArray(results) || results.length <= 1) return results;
+    const maxScore = results.reduce((m, r) => Math.max(m, r.score || 0), 0);
+    const fctx = { ...ctx, maxScore };
+    const scored = results.map((r, i) => ({ r, s: scoreLearned(extractRankerFeatures(r, i, fctx), model) }));
+    scored.sort((a, b) => (b.s - a.s) || ((a.r.chunk?.id ?? '') < (b.r.chunk?.id ?? '') ? -1 : 1));
+    return scored.map(({ r, s }) => ({ ...r, score: s }));
+}
+
 // ─── Graph centrality ──────────────────────────────────────────────────────────
 
 /**