@kolisachint/hoocode-agent 0.4.165 → 0.4.166

package/dist/core/search/rerank.d.ts

@@ -14,15 +14,36 @@
  *     file itself first, which content grep alone cannot do;
  *   - fused prior: the RRF ordering, so retriever consensus still counts.
  *
+ * Not every signal suits every query. A query that names something and a query
+ * that describes behaviour want different evidence, so the name-matching
+ * signal is gated on {@link queryIsProse} — see its comment for the numbers.
+ *
  * Purely lexical-statistical and deterministic — no model, no I/O beyond
  * reading candidate windows. A cross-encoder can later replace the scoring
  * function behind the same signature; that model work belongs to
  * `kolisachint/embeddingsearchtools`, not here.
  */
 import type { FusedCandidate } from "./types.js";
+/**
+ * Is `query` a sentence rather than a name?
+ *
+ * A quoted segment is never prose regardless of its words: the plan collapses
+ * it to one literal term, so the declaration bonus cannot fire on it anyway,
+ * and its path-token split is what lets `"Theme not initialized…"` find
+ * `core/theme.ts`.
+ */
+export declare function queryIsProse(query: string): boolean;
 export interface RerankResult {
     candidates: FusedCandidate[];
     latencyMs: number;
 }
+/**
+ * The exact source text each candidate stands for, as the model should see it.
+ *
+ * Shared with the cross-encoder path so both rerankers score identical text —
+ * otherwise a comparison between them would partly measure which one got a
+ * better view of the candidate.
+ */
+export declare function readCandidateWindows(candidates: readonly FusedCandidate[], cwd: string): Array<string | undefined>;
 export declare function rerankCandidates(query: string, candidates: readonly FusedCandidate[], cwd: string): RerankResult;
 //# sourceMappingURL=rerank.d.ts.map

package/dist/core/search/rerank.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rerank.d.ts","sourceRoot":"","sources":["../../../src/core/search/rerank.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAKH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAsDjD,MAAM,WAAW,YAAY;IAC5B,UAAU,EAAE,cAAc,EAAE,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,cAAc,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,YAAY,CAkFhH","sourcesContent":["/**\n * Deterministic reranker over the fused top-50\n * (docs/hybrid-retrieval-design.md, step 7 of the shipping order).\n *\n * The eval gate showed fused Recall@50 well above Recall@5/10 — the right\n * candidates survive fusion but sit too deep. This reranker re-orders them\n * using evidence that is only cheap to compute *after* fusion, when there\n * are ≤50 candidates instead of thousands of lines:\n *\n *   - term coverage: how many distinct query terms appear in the candidate's\n *     actual expanded window (read from disk);\n *   - path affinity: query terms appearing in the candidate's file path —\n *     this is what lets a query like `core/search/hybrid-search.ts` rank the\n *     file itself first, which content grep alone cannot do;\n *   - fused prior: the RRF ordering, so retriever consensus still counts.\n *\n * Purely lexical-statistical and deterministic — no model, no I/O beyond\n * reading candidate windows. A cross-encoder can later replace the scoring\n * function behind the same signature; that model work belongs to\n * `kolisachint/embeddingsearchtools`, not here.\n */\n\nimport { readFileSync } from \"fs\";\nimport path from \"path\";\nimport { buildLexicalQueryPlan } from \"./lexical-retriever.js\";\nimport type { FusedCandidate } from \"./types.js\";\n\n/** Weights of the scoring blend. The eval harness (`bun run search-eval`) is\n *  the instrument for changing them — don't tune blind. */\nconst WEIGHT_FUSED_PRIOR = 0.4;\nconst WEIGHT_TERM_COVERAGE = 0.35;\nconst WEIGHT_PATH_AFFINITY = 0.25;\n/** Additive bonus when the query *is* the candidate's path (or its suffix):\n *  the caller named the file, so no amount of content evidence elsewhere\n *  should outrank it. */\nconst EXACT_PATH_BONUS = 0.5;\n/**\n * Additive bonus when the window *declares* a query term rather than merely\n * mentioning it.\n *\n * This targets the largest measured gap in the eval: on the 22 exact-symbol\n * queries the definition is in the top 10 about 85% of the time but ranked\n * first only about 20% of the time. Call sites outnumber definitions and\n * contain the identical identifier, so term coverage — which saturates at 1.0\n * for both — cannot separate them. Structure can.\n */\nconst DECLARATION_BONUS = 0.3;\n\n/** Keywords that introduce a definition across the languages this indexes.\n *  Matched against lowercased text, so the term is lowercased too. */\nconst DECLARATION_KEYWORDS =\n\t\"function|class|interface|type|enum|struct|impl|trait|fn|def|const|let|var|namespace|module\";\n\n/** Does `window` declare `term`, as opposed to referencing it? */\nfunction declaresTerm(window: string, term: string): boolean {\n\tconst escaped = term.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n\t// `function foo(`, `class Foo {`, `const foo =` ...\n\tif (new RegExp(`\\\\b(?:${DECLARATION_KEYWORDS})\\\\s+${escaped}\\\\b`).test(window)) return true;\n\t// `foo(...) {` at the start of a line — methods, Go/Rust receivers, Python defs\n\t// already covered above, but this catches object-literal and class members.\n\tif (\n\t\tnew RegExp(`^\\\\s*(?:(?:async|public|private|protected|static|export)\\\\s+)*${escaped}\\\\s*[(<]`, \"m\").test(window)\n\t) {\n\t\treturn true;\n\t}\n\treturn false;\n}\n\n/** Inverse document frequency over the candidate pool.\n *\n * True corpus IDF lives in the BM25 index and is not exposed over the daemon\n * protocol, so this approximates it with the candidate set: a term present in\n * every candidate discriminates nothing, one present in three carries the\n * signal. That is the comparison the reranker actually needs to make, since it\n * only ever orders candidates against each other. */\nfunction inverseDocumentFrequency(documentFrequency: number, total: number): number {\n\treturn Math.log(1 + total / Math.max(1, documentFrequency));\n}\n\nexport interface RerankResult {\n\tcandidates: FusedCandidate[];\n\tlatencyMs: number;\n}\n\nexport function rerankCandidates(query: string, candidates: readonly FusedCandidate[], cwd: string): RerankResult {\n\tconst startedMs = Date.now();\n\tconst plan = buildLexicalQueryPlan(query);\n\tif (!plan || candidates.length < 2) {\n\t\treturn { candidates: [...candidates], latencyMs: Date.now() - startedMs };\n\t}\n\tconst terms = plan.terms;\n\tconst queryPath = query.trim().toLowerCase();\n\n\tconst fileCache = new Map<string, string[] | undefined>();\n\tconst readLines = (rel: string): string[] | undefined => {\n\t\tif (!fileCache.has(rel)) {\n\t\t\ttry {\n\t\t\t\tconst content = readFileSync(path.resolve(cwd, rel), \"utf-8\");\n\t\t\t\tfileCache.set(rel, content.toLowerCase().split(\"\\n\"));\n\t\t\t} catch {\n\t\t\t\tfileCache.set(rel, undefined);\n\t\t\t}\n\t\t}\n\t\treturn fileCache.get(rel);\n\t};\n\n\t// Read every candidate window once: the term/declaration signals and the\n\t// candidate-pool IDF all need them, and files repeat across candidates.\n\tconst windows = candidates.map((candidate) => {\n\t\tconst lines = readLines(candidate.path);\n\t\tif (!lines) return undefined;\n\t\treturn lines.slice(Math.max(0, candidate.startLine - 1), Math.min(lines.length, candidate.endLine)).join(\"\\n\");\n\t});\n\n\t// Candidate-pool document frequency per term, for the IDF weighting below.\n\tconst documentFrequency = new Map<string, number>();\n\tfor (const term of terms) {\n\t\tdocumentFrequency.set(term, windows.filter((w) => w?.includes(term)).length);\n\t}\n\tconst termWeight = new Map(\n\t\tterms.map((t) => [t, inverseDocumentFrequency(documentFrequency.get(t) ?? 0, candidates.length)]),\n\t);\n\tconst totalTermWeight = terms.reduce((sum, t) => sum + (termWeight.get(t) ?? 0), 0);\n\n\t// Fused prior normalized by score, not by position: a candidate both\n\t// retrievers agreed on should outrank one that squeaked in, and a uniform\n\t// 1 - index/length ramp throws that magnitude away.\n\tconst maxRrfScore = Math.max(...candidates.map((c) => c.rrfScore), Number.MIN_VALUE);\n\n\tconst scored = candidates.map((candidate, index) => {\n\t\tconst fusedPrior = candidate.rrfScore / maxRrfScore;\n\n\t\tconst window = windows[index];\n\t\tlet termCoverage = 0;\n\t\tlet declaresAnyTerm = false;\n\t\tif (window && terms.length > 0) {\n\t\t\tconst present = terms.filter((t) => window.includes(t));\n\t\t\ttermCoverage =\n\t\t\t\ttotalTermWeight > 0\n\t\t\t\t\t? present.reduce((sum, t) => sum + (termWeight.get(t) ?? 0), 0) / totalTermWeight\n\t\t\t\t\t: present.length / terms.length;\n\t\t\tdeclaresAnyTerm = present.some((t) => declaresTerm(window, t));\n\t\t}\n\n\t\tconst lowerPath = candidate.path.toLowerCase();\n\t\t// A quoted phrase rarely names a file; split it into path-ish tokens so\n\t\t// `\"token budget exceeded\"` still gets partial path credit.\n\t\tconst pathTerms = terms.length === 1 ? terms[0].split(/[^a-z0-9_$]+/).filter((t) => t.length >= 3) : terms;\n\t\tconst pathAffinity =\n\t\t\tpathTerms.length > 0 ? pathTerms.filter((t) => lowerPath.includes(t)).length / pathTerms.length : 0;\n\n\t\tconst exactPath =\n\t\t\tqueryPath.length >= 3 && (lowerPath === queryPath || lowerPath.endsWith(`/${queryPath}`)) ? 1 : 0;\n\n\t\tconst score =\n\t\t\tWEIGHT_FUSED_PRIOR * fusedPrior +\n\t\t\tWEIGHT_TERM_COVERAGE * termCoverage +\n\t\t\tWEIGHT_PATH_AFFINITY * pathAffinity +\n\t\t\tEXACT_PATH_BONUS * exactPath +\n\t\t\t(declaresAnyTerm ? DECLARATION_BONUS : 0);\n\t\treturn { candidate, index, score };\n\t});\n\n\t// Stable, deterministic: score desc, fused order as tie-break.\n\tscored.sort((a, b) => b.score - a.score || a.index - b.index);\n\treturn { candidates: scored.map((s) => s.candidate), latencyMs: Date.now() - startedMs };\n}\n"]}
+{"version":3,"file":"rerank.d.ts","sourceRoot":"","sources":["../../../src/core/search/rerank.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAKH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AA6GjD;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAQnD;AAiCD,MAAM,WAAW,YAAY;IAC5B,UAAU,EAAE,cAAc,EAAE,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,CAiBlH;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,cAAc,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,YAAY,CAqFhH","sourcesContent":["/**\n * Deterministic reranker over the fused top-50\n * (docs/hybrid-retrieval-design.md, step 7 of the shipping order).\n *\n * The eval gate showed fused Recall@50 well above Recall@5/10 — the right\n * candidates survive fusion but sit too deep. This reranker re-orders them\n * using evidence that is only cheap to compute *after* fusion, when there\n * are ≤50 candidates instead of thousands of lines:\n *\n *   - term coverage: how many distinct query terms appear in the candidate's\n *     actual expanded window (read from disk);\n *   - path affinity: query terms appearing in the candidate's file path —\n *     this is what lets a query like `core/search/hybrid-search.ts` rank the\n *     file itself first, which content grep alone cannot do;\n *   - fused prior: the RRF ordering, so retriever consensus still counts.\n *\n * Not every signal suits every query. A query that names something and a query\n * that describes behaviour want different evidence, so the name-matching\n * signal is gated on {@link queryIsProse} — see its comment for the numbers.\n *\n * Purely lexical-statistical and deterministic — no model, no I/O beyond\n * reading candidate windows. A cross-encoder can later replace the scoring\n * function behind the same signature; that model work belongs to\n * `kolisachint/embeddingsearchtools`, not here.\n */\n\nimport { readFileSync } from \"fs\";\nimport path from \"path\";\nimport { buildLexicalQueryPlan } from \"./lexical-retriever.js\";\nimport type { FusedCandidate } from \"./types.js\";\n\n/** Weights of the scoring blend. The eval harness (`bun run search-eval`) is\n *  the instrument for changing them — don't tune blind. */\nconst WEIGHT_FUSED_PRIOR = 0.4;\nconst WEIGHT_TERM_COVERAGE = 0.35;\nconst WEIGHT_PATH_AFFINITY = 0.25;\n/** Additive bonus when the query *is* the candidate's path (or its suffix):\n *  the caller named the file, so no amount of content evidence elsewhere\n *  should outrank it. */\nconst EXACT_PATH_BONUS = 0.5;\n/**\n * Additive bonus when the window *declares* a query term rather than merely\n * mentioning it.\n *\n * This targets the largest measured gap in the eval: on the 22 exact-symbol\n * queries the definition is in the top 10 about 85% of the time but ranked\n * first only about 20% of the time. Call sites outnumber definitions and\n * contain the identical identifier, so term coverage — which saturates at 1.0\n * for both — cannot separate them. Structure can.\n */\nconst DECLARATION_BONUS = 0.3;\n\n/**\n * Sentence glue. Identifiers and paths never contain these words, so two or\n * more of them means the query is a sentence rather than a name.\n *\n * This gates {@link DECLARATION_BONUS} because that bonus is a *name-matching*\n * signal and a prose query has no name to match. Its plan terms are ordinary\n * words — \"results\", \"search\", \"index\" — so it fires on whichever candidate\n * happens to declare a variable by one of them, at a weight larger than the\n * entire path-affinity term, and buries the fused ordering that did know the\n * answer. On the conceptual class reranking was scoring *below not reranking\n * at all* (MRR 0.246 un-reranked vs 0.124 reranked on `semantic`).\n *\n * Measured on the 62-query gold set, gating it moved conceptual MRR +0.046\n * (`semantic +rr`), +0.040 (`auto +rr`), +0.056 (`bm25+dense +rr`) and left\n * exact-symbol, error-fragment and path bit-identical — the gate never fires\n * on those, by construction. Overall MRR +0.007 to +0.013, which the paired\n * sign test does not call significant (p = 0.11 to 0.29); the classes it\n * protects are what justify it, not the aggregate.\n *\n * Path affinity is deliberately *not* gated. It is a topic signal, not a name\n * signal: \"how does a grep line number become an embedding chunk id\" wants\n * files with `grep` and `chunk` in the path. Gating it too was measured and\n * was strictly worse — same conceptual gain, roughly double the cross-file\n * loss (−0.058 vs −0.032 on `semantic +rr`).\n *\n * Deliberately conservative: two hits, not one, so a terse query like\n * `hybrid search fusion` keeps today's scoring untouched.\n */\nconst PROSE_FUNCTION_WORDS = new Set([\n\t\"a\",\n\t\"after\",\n\t\"all\",\n\t\"an\",\n\t\"and\",\n\t\"any\",\n\t\"are\",\n\t\"as\",\n\t\"at\",\n\t\"be\",\n\t\"been\",\n\t\"before\",\n\t\"between\",\n\t\"but\",\n\t\"by\",\n\t\"can\",\n\t\"does\",\n\t\"do\",\n\t\"each\",\n\t\"for\",\n\t\"from\",\n\t\"had\",\n\t\"has\",\n\t\"have\",\n\t\"how\",\n\t\"if\",\n\t\"in\",\n\t\"into\",\n\t\"is\",\n\t\"it\",\n\t\"its\",\n\t\"of\",\n\t\"on\",\n\t\"one\",\n\t\"or\",\n\t\"should\",\n\t\"so\",\n\t\"than\",\n\t\"that\",\n\t\"the\",\n\t\"then\",\n\t\"this\",\n\t\"to\",\n\t\"under\",\n\t\"was\",\n\t\"were\",\n\t\"what\",\n\t\"when\",\n\t\"where\",\n\t\"which\",\n\t\"why\",\n\t\"with\",\n\t\"would\",\n]);\n/** Function words needed before a query counts as prose. */\nconst PROSE_WORD_THRESHOLD = 2;\n\n/**\n * Is `query` a sentence rather than a name?\n *\n * A quoted segment is never prose regardless of its words: the plan collapses\n * it to one literal term, so the declaration bonus cannot fire on it anyway,\n * and its path-token split is what lets `\"Theme not initialized…\"` find\n * `core/theme.ts`.\n */\nexport function queryIsProse(query: string): boolean {\n\tif (/[\"'`][^\"'`]+[\"'`]/.test(query)) return false;\n\tconst words = query.toLowerCase().match(/[a-z]+/g) ?? [];\n\tlet hits = 0;\n\tfor (const word of words) {\n\t\tif (PROSE_FUNCTION_WORDS.has(word) && ++hits >= PROSE_WORD_THRESHOLD) return true;\n\t}\n\treturn false;\n}\n\n/** Keywords that introduce a definition across the languages this indexes.\n *  Matched against lowercased text, so the term is lowercased too. */\nconst DECLARATION_KEYWORDS =\n\t\"function|class|interface|type|enum|struct|impl|trait|fn|def|const|let|var|namespace|module\";\n\n/** Does `window` declare `term`, as opposed to referencing it? */\nfunction declaresTerm(window: string, term: string): boolean {\n\tconst escaped = term.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n\t// `function foo(`, `class Foo {`, `const foo =` ...\n\tif (new RegExp(`\\\\b(?:${DECLARATION_KEYWORDS})\\\\s+${escaped}\\\\b`).test(window)) return true;\n\t// `foo(...) {` at the start of a line — methods, Go/Rust receivers, Python defs\n\t// already covered above, but this catches object-literal and class members.\n\tif (\n\t\tnew RegExp(`^\\\\s*(?:(?:async|public|private|protected|static|export)\\\\s+)*${escaped}\\\\s*[(<]`, \"m\").test(window)\n\t) {\n\t\treturn true;\n\t}\n\treturn false;\n}\n\n/** Inverse document frequency over the candidate pool.\n *\n * True corpus IDF lives in the BM25 index and is not exposed over the daemon\n * protocol, so this approximates it with the candidate set: a term present in\n * every candidate discriminates nothing, one present in three carries the\n * signal. That is the comparison the reranker actually needs to make, since it\n * only ever orders candidates against each other. */\nfunction inverseDocumentFrequency(documentFrequency: number, total: number): number {\n\treturn Math.log(1 + total / Math.max(1, documentFrequency));\n}\n\nexport interface RerankResult {\n\tcandidates: FusedCandidate[];\n\tlatencyMs: number;\n}\n\n/**\n * The exact source text each candidate stands for, as the model should see it.\n *\n * Shared with the cross-encoder path so both rerankers score identical text —\n * otherwise a comparison between them would partly measure which one got a\n * better view of the candidate.\n */\nexport function readCandidateWindows(candidates: readonly FusedCandidate[], cwd: string): Array<string | undefined> {\n\tconst fileCache = new Map<string, string[] | undefined>();\n\tconst read = (rel: string): string[] | undefined => {\n\t\tif (!fileCache.has(rel)) {\n\t\t\ttry {\n\t\t\t\tfileCache.set(rel, readFileSync(path.resolve(cwd, rel), \"utf-8\").split(\"\\n\"));\n\t\t\t} catch {\n\t\t\t\tfileCache.set(rel, undefined);\n\t\t\t}\n\t\t}\n\t\treturn fileCache.get(rel);\n\t};\n\treturn candidates.map((candidate) => {\n\t\tconst lines = read(candidate.path);\n\t\tif (!lines) return undefined;\n\t\treturn lines.slice(Math.max(0, candidate.startLine - 1), Math.min(lines.length, candidate.endLine)).join(\"\\n\");\n\t});\n}\n\nexport function rerankCandidates(query: string, candidates: readonly FusedCandidate[], cwd: string): RerankResult {\n\tconst startedMs = Date.now();\n\tconst plan = buildLexicalQueryPlan(query);\n\tif (!plan || candidates.length < 2) {\n\t\treturn { candidates: [...candidates], latencyMs: Date.now() - startedMs };\n\t}\n\tconst terms = plan.terms;\n\tconst queryPath = query.trim().toLowerCase();\n\t// Prose asks about behaviour, not about a name, so the one signal that reads\n\t// candidates *as names* is switched off.\n\tconst prose = queryIsProse(query);\n\n\tconst fileCache = new Map<string, string[] | undefined>();\n\tconst readLines = (rel: string): string[] | undefined => {\n\t\tif (!fileCache.has(rel)) {\n\t\t\ttry {\n\t\t\t\tconst content = readFileSync(path.resolve(cwd, rel), \"utf-8\");\n\t\t\t\tfileCache.set(rel, content.toLowerCase().split(\"\\n\"));\n\t\t\t} catch {\n\t\t\t\tfileCache.set(rel, undefined);\n\t\t\t}\n\t\t}\n\t\treturn fileCache.get(rel);\n\t};\n\n\t// Read every candidate window once: the term/declaration signals and the\n\t// candidate-pool IDF all need them, and files repeat across candidates.\n\tconst windows = candidates.map((candidate) => {\n\t\tconst lines = readLines(candidate.path);\n\t\tif (!lines) return undefined;\n\t\treturn lines.slice(Math.max(0, candidate.startLine - 1), Math.min(lines.length, candidate.endLine)).join(\"\\n\");\n\t});\n\n\t// Candidate-pool document frequency per term, for the IDF weighting below.\n\tconst documentFrequency = new Map<string, number>();\n\tfor (const term of terms) {\n\t\tdocumentFrequency.set(term, windows.filter((w) => w?.includes(term)).length);\n\t}\n\tconst termWeight = new Map(\n\t\tterms.map((t) => [t, inverseDocumentFrequency(documentFrequency.get(t) ?? 0, candidates.length)]),\n\t);\n\tconst totalTermWeight = terms.reduce((sum, t) => sum + (termWeight.get(t) ?? 0), 0);\n\n\t// Fused prior normalized by score, not by position: a candidate both\n\t// retrievers agreed on should outrank one that squeaked in, and a uniform\n\t// 1 - index/length ramp throws that magnitude away.\n\tconst maxRrfScore = Math.max(...candidates.map((c) => c.rrfScore), Number.MIN_VALUE);\n\n\tconst scored = candidates.map((candidate, index) => {\n\t\tconst fusedPrior = candidate.rrfScore / maxRrfScore;\n\n\t\tconst window = windows[index];\n\t\tlet termCoverage = 0;\n\t\tlet declaresAnyTerm = false;\n\t\tif (window && terms.length > 0) {\n\t\t\tconst present = terms.filter((t) => window.includes(t));\n\t\t\ttermCoverage =\n\t\t\t\ttotalTermWeight > 0\n\t\t\t\t\t? present.reduce((sum, t) => sum + (termWeight.get(t) ?? 0), 0) / totalTermWeight\n\t\t\t\t\t: present.length / terms.length;\n\t\t\tdeclaresAnyTerm = !prose && present.some((t) => declaresTerm(window, t));\n\t\t}\n\n\t\tconst lowerPath = candidate.path.toLowerCase();\n\t\t// A quoted phrase rarely names a file; split it into path-ish tokens so\n\t\t// `\"token budget exceeded\"` still gets partial path credit.\n\t\tconst pathTerms = terms.length === 1 ? terms[0].split(/[^a-z0-9_$]+/).filter((t) => t.length >= 3) : terms;\n\t\tconst pathAffinity =\n\t\t\tpathTerms.length > 0 ? pathTerms.filter((t) => lowerPath.includes(t)).length / pathTerms.length : 0;\n\n\t\tconst exactPath =\n\t\t\tqueryPath.length >= 3 && (lowerPath === queryPath || lowerPath.endsWith(`/${queryPath}`)) ? 1 : 0;\n\n\t\tconst score =\n\t\t\tWEIGHT_FUSED_PRIOR * fusedPrior +\n\t\t\tWEIGHT_TERM_COVERAGE * termCoverage +\n\t\t\tWEIGHT_PATH_AFFINITY * pathAffinity +\n\t\t\tEXACT_PATH_BONUS * exactPath +\n\t\t\t(declaresAnyTerm ? DECLARATION_BONUS : 0);\n\t\treturn { candidate, index, score };\n\t});\n\n\t// Stable, deterministic: score desc, fused order as tie-break.\n\tscored.sort((a, b) => b.score - a.score || a.index - b.index);\n\treturn { candidates: scored.map((s) => s.candidate), latencyMs: Date.now() - startedMs };\n}\n"]}

package/dist/core/search/rerank.js

@@ -14,6 +14,10 @@
  *     file itself first, which content grep alone cannot do;
  *   - fused prior: the RRF ordering, so retriever consensus still counts.
  *
+ * Not every signal suits every query. A query that names something and a query
+ * that describes behaviour want different evidence, so the name-matching
+ * signal is gated on {@link queryIsProse} — see its comment for the numbers.
+ *
  * Purely lexical-statistical and deterministic — no model, no I/O beyond
  * reading candidate windows. A cross-encoder can later replace the scoring
  * function behind the same signature; that model work belongs to
@@ -42,6 +46,110 @@ const EXACT_PATH_BONUS = 0.5;
  * for both — cannot separate them. Structure can.
  */
 const DECLARATION_BONUS = 0.3;
+/**
+ * Sentence glue. Identifiers and paths never contain these words, so two or
+ * more of them means the query is a sentence rather than a name.
+ *
+ * This gates {@link DECLARATION_BONUS} because that bonus is a *name-matching*
+ * signal and a prose query has no name to match. Its plan terms are ordinary
+ * words — "results", "search", "index" — so it fires on whichever candidate
+ * happens to declare a variable by one of them, at a weight larger than the
+ * entire path-affinity term, and buries the fused ordering that did know the
+ * answer. On the conceptual class reranking was scoring *below not reranking
+ * at all* (MRR 0.246 un-reranked vs 0.124 reranked on `semantic`).
+ *
+ * Measured on the 62-query gold set, gating it moved conceptual MRR +0.046
+ * (`semantic +rr`), +0.040 (`auto +rr`), +0.056 (`bm25+dense +rr`) and left
+ * exact-symbol, error-fragment and path bit-identical — the gate never fires
+ * on those, by construction. Overall MRR +0.007 to +0.013, which the paired
+ * sign test does not call significant (p = 0.11 to 0.29); the classes it
+ * protects are what justify it, not the aggregate.
+ *
+ * Path affinity is deliberately *not* gated. It is a topic signal, not a name
+ * signal: "how does a grep line number become an embedding chunk id" wants
+ * files with `grep` and `chunk` in the path. Gating it too was measured and
+ * was strictly worse — same conceptual gain, roughly double the cross-file
+ * loss (−0.058 vs −0.032 on `semantic +rr`).
+ *
+ * Deliberately conservative: two hits, not one, so a terse query like
+ * `hybrid search fusion` keeps today's scoring untouched.
+ */
+const PROSE_FUNCTION_WORDS = new Set([
+    "a",
+    "after",
+    "all",
+    "an",
+    "and",
+    "any",
+    "are",
+    "as",
+    "at",
+    "be",
+    "been",
+    "before",
+    "between",
+    "but",
+    "by",
+    "can",
+    "does",
+    "do",
+    "each",
+    "for",
+    "from",
+    "had",
+    "has",
+    "have",
+    "how",
+    "if",
+    "in",
+    "into",
+    "is",
+    "it",
+    "its",
+    "of",
+    "on",
+    "one",
+    "or",
+    "should",
+    "so",
+    "than",
+    "that",
+    "the",
+    "then",
+    "this",
+    "to",
+    "under",
+    "was",
+    "were",
+    "what",
+    "when",
+    "where",
+    "which",
+    "why",
+    "with",
+    "would",
+]);
+/** Function words needed before a query counts as prose. */
+const PROSE_WORD_THRESHOLD = 2;
+/**
+ * Is `query` a sentence rather than a name?
+ *
+ * A quoted segment is never prose regardless of its words: the plan collapses
+ * it to one literal term, so the declaration bonus cannot fire on it anyway,
+ * and its path-token split is what lets `"Theme not initialized…"` find
+ * `core/theme.ts`.
+ */
+export function queryIsProse(query) {
+    if (/["'`][^"'`]+["'`]/.test(query))
+        return false;
+    const words = query.toLowerCase().match(/[a-z]+/g) ?? [];
+    let hits = 0;
+    for (const word of words) {
+        if (PROSE_FUNCTION_WORDS.has(word) && ++hits >= PROSE_WORD_THRESHOLD)
+            return true;
+    }
+    return false;
+}
 /** Keywords that introduce a definition across the languages this indexes.
  *  Matched against lowercased text, so the term is lowercased too. */
 const DECLARATION_KEYWORDS = "function|class|interface|type|enum|struct|impl|trait|fn|def|const|let|var|namespace|module";
@@ -68,6 +176,33 @@ function declaresTerm(window, term) {
 function inverseDocumentFrequency(documentFrequency, total) {
     return Math.log(1 + total / Math.max(1, documentFrequency));
 }
+/**
+ * The exact source text each candidate stands for, as the model should see it.
+ *
+ * Shared with the cross-encoder path so both rerankers score identical text —
+ * otherwise a comparison between them would partly measure which one got a
+ * better view of the candidate.
+ */
+export function readCandidateWindows(candidates, cwd) {
+    const fileCache = new Map();
+    const read = (rel) => {
+        if (!fileCache.has(rel)) {
+            try {
+                fileCache.set(rel, readFileSync(path.resolve(cwd, rel), "utf-8").split("\n"));
+            }
+            catch {
+                fileCache.set(rel, undefined);
+            }
+        }
+        return fileCache.get(rel);
+    };
+    return candidates.map((candidate) => {
+        const lines = read(candidate.path);
+        if (!lines)
+            return undefined;
+        return lines.slice(Math.max(0, candidate.startLine - 1), Math.min(lines.length, candidate.endLine)).join("\n");
+    });
+}
 export function rerankCandidates(query, candidates, cwd) {
     const startedMs = Date.now();
     const plan = buildLexicalQueryPlan(query);
@@ -76,6 +211,9 @@ export function rerankCandidates(query, candidates, cwd) {
     }
     const terms = plan.terms;
     const queryPath = query.trim().toLowerCase();
+    // Prose asks about behaviour, not about a name, so the one signal that reads
+    // candidates *as names* is switched off.
+    const prose = queryIsProse(query);
     const fileCache = new Map();
     const readLines = (rel) => {
         if (!fileCache.has(rel)) {
@@ -119,7 +257,7 @@ export function rerankCandidates(query, candidates, cwd) {
                 totalTermWeight > 0
                     ? present.reduce((sum, t) => sum + (termWeight.get(t) ?? 0), 0) / totalTermWeight
                     : present.length / terms.length;
-            declaresAnyTerm = present.some((t) => declaresTerm(window, t));
+            declaresAnyTerm = !prose && present.some((t) => declaresTerm(window, t));
         }
         const lowerPath = candidate.path.toLowerCase();
         // A quoted phrase rarely names a file; split it into path-ish tokens so

package/dist/core/search/rerank.js.map

@@ -1 +1 @@
-{"version":3,"file":"rerank.js","sourceRoot":"","sources":["../../../src/core/search/rerank.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAG/D;6DAC2D;AAC3D,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC;;yBAEyB;AACzB,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAC7B;;;;;;;;;GASG;AACH,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B;sEACsE;AACtE,MAAM,oBAAoB,GACzB,4FAA4F,CAAC;AAE9F,kEAAkE;AAClE,SAAS,YAAY,CAAC,MAAc,EAAE,IAAY,EAAW;IAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IAC5D,oDAAoD;IACpD,IAAI,IAAI,MAAM,CAAC,SAAS,oBAAoB,QAAQ,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5F,kFAAgF;IAChF,4EAA4E;IAC5E,IACC,IAAI,MAAM,CAAC,iEAAiE,OAAO,UAAU,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAC/G,CAAC;QACF,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AAAA,CACb;AAED;;;;;;qDAMqD;AACrD,SAAS,wBAAwB,CAAC,iBAAyB,EAAE,KAAa,EAAU;IACnF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC;AAAA,CAC5D;AAOD,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,UAAqC,EAAE,GAAW,EAAgB;IACjH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,OAAO,EAAE,UAAU,EAAE,CAAC,GAAG,UAAU,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC;IAC3E,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzB,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE7C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAgC,CAAC;IAC1D,MAAM,SAAS,GAAG,CAAC,GAAW,EAAwB,EAAE,CAAC;QACxD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC;gBAC9D,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACR,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YAC/B,CAAC;QACF,CAAC;QACD,OAAO,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAAA,CAC1B,CAAC;IAEF,yEAAyE;IACzE,wEAAwE;IACxE,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK;YAAE,OAAO,SAAS,CAAC;QAC7B,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAAA,CAC/G,CAAC,CAAC;IAEH,2EAA2E;IAC3E,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,iBAAiB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,GAAG,CACzB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,wBAAwB,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CACjG,CAAC;IACF,MAAM,eAAe,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEpF,qEAAqE;IACrE,0EAA0E;IAC1E,oDAAoD;IACpD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAErF,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC;QACnD,MAAM,UAAU,GAAG,SAAS,CAAC,QAAQ,GAAG,WAAW,CAAC;QAEpD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,eAAe,GAAG,KAAK,CAAC;QAC5B,IAAI,MAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACxD,YAAY;gBACX,eAAe,GAAG,CAAC;oBAClB,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,eAAe;oBACjF,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YAClC,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QAC/C,wEAAwE;QACxE,4DAA4D;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC3G,MAAM,YAAY,GACjB,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAErG,MAAM,SAAS,GACd,SAAS,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnG,MAAM,KAAK,GACV,kBAAkB,GAAG,UAAU;YAC/B,oBAAoB,GAAG,YAAY;YACnC,oBAAoB,GAAG,YAAY;YACnC,gBAAgB,GAAG,SAAS;YAC5B,CAAC,eAAe,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3C,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAAA,CACnC,CAAC,CAAC;IAEH,+DAA+D;IAC/D,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC;AAAA,CACzF","sourcesContent":["/**\n * Deterministic reranker over the fused top-50\n * (docs/hybrid-retrieval-design.md, step 7 of the shipping order).\n *\n * The eval gate showed fused Recall@50 well above Recall@5/10 — the right\n * candidates survive fusion but sit too deep. This reranker re-orders them\n * using evidence that is only cheap to compute *after* fusion, when there\n * are ≤50 candidates instead of thousands of lines:\n *\n *   - term coverage: how many distinct query terms appear in the candidate's\n *     actual expanded window (read from disk);\n *   - path affinity: query terms appearing in the candidate's file path —\n *     this is what lets a query like `core/search/hybrid-search.ts` rank the\n *     file itself first, which content grep alone cannot do;\n *   - fused prior: the RRF ordering, so retriever consensus still counts.\n *\n * Purely lexical-statistical and deterministic — no model, no I/O beyond\n * reading candidate windows. A cross-encoder can later replace the scoring\n * function behind the same signature; that model work belongs to\n * `kolisachint/embeddingsearchtools`, not here.\n */\n\nimport { readFileSync } from \"fs\";\nimport path from \"path\";\nimport { buildLexicalQueryPlan } from \"./lexical-retriever.js\";\nimport type { FusedCandidate } from \"./types.js\";\n\n/** Weights of the scoring blend. The eval harness (`bun run search-eval`) is\n *  the instrument for changing them — don't tune blind. */\nconst WEIGHT_FUSED_PRIOR = 0.4;\nconst WEIGHT_TERM_COVERAGE = 0.35;\nconst WEIGHT_PATH_AFFINITY = 0.25;\n/** Additive bonus when the query *is* the candidate's path (or its suffix):\n *  the caller named the file, so no amount of content evidence elsewhere\n *  should outrank it. */\nconst EXACT_PATH_BONUS = 0.5;\n/**\n * Additive bonus when the window *declares* a query term rather than merely\n * mentioning it.\n *\n * This targets the largest measured gap in the eval: on the 22 exact-symbol\n * queries the definition is in the top 10 about 85% of the time but ranked\n * first only about 20% of the time. Call sites outnumber definitions and\n * contain the identical identifier, so term coverage — which saturates at 1.0\n * for both — cannot separate them. Structure can.\n */\nconst DECLARATION_BONUS = 0.3;\n\n/** Keywords that introduce a definition across the languages this indexes.\n *  Matched against lowercased text, so the term is lowercased too. */\nconst DECLARATION_KEYWORDS =\n\t\"function|class|interface|type|enum|struct|impl|trait|fn|def|const|let|var|namespace|module\";\n\n/** Does `window` declare `term`, as opposed to referencing it? */\nfunction declaresTerm(window: string, term: string): boolean {\n\tconst escaped = term.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n\t// `function foo(`, `class Foo {`, `const foo =` ...\n\tif (new RegExp(`\\\\b(?:${DECLARATION_KEYWORDS})\\\\s+${escaped}\\\\b`).test(window)) return true;\n\t// `foo(...) {` at the start of a line — methods, Go/Rust receivers, Python defs\n\t// already covered above, but this catches object-literal and class members.\n\tif (\n\t\tnew RegExp(`^\\\\s*(?:(?:async|public|private|protected|static|export)\\\\s+)*${escaped}\\\\s*[(<]`, \"m\").test(window)\n\t) {\n\t\treturn true;\n\t}\n\treturn false;\n}\n\n/** Inverse document frequency over the candidate pool.\n *\n * True corpus IDF lives in the BM25 index and is not exposed over the daemon\n * protocol, so this approximates it with the candidate set: a term present in\n * every candidate discriminates nothing, one present in three carries the\n * signal. That is the comparison the reranker actually needs to make, since it\n * only ever orders candidates against each other. */\nfunction inverseDocumentFrequency(documentFrequency: number, total: number): number {\n\treturn Math.log(1 + total / Math.max(1, documentFrequency));\n}\n\nexport interface RerankResult {\n\tcandidates: FusedCandidate[];\n\tlatencyMs: number;\n}\n\nexport function rerankCandidates(query: string, candidates: readonly FusedCandidate[], cwd: string): RerankResult {\n\tconst startedMs = Date.now();\n\tconst plan = buildLexicalQueryPlan(query);\n\tif (!plan || candidates.length < 2) {\n\t\treturn { candidates: [...candidates], latencyMs: Date.now() - startedMs };\n\t}\n\tconst terms = plan.terms;\n\tconst queryPath = query.trim().toLowerCase();\n\n\tconst fileCache = new Map<string, string[] | undefined>();\n\tconst readLines = (rel: string): string[] | undefined => {\n\t\tif (!fileCache.has(rel)) {\n\t\t\ttry {\n\t\t\t\tconst content = readFileSync(path.resolve(cwd, rel), \"utf-8\");\n\t\t\t\tfileCache.set(rel, content.toLowerCase().split(\"\\n\"));\n\t\t\t} catch {\n\t\t\t\tfileCache.set(rel, undefined);\n\t\t\t}\n\t\t}\n\t\treturn fileCache.get(rel);\n\t};\n\n\t// Read every candidate window once: the term/declaration signals and the\n\t// candidate-pool IDF all need them, and files repeat across candidates.\n\tconst windows = candidates.map((candidate) => {\n\t\tconst lines = readLines(candidate.path);\n\t\tif (!lines) return undefined;\n\t\treturn lines.slice(Math.max(0, candidate.startLine - 1), Math.min(lines.length, candidate.endLine)).join(\"\\n\");\n\t});\n\n\t// Candidate-pool document frequency per term, for the IDF weighting below.\n\tconst documentFrequency = new Map<string, number>();\n\tfor (const term of terms) {\n\t\tdocumentFrequency.set(term, windows.filter((w) => w?.includes(term)).length);\n\t}\n\tconst termWeight = new Map(\n\t\tterms.map((t) => [t, inverseDocumentFrequency(documentFrequency.get(t) ?? 0, candidates.length)]),\n\t);\n\tconst totalTermWeight = terms.reduce((sum, t) => sum + (termWeight.get(t) ?? 0), 0);\n\n\t// Fused prior normalized by score, not by position: a candidate both\n\t// retrievers agreed on should outrank one that squeaked in, and a uniform\n\t// 1 - index/length ramp throws that magnitude away.\n\tconst maxRrfScore = Math.max(...candidates.map((c) => c.rrfScore), Number.MIN_VALUE);\n\n\tconst scored = candidates.map((candidate, index) => {\n\t\tconst fusedPrior = candidate.rrfScore / maxRrfScore;\n\n\t\tconst window = windows[index];\n\t\tlet termCoverage = 0;\n\t\tlet declaresAnyTerm = false;\n\t\tif (window && terms.length > 0) {\n\t\t\tconst present = terms.filter((t) => window.includes(t));\n\t\t\ttermCoverage =\n\t\t\t\ttotalTermWeight > 0\n\t\t\t\t\t? present.reduce((sum, t) => sum + (termWeight.get(t) ?? 0), 0) / totalTermWeight\n\t\t\t\t\t: present.length / terms.length;\n\t\t\tdeclaresAnyTerm = present.some((t) => declaresTerm(window, t));\n\t\t}\n\n\t\tconst lowerPath = candidate.path.toLowerCase();\n\t\t// A quoted phrase rarely names a file; split it into path-ish tokens so\n\t\t// `\"token budget exceeded\"` still gets partial path credit.\n\t\tconst pathTerms = terms.length === 1 ? terms[0].split(/[^a-z0-9_$]+/).filter((t) => t.length >= 3) : terms;\n\t\tconst pathAffinity =\n\t\t\tpathTerms.length > 0 ? pathTerms.filter((t) => lowerPath.includes(t)).length / pathTerms.length : 0;\n\n\t\tconst exactPath =\n\t\t\tqueryPath.length >= 3 && (lowerPath === queryPath || lowerPath.endsWith(`/${queryPath}`)) ? 1 : 0;\n\n\t\tconst score =\n\t\t\tWEIGHT_FUSED_PRIOR * fusedPrior +\n\t\t\tWEIGHT_TERM_COVERAGE * termCoverage +\n\t\t\tWEIGHT_PATH_AFFINITY * pathAffinity +\n\t\t\tEXACT_PATH_BONUS * exactPath +\n\t\t\t(declaresAnyTerm ? DECLARATION_BONUS : 0);\n\t\treturn { candidate, index, score };\n\t});\n\n\t// Stable, deterministic: score desc, fused order as tie-break.\n\tscored.sort((a, b) => b.score - a.score || a.index - b.index);\n\treturn { candidates: scored.map((s) => s.candidate), latencyMs: Date.now() - startedMs };\n}\n"]}
+{"version":3,"file":"rerank.js","sourceRoot":"","sources":["../../../src/core/search/rerank.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAG/D;6DAC2D;AAC3D,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC;;yBAEyB;AACzB,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAC7B;;;;;;;;;GASG;AACH,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACpC,GAAG;IACH,OAAO;IACP,KAAK;IACL,IAAI;IACJ,KAAK;IACL,KAAK;IACL,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,MAAM;IACN,QAAQ;IACR,SAAS;IACT,KAAK;IACL,IAAI;IACJ,KAAK;IACL,MAAM;IACN,IAAI;IACJ,MAAM;IACN,KAAK;IACL,MAAM;IACN,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,MAAM;IACN,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,QAAQ;IACR,IAAI;IACJ,MAAM;IACN,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,IAAI;IACJ,OAAO;IACP,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,KAAK;IACL,MAAM;IACN,OAAO;CACP,CAAC,CAAC;AACH,4DAA4D;AAC5D,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,KAAa,EAAW;IACpD,IAAI,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAClD,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;IACzD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,IAAI,oBAAoB;YAAE,OAAO,IAAI,CAAC;IACnF,CAAC;IACD,OAAO,KAAK,CAAC;AAAA,CACb;AAED;sEACsE;AACtE,MAAM,oBAAoB,GACzB,4FAA4F,CAAC;AAE9F,kEAAkE;AAClE,SAAS,YAAY,CAAC,MAAc,EAAE,IAAY,EAAW;IAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IAC5D,oDAAoD;IACpD,IAAI,IAAI,MAAM,CAAC,SAAS,oBAAoB,QAAQ,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5F,kFAAgF;IAChF,4EAA4E;IAC5E,IACC,IAAI,MAAM,CAAC,iEAAiE,OAAO,UAAU,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAC/G,CAAC;QACF,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AAAA,CACb;AAED;;;;;;qDAMqD;AACrD,SAAS,wBAAwB,CAAC,iBAAyB,EAAE,KAAa,EAAU;IACnF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC;AAAA,CAC5D;AAOD;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,UAAqC,EAAE,GAAW,EAA6B;IACnH,MAAM,SAAS,GAAG,IAAI,GAAG,EAAgC,CAAC;IAC1D,MAAM,IAAI,GAAG,CAAC,GAAW,EAAwB,EAAE,CAAC;QACnD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC;gBACJ,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/E,CAAC;YAAC,MAAM,CAAC;gBACR,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YAC/B,CAAC;QACF,CAAC;QACD,OAAO,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAAA,CAC1B,CAAC;IACF,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK;YAAE,OAAO,SAAS,CAAC;QAC7B,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAAA,CAC/G,CAAC,CAAC;AAAA,CACH;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,UAAqC,EAAE,GAAW,EAAgB;IACjH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,OAAO,EAAE,UAAU,EAAE,CAAC,GAAG,UAAU,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC;IAC3E,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzB,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC7C,6EAA6E;IAC7E,yCAAyC;IACzC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAElC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAgC,CAAC;IAC1D,MAAM,SAAS,GAAG,CAAC,GAAW,EAAwB,EAAE,CAAC;QACxD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC;gBAC9D,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACR,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YAC/B,CAAC;QACF,CAAC;QACD,OAAO,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAAA,CAC1B,CAAC;IAEF,yEAAyE;IACzE,wEAAwE;IACxE,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK;YAAE,OAAO,SAAS,CAAC;QAC7B,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAAA,CAC/G,CAAC,CAAC;IAEH,2EAA2E;IAC3E,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,iBAAiB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,GAAG,CACzB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,wBAAwB,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CACjG,CAAC;IACF,MAAM,eAAe,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEpF,qEAAqE;IACrE,0EAA0E;IAC1E,oDAAoD;IACpD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAErF,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC;QACnD,MAAM,UAAU,GAAG,SAAS,CAAC,QAAQ,GAAG,WAAW,CAAC;QAEpD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,eAAe,GAAG,KAAK,CAAC;QAC5B,IAAI,MAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACxD,YAAY;gBACX,eAAe,GAAG,CAAC;oBAClB,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,eAAe;oBACjF,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YAClC,eAAe,GAAG,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QAC/C,wEAAwE;QACxE,4DAA4D;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC3G,MAAM,YAAY,GACjB,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAErG,MAAM,SAAS,GACd,SAAS,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnG,MAAM,KAAK,GACV,kBAAkB,GAAG,UAAU;YAC/B,oBAAoB,GAAG,YAAY;YACnC,oBAAoB,GAAG,YAAY;YACnC,gBAAgB,GAAG,SAAS;YAC5B,CAAC,eAAe,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3C,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAAA,CACnC,CAAC,CAAC;IAEH,+DAA+D;IAC/D,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC;AAAA,CACzF","sourcesContent":["/**\n * Deterministic reranker over the fused top-50\n * (docs/hybrid-retrieval-design.md, step 7 of the shipping order).\n *\n * The eval gate showed fused Recall@50 well above Recall@5/10 — the right\n * candidates survive fusion but sit too deep. This reranker re-orders them\n * using evidence that is only cheap to compute *after* fusion, when there\n * are ≤50 candidates instead of thousands of lines:\n *\n *   - term coverage: how many distinct query terms appear in the candidate's\n *     actual expanded window (read from disk);\n *   - path affinity: query terms appearing in the candidate's file path —\n *     this is what lets a query like `core/search/hybrid-search.ts` rank the\n *     file itself first, which content grep alone cannot do;\n *   - fused prior: the RRF ordering, so retriever consensus still counts.\n *\n * Not every signal suits every query. A query that names something and a query\n * that describes behaviour want different evidence, so the name-matching\n * signal is gated on {@link queryIsProse} — see its comment for the numbers.\n *\n * Purely lexical-statistical and deterministic — no model, no I/O beyond\n * reading candidate windows. A cross-encoder can later replace the scoring\n * function behind the same signature; that model work belongs to\n * `kolisachint/embeddingsearchtools`, not here.\n */\n\nimport { readFileSync } from \"fs\";\nimport path from \"path\";\nimport { buildLexicalQueryPlan } from \"./lexical-retriever.js\";\nimport type { FusedCandidate } from \"./types.js\";\n\n/** Weights of the scoring blend. The eval harness (`bun run search-eval`) is\n *  the instrument for changing them — don't tune blind. */\nconst WEIGHT_FUSED_PRIOR = 0.4;\nconst WEIGHT_TERM_COVERAGE = 0.35;\nconst WEIGHT_PATH_AFFINITY = 0.25;\n/** Additive bonus when the query *is* the candidate's path (or its suffix):\n *  the caller named the file, so no amount of content evidence elsewhere\n *  should outrank it. */\nconst EXACT_PATH_BONUS = 0.5;\n/**\n * Additive bonus when the window *declares* a query term rather than merely\n * mentioning it.\n *\n * This targets the largest measured gap in the eval: on the 22 exact-symbol\n * queries the definition is in the top 10 about 85% of the time but ranked\n * first only about 20% of the time. Call sites outnumber definitions and\n * contain the identical identifier, so term coverage — which saturates at 1.0\n * for both — cannot separate them. Structure can.\n */\nconst DECLARATION_BONUS = 0.3;\n\n/**\n * Sentence glue. Identifiers and paths never contain these words, so two or\n * more of them means the query is a sentence rather than a name.\n *\n * This gates {@link DECLARATION_BONUS} because that bonus is a *name-matching*\n * signal and a prose query has no name to match. Its plan terms are ordinary\n * words — \"results\", \"search\", \"index\" — so it fires on whichever candidate\n * happens to declare a variable by one of them, at a weight larger than the\n * entire path-affinity term, and buries the fused ordering that did know the\n * answer. On the conceptual class reranking was scoring *below not reranking\n * at all* (MRR 0.246 un-reranked vs 0.124 reranked on `semantic`).\n *\n * Measured on the 62-query gold set, gating it moved conceptual MRR +0.046\n * (`semantic +rr`), +0.040 (`auto +rr`), +0.056 (`bm25+dense +rr`) and left\n * exact-symbol, error-fragment and path bit-identical — the gate never fires\n * on those, by construction. Overall MRR +0.007 to +0.013, which the paired\n * sign test does not call significant (p = 0.11 to 0.29); the classes it\n * protects are what justify it, not the aggregate.\n *\n * Path affinity is deliberately *not* gated. It is a topic signal, not a name\n * signal: \"how does a grep line number become an embedding chunk id\" wants\n * files with `grep` and `chunk` in the path. Gating it too was measured and\n * was strictly worse — same conceptual gain, roughly double the cross-file\n * loss (−0.058 vs −0.032 on `semantic +rr`).\n *\n * Deliberately conservative: two hits, not one, so a terse query like\n * `hybrid search fusion` keeps today's scoring untouched.\n */\nconst PROSE_FUNCTION_WORDS = new Set([\n\t\"a\",\n\t\"after\",\n\t\"all\",\n\t\"an\",\n\t\"and\",\n\t\"any\",\n\t\"are\",\n\t\"as\",\n\t\"at\",\n\t\"be\",\n\t\"been\",\n\t\"before\",\n\t\"between\",\n\t\"but\",\n\t\"by\",\n\t\"can\",\n\t\"does\",\n\t\"do\",\n\t\"each\",\n\t\"for\",\n\t\"from\",\n\t\"had\",\n\t\"has\",\n\t\"have\",\n\t\"how\",\n\t\"if\",\n\t\"in\",\n\t\"into\",\n\t\"is\",\n\t\"it\",\n\t\"its\",\n\t\"of\",\n\t\"on\",\n\t\"one\",\n\t\"or\",\n\t\"should\",\n\t\"so\",\n\t\"than\",\n\t\"that\",\n\t\"the\",\n\t\"then\",\n\t\"this\",\n\t\"to\",\n\t\"under\",\n\t\"was\",\n\t\"were\",\n\t\"what\",\n\t\"when\",\n\t\"where\",\n\t\"which\",\n\t\"why\",\n\t\"with\",\n\t\"would\",\n]);\n/** Function words needed before a query counts as prose. */\nconst PROSE_WORD_THRESHOLD = 2;\n\n/**\n * Is `query` a sentence rather than a name?\n *\n * A quoted segment is never prose regardless of its words: the plan collapses\n * it to one literal term, so the declaration bonus cannot fire on it anyway,\n * and its path-token split is what lets `\"Theme not initialized…\"` find\n * `core/theme.ts`.\n */\nexport function queryIsProse(query: string): boolean {\n\tif (/[\"'`][^\"'`]+[\"'`]/.test(query)) return false;\n\tconst words = query.toLowerCase().match(/[a-z]+/g) ?? [];\n\tlet hits = 0;\n\tfor (const word of words) {\n\t\tif (PROSE_FUNCTION_WORDS.has(word) && ++hits >= PROSE_WORD_THRESHOLD) return true;\n\t}\n\treturn false;\n}\n\n/** Keywords that introduce a definition across the languages this indexes.\n *  Matched against lowercased text, so the term is lowercased too. */\nconst DECLARATION_KEYWORDS =\n\t\"function|class|interface|type|enum|struct|impl|trait|fn|def|const|let|var|namespace|module\";\n\n/** Does `window` declare `term`, as opposed to referencing it? */\nfunction declaresTerm(window: string, term: string): boolean {\n\tconst escaped = term.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n\t// `function foo(`, `class Foo {`, `const foo =` ...\n\tif (new RegExp(`\\\\b(?:${DECLARATION_KEYWORDS})\\\\s+${escaped}\\\\b`).test(window)) return true;\n\t// `foo(...) {` at the start of a line — methods, Go/Rust receivers, Python defs\n\t// already covered above, but this catches object-literal and class members.\n\tif (\n\t\tnew RegExp(`^\\\\s*(?:(?:async|public|private|protected|static|export)\\\\s+)*${escaped}\\\\s*[(<]`, \"m\").test(window)\n\t) {\n\t\treturn true;\n\t}\n\treturn false;\n}\n\n/** Inverse document frequency over the candidate pool.\n *\n * True corpus IDF lives in the BM25 index and is not exposed over the daemon\n * protocol, so this approximates it with the candidate set: a term present in\n * every candidate discriminates nothing, one present in three carries the\n * signal. That is the comparison the reranker actually needs to make, since it\n * only ever orders candidates against each other. */\nfunction inverseDocumentFrequency(documentFrequency: number, total: number): number {\n\treturn Math.log(1 + total / Math.max(1, documentFrequency));\n}\n\nexport interface RerankResult {\n\tcandidates: FusedCandidate[];\n\tlatencyMs: number;\n}\n\n/**\n * The exact source text each candidate stands for, as the model should see it.\n *\n * Shared with the cross-encoder path so both rerankers score identical text —\n * otherwise a comparison between them would partly measure which one got a\n * better view of the candidate.\n */\nexport function readCandidateWindows(candidates: readonly FusedCandidate[], cwd: string): Array<string | undefined> {\n\tconst fileCache = new Map<string, string[] | undefined>();\n\tconst read = (rel: string): string[] | undefined => {\n\t\tif (!fileCache.has(rel)) {\n\t\t\ttry {\n\t\t\t\tfileCache.set(rel, readFileSync(path.resolve(cwd, rel), \"utf-8\").split(\"\\n\"));\n\t\t\t} catch {\n\t\t\t\tfileCache.set(rel, undefined);\n\t\t\t}\n\t\t}\n\t\treturn fileCache.get(rel);\n\t};\n\treturn candidates.map((candidate) => {\n\t\tconst lines = read(candidate.path);\n\t\tif (!lines) return undefined;\n\t\treturn lines.slice(Math.max(0, candidate.startLine - 1), Math.min(lines.length, candidate.endLine)).join(\"\\n\");\n\t});\n}\n\nexport function rerankCandidates(query: string, candidates: readonly FusedCandidate[], cwd: string): RerankResult {\n\tconst startedMs = Date.now();\n\tconst plan = buildLexicalQueryPlan(query);\n\tif (!plan || candidates.length < 2) {\n\t\treturn { candidates: [...candidates], latencyMs: Date.now() - startedMs };\n\t}\n\tconst terms = plan.terms;\n\tconst queryPath = query.trim().toLowerCase();\n\t// Prose asks about behaviour, not about a name, so the one signal that reads\n\t// candidates *as names* is switched off.\n\tconst prose = queryIsProse(query);\n\n\tconst fileCache = new Map<string, string[] | undefined>();\n\tconst readLines = (rel: string): string[] | undefined => {\n\t\tif (!fileCache.has(rel)) {\n\t\t\ttry {\n\t\t\t\tconst content = readFileSync(path.resolve(cwd, rel), \"utf-8\");\n\t\t\t\tfileCache.set(rel, content.toLowerCase().split(\"\\n\"));\n\t\t\t} catch {\n\t\t\t\tfileCache.set(rel, undefined);\n\t\t\t}\n\t\t}\n\t\treturn fileCache.get(rel);\n\t};\n\n\t// Read every candidate window once: the term/declaration signals and the\n\t// candidate-pool IDF all need them, and files repeat across candidates.\n\tconst windows = candidates.map((candidate) => {\n\t\tconst lines = readLines(candidate.path);\n\t\tif (!lines) return undefined;\n\t\treturn lines.slice(Math.max(0, candidate.startLine - 1), Math.min(lines.length, candidate.endLine)).join(\"\\n\");\n\t});\n\n\t// Candidate-pool document frequency per term, for the IDF weighting below.\n\tconst documentFrequency = new Map<string, number>();\n\tfor (const term of terms) {\n\t\tdocumentFrequency.set(term, windows.filter((w) => w?.includes(term)).length);\n\t}\n\tconst termWeight = new Map(\n\t\tterms.map((t) => [t, inverseDocumentFrequency(documentFrequency.get(t) ?? 0, candidates.length)]),\n\t);\n\tconst totalTermWeight = terms.reduce((sum, t) => sum + (termWeight.get(t) ?? 0), 0);\n\n\t// Fused prior normalized by score, not by position: a candidate both\n\t// retrievers agreed on should outrank one that squeaked in, and a uniform\n\t// 1 - index/length ramp throws that magnitude away.\n\tconst maxRrfScore = Math.max(...candidates.map((c) => c.rrfScore), Number.MIN_VALUE);\n\n\tconst scored = candidates.map((candidate, index) => {\n\t\tconst fusedPrior = candidate.rrfScore / maxRrfScore;\n\n\t\tconst window = windows[index];\n\t\tlet termCoverage = 0;\n\t\tlet declaresAnyTerm = false;\n\t\tif (window && terms.length > 0) {\n\t\t\tconst present = terms.filter((t) => window.includes(t));\n\t\t\ttermCoverage =\n\t\t\t\ttotalTermWeight > 0\n\t\t\t\t\t? present.reduce((sum, t) => sum + (termWeight.get(t) ?? 0), 0) / totalTermWeight\n\t\t\t\t\t: present.length / terms.length;\n\t\t\tdeclaresAnyTerm = !prose && present.some((t) => declaresTerm(window, t));\n\t\t}\n\n\t\tconst lowerPath = candidate.path.toLowerCase();\n\t\t// A quoted phrase rarely names a file; split it into path-ish tokens so\n\t\t// `\"token budget exceeded\"` still gets partial path credit.\n\t\tconst pathTerms = terms.length === 1 ? terms[0].split(/[^a-z0-9_$]+/).filter((t) => t.length >= 3) : terms;\n\t\tconst pathAffinity =\n\t\t\tpathTerms.length > 0 ? pathTerms.filter((t) => lowerPath.includes(t)).length / pathTerms.length : 0;\n\n\t\tconst exactPath =\n\t\t\tqueryPath.length >= 3 && (lowerPath === queryPath || lowerPath.endsWith(`/${queryPath}`)) ? 1 : 0;\n\n\t\tconst score =\n\t\t\tWEIGHT_FUSED_PRIOR * fusedPrior +\n\t\t\tWEIGHT_TERM_COVERAGE * termCoverage +\n\t\t\tWEIGHT_PATH_AFFINITY * pathAffinity +\n\t\t\tEXACT_PATH_BONUS * exactPath +\n\t\t\t(declaresAnyTerm ? DECLARATION_BONUS : 0);\n\t\treturn { candidate, index, score };\n\t});\n\n\t// Stable, deterministic: score desc, fused order as tie-break.\n\tscored.sort((a, b) => b.score - a.score || a.index - b.index);\n\treturn { candidates: scored.map((s) => s.candidate), latencyMs: Date.now() - startedMs };\n}\n"]}

package/examples/extensions/custom-provider-anthropic/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-custom-provider-anthropic",
 	"private": true,
-	"version": "0.2.162",
+	"version": "0.2.163",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/examples/extensions/custom-provider-gitlab-duo/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-custom-provider-gitlab-duo",
 	"private": true,
-	"version": "0.2.162",
+	"version": "0.2.163",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/examples/extensions/sandbox/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-sandbox",
 	"private": true,
-	"version": "0.2.162",
+	"version": "0.2.163",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/examples/extensions/with-deps/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-with-deps",
 	"private": true,
-	"version": "0.2.162",
+	"version": "0.2.163",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@kolisachint/hoocode-agent",
-	"version": "0.4.165",
+	"version": "0.4.166",
 	"description": "Coding agent CLI with read, bash, edit, write tools and session management",
 	"type": "module",
 	"hoocodeConfig": {
@@ -48,9 +48,9 @@
 		"prepublishOnly": "npm run clean && npm run build"
 	},
 	"dependencies": {
-		"@kolisachint/hoocode-agent-core": "^0.4.165",
-		"@kolisachint/hoocode-ai": "^0.4.165",
-		"@kolisachint/hoocode-tui": "^0.4.165",
+		"@kolisachint/hoocode-agent-core": "^0.4.166",
+		"@kolisachint/hoocode-ai": "^0.4.166",
+		"@kolisachint/hoocode-tui": "^0.4.166",
 		"@silvia-odwyer/photon-node": "^0.3.4",
 		"chalk": "^5.5.0",
 		"cli-highlight": "^2.1.11",