@kolisachint/hoocode-agent 0.4.166 → 0.4.168

package/dist/core/search/eval-gold.js

@@ -43,12 +43,27 @@ function findAnchorLines(lines, anchor) {
     }
     return found;
 }
+/** Leading-whitespace width of a line, for matching a block's closer to its
+ *  opener. Tabs count as one; this only ever compares lines in one file, which
+ *  is consistently indented. */
+function indentWidth(line) {
+    return /^[\t ]*/.exec(line)?.[0].length ?? 0;
+}
 /**
  * Extent of the declaration an anchor names.
  *
  * A line ending in an opener (`{`, `(`, `[`) starts a block, which runs to the
- * first line closing at column 0 — the shape every top-level declaration in
- * this codebase has. Anything else is a statement, scored with a small pad.
+ * first closing bracket indented no deeper than the opener. Anything else is a
+ * statement, scored with a small pad.
+ *
+ * The indentation test is load-bearing. This used to close only on a bracket
+ * at *column 0*, which is right for a top-level declaration and wrong for
+ * every class member: a method's own `}` is indented, so the scan walked past
+ * it to the end of the enclosing class. Three of the four boundary-class gold
+ * spans were resolving to 80, 34 and 28 lines for methods that are 7, 4 and 4
+ * lines long, all ending on the same line — the class's closing brace. Gold
+ * that wide scores a hit for retrieving a neighbouring method, which is not
+ * what the query asked for.
  */
 function resolveExtent(lines, anchorIndex) {
     const head = lines[anchorIndex];
@@ -59,11 +74,13 @@ function resolveExtent(lines, anchorIndex) {
             endLine: Math.min(lines.length, anchorIndex + 1 + LINE_ANCHOR_PAD),
         };
     }
+    const anchorIndent = indentWidth(head);
     const limit = Math.min(lines.length, anchorIndex + 1 + MAX_BLOCK_LINES);
     for (let i = anchorIndex + 1; i < limit; i++) {
-        // Column-0 close: `}`, `};`, `});`, `];` — the end of a top-level block.
-        if (/^[}\])]/.test(lines[i]))
+        // `}`, `};`, `});`, `];` at or outside the opener's indentation.
+        if (/^[\t ]*[}\])]/.test(lines[i]) && indentWidth(lines[i]) <= anchorIndent) {
             return { startLine: anchorIndex + 1, endLine: i + 1 };
+        }
     }
     return { startLine: anchorIndex + 1, endLine: Math.min(lines.length, anchorIndex + 1 + UNCLOSED_BLOCK_LINES) };
 }

package/dist/core/search/eval-gold.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-gold.js","sourceRoot":"","sources":["../../../src/core/search/eval-gold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B;;mEAEmE;AACnE,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B;4DAC4D;AAC5D,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,yEAAyE;AACzE,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAQhC,SAAS,SAAS,CAAC,UAAkB,EAAE,GAAW,EAAwB;IACzE,IAAI,CAAC;QACJ,OAAO,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED,2DAA2D;AAC3D,SAAS,eAAe,CAAC,KAAwB,EAAE,MAAc,EAAY;IAC5E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,KAAK,CAAC;AAAA,CACb;AAED;;;;;;GAMG;AACH,SAAS,aAAa,CAAC,KAAwB,EAAE,WAAmB,EAA0C;IAC7G,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;IAChC,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,OAAO;YACN,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,CAAC,GAAG,eAAe,CAAC;YACzD,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,eAAe,CAAC;SAClE,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;IACxE,KAAK,IAAI,CAAC,GAAG,WAAW,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,2EAAyE;QACzE,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;IACrF,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,oBAAoB,CAAC,EAAE,CAAC;AAAA,CAC/G;AAED;oEACoE;AACpE,MAAM,UAAU,eAAe,CAAC,UAAkB,EAAE,IAAkB,EAA4C;IACjH,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,mBAAmB,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;IAErE,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC3B,OAAO,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;IACnE,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,qCAAqC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;IAE7F,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,uBAAuB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;IACvG,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,wBAAwB,OAAO,CAAC,MAAM,gBAAgB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;IAC7G,CAAC;IAED,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;AAAA,CACjD;AAED;oDACoD;AACpD,MAAM,UAAU,cAAc,CAC7B,UAAkB,EAClB,OAA6B,EAC6B;IAC1D,MAAM,MAAM,GAA0B,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACxC,GAAG,KAAK;QACR,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YACjD,IAAI,MAAM,CAAC,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YACjG,OAAO,MAAM,CAAC,IAAI,CAAC;QAAA,CACnB,CAAC;KACF,CAAC,CAAC,CAAC;IACJ,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AAAA,CACrC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,UAAkB,EAAE,OAA6B,EAAyB;IACzG,MAAM,MAAM,GAA0B,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACvG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACtB,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,yBAAyB,EAAE,CAAC,CAAC;QAE9G,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/C,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;gBAC/E,SAAS;YACV,CAAC;YACD,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,mCAAmC,EAAE,CAAC,CAAC;gBAClG,SAAS;YACV,CAAC;YACD,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACxF,MAAM,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,cAAc,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,4BAA4B,KAAK,CAAC,MAAM,SAAS;iBACtG,CAAC,CAAC;gBACH,SAAS;YACV,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM;gBAAE,SAAS;YACpC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,kCAAkC,EAAE,CAAC,CAAC;gBACjG,SAAS;YACV,CAAC;YACD,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,qBAAqB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAClG,CAAC;iBAAM,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,wBAAwB,OAAO,CAAC,MAAM,cAAc,IAAI,CAAC,MAAM,EAAE;iBAC1E,CAAC,CAAC;YACJ,CAAC;iBAAM,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC7E,MAAM,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,wBAAwB,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,4BAA4B,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE;iBAC3G,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC;IACD,OAAO,MAAM,CAAC;AAAA,CACd;AAED,6CAA6C;AAC7C,MAAM,UAAU,WAAW,CAAC,WAAmB,EAAe;IAC7D,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAgB,CAAC;AAAA,CACrE","sourcesContent":["/**\n * Gold-set loading and anchor resolution.\n *\n * A gold span recorded as bare line numbers rots on the next refactor, and a\n * rotted gold set fails *silently* — it just scores lower. So each span also\n * carries an `anchor`: a literal snippet that must sit inside the range. The\n * anchor is the source of truth; the line numbers are a cache of where it was\n * when the fixture was last regenerated.\n *\n * That gives three properties the first gold set lacked:\n *  - `resolveGoldSet` recomputes ranges from anchors, so `--fix` re-pins the\n *    whole set after a refactor instead of someone hand-editing 60 numbers;\n *  - `validateGoldSet` fails loudly when an anchor has moved out of its\n *    recorded range, drifted, or become ambiguous;\n *  - anchors are never used for scoring, so this cannot flatter retrieval —\n *    `spanMatchesGold` still sees only a path and a line range.\n */\n\nimport { readFileSync } from \"node:fs\";\nimport path from \"node:path\";\nimport type { EvalGoldSpan, EvalQuery } from \"./eval.js\";\n\n/** Lines of context kept either side of a non-block anchor (an error literal,\n *  a single statement) — enough to be a real target, tight enough that a\n *  60-line chunk elsewhere in the file does not count as a hit. */\nconst LINE_ANCHOR_PAD = 2;\n/** Upper bound on a resolved block, so an anchor that fails to find its\n *  closing brace cannot silently swallow an entire file. */\nconst MAX_BLOCK_LINES = 120;\n/** Fallback extent when a block anchor never finds its closing brace. */\nconst UNCLOSED_BLOCK_LINES = 15;\n\nexport interface GoldValidationIssue {\n\tqueryId: string;\n\tpath: string;\n\tproblem: string;\n}\n\nfunction readLines(corpusRoot: string, rel: string): string[] | undefined {\n\ttry {\n\t\treturn readFileSync(path.resolve(corpusRoot, rel), \"utf-8\").split(\"\\n\");\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/** Indices (0-based) of every line containing `anchor`. */\nfunction findAnchorLines(lines: readonly string[], anchor: string): number[] {\n\tconst found: number[] = [];\n\tfor (let i = 0; i < lines.length; i++) {\n\t\tif (lines[i].includes(anchor)) found.push(i);\n\t}\n\treturn found;\n}\n\n/**\n * Extent of the declaration an anchor names.\n *\n * A line ending in an opener (`{`, `(`, `[`) starts a block, which runs to the\n * first line closing at column 0 — the shape every top-level declaration in\n * this codebase has. Anything else is a statement, scored with a small pad.\n */\nfunction resolveExtent(lines: readonly string[], anchorIndex: number): { startLine: number; endLine: number } {\n\tconst head = lines[anchorIndex];\n\tconst opensBlock = /[{([]\\s*$/.test(head);\n\tif (!opensBlock) {\n\t\treturn {\n\t\t\tstartLine: Math.max(1, anchorIndex + 1 - LINE_ANCHOR_PAD),\n\t\t\tendLine: Math.min(lines.length, anchorIndex + 1 + LINE_ANCHOR_PAD),\n\t\t};\n\t}\n\tconst limit = Math.min(lines.length, anchorIndex + 1 + MAX_BLOCK_LINES);\n\tfor (let i = anchorIndex + 1; i < limit; i++) {\n\t\t// Column-0 close: `}`, `};`, `});`, `];` — the end of a top-level block.\n\t\tif (/^[}\\])]/.test(lines[i])) return { startLine: anchorIndex + 1, endLine: i + 1 };\n\t}\n\treturn { startLine: anchorIndex + 1, endLine: Math.min(lines.length, anchorIndex + 1 + UNCLOSED_BLOCK_LINES) };\n}\n\n/** Re-pin one gold span's line range from its anchor. Returns the span\n *  unchanged when it is file-scoped or has no anchor to resolve. */\nexport function resolveGoldSpan(corpusRoot: string, span: EvalGoldSpan): { span: EvalGoldSpan; problem?: string } {\n\tconst lines = readLines(corpusRoot, span.path);\n\tif (!lines) return { span, problem: `file not found: ${span.path}` };\n\n\tif (span.scope === \"file\") {\n\t\treturn { span: { ...span, startLine: 1, endLine: lines.length } };\n\t}\n\tif (!span.anchor) return { span, problem: `span-scoped gold needs an anchor: ${span.path}` };\n\n\tconst matches = findAnchorLines(lines, span.anchor);\n\tif (matches.length === 0) return { span, problem: `anchor not found in ${span.path}: ${span.anchor}` };\n\tif (matches.length > 1) {\n\t\treturn { span, problem: `anchor is ambiguous (${matches.length} matches) in ${span.path}: ${span.anchor}` };\n\t}\n\n\tconst { startLine, endLine } = resolveExtent(lines, matches[0]);\n\treturn { span: { ...span, startLine, endLine } };\n}\n\n/** Re-pin every gold span in the set. Problems are collected, not thrown, so\n *  a regeneration run reports all drift at once. */\nexport function resolveGoldSet(\n\tcorpusRoot: string,\n\tdataset: readonly EvalQuery[],\n): { dataset: EvalQuery[]; issues: GoldValidationIssue[] } {\n\tconst issues: GoldValidationIssue[] = [];\n\tconst resolved = dataset.map((query) => ({\n\t\t...query,\n\t\tgold: query.gold.map((span) => {\n\t\t\tconst result = resolveGoldSpan(corpusRoot, span);\n\t\t\tif (result.problem) issues.push({ queryId: query.id, path: span.path, problem: result.problem });\n\t\t\treturn result.span;\n\t\t}),\n\t}));\n\treturn { dataset: resolved, issues };\n}\n\n/**\n * Check the committed fixture against the corpus without rewriting it.\n *\n * Catches the two ways a gold set goes wrong: the anchor no longer exists (the\n * code was deleted or renamed), or it exists but has moved outside the\n * recorded range (the fixture is stale and every score computed from it is\n * wrong).\n */\nexport function validateGoldSet(corpusRoot: string, dataset: readonly EvalQuery[]): GoldValidationIssue[] {\n\tconst issues: GoldValidationIssue[] = [];\n\tconst seenIds = new Set<string>();\n\n\tfor (const query of dataset) {\n\t\tif (seenIds.has(query.id)) issues.push({ queryId: query.id, path: \"\", problem: \"duplicate query id\" });\n\t\tseenIds.add(query.id);\n\t\tif (query.gold.length === 0) issues.push({ queryId: query.id, path: \"\", problem: \"query has no gold spans\" });\n\n\t\tfor (const span of query.gold) {\n\t\t\tconst lines = readLines(corpusRoot, span.path);\n\t\t\tif (!lines) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"file not found\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.startLine === undefined || span.endLine === undefined) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"gold span is missing line numbers\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.startLine < 1 || span.endLine < span.startLine || span.endLine > lines.length) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `line range ${span.startLine}-${span.endLine} out of bounds (file has ${lines.length} lines)`,\n\t\t\t\t});\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.scope === \"file\") continue;\n\t\t\tif (!span.anchor) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"span-scoped gold needs an anchor\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tconst matches = findAnchorLines(lines, span.anchor);\n\t\t\tif (matches.length === 0) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: `anchor not found: ${span.anchor}` });\n\t\t\t} else if (matches.length > 1) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `anchor is ambiguous (${matches.length} matches): ${span.anchor}`,\n\t\t\t\t});\n\t\t\t} else if (matches[0] + 1 < span.startLine || matches[0] + 1 > span.endLine) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `anchor moved to line ${matches[0] + 1}, outside recorded range ${span.startLine}-${span.endLine}`,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n\treturn issues;\n}\n\n/** Load the gold set from a fixture file. */\nexport function loadGoldSet(fixturePath: string): EvalQuery[] {\n\treturn JSON.parse(readFileSync(fixturePath, \"utf-8\")) as EvalQuery[];\n}\n"]}
+{"version":3,"file":"eval-gold.js","sourceRoot":"","sources":["../../../src/core/search/eval-gold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B;;mEAEmE;AACnE,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B;4DAC4D;AAC5D,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,yEAAyE;AACzE,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAQhC,SAAS,SAAS,CAAC,UAAkB,EAAE,GAAW,EAAwB;IACzE,IAAI,CAAC;QACJ,OAAO,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED,2DAA2D;AAC3D,SAAS,eAAe,CAAC,KAAwB,EAAE,MAAc,EAAY;IAC5E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,KAAK,CAAC;AAAA,CACb;AAED;;gCAEgC;AAChC,SAAS,WAAW,CAAC,IAAY,EAAU;IAC1C,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;AAAA,CAC7C;AAED;;;;;;;;;;;;;;;GAeG;AACH,SAAS,aAAa,CAAC,KAAwB,EAAE,WAAmB,EAA0C;IAC7G,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;IAChC,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,OAAO;YACN,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,CAAC,GAAG,eAAe,CAAC;YACzD,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,eAAe,CAAC;SAClE,CAAC;IACH,CAAC;IACD,MAAM,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;IACxE,KAAK,IAAI,CAAC,GAAG,WAAW,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,iEAAiE;QACjE,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,YAAY,EAAE,CAAC;YAC7E,OAAO,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;QACvD,CAAC;IACF,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,oBAAoB,CAAC,EAAE,CAAC;AAAA,CAC/G;AAED;oEACoE;AACpE,MAAM,UAAU,eAAe,CAAC,UAAkB,EAAE,IAAkB,EAA4C;IACjH,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,mBAAmB,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;IAErE,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC3B,OAAO,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;IACnE,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,qCAAqC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;IAE7F,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,uBAAuB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;IACvG,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,wBAAwB,OAAO,CAAC,MAAM,gBAAgB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;IAC7G,CAAC;IAED,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;AAAA,CACjD;AAED;oDACoD;AACpD,MAAM,UAAU,cAAc,CAC7B,UAAkB,EAClB,OAA6B,EAC6B;IAC1D,MAAM,MAAM,GAA0B,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACxC,GAAG,KAAK;QACR,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YACjD,IAAI,MAAM,CAAC,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YACjG,OAAO,MAAM,CAAC,IAAI,CAAC;QAAA,CACnB,CAAC;KACF,CAAC,CAAC,CAAC;IACJ,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AAAA,CACrC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,UAAkB,EAAE,OAA6B,EAAyB;IACzG,MAAM,MAAM,GAA0B,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACvG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACtB,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,yBAAyB,EAAE,CAAC,CAAC;QAE9G,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/C,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;gBAC/E,SAAS;YACV,CAAC;YACD,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,mCAAmC,EAAE,CAAC,CAAC;gBAClG,SAAS;YACV,CAAC;YACD,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACxF,MAAM,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,cAAc,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,4BAA4B,KAAK,CAAC,MAAM,SAAS;iBACtG,CAAC,CAAC;gBACH,SAAS;YACV,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM;gBAAE,SAAS;YACpC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,kCAAkC,EAAE,CAAC,CAAC;gBACjG,SAAS;YACV,CAAC;YACD,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,qBAAqB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAClG,CAAC;iBAAM,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,wBAAwB,OAAO,CAAC,MAAM,cAAc,IAAI,CAAC,MAAM,EAAE;iBAC1E,CAAC,CAAC;YACJ,CAAC;iBAAM,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC7E,MAAM,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,wBAAwB,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,4BAA4B,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE;iBAC3G,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC;IACD,OAAO,MAAM,CAAC;AAAA,CACd;AAED,6CAA6C;AAC7C,MAAM,UAAU,WAAW,CAAC,WAAmB,EAAe;IAC7D,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAgB,CAAC;AAAA,CACrE","sourcesContent":["/**\n * Gold-set loading and anchor resolution.\n *\n * A gold span recorded as bare line numbers rots on the next refactor, and a\n * rotted gold set fails *silently* — it just scores lower. So each span also\n * carries an `anchor`: a literal snippet that must sit inside the range. The\n * anchor is the source of truth; the line numbers are a cache of where it was\n * when the fixture was last regenerated.\n *\n * That gives three properties the first gold set lacked:\n *  - `resolveGoldSet` recomputes ranges from anchors, so `--fix` re-pins the\n *    whole set after a refactor instead of someone hand-editing 60 numbers;\n *  - `validateGoldSet` fails loudly when an anchor has moved out of its\n *    recorded range, drifted, or become ambiguous;\n *  - anchors are never used for scoring, so this cannot flatter retrieval —\n *    `spanMatchesGold` still sees only a path and a line range.\n */\n\nimport { readFileSync } from \"node:fs\";\nimport path from \"node:path\";\nimport type { EvalGoldSpan, EvalQuery } from \"./eval.js\";\n\n/** Lines of context kept either side of a non-block anchor (an error literal,\n *  a single statement) — enough to be a real target, tight enough that a\n *  60-line chunk elsewhere in the file does not count as a hit. */\nconst LINE_ANCHOR_PAD = 2;\n/** Upper bound on a resolved block, so an anchor that fails to find its\n *  closing brace cannot silently swallow an entire file. */\nconst MAX_BLOCK_LINES = 120;\n/** Fallback extent when a block anchor never finds its closing brace. */\nconst UNCLOSED_BLOCK_LINES = 15;\n\nexport interface GoldValidationIssue {\n\tqueryId: string;\n\tpath: string;\n\tproblem: string;\n}\n\nfunction readLines(corpusRoot: string, rel: string): string[] | undefined {\n\ttry {\n\t\treturn readFileSync(path.resolve(corpusRoot, rel), \"utf-8\").split(\"\\n\");\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/** Indices (0-based) of every line containing `anchor`. */\nfunction findAnchorLines(lines: readonly string[], anchor: string): number[] {\n\tconst found: number[] = [];\n\tfor (let i = 0; i < lines.length; i++) {\n\t\tif (lines[i].includes(anchor)) found.push(i);\n\t}\n\treturn found;\n}\n\n/** Leading-whitespace width of a line, for matching a block's closer to its\n *  opener. Tabs count as one; this only ever compares lines in one file, which\n *  is consistently indented. */\nfunction indentWidth(line: string): number {\n\treturn /^[\\t ]*/.exec(line)?.[0].length ?? 0;\n}\n\n/**\n * Extent of the declaration an anchor names.\n *\n * A line ending in an opener (`{`, `(`, `[`) starts a block, which runs to the\n * first closing bracket indented no deeper than the opener. Anything else is a\n * statement, scored with a small pad.\n *\n * The indentation test is load-bearing. This used to close only on a bracket\n * at *column 0*, which is right for a top-level declaration and wrong for\n * every class member: a method's own `}` is indented, so the scan walked past\n * it to the end of the enclosing class. Three of the four boundary-class gold\n * spans were resolving to 80, 34 and 28 lines for methods that are 7, 4 and 4\n * lines long, all ending on the same line — the class's closing brace. Gold\n * that wide scores a hit for retrieving a neighbouring method, which is not\n * what the query asked for.\n */\nfunction resolveExtent(lines: readonly string[], anchorIndex: number): { startLine: number; endLine: number } {\n\tconst head = lines[anchorIndex];\n\tconst opensBlock = /[{([]\\s*$/.test(head);\n\tif (!opensBlock) {\n\t\treturn {\n\t\t\tstartLine: Math.max(1, anchorIndex + 1 - LINE_ANCHOR_PAD),\n\t\t\tendLine: Math.min(lines.length, anchorIndex + 1 + LINE_ANCHOR_PAD),\n\t\t};\n\t}\n\tconst anchorIndent = indentWidth(head);\n\tconst limit = Math.min(lines.length, anchorIndex + 1 + MAX_BLOCK_LINES);\n\tfor (let i = anchorIndex + 1; i < limit; i++) {\n\t\t// `}`, `};`, `});`, `];` at or outside the opener's indentation.\n\t\tif (/^[\\t ]*[}\\])]/.test(lines[i]) && indentWidth(lines[i]) <= anchorIndent) {\n\t\t\treturn { startLine: anchorIndex + 1, endLine: i + 1 };\n\t\t}\n\t}\n\treturn { startLine: anchorIndex + 1, endLine: Math.min(lines.length, anchorIndex + 1 + UNCLOSED_BLOCK_LINES) };\n}\n\n/** Re-pin one gold span's line range from its anchor. Returns the span\n *  unchanged when it is file-scoped or has no anchor to resolve. */\nexport function resolveGoldSpan(corpusRoot: string, span: EvalGoldSpan): { span: EvalGoldSpan; problem?: string } {\n\tconst lines = readLines(corpusRoot, span.path);\n\tif (!lines) return { span, problem: `file not found: ${span.path}` };\n\n\tif (span.scope === \"file\") {\n\t\treturn { span: { ...span, startLine: 1, endLine: lines.length } };\n\t}\n\tif (!span.anchor) return { span, problem: `span-scoped gold needs an anchor: ${span.path}` };\n\n\tconst matches = findAnchorLines(lines, span.anchor);\n\tif (matches.length === 0) return { span, problem: `anchor not found in ${span.path}: ${span.anchor}` };\n\tif (matches.length > 1) {\n\t\treturn { span, problem: `anchor is ambiguous (${matches.length} matches) in ${span.path}: ${span.anchor}` };\n\t}\n\n\tconst { startLine, endLine } = resolveExtent(lines, matches[0]);\n\treturn { span: { ...span, startLine, endLine } };\n}\n\n/** Re-pin every gold span in the set. Problems are collected, not thrown, so\n *  a regeneration run reports all drift at once. */\nexport function resolveGoldSet(\n\tcorpusRoot: string,\n\tdataset: readonly EvalQuery[],\n): { dataset: EvalQuery[]; issues: GoldValidationIssue[] } {\n\tconst issues: GoldValidationIssue[] = [];\n\tconst resolved = dataset.map((query) => ({\n\t\t...query,\n\t\tgold: query.gold.map((span) => {\n\t\t\tconst result = resolveGoldSpan(corpusRoot, span);\n\t\t\tif (result.problem) issues.push({ queryId: query.id, path: span.path, problem: result.problem });\n\t\t\treturn result.span;\n\t\t}),\n\t}));\n\treturn { dataset: resolved, issues };\n}\n\n/**\n * Check the committed fixture against the corpus without rewriting it.\n *\n * Catches the two ways a gold set goes wrong: the anchor no longer exists (the\n * code was deleted or renamed), or it exists but has moved outside the\n * recorded range (the fixture is stale and every score computed from it is\n * wrong).\n */\nexport function validateGoldSet(corpusRoot: string, dataset: readonly EvalQuery[]): GoldValidationIssue[] {\n\tconst issues: GoldValidationIssue[] = [];\n\tconst seenIds = new Set<string>();\n\n\tfor (const query of dataset) {\n\t\tif (seenIds.has(query.id)) issues.push({ queryId: query.id, path: \"\", problem: \"duplicate query id\" });\n\t\tseenIds.add(query.id);\n\t\tif (query.gold.length === 0) issues.push({ queryId: query.id, path: \"\", problem: \"query has no gold spans\" });\n\n\t\tfor (const span of query.gold) {\n\t\t\tconst lines = readLines(corpusRoot, span.path);\n\t\t\tif (!lines) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"file not found\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.startLine === undefined || span.endLine === undefined) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"gold span is missing line numbers\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.startLine < 1 || span.endLine < span.startLine || span.endLine > lines.length) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `line range ${span.startLine}-${span.endLine} out of bounds (file has ${lines.length} lines)`,\n\t\t\t\t});\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.scope === \"file\") continue;\n\t\t\tif (!span.anchor) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"span-scoped gold needs an anchor\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tconst matches = findAnchorLines(lines, span.anchor);\n\t\t\tif (matches.length === 0) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: `anchor not found: ${span.anchor}` });\n\t\t\t} else if (matches.length > 1) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `anchor is ambiguous (${matches.length} matches): ${span.anchor}`,\n\t\t\t\t});\n\t\t\t} else if (matches[0] + 1 < span.startLine || matches[0] + 1 > span.endLine) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `anchor moved to line ${matches[0] + 1}, outside recorded range ${span.startLine}-${span.endLine}`,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n\treturn issues;\n}\n\n/** Load the gold set from a fixture file. */\nexport function loadGoldSet(fixturePath: string): EvalQuery[] {\n\treturn JSON.parse(readFileSync(fixturePath, \"utf-8\")) as EvalQuery[];\n}\n"]}

package/dist/core/search/eval-harness.d.ts

@@ -48,6 +48,10 @@ export interface EvalProvenance {
     /** Uncommitted changes present at run time. Only meaningful (and only
      *  possible) when `corpusFromWorkingTree` is true. */
     corpusDirty: boolean;
+    /** Files removed from the corpus before indexing — see
+     *  {@link CORPUS_EXCLUSIONS}. A score against a different exclusion list is
+     *  a score against a different corpus, so it is recorded, not assumed. */
+    corpusExcluded: string[];
     /** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,
      *  but differs when pinning an old corpus with today's code. */
     harnessSha: string;
@@ -109,9 +113,32 @@ export interface PinnedCorpus {
     sha: string;
     fromWorkingTree: boolean;
     dirty: boolean;
+    /** Files removed from the corpus before indexing. Empty when the corpus is
+     *  the live working tree, which is never mutated. */
+    excluded: string[];
     /** Removes the worktree, if one was created. */
     dispose: () => void;
 }
+/**
+ * Files that describe this eval rather than being searched by it.
+ *
+ * The fixtures hold all 62 query strings verbatim, so every query is a perfect
+ * lexical match against its own entry, and the design note quotes the same
+ * queries while discussing the classes they belong to. Measured before this
+ * exclusion existed: **56 of 62 queries had one of these files in the top 10,
+ * 27 of 62 had one as the #1 result, and they consumed 133 of the 620
+ * top-10 slots** — a fifth of the window, spent on the eval reading itself.
+ *
+ * That is not a ranking artifact a reranker can fix: it displaces real answers
+ * out of the window entirely, which is why two boundary-class queries were
+ * absent from the top *50* rather than merely buried. Retrieving your own
+ * question is not retrieval, so the corpus is scored without them.
+ *
+ * Removed from the pinned worktree before indexing, never from the repo — and
+ * recorded in the run's provenance so a score is never silently taken against
+ * a different corpus than it claims.
+ */
+export declare const CORPUS_EXCLUSIONS: readonly string[];
 /**
  * Materialize the corpus to evaluate.
  *

package/dist/core/search/eval-harness.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-harness.d.ts","sourceRoot":"","sources":["../../../src/core/search/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,KAAK,eAAe,EAAiB,MAAM,WAAW,CAAC;AAEjG,+DAA+D;AAC/D,MAAM,WAAW,aAAa;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,wCAAwC;IACxC,CAAC,EAAE,MAAM,CAAC;IACV,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED,2EAA2E;AAC3E,MAAM,WAAW,cAAc;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB;iEAC2D;IAC3D,qBAAqB,EAAE,OAAO,CAAC;IAC/B;0DACsD;IACtD,WAAW,EAAE,OAAO,CAAC;IACrB;oEACgE;IAChE,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;;OAMG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;yDACqD;IACrD,QAAQ,EAAE;QACT,SAAS,EAAE,OAAO,CAAC;QACnB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,0DAA0D;QAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;QACd;;8EAEsE;QACtE,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,aAAa,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IACF;kDAC8C;IAC9C,YAAY,CAAC,EAAE;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,WAAW,aAAa;IAC7B,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,OAAO,EAAE,SAAS,UAAU,EAAE,CAAC;IAC/B,UAAU,EAAE,aAAa,EAAE,CAAC;IAC5B,QAAQ,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,eAAe,EAAE,CAAA;KAAE,CAAC,CAAC;CAC3E;AAMD;0EAC0E;AAC1E,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CA+B5D;AAED,MAAM,WAAW,YAAY;IAC5B,qCAAqC;IACrC,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,OAAO,CAAC;IACzB,KAAK,EAAE,OAAO,CAAC;IACf,gDAAgD;IAChD,OAAO,EAAE,MAAM,IAAI,CAAC;CACpB;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,YAAY,CAyCjF;AAYD,wBAAgB,iBAAiB,CAChC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,YAAY,EACpB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,gBAAgB,GAAG,SAAS,EACrC,eAAe,CAAC,EAAE,MAAM,EACxB,aAAa,CAAC,EAAE,gBAAgB,GAC9B,cAAc,CA2BhB;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,SAAS,SAAS,EAAE,GAAG,aAAa,CAAC,SAAS,CAAC,CAQxF;AAED,MAAM,WAAW,mBAAmB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,SAAS,SAAS,EAAE,CAAC;IAC9B,OAAO,EAAE,SAAS,UAAU,EAAE,CAAC;IAC/B,OAAO,CAAC,EAAE,gBAAgB,CAAC;IAC3B;wEACoE;IACpE,aAAa,CAAC,EAAE,gBAAgB,CAAC;IACjC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,KAAK,IAAI,CAAC;CACpD;AAED,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC;IACzE,UAAU,EAAE,aAAa,EAAE,CAAC;IAC5B,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACpC,CAAC,CA4CD;AAED,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,GAAG,MAAM,CAcjF","sourcesContent":["/**\n * Eval harness: corpus pinning, provenance capture, and run records.\n *\n * The scoring math lives in `eval.ts`; this module is everything around it\n * that makes a number *comparable to a later number*. Three problems it\n * exists to solve, all of which bit the first eval round\n * (docs/hybrid-retrieval-design.md, \"Eval results\"):\n *\n *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,\n *     so every commit moves the thing being measured. A baseline taken today\n *     and a rerun taken after a retrieval change differ by both the change\n *     and the intervening commits, and nothing in the output says so. Fix:\n *     run against a detached git worktree pinned to an explicit SHA, and put\n *     that SHA in the record.\n *  2. **Nothing was recorded.** Results were printed to a terminal and\n *     hand-copied into a markdown table with no repo SHA, no embedder\n *     identity, and no index state. Fix: emit a machine-readable run record.\n *  3. **A degraded run looks like a real one.** With no embsearch binary the\n *     semantic and hybrid rows silently degrade to lexical, producing a table\n *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the\n *     record, plus a per-row degraded count that the writer refuses to hide.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readdirSync, readFileSync, statSync } from \"node:fs\";\nimport { execFileSync } from \"child_process\";\nimport { rmSync } from \"fs\";\nimport { tmpdir } from \"os\";\nimport path from \"path\";\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { type EvalConfig, type EvalQuery, type EvalQueryResult, evaluateQuery } from \"./eval.js\";\n\n/** Metrics aggregated per config across the whole gold set. */\nexport interface EvalAggregate {\n\tlabel: string;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n\t/** Queries scored under this config. */\n\tn: number;\n\t/** How many of them ran degraded (requested retriever unavailable). */\n\tdegraded: number;\n}\n\n/** Everything needed to decide whether two run records may be compared. */\nexport interface EvalProvenance {\n\ttimestampMs: number;\n\t/** SHA of the corpus actually indexed and searched. */\n\tcorpusSha: string;\n\t/** Ref the caller asked for, before resolution (e.g. \"HEAD\"). */\n\tcorpusRef: string;\n\t/** True when the corpus came from the live working tree rather than a\n\t *  pinned worktree — results are then not reproducible. */\n\tcorpusFromWorkingTree: boolean;\n\t/** Uncommitted changes present at run time. Only meaningful (and only\n\t *  possible) when `corpusFromWorkingTree` is true. */\n\tcorpusDirty: boolean;\n\t/** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,\n\t *  but differs when pinning an old corpus with today's code. */\n\tharnessSha: string;\n\t/**\n\t * Content hash of `src/core/search` + the chunker. Every tuning constant\n\t * that shapes a result — the fusion cap, top-k depths, rerank weights,\n\t * chunk sizing — lives in those files, so a changed hash means the\n\t * numbers are not comparable, without this module having to maintain a\n\t * hand-copied (and inevitably stale) list of constants.\n\t */\n\tretrievalSourceHash: string;\n\t/** Embedding backend state. `available: false` means every semantic and\n\t *  hybrid row in this record degraded to lexical. */\n\tembedder: {\n\t\tavailable: boolean;\n\t\treason?: string;\n\t\t/** Indexed chunk count when the index reached `ready`. */\n\t\tchunkCount?: number;\n\t\tphase: string;\n\t\t/** Binary that served the embeddings, and its self-reported version.\n\t\t *  The embedding model is baked into the binary at build time, so this\n\t\t *  is the only thing that identifies which model produced a score. */\n\t\tbinaryPath?: string;\n\t\tbinaryVersion?: string;\n\t};\n\t/** Daemon-side BM25 hybrid store, when the run included one. Absent means\n\t *  the record has no `daemon-hybrid` rows. */\n\tdaemonHybrid?: { available: boolean; phase: string };\n\truntime: { node: string; platform: string; arch: string };\n}\n\nexport interface EvalRunRecord {\n\tprovenance: EvalProvenance;\n\tgoldSet: { queryCount: number; byClass: Record<string, number>; goldSpanCount: number };\n\tconfigs: readonly EvalConfig[];\n\taggregates: EvalAggregate[];\n\tperQuery: Array<{ id: string; class: string; results: EvalQueryResult[] }>;\n}\n\nfunction git(repoRoot: string, args: string[]): string {\n\treturn execFileSync(\"git\", [\"-C\", repoRoot, ...args], { encoding: \"utf-8\" }).trim();\n}\n\n/** Hash every retrieval-shaping source file, so a tuning change is visible as\n *  a changed provenance field rather than an unexplained metric shift. */\nexport function hashRetrievalSource(repoRoot: string): string {\n\tconst roots = [\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/search\"),\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/embsearch/chunker.ts\"),\n\t];\n\tconst files: string[] = [];\n\tconst walk = (target: string): void => {\n\t\tlet stat: ReturnType<typeof statSync>;\n\t\ttry {\n\t\t\tstat = statSync(target);\n\t\t} catch {\n\t\t\treturn;\n\t\t}\n\t\tif (stat.isDirectory()) {\n\t\t\tfor (const entry of readdirSync(target).sort()) walk(path.join(target, entry));\n\t\t} else if (target.endsWith(\".ts\")) {\n\t\t\tfiles.push(target);\n\t\t}\n\t};\n\tfor (const root of roots) walk(root);\n\n\tconst hash = createHash(\"sha256\");\n\tfor (const file of files) {\n\t\t// Eval-only modules are excluded: changing how we measure must not look\n\t\t// like changing what we measure.\n\t\tconst base = path.basename(file);\n\t\tif (base.startsWith(\"eval\")) continue;\n\t\thash.update(path.relative(repoRoot, file).replace(/\\\\/g, \"/\"));\n\t\thash.update(readFileSync(file));\n\t}\n\treturn hash.digest(\"hex\").slice(0, 16);\n}\n\nexport interface PinnedCorpus {\n\t/** Directory to index and search. */\n\tcwd: string;\n\tsha: string;\n\tfromWorkingTree: boolean;\n\tdirty: boolean;\n\t/** Removes the worktree, if one was created. */\n\tdispose: () => void;\n}\n\n/**\n * Materialize the corpus to evaluate.\n *\n * With a `ref`, checks out a detached worktree at that commit so the corpus is\n * byte-identical on every rerun. Without one, falls back to the live working\n * tree and reports `dirty` so the record shows the run was not reproducible.\n */\nexport function pinCorpus(repoRoot: string, ref: string | undefined): PinnedCorpus {\n\tconst dirty = git(repoRoot, [\"status\", \"--porcelain\"]).length > 0;\n\tif (!ref) {\n\t\treturn {\n\t\t\tcwd: repoRoot,\n\t\t\tsha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\t\tfromWorkingTree: true,\n\t\t\tdirty,\n\t\t\tdispose: () => {},\n\t\t};\n\t}\n\n\tconst sha = git(repoRoot, [\"rev-parse\", ref]);\n\t// Deterministic path, not mkdtemp: the embedding store is keyed by a hash of\n\t// the corpus directory, so a fresh temp path every run would re-embed all\n\t// ~17k chunks (minutes) instead of reusing the store built for this exact\n\t// SHA. The worktree is still removed afterwards; only the store persists.\n\tconst dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}`);\n\tif (existsSync(dir)) {\n\t\t// Left behind by an interrupted run — drop it so `worktree add` succeeds.\n\t\ttry {\n\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t} catch {\n\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\tgit(repoRoot, [\"worktree\", \"prune\"]);\n\t\t}\n\t}\n\tgit(repoRoot, [\"worktree\", \"add\", \"--detach\", dir, sha]);\n\treturn {\n\t\tcwd: dir,\n\t\tsha,\n\t\tfromWorkingTree: false,\n\t\tdirty: false,\n\t\tdispose: () => {\n\t\t\ttry {\n\t\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t\t} catch {\n\t\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\t}\n\t\t},\n\t};\n}\n\n/** `<binary> --version`, or undefined when it cannot be run. */\nfunction probeBinaryVersion(binaryPath: string | undefined): string | undefined {\n\tif (!binaryPath) return undefined;\n\ttry {\n\t\treturn execFileSync(binaryPath, [\"--version\"], { encoding: \"utf-8\" }).trim();\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\nexport function collectProvenance(\n\trepoRoot: string,\n\tcorpus: PinnedCorpus,\n\tcorpusRef: string,\n\tservice: EmbsearchService | undefined,\n\tembsearchBinary?: string,\n\thybridService?: EmbsearchService,\n): EvalProvenance {\n\tconst state = service?.getState();\n\tconst phase = state?.phase ?? \"absent\";\n\treturn {\n\t\ttimestampMs: Date.now(),\n\t\tcorpusSha: corpus.sha,\n\t\tcorpusRef,\n\t\tcorpusFromWorkingTree: corpus.fromWorkingTree,\n\t\tcorpusDirty: corpus.dirty,\n\t\tharnessSha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\tretrievalSourceHash: hashRetrievalSource(repoRoot),\n\t\tembedder: {\n\t\t\t// `ready` is the only phase the service reaches with a real embedder:\n\t\t\t// it rejects the mock backend at startup, so availability here also\n\t\t\t// certifies the numbers came from a genuine ONNX build.\n\t\t\tavailable: service?.isAvailable() ?? false,\n\t\t\treason: state && \"reason\" in state ? state.reason : undefined,\n\t\t\tchunkCount: state?.phase === \"ready\" ? state.chunkCount : undefined,\n\t\t\tphase,\n\t\t\tbinaryPath: embsearchBinary,\n\t\t\tbinaryVersion: probeBinaryVersion(embsearchBinary),\n\t\t},\n\t\tdaemonHybrid: hybridService\n\t\t\t? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }\n\t\t\t: undefined,\n\t\truntime: { node: process.version, platform: process.platform, arch: process.arch },\n\t};\n}\n\nexport function summarizeGoldSet(dataset: readonly EvalQuery[]): EvalRunRecord[\"goldSet\"] {\n\tconst byClass: Record<string, number> = {};\n\tlet goldSpanCount = 0;\n\tfor (const query of dataset) {\n\t\tbyClass[query.class] = (byClass[query.class] ?? 0) + 1;\n\t\tgoldSpanCount += query.gold.length;\n\t}\n\treturn { queryCount: dataset.length, byClass, goldSpanCount };\n}\n\nexport interface RunEvalSuiteOptions {\n\tcwd: string;\n\tdataset: readonly EvalQuery[];\n\tconfigs: readonly EvalConfig[];\n\tservice?: EmbsearchService;\n\t/** Second service backed by a daemon-side BM25 hybrid store, for the\n\t *  `daemon-hybrid` configs. Absent means those rows are omitted. */\n\thybridService?: EmbsearchService;\n\tonQuery?: (index: number, query: EvalQuery) => void;\n}\n\nexport async function runEvalSuite(options: RunEvalSuiteOptions): Promise<{\n\taggregates: EvalAggregate[];\n\tperQuery: EvalRunRecord[\"perQuery\"];\n}> {\n\tconst { cwd, dataset, configs, service } = options;\n\tconst totals = new Map<string, EvalAggregate>();\n\tconst perQuery: EvalRunRecord[\"perQuery\"] = [];\n\n\tfor (const [index, evalQuery] of dataset.entries()) {\n\t\toptions.onQuery?.(index, evalQuery);\n\t\tconst results = await evaluateQuery(cwd, evalQuery, configs, service, options.hybridService);\n\t\tperQuery.push({ id: evalQuery.id, class: evalQuery.class, results });\n\t\tfor (const result of results) {\n\t\t\tconst total = totals.get(result.label) ?? {\n\t\t\t\tlabel: result.label,\n\t\t\t\trecallAt1: 0,\n\t\t\t\trecallAt5: 0,\n\t\t\t\trecallAt10: 0,\n\t\t\t\trecallAt50: 0,\n\t\t\t\tmrr: 0,\n\t\t\t\tn: 0,\n\t\t\t\tdegraded: 0,\n\t\t\t};\n\t\t\ttotal.recallAt1 += result.recallAt1;\n\t\t\ttotal.recallAt5 += result.recallAt5;\n\t\t\ttotal.recallAt10 += result.recallAt10;\n\t\t\ttotal.recallAt50 += result.recallAt50;\n\t\t\ttotal.mrr += result.mrr;\n\t\t\ttotal.n++;\n\t\t\tif (result.degraded) total.degraded++;\n\t\t\ttotals.set(result.label, total);\n\t\t}\n\t}\n\n\tconst aggregates = configs\n\t\t.map((config) => totals.get(config.label))\n\t\t.filter((total): total is EvalAggregate => total !== undefined)\n\t\t.map((total) => ({\n\t\t\t...total,\n\t\t\trecallAt1: total.recallAt1 / total.n,\n\t\t\trecallAt5: total.recallAt5 / total.n,\n\t\t\trecallAt10: total.recallAt10 / total.n,\n\t\t\trecallAt50: total.recallAt50 / total.n,\n\t\t\tmrr: total.mrr / total.n,\n\t\t}));\n\n\treturn { aggregates, perQuery };\n}\n\nexport function formatAggregateTable(aggregates: readonly EvalAggregate[]): string {\n\tconst pct = (x: number) => `${Math.round(x * 100)}%`.padStart(5);\n\tconst lines = [\n\t\t\"config           |  R@1  |  R@5  | R@10  | R@50  |  MRR  | notes\",\n\t\t\"-----------------|-------|-------|-------|-------|-------|------\",\n\t];\n\tfor (const a of aggregates) {\n\t\tconst notes = a.degraded === a.n ? \"degraded to lexical\" : a.degraded > 0 ? `${a.degraded}/${a.n} degraded` : \"\";\n\t\tlines.push(\n\t\t\t`${a.label.padEnd(16)} | ${pct(a.recallAt1)} | ${pct(a.recallAt5)} | ${pct(a.recallAt10)} | ` +\n\t\t\t\t`${pct(a.recallAt50)} | ${a.mrr.toFixed(3)} | ${notes}`,\n\t\t);\n\t}\n\treturn lines.join(\"\\n\");\n}\n"]}
+{"version":3,"file":"eval-harness.d.ts","sourceRoot":"","sources":["../../../src/core/search/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,KAAK,eAAe,EAAiB,MAAM,WAAW,CAAC;AAEjG,+DAA+D;AAC/D,MAAM,WAAW,aAAa;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,wCAAwC;IACxC,CAAC,EAAE,MAAM,CAAC;IACV,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED,2EAA2E;AAC3E,MAAM,WAAW,cAAc;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB;iEAC2D;IAC3D,qBAAqB,EAAE,OAAO,CAAC;IAC/B;0DACsD;IACtD,WAAW,EAAE,OAAO,CAAC;IACrB;;8EAE0E;IAC1E,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB;oEACgE;IAChE,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;;OAMG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;yDACqD;IACrD,QAAQ,EAAE;QACT,SAAS,EAAE,OAAO,CAAC;QACnB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,0DAA0D;QAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;QACd;;8EAEsE;QACtE,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,aAAa,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IACF;kDAC8C;IAC9C,YAAY,CAAC,EAAE;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,WAAW,aAAa;IAC7B,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,OAAO,EAAE,SAAS,UAAU,EAAE,CAAC;IAC/B,UAAU,EAAE,aAAa,EAAE,CAAC;IAC5B,QAAQ,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,eAAe,EAAE,CAAA;KAAE,CAAC,CAAC;CAC3E;AAMD;0EAC0E;AAC1E,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CA+B5D;AAED,MAAM,WAAW,YAAY;IAC5B,qCAAqC;IACrC,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,OAAO,CAAC;IACzB,KAAK,EAAE,OAAO,CAAC;IACf;yDACqD;IACrD,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,gDAAgD;IAChD,OAAO,EAAE,MAAM,IAAI,CAAC;CACpB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,iBAAiB,EAAE,SAAS,MAAM,EAK9C,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,YAAY,CAwDjF;AAYD,wBAAgB,iBAAiB,CAChC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,YAAY,EACpB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,gBAAgB,GAAG,SAAS,EACrC,eAAe,CAAC,EAAE,MAAM,EACxB,aAAa,CAAC,EAAE,gBAAgB,GAC9B,cAAc,CA4BhB;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,SAAS,SAAS,EAAE,GAAG,aAAa,CAAC,SAAS,CAAC,CAQxF;AAED,MAAM,WAAW,mBAAmB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,SAAS,SAAS,EAAE,CAAC;IAC9B,OAAO,EAAE,SAAS,UAAU,EAAE,CAAC;IAC/B,OAAO,CAAC,EAAE,gBAAgB,CAAC;IAC3B;wEACoE;IACpE,aAAa,CAAC,EAAE,gBAAgB,CAAC;IACjC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,KAAK,IAAI,CAAC;CACpD;AAED,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC;IACzE,UAAU,EAAE,aAAa,EAAE,CAAC;IAC5B,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACpC,CAAC,CA4CD;AAED,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,GAAG,MAAM,CAcjF","sourcesContent":["/**\n * Eval harness: corpus pinning, provenance capture, and run records.\n *\n * The scoring math lives in `eval.ts`; this module is everything around it\n * that makes a number *comparable to a later number*. Three problems it\n * exists to solve, all of which bit the first eval round\n * (docs/hybrid-retrieval-design.md, \"Eval results\"):\n *\n *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,\n *     so every commit moves the thing being measured. A baseline taken today\n *     and a rerun taken after a retrieval change differ by both the change\n *     and the intervening commits, and nothing in the output says so. Fix:\n *     run against a detached git worktree pinned to an explicit SHA, and put\n *     that SHA in the record.\n *  2. **Nothing was recorded.** Results were printed to a terminal and\n *     hand-copied into a markdown table with no repo SHA, no embedder\n *     identity, and no index state. Fix: emit a machine-readable run record.\n *  3. **A degraded run looks like a real one.** With no embsearch binary the\n *     semantic and hybrid rows silently degrade to lexical, producing a table\n *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the\n *     record, plus a per-row degraded count that the writer refuses to hide.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readdirSync, readFileSync, statSync } from \"node:fs\";\nimport { execFileSync } from \"child_process\";\nimport { rmSync } from \"fs\";\nimport { tmpdir } from \"os\";\nimport path from \"path\";\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { type EvalConfig, type EvalQuery, type EvalQueryResult, evaluateQuery } from \"./eval.js\";\n\n/** Metrics aggregated per config across the whole gold set. */\nexport interface EvalAggregate {\n\tlabel: string;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n\t/** Queries scored under this config. */\n\tn: number;\n\t/** How many of them ran degraded (requested retriever unavailable). */\n\tdegraded: number;\n}\n\n/** Everything needed to decide whether two run records may be compared. */\nexport interface EvalProvenance {\n\ttimestampMs: number;\n\t/** SHA of the corpus actually indexed and searched. */\n\tcorpusSha: string;\n\t/** Ref the caller asked for, before resolution (e.g. \"HEAD\"). */\n\tcorpusRef: string;\n\t/** True when the corpus came from the live working tree rather than a\n\t *  pinned worktree — results are then not reproducible. */\n\tcorpusFromWorkingTree: boolean;\n\t/** Uncommitted changes present at run time. Only meaningful (and only\n\t *  possible) when `corpusFromWorkingTree` is true. */\n\tcorpusDirty: boolean;\n\t/** Files removed from the corpus before indexing — see\n\t *  {@link CORPUS_EXCLUSIONS}. A score against a different exclusion list is\n\t *  a score against a different corpus, so it is recorded, not assumed. */\n\tcorpusExcluded: string[];\n\t/** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,\n\t *  but differs when pinning an old corpus with today's code. */\n\tharnessSha: string;\n\t/**\n\t * Content hash of `src/core/search` + the chunker. Every tuning constant\n\t * that shapes a result — the fusion cap, top-k depths, rerank weights,\n\t * chunk sizing — lives in those files, so a changed hash means the\n\t * numbers are not comparable, without this module having to maintain a\n\t * hand-copied (and inevitably stale) list of constants.\n\t */\n\tretrievalSourceHash: string;\n\t/** Embedding backend state. `available: false` means every semantic and\n\t *  hybrid row in this record degraded to lexical. */\n\tembedder: {\n\t\tavailable: boolean;\n\t\treason?: string;\n\t\t/** Indexed chunk count when the index reached `ready`. */\n\t\tchunkCount?: number;\n\t\tphase: string;\n\t\t/** Binary that served the embeddings, and its self-reported version.\n\t\t *  The embedding model is baked into the binary at build time, so this\n\t\t *  is the only thing that identifies which model produced a score. */\n\t\tbinaryPath?: string;\n\t\tbinaryVersion?: string;\n\t};\n\t/** Daemon-side BM25 hybrid store, when the run included one. Absent means\n\t *  the record has no `daemon-hybrid` rows. */\n\tdaemonHybrid?: { available: boolean; phase: string };\n\truntime: { node: string; platform: string; arch: string };\n}\n\nexport interface EvalRunRecord {\n\tprovenance: EvalProvenance;\n\tgoldSet: { queryCount: number; byClass: Record<string, number>; goldSpanCount: number };\n\tconfigs: readonly EvalConfig[];\n\taggregates: EvalAggregate[];\n\tperQuery: Array<{ id: string; class: string; results: EvalQueryResult[] }>;\n}\n\nfunction git(repoRoot: string, args: string[]): string {\n\treturn execFileSync(\"git\", [\"-C\", repoRoot, ...args], { encoding: \"utf-8\" }).trim();\n}\n\n/** Hash every retrieval-shaping source file, so a tuning change is visible as\n *  a changed provenance field rather than an unexplained metric shift. */\nexport function hashRetrievalSource(repoRoot: string): string {\n\tconst roots = [\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/search\"),\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/embsearch/chunker.ts\"),\n\t];\n\tconst files: string[] = [];\n\tconst walk = (target: string): void => {\n\t\tlet stat: ReturnType<typeof statSync>;\n\t\ttry {\n\t\t\tstat = statSync(target);\n\t\t} catch {\n\t\t\treturn;\n\t\t}\n\t\tif (stat.isDirectory()) {\n\t\t\tfor (const entry of readdirSync(target).sort()) walk(path.join(target, entry));\n\t\t} else if (target.endsWith(\".ts\")) {\n\t\t\tfiles.push(target);\n\t\t}\n\t};\n\tfor (const root of roots) walk(root);\n\n\tconst hash = createHash(\"sha256\");\n\tfor (const file of files) {\n\t\t// Eval-only modules are excluded: changing how we measure must not look\n\t\t// like changing what we measure.\n\t\tconst base = path.basename(file);\n\t\tif (base.startsWith(\"eval\")) continue;\n\t\thash.update(path.relative(repoRoot, file).replace(/\\\\/g, \"/\"));\n\t\thash.update(readFileSync(file));\n\t}\n\treturn hash.digest(\"hex\").slice(0, 16);\n}\n\nexport interface PinnedCorpus {\n\t/** Directory to index and search. */\n\tcwd: string;\n\tsha: string;\n\tfromWorkingTree: boolean;\n\tdirty: boolean;\n\t/** Files removed from the corpus before indexing. Empty when the corpus is\n\t *  the live working tree, which is never mutated. */\n\texcluded: string[];\n\t/** Removes the worktree, if one was created. */\n\tdispose: () => void;\n}\n\n/**\n * Files that describe this eval rather than being searched by it.\n *\n * The fixtures hold all 62 query strings verbatim, so every query is a perfect\n * lexical match against its own entry, and the design note quotes the same\n * queries while discussing the classes they belong to. Measured before this\n * exclusion existed: **56 of 62 queries had one of these files in the top 10,\n * 27 of 62 had one as the #1 result, and they consumed 133 of the 620\n * top-10 slots** — a fifth of the window, spent on the eval reading itself.\n *\n * That is not a ranking artifact a reranker can fix: it displaces real answers\n * out of the window entirely, which is why two boundary-class queries were\n * absent from the top *50* rather than merely buried. Retrieving your own\n * question is not retrieval, so the corpus is scored without them.\n *\n * Removed from the pinned worktree before indexing, never from the repo — and\n * recorded in the run's provenance so a score is never silently taken against\n * a different corpus than it claims.\n */\nexport const CORPUS_EXCLUSIONS: readonly string[] = [\n\t\"packages/coding-agent/test/fixtures/search-eval.json\",\n\t\"packages/coding-agent/test/fixtures/search-eval-live.json\",\n\t\"packages/coding-agent/test/fixtures/search-eval-baseline.json\",\n\t\"docs/hybrid-retrieval-design.md\",\n];\n\n/**\n * Materialize the corpus to evaluate.\n *\n * With a `ref`, checks out a detached worktree at that commit so the corpus is\n * byte-identical on every rerun. Without one, falls back to the live working\n * tree and reports `dirty` so the record shows the run was not reproducible.\n */\nexport function pinCorpus(repoRoot: string, ref: string | undefined): PinnedCorpus {\n\tconst dirty = git(repoRoot, [\"status\", \"--porcelain\"]).length > 0;\n\tif (!ref) {\n\t\t// The live working tree is the user's checkout; deleting files from it to\n\t\t// tidy a measurement would be an unforgivable trade. Working-tree runs\n\t\t// are already stamped non-reproducible, so they carry the contamination.\n\t\treturn {\n\t\t\tcwd: repoRoot,\n\t\t\tsha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\t\tfromWorkingTree: true,\n\t\t\tdirty,\n\t\t\texcluded: [],\n\t\t\tdispose: () => {},\n\t\t};\n\t}\n\n\tconst sha = git(repoRoot, [\"rev-parse\", ref]);\n\t// Deterministic path, not mkdtemp: the embedding store is keyed by a hash of\n\t// the corpus directory, so a fresh temp path every run would re-embed all\n\t// ~17k chunks (minutes) instead of reusing the store built for this exact\n\t// SHA. The worktree is still removed afterwards; only the store persists.\n\tconst dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}`);\n\tif (existsSync(dir)) {\n\t\t// Left behind by an interrupted run — drop it so `worktree add` succeeds.\n\t\ttry {\n\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t} catch {\n\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\tgit(repoRoot, [\"worktree\", \"prune\"]);\n\t\t}\n\t}\n\tgit(repoRoot, [\"worktree\", \"add\", \"--detach\", dir, sha]);\n\n\tconst excluded: string[] = [];\n\tfor (const rel of CORPUS_EXCLUSIONS) {\n\t\tconst target = path.join(dir, rel);\n\t\tif (existsSync(target)) {\n\t\t\trmSync(target, { force: true });\n\t\t\texcluded.push(rel);\n\t\t}\n\t}\n\n\treturn {\n\t\tcwd: dir,\n\t\tsha,\n\t\tfromWorkingTree: false,\n\t\tdirty: false,\n\t\texcluded,\n\t\tdispose: () => {\n\t\t\ttry {\n\t\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t\t} catch {\n\t\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\t}\n\t\t},\n\t};\n}\n\n/** `<binary> --version`, or undefined when it cannot be run. */\nfunction probeBinaryVersion(binaryPath: string | undefined): string | undefined {\n\tif (!binaryPath) return undefined;\n\ttry {\n\t\treturn execFileSync(binaryPath, [\"--version\"], { encoding: \"utf-8\" }).trim();\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\nexport function collectProvenance(\n\trepoRoot: string,\n\tcorpus: PinnedCorpus,\n\tcorpusRef: string,\n\tservice: EmbsearchService | undefined,\n\tembsearchBinary?: string,\n\thybridService?: EmbsearchService,\n): EvalProvenance {\n\tconst state = service?.getState();\n\tconst phase = state?.phase ?? \"absent\";\n\treturn {\n\t\ttimestampMs: Date.now(),\n\t\tcorpusSha: corpus.sha,\n\t\tcorpusRef,\n\t\tcorpusFromWorkingTree: corpus.fromWorkingTree,\n\t\tcorpusDirty: corpus.dirty,\n\t\tcorpusExcluded: corpus.excluded,\n\t\tharnessSha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\tretrievalSourceHash: hashRetrievalSource(repoRoot),\n\t\tembedder: {\n\t\t\t// `ready` is the only phase the service reaches with a real embedder:\n\t\t\t// it rejects the mock backend at startup, so availability here also\n\t\t\t// certifies the numbers came from a genuine ONNX build.\n\t\t\tavailable: service?.isAvailable() ?? false,\n\t\t\treason: state && \"reason\" in state ? state.reason : undefined,\n\t\t\tchunkCount: state?.phase === \"ready\" ? state.chunkCount : undefined,\n\t\t\tphase,\n\t\t\tbinaryPath: embsearchBinary,\n\t\t\tbinaryVersion: probeBinaryVersion(embsearchBinary),\n\t\t},\n\t\tdaemonHybrid: hybridService\n\t\t\t? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }\n\t\t\t: undefined,\n\t\truntime: { node: process.version, platform: process.platform, arch: process.arch },\n\t};\n}\n\nexport function summarizeGoldSet(dataset: readonly EvalQuery[]): EvalRunRecord[\"goldSet\"] {\n\tconst byClass: Record<string, number> = {};\n\tlet goldSpanCount = 0;\n\tfor (const query of dataset) {\n\t\tbyClass[query.class] = (byClass[query.class] ?? 0) + 1;\n\t\tgoldSpanCount += query.gold.length;\n\t}\n\treturn { queryCount: dataset.length, byClass, goldSpanCount };\n}\n\nexport interface RunEvalSuiteOptions {\n\tcwd: string;\n\tdataset: readonly EvalQuery[];\n\tconfigs: readonly EvalConfig[];\n\tservice?: EmbsearchService;\n\t/** Second service backed by a daemon-side BM25 hybrid store, for the\n\t *  `daemon-hybrid` configs. Absent means those rows are omitted. */\n\thybridService?: EmbsearchService;\n\tonQuery?: (index: number, query: EvalQuery) => void;\n}\n\nexport async function runEvalSuite(options: RunEvalSuiteOptions): Promise<{\n\taggregates: EvalAggregate[];\n\tperQuery: EvalRunRecord[\"perQuery\"];\n}> {\n\tconst { cwd, dataset, configs, service } = options;\n\tconst totals = new Map<string, EvalAggregate>();\n\tconst perQuery: EvalRunRecord[\"perQuery\"] = [];\n\n\tfor (const [index, evalQuery] of dataset.entries()) {\n\t\toptions.onQuery?.(index, evalQuery);\n\t\tconst results = await evaluateQuery(cwd, evalQuery, configs, service, options.hybridService);\n\t\tperQuery.push({ id: evalQuery.id, class: evalQuery.class, results });\n\t\tfor (const result of results) {\n\t\t\tconst total = totals.get(result.label) ?? {\n\t\t\t\tlabel: result.label,\n\t\t\t\trecallAt1: 0,\n\t\t\t\trecallAt5: 0,\n\t\t\t\trecallAt10: 0,\n\t\t\t\trecallAt50: 0,\n\t\t\t\tmrr: 0,\n\t\t\t\tn: 0,\n\t\t\t\tdegraded: 0,\n\t\t\t};\n\t\t\ttotal.recallAt1 += result.recallAt1;\n\t\t\ttotal.recallAt5 += result.recallAt5;\n\t\t\ttotal.recallAt10 += result.recallAt10;\n\t\t\ttotal.recallAt50 += result.recallAt50;\n\t\t\ttotal.mrr += result.mrr;\n\t\t\ttotal.n++;\n\t\t\tif (result.degraded) total.degraded++;\n\t\t\ttotals.set(result.label, total);\n\t\t}\n\t}\n\n\tconst aggregates = configs\n\t\t.map((config) => totals.get(config.label))\n\t\t.filter((total): total is EvalAggregate => total !== undefined)\n\t\t.map((total) => ({\n\t\t\t...total,\n\t\t\trecallAt1: total.recallAt1 / total.n,\n\t\t\trecallAt5: total.recallAt5 / total.n,\n\t\t\trecallAt10: total.recallAt10 / total.n,\n\t\t\trecallAt50: total.recallAt50 / total.n,\n\t\t\tmrr: total.mrr / total.n,\n\t\t}));\n\n\treturn { aggregates, perQuery };\n}\n\nexport function formatAggregateTable(aggregates: readonly EvalAggregate[]): string {\n\tconst pct = (x: number) => `${Math.round(x * 100)}%`.padStart(5);\n\tconst lines = [\n\t\t\"config           |  R@1  |  R@5  | R@10  | R@50  |  MRR  | notes\",\n\t\t\"-----------------|-------|-------|-------|-------|-------|------\",\n\t];\n\tfor (const a of aggregates) {\n\t\tconst notes = a.degraded === a.n ? \"degraded to lexical\" : a.degraded > 0 ? `${a.degraded}/${a.n} degraded` : \"\";\n\t\tlines.push(\n\t\t\t`${a.label.padEnd(16)} | ${pct(a.recallAt1)} | ${pct(a.recallAt5)} | ${pct(a.recallAt10)} | ` +\n\t\t\t\t`${pct(a.recallAt50)} | ${a.mrr.toFixed(3)} | ${notes}`,\n\t\t);\n\t}\n\treturn lines.join(\"\\n\");\n}\n"]}

package/dist/core/search/eval-harness.js

@@ -68,6 +68,31 @@ export function hashRetrievalSource(repoRoot) {
     }
     return hash.digest("hex").slice(0, 16);
 }
+/**
+ * Files that describe this eval rather than being searched by it.
+ *
+ * The fixtures hold all 62 query strings verbatim, so every query is a perfect
+ * lexical match against its own entry, and the design note quotes the same
+ * queries while discussing the classes they belong to. Measured before this
+ * exclusion existed: **56 of 62 queries had one of these files in the top 10,
+ * 27 of 62 had one as the #1 result, and they consumed 133 of the 620
+ * top-10 slots** — a fifth of the window, spent on the eval reading itself.
+ *
+ * That is not a ranking artifact a reranker can fix: it displaces real answers
+ * out of the window entirely, which is why two boundary-class queries were
+ * absent from the top *50* rather than merely buried. Retrieving your own
+ * question is not retrieval, so the corpus is scored without them.
+ *
+ * Removed from the pinned worktree before indexing, never from the repo — and
+ * recorded in the run's provenance so a score is never silently taken against
+ * a different corpus than it claims.
+ */
+export const CORPUS_EXCLUSIONS = [
+    "packages/coding-agent/test/fixtures/search-eval.json",
+    "packages/coding-agent/test/fixtures/search-eval-live.json",
+    "packages/coding-agent/test/fixtures/search-eval-baseline.json",
+    "docs/hybrid-retrieval-design.md",
+];
 /**
  * Materialize the corpus to evaluate.
  *
@@ -78,11 +103,15 @@ export function hashRetrievalSource(repoRoot) {
 export function pinCorpus(repoRoot, ref) {
     const dirty = git(repoRoot, ["status", "--porcelain"]).length > 0;
     if (!ref) {
+        // The live working tree is the user's checkout; deleting files from it to
+        // tidy a measurement would be an unforgivable trade. Working-tree runs
+        // are already stamped non-reproducible, so they carry the contamination.
         return {
             cwd: repoRoot,
             sha: git(repoRoot, ["rev-parse", "HEAD"]),
             fromWorkingTree: true,
             dirty,
+            excluded: [],
             dispose: () => { },
         };
     }
@@ -103,11 +132,20 @@ export function pinCorpus(repoRoot, ref) {
         }
     }
     git(repoRoot, ["worktree", "add", "--detach", dir, sha]);
+    const excluded = [];
+    for (const rel of CORPUS_EXCLUSIONS) {
+        const target = path.join(dir, rel);
+        if (existsSync(target)) {
+            rmSync(target, { force: true });
+            excluded.push(rel);
+        }
+    }
     return {
         cwd: dir,
         sha,
         fromWorkingTree: false,
         dirty: false,
+        excluded,
         dispose: () => {
             try {
                 git(repoRoot, ["worktree", "remove", "--force", dir]);
@@ -138,6 +176,7 @@ export function collectProvenance(repoRoot, corpus, corpusRef, service, embsearc
         corpusRef,
         corpusFromWorkingTree: corpus.fromWorkingTree,
         corpusDirty: corpus.dirty,
+        corpusExcluded: corpus.excluded,
         harnessSha: git(repoRoot, ["rev-parse", "HEAD"]),
         retrievalSourceHash: hashRetrievalSource(repoRoot),
         embedder: {

package/dist/core/search/eval-harness.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-harness.js","sourceRoot":"","sources":["../../../src/core/search/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAyD,aAAa,EAAE,MAAM,WAAW,CAAC;AAoEjG,SAAS,GAAG,CAAC,QAAgB,EAAE,IAAc,EAAU;IACtD,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAAA,CACpF;AAED;0EAC0E;AAC1E,MAAM,UAAU,mBAAmB,CAAC,QAAgB,EAAU;IAC7D,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,uCAAuC,CAAC;QAC5D,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,qDAAqD,CAAC;KAC1E,CAAC;IACF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,MAAc,EAAQ,EAAE,CAAC;QACtC,IAAI,IAAiC,CAAC;QACtC,IAAI,CAAC;YACJ,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACR,OAAO;QACR,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE;gBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;IAAA,CACD,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,wEAAwE;QACxE,iCAAiC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACtC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAAA,CACvC;AAYD;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,GAAuB,EAAgB;IAClF,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAClE,IAAI,CAAC,GAAG,EAAE,CAAC;QACV,OAAO;YACN,GAAG,EAAE,QAAQ;YACb,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACzC,eAAe,EAAE,IAAI;YACrB,KAAK;YACL,OAAO,EAAE,GAAG,EAAE,CAAC,EAAC,CAAC;SACjB,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9C,6EAA6E;IAC7E,0EAA0E;IAC1E,0EAA0E;IAC1E,0EAA0E;IAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,4EAA0E;QAC1E,IAAI,CAAC;YACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;QACtC,CAAC;IACF,CAAC;IACD,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IACzD,OAAO;QACN,GAAG,EAAE,GAAG;QACR,GAAG;QACH,eAAe,EAAE,KAAK;QACtB,KAAK,EAAE,KAAK;QACZ,OAAO,EAAE,GAAG,EAAE,CAAC;YACd,IAAI,CAAC;gBACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/C,CAAC;QAAA,CACD;KACD,CAAC;AAAA,CACF;AAED,gEAAgE;AAChE,SAAS,kBAAkB,CAAC,UAA8B,EAAsB;IAC/E,IAAI,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAClC,IAAI,CAAC;QACJ,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED,MAAM,UAAU,iBAAiB,CAChC,QAAgB,EAChB,MAAoB,EACpB,SAAiB,EACjB,OAAqC,EACrC,eAAwB,EACxB,aAAgC,EACf;IACjB,MAAM,KAAK,GAAG,OAAO,EAAE,QAAQ,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,KAAK,EAAE,KAAK,IAAI,QAAQ,CAAC;IACvC,OAAO;QACN,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;QACvB,SAAS,EAAE,MAAM,CAAC,GAAG;QACrB,SAAS;QACT,qBAAqB,EAAE,MAAM,CAAC,eAAe;QAC7C,WAAW,EAAE,MAAM,CAAC,KAAK;QACzB,UAAU,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAChD,mBAAmB,EAAE,mBAAmB,CAAC,QAAQ,CAAC;QAClD,QAAQ,EAAE;YACT,sEAAsE;YACtE,oEAAoE;YACpE,wDAAwD;YACxD,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,KAAK;YAC1C,MAAM,EAAE,KAAK,IAAI,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YAC7D,UAAU,EAAE,KAAK,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YACnE,KAAK;YACL,UAAU,EAAE,eAAe;YAC3B,aAAa,EAAE,kBAAkB,CAAC,eAAe,CAAC;SAClD;QACD,YAAY,EAAE,aAAa;YAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,aAAa,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,aAAa,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE;YACnF,CAAC,CAAC,SAAS;QACZ,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE;KAClF,CAAC;AAAA,CACF;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA6B,EAA4B;IACzF,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACvD,aAAa,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;IACpC,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AAAA,CAC9D;AAaD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B,EAG5D;IACF,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAChD,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAE/C,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC7F,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACrE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI;gBACzC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC;gBACb,UAAU,EAAE,CAAC;gBACb,GAAG,EAAE,CAAC;gBACN,CAAC,EAAE,CAAC;gBACJ,QAAQ,EAAE,CAAC;aACX,CAAC;YACF,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC;YACxB,KAAK,CAAC,CAAC,EAAE,CAAC;YACV,IAAI,MAAM,CAAC,QAAQ;gBAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACjC,CAAC;IACF,CAAC;IAED,MAAM,UAAU,GAAG,OAAO;SACxB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;SACzC,MAAM,CAAC,CAAC,KAAK,EAA0B,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC;SAC9D,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAChB,GAAG,KAAK;QACR,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;KACxB,CAAC,CAAC,CAAC;IAEL,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAAA,CAChC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAAoC,EAAU;IAClF,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG;QACb,kEAAkE;QAClE,kEAAkE;KAClE,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;QACjH,KAAK,CAAC,IAAI,CACT,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK;YAC5F,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,EAAE,CACxD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAAA,CACxB","sourcesContent":["/**\n * Eval harness: corpus pinning, provenance capture, and run records.\n *\n * The scoring math lives in `eval.ts`; this module is everything around it\n * that makes a number *comparable to a later number*. Three problems it\n * exists to solve, all of which bit the first eval round\n * (docs/hybrid-retrieval-design.md, \"Eval results\"):\n *\n *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,\n *     so every commit moves the thing being measured. A baseline taken today\n *     and a rerun taken after a retrieval change differ by both the change\n *     and the intervening commits, and nothing in the output says so. Fix:\n *     run against a detached git worktree pinned to an explicit SHA, and put\n *     that SHA in the record.\n *  2. **Nothing was recorded.** Results were printed to a terminal and\n *     hand-copied into a markdown table with no repo SHA, no embedder\n *     identity, and no index state. Fix: emit a machine-readable run record.\n *  3. **A degraded run looks like a real one.** With no embsearch binary the\n *     semantic and hybrid rows silently degrade to lexical, producing a table\n *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the\n *     record, plus a per-row degraded count that the writer refuses to hide.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readdirSync, readFileSync, statSync } from \"node:fs\";\nimport { execFileSync } from \"child_process\";\nimport { rmSync } from \"fs\";\nimport { tmpdir } from \"os\";\nimport path from \"path\";\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { type EvalConfig, type EvalQuery, type EvalQueryResult, evaluateQuery } from \"./eval.js\";\n\n/** Metrics aggregated per config across the whole gold set. */\nexport interface EvalAggregate {\n\tlabel: string;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n\t/** Queries scored under this config. */\n\tn: number;\n\t/** How many of them ran degraded (requested retriever unavailable). */\n\tdegraded: number;\n}\n\n/** Everything needed to decide whether two run records may be compared. */\nexport interface EvalProvenance {\n\ttimestampMs: number;\n\t/** SHA of the corpus actually indexed and searched. */\n\tcorpusSha: string;\n\t/** Ref the caller asked for, before resolution (e.g. \"HEAD\"). */\n\tcorpusRef: string;\n\t/** True when the corpus came from the live working tree rather than a\n\t *  pinned worktree — results are then not reproducible. */\n\tcorpusFromWorkingTree: boolean;\n\t/** Uncommitted changes present at run time. Only meaningful (and only\n\t *  possible) when `corpusFromWorkingTree` is true. */\n\tcorpusDirty: boolean;\n\t/** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,\n\t *  but differs when pinning an old corpus with today's code. */\n\tharnessSha: string;\n\t/**\n\t * Content hash of `src/core/search` + the chunker. Every tuning constant\n\t * that shapes a result — the fusion cap, top-k depths, rerank weights,\n\t * chunk sizing — lives in those files, so a changed hash means the\n\t * numbers are not comparable, without this module having to maintain a\n\t * hand-copied (and inevitably stale) list of constants.\n\t */\n\tretrievalSourceHash: string;\n\t/** Embedding backend state. `available: false` means every semantic and\n\t *  hybrid row in this record degraded to lexical. */\n\tembedder: {\n\t\tavailable: boolean;\n\t\treason?: string;\n\t\t/** Indexed chunk count when the index reached `ready`. */\n\t\tchunkCount?: number;\n\t\tphase: string;\n\t\t/** Binary that served the embeddings, and its self-reported version.\n\t\t *  The embedding model is baked into the binary at build time, so this\n\t\t *  is the only thing that identifies which model produced a score. */\n\t\tbinaryPath?: string;\n\t\tbinaryVersion?: string;\n\t};\n\t/** Daemon-side BM25 hybrid store, when the run included one. Absent means\n\t *  the record has no `daemon-hybrid` rows. */\n\tdaemonHybrid?: { available: boolean; phase: string };\n\truntime: { node: string; platform: string; arch: string };\n}\n\nexport interface EvalRunRecord {\n\tprovenance: EvalProvenance;\n\tgoldSet: { queryCount: number; byClass: Record<string, number>; goldSpanCount: number };\n\tconfigs: readonly EvalConfig[];\n\taggregates: EvalAggregate[];\n\tperQuery: Array<{ id: string; class: string; results: EvalQueryResult[] }>;\n}\n\nfunction git(repoRoot: string, args: string[]): string {\n\treturn execFileSync(\"git\", [\"-C\", repoRoot, ...args], { encoding: \"utf-8\" }).trim();\n}\n\n/** Hash every retrieval-shaping source file, so a tuning change is visible as\n *  a changed provenance field rather than an unexplained metric shift. */\nexport function hashRetrievalSource(repoRoot: string): string {\n\tconst roots = [\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/search\"),\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/embsearch/chunker.ts\"),\n\t];\n\tconst files: string[] = [];\n\tconst walk = (target: string): void => {\n\t\tlet stat: ReturnType<typeof statSync>;\n\t\ttry {\n\t\t\tstat = statSync(target);\n\t\t} catch {\n\t\t\treturn;\n\t\t}\n\t\tif (stat.isDirectory()) {\n\t\t\tfor (const entry of readdirSync(target).sort()) walk(path.join(target, entry));\n\t\t} else if (target.endsWith(\".ts\")) {\n\t\t\tfiles.push(target);\n\t\t}\n\t};\n\tfor (const root of roots) walk(root);\n\n\tconst hash = createHash(\"sha256\");\n\tfor (const file of files) {\n\t\t// Eval-only modules are excluded: changing how we measure must not look\n\t\t// like changing what we measure.\n\t\tconst base = path.basename(file);\n\t\tif (base.startsWith(\"eval\")) continue;\n\t\thash.update(path.relative(repoRoot, file).replace(/\\\\/g, \"/\"));\n\t\thash.update(readFileSync(file));\n\t}\n\treturn hash.digest(\"hex\").slice(0, 16);\n}\n\nexport interface PinnedCorpus {\n\t/** Directory to index and search. */\n\tcwd: string;\n\tsha: string;\n\tfromWorkingTree: boolean;\n\tdirty: boolean;\n\t/** Removes the worktree, if one was created. */\n\tdispose: () => void;\n}\n\n/**\n * Materialize the corpus to evaluate.\n *\n * With a `ref`, checks out a detached worktree at that commit so the corpus is\n * byte-identical on every rerun. Without one, falls back to the live working\n * tree and reports `dirty` so the record shows the run was not reproducible.\n */\nexport function pinCorpus(repoRoot: string, ref: string | undefined): PinnedCorpus {\n\tconst dirty = git(repoRoot, [\"status\", \"--porcelain\"]).length > 0;\n\tif (!ref) {\n\t\treturn {\n\t\t\tcwd: repoRoot,\n\t\t\tsha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\t\tfromWorkingTree: true,\n\t\t\tdirty,\n\t\t\tdispose: () => {},\n\t\t};\n\t}\n\n\tconst sha = git(repoRoot, [\"rev-parse\", ref]);\n\t// Deterministic path, not mkdtemp: the embedding store is keyed by a hash of\n\t// the corpus directory, so a fresh temp path every run would re-embed all\n\t// ~17k chunks (minutes) instead of reusing the store built for this exact\n\t// SHA. The worktree is still removed afterwards; only the store persists.\n\tconst dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}`);\n\tif (existsSync(dir)) {\n\t\t// Left behind by an interrupted run — drop it so `worktree add` succeeds.\n\t\ttry {\n\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t} catch {\n\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\tgit(repoRoot, [\"worktree\", \"prune\"]);\n\t\t}\n\t}\n\tgit(repoRoot, [\"worktree\", \"add\", \"--detach\", dir, sha]);\n\treturn {\n\t\tcwd: dir,\n\t\tsha,\n\t\tfromWorkingTree: false,\n\t\tdirty: false,\n\t\tdispose: () => {\n\t\t\ttry {\n\t\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t\t} catch {\n\t\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\t}\n\t\t},\n\t};\n}\n\n/** `<binary> --version`, or undefined when it cannot be run. */\nfunction probeBinaryVersion(binaryPath: string | undefined): string | undefined {\n\tif (!binaryPath) return undefined;\n\ttry {\n\t\treturn execFileSync(binaryPath, [\"--version\"], { encoding: \"utf-8\" }).trim();\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\nexport function collectProvenance(\n\trepoRoot: string,\n\tcorpus: PinnedCorpus,\n\tcorpusRef: string,\n\tservice: EmbsearchService | undefined,\n\tembsearchBinary?: string,\n\thybridService?: EmbsearchService,\n): EvalProvenance {\n\tconst state = service?.getState();\n\tconst phase = state?.phase ?? \"absent\";\n\treturn {\n\t\ttimestampMs: Date.now(),\n\t\tcorpusSha: corpus.sha,\n\t\tcorpusRef,\n\t\tcorpusFromWorkingTree: corpus.fromWorkingTree,\n\t\tcorpusDirty: corpus.dirty,\n\t\tharnessSha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\tretrievalSourceHash: hashRetrievalSource(repoRoot),\n\t\tembedder: {\n\t\t\t// `ready` is the only phase the service reaches with a real embedder:\n\t\t\t// it rejects the mock backend at startup, so availability here also\n\t\t\t// certifies the numbers came from a genuine ONNX build.\n\t\t\tavailable: service?.isAvailable() ?? false,\n\t\t\treason: state && \"reason\" in state ? state.reason : undefined,\n\t\t\tchunkCount: state?.phase === \"ready\" ? state.chunkCount : undefined,\n\t\t\tphase,\n\t\t\tbinaryPath: embsearchBinary,\n\t\t\tbinaryVersion: probeBinaryVersion(embsearchBinary),\n\t\t},\n\t\tdaemonHybrid: hybridService\n\t\t\t? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }\n\t\t\t: undefined,\n\t\truntime: { node: process.version, platform: process.platform, arch: process.arch },\n\t};\n}\n\nexport function summarizeGoldSet(dataset: readonly EvalQuery[]): EvalRunRecord[\"goldSet\"] {\n\tconst byClass: Record<string, number> = {};\n\tlet goldSpanCount = 0;\n\tfor (const query of dataset) {\n\t\tbyClass[query.class] = (byClass[query.class] ?? 0) + 1;\n\t\tgoldSpanCount += query.gold.length;\n\t}\n\treturn { queryCount: dataset.length, byClass, goldSpanCount };\n}\n\nexport interface RunEvalSuiteOptions {\n\tcwd: string;\n\tdataset: readonly EvalQuery[];\n\tconfigs: readonly EvalConfig[];\n\tservice?: EmbsearchService;\n\t/** Second service backed by a daemon-side BM25 hybrid store, for the\n\t *  `daemon-hybrid` configs. Absent means those rows are omitted. */\n\thybridService?: EmbsearchService;\n\tonQuery?: (index: number, query: EvalQuery) => void;\n}\n\nexport async function runEvalSuite(options: RunEvalSuiteOptions): Promise<{\n\taggregates: EvalAggregate[];\n\tperQuery: EvalRunRecord[\"perQuery\"];\n}> {\n\tconst { cwd, dataset, configs, service } = options;\n\tconst totals = new Map<string, EvalAggregate>();\n\tconst perQuery: EvalRunRecord[\"perQuery\"] = [];\n\n\tfor (const [index, evalQuery] of dataset.entries()) {\n\t\toptions.onQuery?.(index, evalQuery);\n\t\tconst results = await evaluateQuery(cwd, evalQuery, configs, service, options.hybridService);\n\t\tperQuery.push({ id: evalQuery.id, class: evalQuery.class, results });\n\t\tfor (const result of results) {\n\t\t\tconst total = totals.get(result.label) ?? {\n\t\t\t\tlabel: result.label,\n\t\t\t\trecallAt1: 0,\n\t\t\t\trecallAt5: 0,\n\t\t\t\trecallAt10: 0,\n\t\t\t\trecallAt50: 0,\n\t\t\t\tmrr: 0,\n\t\t\t\tn: 0,\n\t\t\t\tdegraded: 0,\n\t\t\t};\n\t\t\ttotal.recallAt1 += result.recallAt1;\n\t\t\ttotal.recallAt5 += result.recallAt5;\n\t\t\ttotal.recallAt10 += result.recallAt10;\n\t\t\ttotal.recallAt50 += result.recallAt50;\n\t\t\ttotal.mrr += result.mrr;\n\t\t\ttotal.n++;\n\t\t\tif (result.degraded) total.degraded++;\n\t\t\ttotals.set(result.label, total);\n\t\t}\n\t}\n\n\tconst aggregates = configs\n\t\t.map((config) => totals.get(config.label))\n\t\t.filter((total): total is EvalAggregate => total !== undefined)\n\t\t.map((total) => ({\n\t\t\t...total,\n\t\t\trecallAt1: total.recallAt1 / total.n,\n\t\t\trecallAt5: total.recallAt5 / total.n,\n\t\t\trecallAt10: total.recallAt10 / total.n,\n\t\t\trecallAt50: total.recallAt50 / total.n,\n\t\t\tmrr: total.mrr / total.n,\n\t\t}));\n\n\treturn { aggregates, perQuery };\n}\n\nexport function formatAggregateTable(aggregates: readonly EvalAggregate[]): string {\n\tconst pct = (x: number) => `${Math.round(x * 100)}%`.padStart(5);\n\tconst lines = [\n\t\t\"config           |  R@1  |  R@5  | R@10  | R@50  |  MRR  | notes\",\n\t\t\"-----------------|-------|-------|-------|-------|-------|------\",\n\t];\n\tfor (const a of aggregates) {\n\t\tconst notes = a.degraded === a.n ? \"degraded to lexical\" : a.degraded > 0 ? `${a.degraded}/${a.n} degraded` : \"\";\n\t\tlines.push(\n\t\t\t`${a.label.padEnd(16)} | ${pct(a.recallAt1)} | ${pct(a.recallAt5)} | ${pct(a.recallAt10)} | ` +\n\t\t\t\t`${pct(a.recallAt50)} | ${a.mrr.toFixed(3)} | ${notes}`,\n\t\t);\n\t}\n\treturn lines.join(\"\\n\");\n}\n"]}
+{"version":3,"file":"eval-harness.js","sourceRoot":"","sources":["../../../src/core/search/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAyD,aAAa,EAAE,MAAM,WAAW,CAAC;AAwEjG,SAAS,GAAG,CAAC,QAAgB,EAAE,IAAc,EAAU;IACtD,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAAA,CACpF;AAED;0EAC0E;AAC1E,MAAM,UAAU,mBAAmB,CAAC,QAAgB,EAAU;IAC7D,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,uCAAuC,CAAC;QAC5D,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,qDAAqD,CAAC;KAC1E,CAAC;IACF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,MAAc,EAAQ,EAAE,CAAC;QACtC,IAAI,IAAiC,CAAC;QACtC,IAAI,CAAC;YACJ,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACR,OAAO;QACR,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE;gBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;IAAA,CACD,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,wEAAwE;QACxE,iCAAiC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACtC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAAA,CACvC;AAeD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAsB;IACnD,sDAAsD;IACtD,2DAA2D;IAC3D,+DAA+D;IAC/D,iCAAiC;CACjC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,GAAuB,EAAgB;IAClF,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAClE,IAAI,CAAC,GAAG,EAAE,CAAC;QACV,0EAA0E;QAC1E,uEAAuE;QACvE,yEAAyE;QACzE,OAAO;YACN,GAAG,EAAE,QAAQ;YACb,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACzC,eAAe,EAAE,IAAI;YACrB,KAAK;YACL,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,GAAG,EAAE,CAAC,EAAC,CAAC;SACjB,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9C,6EAA6E;IAC7E,0EAA0E;IAC1E,0EAA0E;IAC1E,0EAA0E;IAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,4EAA0E;QAC1E,IAAI,CAAC;YACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;QACtC,CAAC;IACF,CAAC;IACD,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IAEzD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACnC,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAChC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACF,CAAC;IAED,OAAO;QACN,GAAG,EAAE,GAAG;QACR,GAAG;QACH,eAAe,EAAE,KAAK;QACtB,KAAK,EAAE,KAAK;QACZ,QAAQ;QACR,OAAO,EAAE,GAAG,EAAE,CAAC;YACd,IAAI,CAAC;gBACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/C,CAAC;QAAA,CACD;KACD,CAAC;AAAA,CACF;AAED,gEAAgE;AAChE,SAAS,kBAAkB,CAAC,UAA8B,EAAsB;IAC/E,IAAI,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAClC,IAAI,CAAC;QACJ,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED,MAAM,UAAU,iBAAiB,CAChC,QAAgB,EAChB,MAAoB,EACpB,SAAiB,EACjB,OAAqC,EACrC,eAAwB,EACxB,aAAgC,EACf;IACjB,MAAM,KAAK,GAAG,OAAO,EAAE,QAAQ,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,KAAK,EAAE,KAAK,IAAI,QAAQ,CAAC;IACvC,OAAO;QACN,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;QACvB,SAAS,EAAE,MAAM,CAAC,GAAG;QACrB,SAAS;QACT,qBAAqB,EAAE,MAAM,CAAC,eAAe;QAC7C,WAAW,EAAE,MAAM,CAAC,KAAK;QACzB,cAAc,EAAE,MAAM,CAAC,QAAQ;QAC/B,UAAU,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAChD,mBAAmB,EAAE,mBAAmB,CAAC,QAAQ,CAAC;QAClD,QAAQ,EAAE;YACT,sEAAsE;YACtE,oEAAoE;YACpE,wDAAwD;YACxD,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,KAAK;YAC1C,MAAM,EAAE,KAAK,IAAI,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YAC7D,UAAU,EAAE,KAAK,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YACnE,KAAK;YACL,UAAU,EAAE,eAAe;YAC3B,aAAa,EAAE,kBAAkB,CAAC,eAAe,CAAC;SAClD;QACD,YAAY,EAAE,aAAa;YAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,aAAa,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,aAAa,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE;YACnF,CAAC,CAAC,SAAS;QACZ,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE;KAClF,CAAC;AAAA,CACF;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA6B,EAA4B;IACzF,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACvD,aAAa,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;IACpC,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AAAA,CAC9D;AAaD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B,EAG5D;IACF,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAChD,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAE/C,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC7F,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACrE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI;gBACzC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC;gBACb,UAAU,EAAE,CAAC;gBACb,GAAG,EAAE,CAAC;gBACN,CAAC,EAAE,CAAC;gBACJ,QAAQ,EAAE,CAAC;aACX,CAAC;YACF,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC;YACxB,KAAK,CAAC,CAAC,EAAE,CAAC;YACV,IAAI,MAAM,CAAC,QAAQ;gBAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACjC,CAAC;IACF,CAAC;IAED,MAAM,UAAU,GAAG,OAAO;SACxB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;SACzC,MAAM,CAAC,CAAC,KAAK,EAA0B,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC;SAC9D,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAChB,GAAG,KAAK;QACR,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;KACxB,CAAC,CAAC,CAAC;IAEL,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAAA,CAChC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAAoC,EAAU;IAClF,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG;QACb,kEAAkE;QAClE,kEAAkE;KAClE,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;QACjH,KAAK,CAAC,IAAI,CACT,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK;YAC5F,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,EAAE,CACxD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAAA,CACxB","sourcesContent":["/**\n * Eval harness: corpus pinning, provenance capture, and run records.\n *\n * The scoring math lives in `eval.ts`; this module is everything around it\n * that makes a number *comparable to a later number*. Three problems it\n * exists to solve, all of which bit the first eval round\n * (docs/hybrid-retrieval-design.md, \"Eval results\"):\n *\n *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,\n *     so every commit moves the thing being measured. A baseline taken today\n *     and a rerun taken after a retrieval change differ by both the change\n *     and the intervening commits, and nothing in the output says so. Fix:\n *     run against a detached git worktree pinned to an explicit SHA, and put\n *     that SHA in the record.\n *  2. **Nothing was recorded.** Results were printed to a terminal and\n *     hand-copied into a markdown table with no repo SHA, no embedder\n *     identity, and no index state. Fix: emit a machine-readable run record.\n *  3. **A degraded run looks like a real one.** With no embsearch binary the\n *     semantic and hybrid rows silently degrade to lexical, producing a table\n *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the\n *     record, plus a per-row degraded count that the writer refuses to hide.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readdirSync, readFileSync, statSync } from \"node:fs\";\nimport { execFileSync } from \"child_process\";\nimport { rmSync } from \"fs\";\nimport { tmpdir } from \"os\";\nimport path from \"path\";\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { type EvalConfig, type EvalQuery, type EvalQueryResult, evaluateQuery } from \"./eval.js\";\n\n/** Metrics aggregated per config across the whole gold set. */\nexport interface EvalAggregate {\n\tlabel: string;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n\t/** Queries scored under this config. */\n\tn: number;\n\t/** How many of them ran degraded (requested retriever unavailable). */\n\tdegraded: number;\n}\n\n/** Everything needed to decide whether two run records may be compared. */\nexport interface EvalProvenance {\n\ttimestampMs: number;\n\t/** SHA of the corpus actually indexed and searched. */\n\tcorpusSha: string;\n\t/** Ref the caller asked for, before resolution (e.g. \"HEAD\"). */\n\tcorpusRef: string;\n\t/** True when the corpus came from the live working tree rather than a\n\t *  pinned worktree — results are then not reproducible. */\n\tcorpusFromWorkingTree: boolean;\n\t/** Uncommitted changes present at run time. Only meaningful (and only\n\t *  possible) when `corpusFromWorkingTree` is true. */\n\tcorpusDirty: boolean;\n\t/** Files removed from the corpus before indexing — see\n\t *  {@link CORPUS_EXCLUSIONS}. A score against a different exclusion list is\n\t *  a score against a different corpus, so it is recorded, not assumed. */\n\tcorpusExcluded: string[];\n\t/** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,\n\t *  but differs when pinning an old corpus with today's code. */\n\tharnessSha: string;\n\t/**\n\t * Content hash of `src/core/search` + the chunker. Every tuning constant\n\t * that shapes a result — the fusion cap, top-k depths, rerank weights,\n\t * chunk sizing — lives in those files, so a changed hash means the\n\t * numbers are not comparable, without this module having to maintain a\n\t * hand-copied (and inevitably stale) list of constants.\n\t */\n\tretrievalSourceHash: string;\n\t/** Embedding backend state. `available: false` means every semantic and\n\t *  hybrid row in this record degraded to lexical. */\n\tembedder: {\n\t\tavailable: boolean;\n\t\treason?: string;\n\t\t/** Indexed chunk count when the index reached `ready`. */\n\t\tchunkCount?: number;\n\t\tphase: string;\n\t\t/** Binary that served the embeddings, and its self-reported version.\n\t\t *  The embedding model is baked into the binary at build time, so this\n\t\t *  is the only thing that identifies which model produced a score. */\n\t\tbinaryPath?: string;\n\t\tbinaryVersion?: string;\n\t};\n\t/** Daemon-side BM25 hybrid store, when the run included one. Absent means\n\t *  the record has no `daemon-hybrid` rows. */\n\tdaemonHybrid?: { available: boolean; phase: string };\n\truntime: { node: string; platform: string; arch: string };\n}\n\nexport interface EvalRunRecord {\n\tprovenance: EvalProvenance;\n\tgoldSet: { queryCount: number; byClass: Record<string, number>; goldSpanCount: number };\n\tconfigs: readonly EvalConfig[];\n\taggregates: EvalAggregate[];\n\tperQuery: Array<{ id: string; class: string; results: EvalQueryResult[] }>;\n}\n\nfunction git(repoRoot: string, args: string[]): string {\n\treturn execFileSync(\"git\", [\"-C\", repoRoot, ...args], { encoding: \"utf-8\" }).trim();\n}\n\n/** Hash every retrieval-shaping source file, so a tuning change is visible as\n *  a changed provenance field rather than an unexplained metric shift. */\nexport function hashRetrievalSource(repoRoot: string): string {\n\tconst roots = [\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/search\"),\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/embsearch/chunker.ts\"),\n\t];\n\tconst files: string[] = [];\n\tconst walk = (target: string): void => {\n\t\tlet stat: ReturnType<typeof statSync>;\n\t\ttry {\n\t\t\tstat = statSync(target);\n\t\t} catch {\n\t\t\treturn;\n\t\t}\n\t\tif (stat.isDirectory()) {\n\t\t\tfor (const entry of readdirSync(target).sort()) walk(path.join(target, entry));\n\t\t} else if (target.endsWith(\".ts\")) {\n\t\t\tfiles.push(target);\n\t\t}\n\t};\n\tfor (const root of roots) walk(root);\n\n\tconst hash = createHash(\"sha256\");\n\tfor (const file of files) {\n\t\t// Eval-only modules are excluded: changing how we measure must not look\n\t\t// like changing what we measure.\n\t\tconst base = path.basename(file);\n\t\tif (base.startsWith(\"eval\")) continue;\n\t\thash.update(path.relative(repoRoot, file).replace(/\\\\/g, \"/\"));\n\t\thash.update(readFileSync(file));\n\t}\n\treturn hash.digest(\"hex\").slice(0, 16);\n}\n\nexport interface PinnedCorpus {\n\t/** Directory to index and search. */\n\tcwd: string;\n\tsha: string;\n\tfromWorkingTree: boolean;\n\tdirty: boolean;\n\t/** Files removed from the corpus before indexing. Empty when the corpus is\n\t *  the live working tree, which is never mutated. */\n\texcluded: string[];\n\t/** Removes the worktree, if one was created. */\n\tdispose: () => void;\n}\n\n/**\n * Files that describe this eval rather than being searched by it.\n *\n * The fixtures hold all 62 query strings verbatim, so every query is a perfect\n * lexical match against its own entry, and the design note quotes the same\n * queries while discussing the classes they belong to. Measured before this\n * exclusion existed: **56 of 62 queries had one of these files in the top 10,\n * 27 of 62 had one as the #1 result, and they consumed 133 of the 620\n * top-10 slots** — a fifth of the window, spent on the eval reading itself.\n *\n * That is not a ranking artifact a reranker can fix: it displaces real answers\n * out of the window entirely, which is why two boundary-class queries were\n * absent from the top *50* rather than merely buried. Retrieving your own\n * question is not retrieval, so the corpus is scored without them.\n *\n * Removed from the pinned worktree before indexing, never from the repo — and\n * recorded in the run's provenance so a score is never silently taken against\n * a different corpus than it claims.\n */\nexport const CORPUS_EXCLUSIONS: readonly string[] = [\n\t\"packages/coding-agent/test/fixtures/search-eval.json\",\n\t\"packages/coding-agent/test/fixtures/search-eval-live.json\",\n\t\"packages/coding-agent/test/fixtures/search-eval-baseline.json\",\n\t\"docs/hybrid-retrieval-design.md\",\n];\n\n/**\n * Materialize the corpus to evaluate.\n *\n * With a `ref`, checks out a detached worktree at that commit so the corpus is\n * byte-identical on every rerun. Without one, falls back to the live working\n * tree and reports `dirty` so the record shows the run was not reproducible.\n */\nexport function pinCorpus(repoRoot: string, ref: string | undefined): PinnedCorpus {\n\tconst dirty = git(repoRoot, [\"status\", \"--porcelain\"]).length > 0;\n\tif (!ref) {\n\t\t// The live working tree is the user's checkout; deleting files from it to\n\t\t// tidy a measurement would be an unforgivable trade. Working-tree runs\n\t\t// are already stamped non-reproducible, so they carry the contamination.\n\t\treturn {\n\t\t\tcwd: repoRoot,\n\t\t\tsha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\t\tfromWorkingTree: true,\n\t\t\tdirty,\n\t\t\texcluded: [],\n\t\t\tdispose: () => {},\n\t\t};\n\t}\n\n\tconst sha = git(repoRoot, [\"rev-parse\", ref]);\n\t// Deterministic path, not mkdtemp: the embedding store is keyed by a hash of\n\t// the corpus directory, so a fresh temp path every run would re-embed all\n\t// ~17k chunks (minutes) instead of reusing the store built for this exact\n\t// SHA. The worktree is still removed afterwards; only the store persists.\n\tconst dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}`);\n\tif (existsSync(dir)) {\n\t\t// Left behind by an interrupted run — drop it so `worktree add` succeeds.\n\t\ttry {\n\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t} catch {\n\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\tgit(repoRoot, [\"worktree\", \"prune\"]);\n\t\t}\n\t}\n\tgit(repoRoot, [\"worktree\", \"add\", \"--detach\", dir, sha]);\n\n\tconst excluded: string[] = [];\n\tfor (const rel of CORPUS_EXCLUSIONS) {\n\t\tconst target = path.join(dir, rel);\n\t\tif (existsSync(target)) {\n\t\t\trmSync(target, { force: true });\n\t\t\texcluded.push(rel);\n\t\t}\n\t}\n\n\treturn {\n\t\tcwd: dir,\n\t\tsha,\n\t\tfromWorkingTree: false,\n\t\tdirty: false,\n\t\texcluded,\n\t\tdispose: () => {\n\t\t\ttry {\n\t\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t\t} catch {\n\t\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\t}\n\t\t},\n\t};\n}\n\n/** `<binary> --version`, or undefined when it cannot be run. */\nfunction probeBinaryVersion(binaryPath: string | undefined): string | undefined {\n\tif (!binaryPath) return undefined;\n\ttry {\n\t\treturn execFileSync(binaryPath, [\"--version\"], { encoding: \"utf-8\" }).trim();\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\nexport function collectProvenance(\n\trepoRoot: string,\n\tcorpus: PinnedCorpus,\n\tcorpusRef: string,\n\tservice: EmbsearchService | undefined,\n\tembsearchBinary?: string,\n\thybridService?: EmbsearchService,\n): EvalProvenance {\n\tconst state = service?.getState();\n\tconst phase = state?.phase ?? \"absent\";\n\treturn {\n\t\ttimestampMs: Date.now(),\n\t\tcorpusSha: corpus.sha,\n\t\tcorpusRef,\n\t\tcorpusFromWorkingTree: corpus.fromWorkingTree,\n\t\tcorpusDirty: corpus.dirty,\n\t\tcorpusExcluded: corpus.excluded,\n\t\tharnessSha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\tretrievalSourceHash: hashRetrievalSource(repoRoot),\n\t\tembedder: {\n\t\t\t// `ready` is the only phase the service reaches with a real embedder:\n\t\t\t// it rejects the mock backend at startup, so availability here also\n\t\t\t// certifies the numbers came from a genuine ONNX build.\n\t\t\tavailable: service?.isAvailable() ?? false,\n\t\t\treason: state && \"reason\" in state ? state.reason : undefined,\n\t\t\tchunkCount: state?.phase === \"ready\" ? state.chunkCount : undefined,\n\t\t\tphase,\n\t\t\tbinaryPath: embsearchBinary,\n\t\t\tbinaryVersion: probeBinaryVersion(embsearchBinary),\n\t\t},\n\t\tdaemonHybrid: hybridService\n\t\t\t? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }\n\t\t\t: undefined,\n\t\truntime: { node: process.version, platform: process.platform, arch: process.arch },\n\t};\n}\n\nexport function summarizeGoldSet(dataset: readonly EvalQuery[]): EvalRunRecord[\"goldSet\"] {\n\tconst byClass: Record<string, number> = {};\n\tlet goldSpanCount = 0;\n\tfor (const query of dataset) {\n\t\tbyClass[query.class] = (byClass[query.class] ?? 0) + 1;\n\t\tgoldSpanCount += query.gold.length;\n\t}\n\treturn { queryCount: dataset.length, byClass, goldSpanCount };\n}\n\nexport interface RunEvalSuiteOptions {\n\tcwd: string;\n\tdataset: readonly EvalQuery[];\n\tconfigs: readonly EvalConfig[];\n\tservice?: EmbsearchService;\n\t/** Second service backed by a daemon-side BM25 hybrid store, for the\n\t *  `daemon-hybrid` configs. Absent means those rows are omitted. */\n\thybridService?: EmbsearchService;\n\tonQuery?: (index: number, query: EvalQuery) => void;\n}\n\nexport async function runEvalSuite(options: RunEvalSuiteOptions): Promise<{\n\taggregates: EvalAggregate[];\n\tperQuery: EvalRunRecord[\"perQuery\"];\n}> {\n\tconst { cwd, dataset, configs, service } = options;\n\tconst totals = new Map<string, EvalAggregate>();\n\tconst perQuery: EvalRunRecord[\"perQuery\"] = [];\n\n\tfor (const [index, evalQuery] of dataset.entries()) {\n\t\toptions.onQuery?.(index, evalQuery);\n\t\tconst results = await evaluateQuery(cwd, evalQuery, configs, service, options.hybridService);\n\t\tperQuery.push({ id: evalQuery.id, class: evalQuery.class, results });\n\t\tfor (const result of results) {\n\t\t\tconst total = totals.get(result.label) ?? {\n\t\t\t\tlabel: result.label,\n\t\t\t\trecallAt1: 0,\n\t\t\t\trecallAt5: 0,\n\t\t\t\trecallAt10: 0,\n\t\t\t\trecallAt50: 0,\n\t\t\t\tmrr: 0,\n\t\t\t\tn: 0,\n\t\t\t\tdegraded: 0,\n\t\t\t};\n\t\t\ttotal.recallAt1 += result.recallAt1;\n\t\t\ttotal.recallAt5 += result.recallAt5;\n\t\t\ttotal.recallAt10 += result.recallAt10;\n\t\t\ttotal.recallAt50 += result.recallAt50;\n\t\t\ttotal.mrr += result.mrr;\n\t\t\ttotal.n++;\n\t\t\tif (result.degraded) total.degraded++;\n\t\t\ttotals.set(result.label, total);\n\t\t}\n\t}\n\n\tconst aggregates = configs\n\t\t.map((config) => totals.get(config.label))\n\t\t.filter((total): total is EvalAggregate => total !== undefined)\n\t\t.map((total) => ({\n\t\t\t...total,\n\t\t\trecallAt1: total.recallAt1 / total.n,\n\t\t\trecallAt5: total.recallAt5 / total.n,\n\t\t\trecallAt10: total.recallAt10 / total.n,\n\t\t\trecallAt50: total.recallAt50 / total.n,\n\t\t\tmrr: total.mrr / total.n,\n\t\t}));\n\n\treturn { aggregates, perQuery };\n}\n\nexport function formatAggregateTable(aggregates: readonly EvalAggregate[]): string {\n\tconst pct = (x: number) => `${Math.round(x * 100)}%`.padStart(5);\n\tconst lines = [\n\t\t\"config           |  R@1  |  R@5  | R@10  | R@50  |  MRR  | notes\",\n\t\t\"-----------------|-------|-------|-------|-------|-------|------\",\n\t];\n\tfor (const a of aggregates) {\n\t\tconst notes = a.degraded === a.n ? \"degraded to lexical\" : a.degraded > 0 ? `${a.degraded}/${a.n} degraded` : \"\";\n\t\tlines.push(\n\t\t\t`${a.label.padEnd(16)} | ${pct(a.recallAt1)} | ${pct(a.recallAt5)} | ${pct(a.recallAt10)} | ` +\n\t\t\t\t`${pct(a.recallAt50)} | ${a.mrr.toFixed(3)} | ${notes}`,\n\t\t);\n\t}\n\treturn lines.join(\"\\n\");\n}\n"]}

package/dist/core/search/eval.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../../src/core/search/eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAEhF,MAAM,WAAW,YAAY;IAC5B,gCAAgC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,mEAAmE;IACnE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,SAAS;IACzB,EAAE,EAAE,MAAM,CAAC;IACX;6DACyD;IACzD,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,YAAY,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;iEAC6D;IAC7D,YAAY,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,YAAY,EAAE,SAAS,UAAU,EA6B7C,CAAC;AAKF,wBAAgB,eAAe,CAAC,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,YAAY,GAAG,OAAO,CAKhF;AAED,gFAAgF;AAChF,wBAAgB,SAAS,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,YAAY,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAQhH;AAED;;;;;;;;GAQG;AACH,wBAAgB,GAAG,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,YAAY,EAAE,GAAG,MAAM,CAQ/F;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,kBAAkB,CAAC;IACjC,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;CACZ;AAED,wBAAsB,aAAa,CAClC,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,SAAS,EACpB,OAAO,GAAE,SAAS,UAAU,EAAiB,EAC7C,OAAO,CAAC,EAAE,gBAAgB,EAC1B,aAAa,CAAC,EAAE,gBAAgB,GAC9B,OAAO,CAAC,eAAe,EAAE,CAAC,CAiC5B","sourcesContent":["/**\n * Retrieval evaluation gate (docs/hybrid-retrieval-design.md, step 6 of the\n * shipping order).\n *\n * Measures Recall@K and MRR for lexical, semantic, and hybrid retrieval across\n * an RRF `k` sweep, against a gold set keyed by **path + line range matched by\n * span overlap** — never by chunkId, which is only stable per index build.\n * Recall@50 doubles as the reranker gate: a gold span that never reaches the\n * fused top-50 cannot be rescued by any reranker.\n *\n * Rank-sensitive metrics are the point of Recall@1 and MRR: an agent reads the\n * top result or two, so \"the gold span is somewhere in the top 5\" understates\n * how much ordering matters. Recall@5/10/50 are kept for continuity with the\n * numbers already published in the design note.\n *\n * Driven by `scripts/search-eval.ts` via `eval-harness.ts`; the scoring logic\n * lives here so it is unit-testable without an embedding index.\n */\n\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { retrieveCandidates } from \"./hybrid-search.js\";\nimport type { CandidateSpan, ResolvedSearchMode, SearchMode } from \"./types.js\";\n\nexport interface EvalGoldSpan {\n\t/** Repo-relative POSIX path. */\n\tpath: string;\n\t/** 1-based inclusive; omit both to accept any span in the file. */\n\tstartLine?: number;\n\tendLine?: number;\n\t/**\n\t * Literal source text that must occur inside `[startLine, endLine]`. Not\n\t * used for scoring — it is how the gold set survives the corpus moving\n\t * underneath it: `scripts/search-eval-gold.ts` re-resolves the line range\n\t * from this anchor, and the fixture test fails when an anchor no longer\n\t * sits inside its recorded range. Omitted for `scope: \"file\"` entries.\n\t */\n\tanchor?: string;\n\t/**\n\t * `\"span\"` (default) scores by overlap with the recorded line range;\n\t * `\"file\"` accepts any span in the file. File scope is deliberate for\n\t * path-class queries, where the whole file *is* the answer — it is\n\t * recorded explicitly so a file-level score is never mistaken for a\n\t * span-level one.\n\t */\n\tscope?: \"span\" | \"file\";\n}\n\nexport interface EvalQuery {\n\tid: string;\n\t/** Query class from the design doc (exact-symbol, path, error-fragment,\n\t *  conceptual, cross-file, boundary). Reporting only. */\n\tclass: string;\n\tquery: string;\n\tgold: EvalGoldSpan[];\n}\n\nexport interface EvalConfig {\n\tlabel: string;\n\tmode: SearchMode;\n\trrfK?: number;\n\trerank?: boolean;\n\t/**\n\t * Score this config against a daemon-side hybrid store (BM25 fused with\n\t * vectors inside the Rust daemon) instead of the dense-only index. Skipped\n\t * when the harness has no hybrid service, so records without one simply\n\t * omit the row rather than silently scoring it as plain semantic.\n\t */\n\tdaemonHybrid?: boolean;\n\t/**\n\t * Fetch the daemon's BM25 index as a separate leg and fuse it here with\n\t * dense and grep. Like `daemonHybrid` it needs a hybrid store, so it is\n\t * skipped when the harness has no hybrid service.\n\t */\n\tbm25Leg?: boolean;\n\t/** Rerank with the daemon's cross-encoder rather than the deterministic\n\t *  scorer. Needs a service, so it is skipped without one. */\n\tcrossEncoder?: boolean;\n}\n\n/**\n * The sweep from the design doc — single retrievers, hybrid across k, the\n * routed auto mode — plus reranked (`+rr`) variants for the step 7 gate.\n *\n * Exported because `scripts/search-eval.ts` and `test/search-eval.test.ts`\n * both consume it. It previously lost its `export` to a knip-driven\n * dead-export sweep (02efaab): the only importer was a `.mjs` script reading\n * from `dist/`, which static analysis cannot see, and the eval gate silently\n * stopped running. The TypeScript importers are the fix — do not \"clean up\"\n * this export without checking them.\n */\nexport const EVAL_CONFIGS: readonly EvalConfig[] = [\n\t{ label: \"lexical\", mode: \"lexical\" },\n\t{ label: \"semantic\", mode: \"semantic\" },\n\t{ label: \"hybrid k=0\", mode: \"hybrid\", rrfK: 0 },\n\t{ label: \"hybrid k=2\", mode: \"hybrid\", rrfK: 2 },\n\t{ label: \"hybrid k=10\", mode: \"hybrid\", rrfK: 10 },\n\t{ label: \"hybrid k=60\", mode: \"hybrid\", rrfK: 60 },\n\t{ label: \"auto\", mode: \"auto\" },\n\t{ label: \"lexical +rr\", mode: \"lexical\", rerank: true },\n\t{ label: \"semantic +rr\", mode: \"semantic\", rerank: true },\n\t{ label: \"hybrid k=2 +rr\", mode: \"hybrid\", rrfK: 2, rerank: true },\n\t{ label: \"hybrid k=60 +rr\", mode: \"hybrid\", rrfK: 60, rerank: true },\n\t{ label: \"auto +rr\", mode: \"auto\", rerank: true },\n\t// Is BM25 a better lexical leg than ripgrep? These two run the daemon's own\n\t// vector+BM25 fusion and no ripgrep at all, so comparing them against\n\t// \"semantic\" isolates what BM25 adds, and against \"hybrid k=2\" compares the\n\t// two lexical legs at the system level.\n\t{ label: \"daemon-hybrid\", mode: \"semantic\", daemonHybrid: true },\n\t{ label: \"daemon-hybrid +rr\", mode: \"semantic\", rerank: true, daemonHybrid: true },\n\t// Three-way fusion: dense + BM25 + grep, all fused here so `k` is ours and\n\t// the per-leg ranks survive into the trace. `bm25+dense` isolates what the\n\t// grep leg still contributes once BM25 is present.\n\t{ label: \"bm25+dense +rr\", mode: \"semantic\", rerank: true, bm25Leg: true },\n\t{ label: \"3-way +rr\", mode: \"hybrid\", rerank: true, bm25Leg: true },\n\t// Cross-encoder reranking, against the same retrieval each deterministic\n\t// `+rr` row uses — so the delta is the reranker and nothing else.\n\t{ label: \"semantic +ce\", mode: \"semantic\", crossEncoder: true },\n\t{ label: \"auto +ce\", mode: \"auto\", crossEncoder: true },\n\t{ label: \"bm25+dense +ce\", mode: \"semantic\", bm25Leg: true, crossEncoder: true },\n];\n\n/** Candidates fetched per eval query — deep enough for the reranker gate. */\nconst EVAL_FETCH_LIMIT = 50;\n\nexport function spanMatchesGold(span: CandidateSpan, gold: EvalGoldSpan): boolean {\n\tif (span.path !== gold.path) return false;\n\tif (gold.scope === \"file\") return true;\n\tif (gold.startLine === undefined || gold.endLine === undefined) return true;\n\treturn span.startLine <= gold.endLine && span.endLine >= gold.startLine;\n}\n\n/** Fraction of gold spans matched by at least one of the top-`k` candidates. */\nexport function recallAtK(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[], k: number): number {\n\tif (gold.length === 0) return 0;\n\tconst top = candidates.slice(0, k);\n\tlet matched = 0;\n\tfor (const g of gold) {\n\t\tif (top.some((c) => spanMatchesGold(c, g))) matched++;\n\t}\n\treturn matched / gold.length;\n}\n\n/**\n * Mean reciprocal rank, averaged over gold spans: for each gold span, `1/rank`\n * of the first candidate matching it (0 when it never appears).\n *\n * Averaging per gold span rather than taking the single first hit keeps\n * multi-span queries (the cross-file class) honest — finding one of two\n * required sites should not score like finding both. For single-span queries\n * this reduces to textbook MRR.\n */\nexport function mrr(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[]): number {\n\tif (gold.length === 0) return 0;\n\tlet total = 0;\n\tfor (const g of gold) {\n\t\tconst index = candidates.findIndex((c) => spanMatchesGold(c, g));\n\t\tif (index >= 0) total += 1 / (index + 1);\n\t}\n\treturn total / gold.length;\n}\n\nexport interface EvalQueryResult {\n\tlabel: string;\n\tresolvedMode: ResolvedSearchMode;\n\tdegraded: boolean;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n}\n\nexport async function evaluateQuery(\n\tcwd: string,\n\tevalQuery: EvalQuery,\n\tconfigs: readonly EvalConfig[] = EVAL_CONFIGS,\n\tservice?: EmbsearchService,\n\thybridService?: EmbsearchService,\n): Promise<EvalQueryResult[]> {\n\tconst results: EvalQueryResult[] = [];\n\tfor (const config of configs) {\n\t\t// A daemon-hybrid config against the plain store would error in the\n\t\t// daemon (`query_hybrid requires a hybrid store`); omit the row instead.\n\t\tif ((config.daemonHybrid || config.bm25Leg) && !hybridService) continue;\n\t\t// Cross-encoder rows need a live daemon; omit rather than silently\n\t\t// falling back to the deterministic reranker under a \"+ce\" label.\n\t\tif (config.crossEncoder && !(config.bm25Leg ? hybridService : service)?.supportsCrossEncoder()) continue;\n\t\tconst retrieved = await retrieveCandidates({\n\t\t\tcwd,\n\t\t\tquery: evalQuery.query,\n\t\t\tmode: config.mode,\n\t\t\trrfK: config.rrfK,\n\t\t\trerank: config.rerank ?? false,\n\t\t\tlimit: EVAL_FETCH_LIMIT,\n\t\t\tservice: config.daemonHybrid || config.bm25Leg ? hybridService : service,\n\t\t\tdaemonHybrid: config.daemonHybrid,\n\t\t\tbm25Leg: config.bm25Leg,\n\t\t\tcrossEncoder: config.crossEncoder,\n\t\t});\n\t\tresults.push({\n\t\t\tlabel: config.label,\n\t\t\tresolvedMode: retrieved.resolvedMode,\n\t\t\tdegraded: retrieved.degradedReason !== undefined,\n\t\t\trecallAt1: recallAtK(retrieved.candidates, evalQuery.gold, 1),\n\t\t\trecallAt5: recallAtK(retrieved.candidates, evalQuery.gold, 5),\n\t\t\trecallAt10: recallAtK(retrieved.candidates, evalQuery.gold, 10),\n\t\t\trecallAt50: recallAtK(retrieved.candidates, evalQuery.gold, 50),\n\t\t\tmrr: mrr(retrieved.candidates, evalQuery.gold),\n\t\t});\n\t}\n\treturn results;\n}\n"]}
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../../src/core/search/eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAEhF,MAAM,WAAW,YAAY;IAC5B,gCAAgC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,mEAAmE;IACnE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,SAAS;IACzB,EAAE,EAAE,MAAM,CAAC;IACX;6DACyD;IACzD,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,YAAY,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;iEAC6D;IAC7D,YAAY,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,YAAY,EAAE,SAAS,UAAU,EAoC7C,CAAC;AAKF,wBAAgB,eAAe,CAAC,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,YAAY,GAAG,OAAO,CAKhF;AAED,gFAAgF;AAChF,wBAAgB,SAAS,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,YAAY,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAQhH;AAED;;;;;;;;GAQG;AACH,wBAAgB,GAAG,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,YAAY,EAAE,GAAG,MAAM,CAQ/F;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,kBAAkB,CAAC;IACjC,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;CACZ;AAED,wBAAsB,aAAa,CAClC,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,SAAS,EACpB,OAAO,GAAE,SAAS,UAAU,EAAiB,EAC7C,OAAO,CAAC,EAAE,gBAAgB,EAC1B,aAAa,CAAC,EAAE,gBAAgB,GAC9B,OAAO,CAAC,eAAe,EAAE,CAAC,CAiC5B","sourcesContent":["/**\n * Retrieval evaluation gate (docs/hybrid-retrieval-design.md, step 6 of the\n * shipping order).\n *\n * Measures Recall@K and MRR for lexical, semantic, and hybrid retrieval across\n * an RRF `k` sweep, against a gold set keyed by **path + line range matched by\n * span overlap** — never by chunkId, which is only stable per index build.\n * Recall@50 doubles as the reranker gate: a gold span that never reaches the\n * fused top-50 cannot be rescued by any reranker.\n *\n * Rank-sensitive metrics are the point of Recall@1 and MRR: an agent reads the\n * top result or two, so \"the gold span is somewhere in the top 5\" understates\n * how much ordering matters. Recall@5/10/50 are kept for continuity with the\n * numbers already published in the design note.\n *\n * Driven by `scripts/search-eval.ts` via `eval-harness.ts`; the scoring logic\n * lives here so it is unit-testable without an embedding index.\n */\n\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { retrieveCandidates } from \"./hybrid-search.js\";\nimport type { CandidateSpan, ResolvedSearchMode, SearchMode } from \"./types.js\";\n\nexport interface EvalGoldSpan {\n\t/** Repo-relative POSIX path. */\n\tpath: string;\n\t/** 1-based inclusive; omit both to accept any span in the file. */\n\tstartLine?: number;\n\tendLine?: number;\n\t/**\n\t * Literal source text that must occur inside `[startLine, endLine]`. Not\n\t * used for scoring — it is how the gold set survives the corpus moving\n\t * underneath it: `scripts/search-eval-gold.ts` re-resolves the line range\n\t * from this anchor, and the fixture test fails when an anchor no longer\n\t * sits inside its recorded range. Omitted for `scope: \"file\"` entries.\n\t */\n\tanchor?: string;\n\t/**\n\t * `\"span\"` (default) scores by overlap with the recorded line range;\n\t * `\"file\"` accepts any span in the file. File scope is deliberate for\n\t * path-class queries, where the whole file *is* the answer — it is\n\t * recorded explicitly so a file-level score is never mistaken for a\n\t * span-level one.\n\t */\n\tscope?: \"span\" | \"file\";\n}\n\nexport interface EvalQuery {\n\tid: string;\n\t/** Query class from the design doc (exact-symbol, path, error-fragment,\n\t *  conceptual, cross-file, boundary). Reporting only. */\n\tclass: string;\n\tquery: string;\n\tgold: EvalGoldSpan[];\n}\n\nexport interface EvalConfig {\n\tlabel: string;\n\tmode: SearchMode;\n\trrfK?: number;\n\trerank?: boolean;\n\t/**\n\t * Score this config against a daemon-side hybrid store (BM25 fused with\n\t * vectors inside the Rust daemon) instead of the dense-only index. Skipped\n\t * when the harness has no hybrid service, so records without one simply\n\t * omit the row rather than silently scoring it as plain semantic.\n\t */\n\tdaemonHybrid?: boolean;\n\t/**\n\t * Fetch the daemon's BM25 index as a separate leg and fuse it here with\n\t * dense and grep. Like `daemonHybrid` it needs a hybrid store, so it is\n\t * skipped when the harness has no hybrid service.\n\t */\n\tbm25Leg?: boolean;\n\t/** Rerank with the daemon's cross-encoder rather than the deterministic\n\t *  scorer. Needs a service, so it is skipped without one. */\n\tcrossEncoder?: boolean;\n}\n\n/**\n * The sweep from the design doc — single retrievers, hybrid across k, the\n * routed auto mode — plus reranked (`+rr`) variants for the step 7 gate.\n *\n * Exported because `scripts/search-eval.ts` and `test/search-eval.test.ts`\n * both consume it. It previously lost its `export` to a knip-driven\n * dead-export sweep (02efaab): the only importer was a `.mjs` script reading\n * from `dist/`, which static analysis cannot see, and the eval gate silently\n * stopped running. The TypeScript importers are the fix — do not \"clean up\"\n * this export without checking them.\n */\nexport const EVAL_CONFIGS: readonly EvalConfig[] = [\n\t{ label: \"lexical\", mode: \"lexical\" },\n\t{ label: \"semantic\", mode: \"semantic\" },\n\t{ label: \"hybrid k=0\", mode: \"hybrid\", rrfK: 0 },\n\t{ label: \"hybrid k=2\", mode: \"hybrid\", rrfK: 2 },\n\t{ label: \"hybrid k=10\", mode: \"hybrid\", rrfK: 10 },\n\t{ label: \"hybrid k=60\", mode: \"hybrid\", rrfK: 60 },\n\t{ label: \"auto\", mode: \"auto\" },\n\t{ label: \"lexical +rr\", mode: \"lexical\", rerank: true },\n\t{ label: \"semantic +rr\", mode: \"semantic\", rerank: true },\n\t{ label: \"hybrid k=2 +rr\", mode: \"hybrid\", rrfK: 2, rerank: true },\n\t{ label: \"hybrid k=60 +rr\", mode: \"hybrid\", rrfK: 60, rerank: true },\n\t{ label: \"auto +rr\", mode: \"auto\", rerank: true },\n\t// Is BM25 a better lexical leg than ripgrep? These two run the daemon's own\n\t// vector+BM25 fusion and no ripgrep at all, so comparing them against\n\t// \"semantic\" isolates what BM25 adds, and against \"hybrid k=2\" compares the\n\t// two lexical legs at the system level.\n\t{ label: \"daemon-hybrid\", mode: \"semantic\", daemonHybrid: true },\n\t{ label: \"daemon-hybrid +rr\", mode: \"semantic\", rerank: true, daemonHybrid: true },\n\t// Three-way fusion: dense + BM25 + grep, all fused here so `k` is ours and\n\t// the per-leg ranks survive into the trace. `bm25+dense` isolates what the\n\t// grep leg still contributes once BM25 is present.\n\t//\n\t// `3-way` used to lose badly to `bm25+dense` (MRR 0.608 -> 0.552, 18 of 62\n\t// queries worse) because two lexical views of the same corpus dilute each\n\t// other in fusion. Its grep leg is now scoped to files the index has not\n\t// read, so the two rows are identical on this corpus by construction —\n\t// the difference only appears under `--live-edits`, where `3-way` scores\n\t// 1.000 and `bm25+dense` scores 0.000.\n\t{ label: \"bm25+dense +rr\", mode: \"semantic\", rerank: true, bm25Leg: true },\n\t{ label: \"3-way +rr\", mode: \"hybrid\", rerank: true, bm25Leg: true },\n\t// Cross-encoder reranking, against the same retrieval each deterministic\n\t// `+rr` row uses — so the delta is the reranker and nothing else.\n\t{ label: \"semantic +ce\", mode: \"semantic\", crossEncoder: true },\n\t{ label: \"auto +ce\", mode: \"auto\", crossEncoder: true },\n\t{ label: \"bm25+dense +ce\", mode: \"semantic\", bm25Leg: true, crossEncoder: true },\n];\n\n/** Candidates fetched per eval query — deep enough for the reranker gate. */\nconst EVAL_FETCH_LIMIT = 50;\n\nexport function spanMatchesGold(span: CandidateSpan, gold: EvalGoldSpan): boolean {\n\tif (span.path !== gold.path) return false;\n\tif (gold.scope === \"file\") return true;\n\tif (gold.startLine === undefined || gold.endLine === undefined) return true;\n\treturn span.startLine <= gold.endLine && span.endLine >= gold.startLine;\n}\n\n/** Fraction of gold spans matched by at least one of the top-`k` candidates. */\nexport function recallAtK(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[], k: number): number {\n\tif (gold.length === 0) return 0;\n\tconst top = candidates.slice(0, k);\n\tlet matched = 0;\n\tfor (const g of gold) {\n\t\tif (top.some((c) => spanMatchesGold(c, g))) matched++;\n\t}\n\treturn matched / gold.length;\n}\n\n/**\n * Mean reciprocal rank, averaged over gold spans: for each gold span, `1/rank`\n * of the first candidate matching it (0 when it never appears).\n *\n * Averaging per gold span rather than taking the single first hit keeps\n * multi-span queries (the cross-file class) honest — finding one of two\n * required sites should not score like finding both. For single-span queries\n * this reduces to textbook MRR.\n */\nexport function mrr(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[]): number {\n\tif (gold.length === 0) return 0;\n\tlet total = 0;\n\tfor (const g of gold) {\n\t\tconst index = candidates.findIndex((c) => spanMatchesGold(c, g));\n\t\tif (index >= 0) total += 1 / (index + 1);\n\t}\n\treturn total / gold.length;\n}\n\nexport interface EvalQueryResult {\n\tlabel: string;\n\tresolvedMode: ResolvedSearchMode;\n\tdegraded: boolean;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n}\n\nexport async function evaluateQuery(\n\tcwd: string,\n\tevalQuery: EvalQuery,\n\tconfigs: readonly EvalConfig[] = EVAL_CONFIGS,\n\tservice?: EmbsearchService,\n\thybridService?: EmbsearchService,\n): Promise<EvalQueryResult[]> {\n\tconst results: EvalQueryResult[] = [];\n\tfor (const config of configs) {\n\t\t// A daemon-hybrid config against the plain store would error in the\n\t\t// daemon (`query_hybrid requires a hybrid store`); omit the row instead.\n\t\tif ((config.daemonHybrid || config.bm25Leg) && !hybridService) continue;\n\t\t// Cross-encoder rows need a live daemon; omit rather than silently\n\t\t// falling back to the deterministic reranker under a \"+ce\" label.\n\t\tif (config.crossEncoder && !(config.bm25Leg ? hybridService : service)?.supportsCrossEncoder()) continue;\n\t\tconst retrieved = await retrieveCandidates({\n\t\t\tcwd,\n\t\t\tquery: evalQuery.query,\n\t\t\tmode: config.mode,\n\t\t\trrfK: config.rrfK,\n\t\t\trerank: config.rerank ?? false,\n\t\t\tlimit: EVAL_FETCH_LIMIT,\n\t\t\tservice: config.daemonHybrid || config.bm25Leg ? hybridService : service,\n\t\t\tdaemonHybrid: config.daemonHybrid,\n\t\t\tbm25Leg: config.bm25Leg,\n\t\t\tcrossEncoder: config.crossEncoder,\n\t\t});\n\t\tresults.push({\n\t\t\tlabel: config.label,\n\t\t\tresolvedMode: retrieved.resolvedMode,\n\t\t\tdegraded: retrieved.degradedReason !== undefined,\n\t\t\trecallAt1: recallAtK(retrieved.candidates, evalQuery.gold, 1),\n\t\t\trecallAt5: recallAtK(retrieved.candidates, evalQuery.gold, 5),\n\t\t\trecallAt10: recallAtK(retrieved.candidates, evalQuery.gold, 10),\n\t\t\trecallAt50: recallAtK(retrieved.candidates, evalQuery.gold, 50),\n\t\t\tmrr: mrr(retrieved.candidates, evalQuery.gold),\n\t\t});\n\t}\n\treturn results;\n}\n"]}

package/dist/core/search/eval.js

@@ -50,6 +50,13 @@ export const EVAL_CONFIGS = [
     // Three-way fusion: dense + BM25 + grep, all fused here so `k` is ours and
     // the per-leg ranks survive into the trace. `bm25+dense` isolates what the
     // grep leg still contributes once BM25 is present.
+    //
+    // `3-way` used to lose badly to `bm25+dense` (MRR 0.608 -> 0.552, 18 of 62
+    // queries worse) because two lexical views of the same corpus dilute each
+    // other in fusion. Its grep leg is now scoped to files the index has not
+    // read, so the two rows are identical on this corpus by construction —
+    // the difference only appears under `--live-edits`, where `3-way` scores
+    // 1.000 and `bm25+dense` scores 0.000.
     { label: "bm25+dense +rr", mode: "semantic", rerank: true, bm25Leg: true },
     { label: "3-way +rr", mode: "hybrid", rerank: true, bm25Leg: true },
     // Cross-encoder reranking, against the same retrieval each deterministic

package/dist/core/search/eval.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../../src/core/search/eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AA2DxD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,YAAY,GAA0B;IAClD,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE;IACrC,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE;IACvC,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE;IAChD,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE;IAChD,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE;IAClD,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE;IAClD,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;IAC/B,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE;IACvD,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE;IACzD,EAAE,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;IAClE,EAAE,KAAK,EAAE,iBAAiB,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;IACpE,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE;IACjD,4EAA4E;IAC5E,sEAAsE;IACtE,4EAA4E;IAC5E,wCAAwC;IACxC,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,IAAI,EAAE;IAChE,EAAE,KAAK,EAAE,mBAAmB,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE;IAClF,2EAA2E;IAC3E,2EAA2E;IAC3E,mDAAmD;IACnD,EAAE,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE;IAC1E,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE;IACnE,yEAAyE;IACzE,oEAAkE;IAClE,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,IAAI,EAAE;IAC/D,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE;IACvD,EAAE,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE;CAChF,CAAC;AAEF,+EAA6E;AAC7E,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,MAAM,UAAU,eAAe,CAAC,IAAmB,EAAE,IAAkB,EAAW;IACjF,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACvC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5E,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC;AAAA,CACxE;AAED,gFAAgF;AAChF,MAAM,UAAU,SAAS,CAAC,UAAoC,EAAE,IAA6B,EAAE,CAAS,EAAU;IACjH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACtB,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;IACvD,CAAC;IACD,OAAO,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;AAAA,CAC7B;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,GAAG,CAAC,UAAoC,EAAE,IAA6B,EAAU;IAChG,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjE,IAAI,KAAK,IAAI,CAAC;YAAE,KAAK,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;AAAA,CAC3B;AAaD,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,GAAW,EACX,SAAoB,EACpB,OAAO,GAA0B,YAAY,EAC7C,OAA0B,EAC1B,aAAgC,EACH;IAC7B,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,oEAAoE;QACpE,yEAAyE;QACzE,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa;YAAE,SAAS;QACxE,mEAAmE;QACnE,kEAAkE;QAClE,IAAI,MAAM,CAAC,YAAY,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,oBAAoB,EAAE;YAAE,SAAS;QACzG,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC;YAC1C,GAAG;YACH,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,KAAK;YAC9B,KAAK,EAAE,gBAAgB;YACvB,OAAO,EAAE,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO;YACxE,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,YAAY,EAAE,MAAM,CAAC,YAAY;SACjC,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC;YACZ,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,YAAY,EAAE,SAAS,CAAC,YAAY;YACpC,QAAQ,EAAE,SAAS,CAAC,cAAc,KAAK,SAAS;YAChD,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7D,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7D,UAAU,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/D,UAAU,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/D,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,CAAC;SAC9C,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AAAA,CACf","sourcesContent":["/**\n * Retrieval evaluation gate (docs/hybrid-retrieval-design.md, step 6 of the\n * shipping order).\n *\n * Measures Recall@K and MRR for lexical, semantic, and hybrid retrieval across\n * an RRF `k` sweep, against a gold set keyed by **path + line range matched by\n * span overlap** — never by chunkId, which is only stable per index build.\n * Recall@50 doubles as the reranker gate: a gold span that never reaches the\n * fused top-50 cannot be rescued by any reranker.\n *\n * Rank-sensitive metrics are the point of Recall@1 and MRR: an agent reads the\n * top result or two, so \"the gold span is somewhere in the top 5\" understates\n * how much ordering matters. Recall@5/10/50 are kept for continuity with the\n * numbers already published in the design note.\n *\n * Driven by `scripts/search-eval.ts` via `eval-harness.ts`; the scoring logic\n * lives here so it is unit-testable without an embedding index.\n */\n\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { retrieveCandidates } from \"./hybrid-search.js\";\nimport type { CandidateSpan, ResolvedSearchMode, SearchMode } from \"./types.js\";\n\nexport interface EvalGoldSpan {\n\t/** Repo-relative POSIX path. */\n\tpath: string;\n\t/** 1-based inclusive; omit both to accept any span in the file. */\n\tstartLine?: number;\n\tendLine?: number;\n\t/**\n\t * Literal source text that must occur inside `[startLine, endLine]`. Not\n\t * used for scoring — it is how the gold set survives the corpus moving\n\t * underneath it: `scripts/search-eval-gold.ts` re-resolves the line range\n\t * from this anchor, and the fixture test fails when an anchor no longer\n\t * sits inside its recorded range. Omitted for `scope: \"file\"` entries.\n\t */\n\tanchor?: string;\n\t/**\n\t * `\"span\"` (default) scores by overlap with the recorded line range;\n\t * `\"file\"` accepts any span in the file. File scope is deliberate for\n\t * path-class queries, where the whole file *is* the answer — it is\n\t * recorded explicitly so a file-level score is never mistaken for a\n\t * span-level one.\n\t */\n\tscope?: \"span\" | \"file\";\n}\n\nexport interface EvalQuery {\n\tid: string;\n\t/** Query class from the design doc (exact-symbol, path, error-fragment,\n\t *  conceptual, cross-file, boundary). Reporting only. */\n\tclass: string;\n\tquery: string;\n\tgold: EvalGoldSpan[];\n}\n\nexport interface EvalConfig {\n\tlabel: string;\n\tmode: SearchMode;\n\trrfK?: number;\n\trerank?: boolean;\n\t/**\n\t * Score this config against a daemon-side hybrid store (BM25 fused with\n\t * vectors inside the Rust daemon) instead of the dense-only index. Skipped\n\t * when the harness has no hybrid service, so records without one simply\n\t * omit the row rather than silently scoring it as plain semantic.\n\t */\n\tdaemonHybrid?: boolean;\n\t/**\n\t * Fetch the daemon's BM25 index as a separate leg and fuse it here with\n\t * dense and grep. Like `daemonHybrid` it needs a hybrid store, so it is\n\t * skipped when the harness has no hybrid service.\n\t */\n\tbm25Leg?: boolean;\n\t/** Rerank with the daemon's cross-encoder rather than the deterministic\n\t *  scorer. Needs a service, so it is skipped without one. */\n\tcrossEncoder?: boolean;\n}\n\n/**\n * The sweep from the design doc — single retrievers, hybrid across k, the\n * routed auto mode — plus reranked (`+rr`) variants for the step 7 gate.\n *\n * Exported because `scripts/search-eval.ts` and `test/search-eval.test.ts`\n * both consume it. It previously lost its `export` to a knip-driven\n * dead-export sweep (02efaab): the only importer was a `.mjs` script reading\n * from `dist/`, which static analysis cannot see, and the eval gate silently\n * stopped running. The TypeScript importers are the fix — do not \"clean up\"\n * this export without checking them.\n */\nexport const EVAL_CONFIGS: readonly EvalConfig[] = [\n\t{ label: \"lexical\", mode: \"lexical\" },\n\t{ label: \"semantic\", mode: \"semantic\" },\n\t{ label: \"hybrid k=0\", mode: \"hybrid\", rrfK: 0 },\n\t{ label: \"hybrid k=2\", mode: \"hybrid\", rrfK: 2 },\n\t{ label: \"hybrid k=10\", mode: \"hybrid\", rrfK: 10 },\n\t{ label: \"hybrid k=60\", mode: \"hybrid\", rrfK: 60 },\n\t{ label: \"auto\", mode: \"auto\" },\n\t{ label: \"lexical +rr\", mode: \"lexical\", rerank: true },\n\t{ label: \"semantic +rr\", mode: \"semantic\", rerank: true },\n\t{ label: \"hybrid k=2 +rr\", mode: \"hybrid\", rrfK: 2, rerank: true },\n\t{ label: \"hybrid k=60 +rr\", mode: \"hybrid\", rrfK: 60, rerank: true },\n\t{ label: \"auto +rr\", mode: \"auto\", rerank: true },\n\t// Is BM25 a better lexical leg than ripgrep? These two run the daemon's own\n\t// vector+BM25 fusion and no ripgrep at all, so comparing them against\n\t// \"semantic\" isolates what BM25 adds, and against \"hybrid k=2\" compares the\n\t// two lexical legs at the system level.\n\t{ label: \"daemon-hybrid\", mode: \"semantic\", daemonHybrid: true },\n\t{ label: \"daemon-hybrid +rr\", mode: \"semantic\", rerank: true, daemonHybrid: true },\n\t// Three-way fusion: dense + BM25 + grep, all fused here so `k` is ours and\n\t// the per-leg ranks survive into the trace. `bm25+dense` isolates what the\n\t// grep leg still contributes once BM25 is present.\n\t{ label: \"bm25+dense +rr\", mode: \"semantic\", rerank: true, bm25Leg: true },\n\t{ label: \"3-way +rr\", mode: \"hybrid\", rerank: true, bm25Leg: true },\n\t// Cross-encoder reranking, against the same retrieval each deterministic\n\t// `+rr` row uses — so the delta is the reranker and nothing else.\n\t{ label: \"semantic +ce\", mode: \"semantic\", crossEncoder: true },\n\t{ label: \"auto +ce\", mode: \"auto\", crossEncoder: true },\n\t{ label: \"bm25+dense +ce\", mode: \"semantic\", bm25Leg: true, crossEncoder: true },\n];\n\n/** Candidates fetched per eval query — deep enough for the reranker gate. */\nconst EVAL_FETCH_LIMIT = 50;\n\nexport function spanMatchesGold(span: CandidateSpan, gold: EvalGoldSpan): boolean {\n\tif (span.path !== gold.path) return false;\n\tif (gold.scope === \"file\") return true;\n\tif (gold.startLine === undefined || gold.endLine === undefined) return true;\n\treturn span.startLine <= gold.endLine && span.endLine >= gold.startLine;\n}\n\n/** Fraction of gold spans matched by at least one of the top-`k` candidates. */\nexport function recallAtK(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[], k: number): number {\n\tif (gold.length === 0) return 0;\n\tconst top = candidates.slice(0, k);\n\tlet matched = 0;\n\tfor (const g of gold) {\n\t\tif (top.some((c) => spanMatchesGold(c, g))) matched++;\n\t}\n\treturn matched / gold.length;\n}\n\n/**\n * Mean reciprocal rank, averaged over gold spans: for each gold span, `1/rank`\n * of the first candidate matching it (0 when it never appears).\n *\n * Averaging per gold span rather than taking the single first hit keeps\n * multi-span queries (the cross-file class) honest — finding one of two\n * required sites should not score like finding both. For single-span queries\n * this reduces to textbook MRR.\n */\nexport function mrr(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[]): number {\n\tif (gold.length === 0) return 0;\n\tlet total = 0;\n\tfor (const g of gold) {\n\t\tconst index = candidates.findIndex((c) => spanMatchesGold(c, g));\n\t\tif (index >= 0) total += 1 / (index + 1);\n\t}\n\treturn total / gold.length;\n}\n\nexport interface EvalQueryResult {\n\tlabel: string;\n\tresolvedMode: ResolvedSearchMode;\n\tdegraded: boolean;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n}\n\nexport async function evaluateQuery(\n\tcwd: string,\n\tevalQuery: EvalQuery,\n\tconfigs: readonly EvalConfig[] = EVAL_CONFIGS,\n\tservice?: EmbsearchService,\n\thybridService?: EmbsearchService,\n): Promise<EvalQueryResult[]> {\n\tconst results: EvalQueryResult[] = [];\n\tfor (const config of configs) {\n\t\t// A daemon-hybrid config against the plain store would error in the\n\t\t// daemon (`query_hybrid requires a hybrid store`); omit the row instead.\n\t\tif ((config.daemonHybrid || config.bm25Leg) && !hybridService) continue;\n\t\t// Cross-encoder rows need a live daemon; omit rather than silently\n\t\t// falling back to the deterministic reranker under a \"+ce\" label.\n\t\tif (config.crossEncoder && !(config.bm25Leg ? hybridService : service)?.supportsCrossEncoder()) continue;\n\t\tconst retrieved = await retrieveCandidates({\n\t\t\tcwd,\n\t\t\tquery: evalQuery.query,\n\t\t\tmode: config.mode,\n\t\t\trrfK: config.rrfK,\n\t\t\trerank: config.rerank ?? false,\n\t\t\tlimit: EVAL_FETCH_LIMIT,\n\t\t\tservice: config.daemonHybrid || config.bm25Leg ? hybridService : service,\n\t\t\tdaemonHybrid: config.daemonHybrid,\n\t\t\tbm25Leg: config.bm25Leg,\n\t\t\tcrossEncoder: config.crossEncoder,\n\t\t});\n\t\tresults.push({\n\t\t\tlabel: config.label,\n\t\t\tresolvedMode: retrieved.resolvedMode,\n\t\t\tdegraded: retrieved.degradedReason !== undefined,\n\t\t\trecallAt1: recallAtK(retrieved.candidates, evalQuery.gold, 1),\n\t\t\trecallAt5: recallAtK(retrieved.candidates, evalQuery.gold, 5),\n\t\t\trecallAt10: recallAtK(retrieved.candidates, evalQuery.gold, 10),\n\t\t\trecallAt50: recallAtK(retrieved.candidates, evalQuery.gold, 50),\n\t\t\tmrr: mrr(retrieved.candidates, evalQuery.gold),\n\t\t});\n\t}\n\treturn results;\n}\n"]}
+{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../../src/core/search/eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AA2DxD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,YAAY,GAA0B;IAClD,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE;IACrC,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE;IACvC,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE;IAChD,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE;IAChD,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE;IAClD,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE;IAClD,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;IAC/B,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE;IACvD,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE;IACzD,EAAE,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;IAClE,EAAE,KAAK,EAAE,iBAAiB,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;IACpE,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE;IACjD,4EAA4E;IAC5E,sEAAsE;IACtE,4EAA4E;IAC5E,wCAAwC;IACxC,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,IAAI,EAAE;IAChE,EAAE,KAAK,EAAE,mBAAmB,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE;IAClF,2EAA2E;IAC3E,2EAA2E;IAC3E,mDAAmD;IACnD,EAAE;IACF,2EAA2E;IAC3E,0EAA0E;IAC1E,yEAAyE;IACzE,yEAAuE;IACvE,yEAAyE;IACzE,uCAAuC;IACvC,EAAE,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE;IAC1E,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE;IACnE,yEAAyE;IACzE,oEAAkE;IAClE,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,IAAI,EAAE;IAC/D,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE;IACvD,EAAE,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE;CAChF,CAAC;AAEF,+EAA6E;AAC7E,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,MAAM,UAAU,eAAe,CAAC,IAAmB,EAAE,IAAkB,EAAW;IACjF,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACvC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5E,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC;AAAA,CACxE;AAED,gFAAgF;AAChF,MAAM,UAAU,SAAS,CAAC,UAAoC,EAAE,IAA6B,EAAE,CAAS,EAAU;IACjH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACtB,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;IACvD,CAAC;IACD,OAAO,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;AAAA,CAC7B;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,GAAG,CAAC,UAAoC,EAAE,IAA6B,EAAU;IAChG,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjE,IAAI,KAAK,IAAI,CAAC;YAAE,KAAK,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;AAAA,CAC3B;AAaD,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,GAAW,EACX,SAAoB,EACpB,OAAO,GAA0B,YAAY,EAC7C,OAA0B,EAC1B,aAAgC,EACH;IAC7B,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,oEAAoE;QACpE,yEAAyE;QACzE,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa;YAAE,SAAS;QACxE,mEAAmE;QACnE,kEAAkE;QAClE,IAAI,MAAM,CAAC,YAAY,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,oBAAoB,EAAE;YAAE,SAAS;QACzG,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC;YAC1C,GAAG;YACH,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,KAAK;YAC9B,KAAK,EAAE,gBAAgB;YACvB,OAAO,EAAE,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO;YACxE,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,YAAY,EAAE,MAAM,CAAC,YAAY;SACjC,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC;YACZ,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,YAAY,EAAE,SAAS,CAAC,YAAY;YACpC,QAAQ,EAAE,SAAS,CAAC,cAAc,KAAK,SAAS;YAChD,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7D,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7D,UAAU,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/D,UAAU,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/D,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,CAAC;SAC9C,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AAAA,CACf","sourcesContent":["/**\n * Retrieval evaluation gate (docs/hybrid-retrieval-design.md, step 6 of the\n * shipping order).\n *\n * Measures Recall@K and MRR for lexical, semantic, and hybrid retrieval across\n * an RRF `k` sweep, against a gold set keyed by **path + line range matched by\n * span overlap** — never by chunkId, which is only stable per index build.\n * Recall@50 doubles as the reranker gate: a gold span that never reaches the\n * fused top-50 cannot be rescued by any reranker.\n *\n * Rank-sensitive metrics are the point of Recall@1 and MRR: an agent reads the\n * top result or two, so \"the gold span is somewhere in the top 5\" understates\n * how much ordering matters. Recall@5/10/50 are kept for continuity with the\n * numbers already published in the design note.\n *\n * Driven by `scripts/search-eval.ts` via `eval-harness.ts`; the scoring logic\n * lives here so it is unit-testable without an embedding index.\n */\n\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { retrieveCandidates } from \"./hybrid-search.js\";\nimport type { CandidateSpan, ResolvedSearchMode, SearchMode } from \"./types.js\";\n\nexport interface EvalGoldSpan {\n\t/** Repo-relative POSIX path. */\n\tpath: string;\n\t/** 1-based inclusive; omit both to accept any span in the file. */\n\tstartLine?: number;\n\tendLine?: number;\n\t/**\n\t * Literal source text that must occur inside `[startLine, endLine]`. Not\n\t * used for scoring — it is how the gold set survives the corpus moving\n\t * underneath it: `scripts/search-eval-gold.ts` re-resolves the line range\n\t * from this anchor, and the fixture test fails when an anchor no longer\n\t * sits inside its recorded range. Omitted for `scope: \"file\"` entries.\n\t */\n\tanchor?: string;\n\t/**\n\t * `\"span\"` (default) scores by overlap with the recorded line range;\n\t * `\"file\"` accepts any span in the file. File scope is deliberate for\n\t * path-class queries, where the whole file *is* the answer — it is\n\t * recorded explicitly so a file-level score is never mistaken for a\n\t * span-level one.\n\t */\n\tscope?: \"span\" | \"file\";\n}\n\nexport interface EvalQuery {\n\tid: string;\n\t/** Query class from the design doc (exact-symbol, path, error-fragment,\n\t *  conceptual, cross-file, boundary). Reporting only. */\n\tclass: string;\n\tquery: string;\n\tgold: EvalGoldSpan[];\n}\n\nexport interface EvalConfig {\n\tlabel: string;\n\tmode: SearchMode;\n\trrfK?: number;\n\trerank?: boolean;\n\t/**\n\t * Score this config against a daemon-side hybrid store (BM25 fused with\n\t * vectors inside the Rust daemon) instead of the dense-only index. Skipped\n\t * when the harness has no hybrid service, so records without one simply\n\t * omit the row rather than silently scoring it as plain semantic.\n\t */\n\tdaemonHybrid?: boolean;\n\t/**\n\t * Fetch the daemon's BM25 index as a separate leg and fuse it here with\n\t * dense and grep. Like `daemonHybrid` it needs a hybrid store, so it is\n\t * skipped when the harness has no hybrid service.\n\t */\n\tbm25Leg?: boolean;\n\t/** Rerank with the daemon's cross-encoder rather than the deterministic\n\t *  scorer. Needs a service, so it is skipped without one. */\n\tcrossEncoder?: boolean;\n}\n\n/**\n * The sweep from the design doc — single retrievers, hybrid across k, the\n * routed auto mode — plus reranked (`+rr`) variants for the step 7 gate.\n *\n * Exported because `scripts/search-eval.ts` and `test/search-eval.test.ts`\n * both consume it. It previously lost its `export` to a knip-driven\n * dead-export sweep (02efaab): the only importer was a `.mjs` script reading\n * from `dist/`, which static analysis cannot see, and the eval gate silently\n * stopped running. The TypeScript importers are the fix — do not \"clean up\"\n * this export without checking them.\n */\nexport const EVAL_CONFIGS: readonly EvalConfig[] = [\n\t{ label: \"lexical\", mode: \"lexical\" },\n\t{ label: \"semantic\", mode: \"semantic\" },\n\t{ label: \"hybrid k=0\", mode: \"hybrid\", rrfK: 0 },\n\t{ label: \"hybrid k=2\", mode: \"hybrid\", rrfK: 2 },\n\t{ label: \"hybrid k=10\", mode: \"hybrid\", rrfK: 10 },\n\t{ label: \"hybrid k=60\", mode: \"hybrid\", rrfK: 60 },\n\t{ label: \"auto\", mode: \"auto\" },\n\t{ label: \"lexical +rr\", mode: \"lexical\", rerank: true },\n\t{ label: \"semantic +rr\", mode: \"semantic\", rerank: true },\n\t{ label: \"hybrid k=2 +rr\", mode: \"hybrid\", rrfK: 2, rerank: true },\n\t{ label: \"hybrid k=60 +rr\", mode: \"hybrid\", rrfK: 60, rerank: true },\n\t{ label: \"auto +rr\", mode: \"auto\", rerank: true },\n\t// Is BM25 a better lexical leg than ripgrep? These two run the daemon's own\n\t// vector+BM25 fusion and no ripgrep at all, so comparing them against\n\t// \"semantic\" isolates what BM25 adds, and against \"hybrid k=2\" compares the\n\t// two lexical legs at the system level.\n\t{ label: \"daemon-hybrid\", mode: \"semantic\", daemonHybrid: true },\n\t{ label: \"daemon-hybrid +rr\", mode: \"semantic\", rerank: true, daemonHybrid: true },\n\t// Three-way fusion: dense + BM25 + grep, all fused here so `k` is ours and\n\t// the per-leg ranks survive into the trace. `bm25+dense` isolates what the\n\t// grep leg still contributes once BM25 is present.\n\t//\n\t// `3-way` used to lose badly to `bm25+dense` (MRR 0.608 -> 0.552, 18 of 62\n\t// queries worse) because two lexical views of the same corpus dilute each\n\t// other in fusion. Its grep leg is now scoped to files the index has not\n\t// read, so the two rows are identical on this corpus by construction —\n\t// the difference only appears under `--live-edits`, where `3-way` scores\n\t// 1.000 and `bm25+dense` scores 0.000.\n\t{ label: \"bm25+dense +rr\", mode: \"semantic\", rerank: true, bm25Leg: true },\n\t{ label: \"3-way +rr\", mode: \"hybrid\", rerank: true, bm25Leg: true },\n\t// Cross-encoder reranking, against the same retrieval each deterministic\n\t// `+rr` row uses — so the delta is the reranker and nothing else.\n\t{ label: \"semantic +ce\", mode: \"semantic\", crossEncoder: true },\n\t{ label: \"auto +ce\", mode: \"auto\", crossEncoder: true },\n\t{ label: \"bm25+dense +ce\", mode: \"semantic\", bm25Leg: true, crossEncoder: true },\n];\n\n/** Candidates fetched per eval query — deep enough for the reranker gate. */\nconst EVAL_FETCH_LIMIT = 50;\n\nexport function spanMatchesGold(span: CandidateSpan, gold: EvalGoldSpan): boolean {\n\tif (span.path !== gold.path) return false;\n\tif (gold.scope === \"file\") return true;\n\tif (gold.startLine === undefined || gold.endLine === undefined) return true;\n\treturn span.startLine <= gold.endLine && span.endLine >= gold.startLine;\n}\n\n/** Fraction of gold spans matched by at least one of the top-`k` candidates. */\nexport function recallAtK(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[], k: number): number {\n\tif (gold.length === 0) return 0;\n\tconst top = candidates.slice(0, k);\n\tlet matched = 0;\n\tfor (const g of gold) {\n\t\tif (top.some((c) => spanMatchesGold(c, g))) matched++;\n\t}\n\treturn matched / gold.length;\n}\n\n/**\n * Mean reciprocal rank, averaged over gold spans: for each gold span, `1/rank`\n * of the first candidate matching it (0 when it never appears).\n *\n * Averaging per gold span rather than taking the single first hit keeps\n * multi-span queries (the cross-file class) honest — finding one of two\n * required sites should not score like finding both. For single-span queries\n * this reduces to textbook MRR.\n */\nexport function mrr(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[]): number {\n\tif (gold.length === 0) return 0;\n\tlet total = 0;\n\tfor (const g of gold) {\n\t\tconst index = candidates.findIndex((c) => spanMatchesGold(c, g));\n\t\tif (index >= 0) total += 1 / (index + 1);\n\t}\n\treturn total / gold.length;\n}\n\nexport interface EvalQueryResult {\n\tlabel: string;\n\tresolvedMode: ResolvedSearchMode;\n\tdegraded: boolean;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n}\n\nexport async function evaluateQuery(\n\tcwd: string,\n\tevalQuery: EvalQuery,\n\tconfigs: readonly EvalConfig[] = EVAL_CONFIGS,\n\tservice?: EmbsearchService,\n\thybridService?: EmbsearchService,\n): Promise<EvalQueryResult[]> {\n\tconst results: EvalQueryResult[] = [];\n\tfor (const config of configs) {\n\t\t// A daemon-hybrid config against the plain store would error in the\n\t\t// daemon (`query_hybrid requires a hybrid store`); omit the row instead.\n\t\tif ((config.daemonHybrid || config.bm25Leg) && !hybridService) continue;\n\t\t// Cross-encoder rows need a live daemon; omit rather than silently\n\t\t// falling back to the deterministic reranker under a \"+ce\" label.\n\t\tif (config.crossEncoder && !(config.bm25Leg ? hybridService : service)?.supportsCrossEncoder()) continue;\n\t\tconst retrieved = await retrieveCandidates({\n\t\t\tcwd,\n\t\t\tquery: evalQuery.query,\n\t\t\tmode: config.mode,\n\t\t\trrfK: config.rrfK,\n\t\t\trerank: config.rerank ?? false,\n\t\t\tlimit: EVAL_FETCH_LIMIT,\n\t\t\tservice: config.daemonHybrid || config.bm25Leg ? hybridService : service,\n\t\t\tdaemonHybrid: config.daemonHybrid,\n\t\t\tbm25Leg: config.bm25Leg,\n\t\t\tcrossEncoder: config.crossEncoder,\n\t\t});\n\t\tresults.push({\n\t\t\tlabel: config.label,\n\t\t\tresolvedMode: retrieved.resolvedMode,\n\t\t\tdegraded: retrieved.degradedReason !== undefined,\n\t\t\trecallAt1: recallAtK(retrieved.candidates, evalQuery.gold, 1),\n\t\t\trecallAt5: recallAtK(retrieved.candidates, evalQuery.gold, 5),\n\t\t\trecallAt10: recallAtK(retrieved.candidates, evalQuery.gold, 10),\n\t\t\trecallAt50: recallAtK(retrieved.candidates, evalQuery.gold, 50),\n\t\t\tmrr: mrr(retrieved.candidates, evalQuery.gold),\n\t\t});\n\t}\n\treturn results;\n}\n"]}

package/dist/core/search/hybrid-search.d.ts

@@ -80,6 +80,23 @@ export interface RunSearchResult {
         total: number;
     };
 }
+/**
+ * Move candidates from files the index has not read to the front.
+ *
+ * They are there because grep found them and nothing else could: the index is
+ * ranking a stale copy of the file, or has never seen it. Left to fuse, they
+ * lose — RRF rewards agreement, and one leg reporting a single document is
+ * outvoted by two legs agreeing on hundreds. Measured: scoping grep to stale
+ * files without this hoist scored 25% on the live-edit set where unscoped grep
+ * scored 100%, because the fused window filled with consensus hits about the
+ * indexed copy.
+ *
+ * Capped, because "stale" scales with how far behind the index is. A few
+ * edited files is the case this exists for; a fresh checkout makes everything
+ * stale, and hoisting all of it would quietly turn hybrid search back into
+ * grep. Past the cap the remainder keeps its fused position.
+ */
+export declare function hoistStaleCandidates(candidates: readonly FusedCandidate[], staleFiles: ReadonlySet<string>): FusedCandidate[];
 export declare function retrieveCandidates(options: RetrieveOptions): Promise<RetrieveResult>;
 export declare function runSearch(options: RunSearchOptions): Promise<RunSearchResult>;
 //# sourceMappingURL=hybrid-search.d.ts.map