@kolisachint/hoocode-agent 0.4.166 → 0.4.167

package/CHANGELOG.md

@@ -1,5 +1,7 @@
 # Changelog
 
+## [0.4.167] - 2026-08-08
+
 ## [0.4.166] - 2026-08-08
 
 ## [0.4.165] - 2026-08-08

package/dist/core/search/eval-gold.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-gold.d.ts","sourceRoot":"","sources":["../../../src/core/search/eval-gold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAYzD,MAAM,WAAW,mBAAmB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CAChB;AA2CD;oEACoE;AACpE,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,CAiBhH;AAED;oDACoD;AACpD,wBAAgB,cAAc,CAC7B,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,SAAS,SAAS,EAAE,GAC3B;IAAE,OAAO,EAAE,SAAS,EAAE,CAAC;IAAC,MAAM,EAAE,mBAAmB,EAAE,CAAA;CAAE,CAWzD;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,SAAS,EAAE,GAAG,mBAAmB,EAAE,CAmDxG;AAED,6CAA6C;AAC7C,wBAAgB,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,SAAS,EAAE,CAE5D","sourcesContent":["/**\n * Gold-set loading and anchor resolution.\n *\n * A gold span recorded as bare line numbers rots on the next refactor, and a\n * rotted gold set fails *silently* — it just scores lower. So each span also\n * carries an `anchor`: a literal snippet that must sit inside the range. The\n * anchor is the source of truth; the line numbers are a cache of where it was\n * when the fixture was last regenerated.\n *\n * That gives three properties the first gold set lacked:\n *  - `resolveGoldSet` recomputes ranges from anchors, so `--fix` re-pins the\n *    whole set after a refactor instead of someone hand-editing 60 numbers;\n *  - `validateGoldSet` fails loudly when an anchor has moved out of its\n *    recorded range, drifted, or become ambiguous;\n *  - anchors are never used for scoring, so this cannot flatter retrieval —\n *    `spanMatchesGold` still sees only a path and a line range.\n */\n\nimport { readFileSync } from \"node:fs\";\nimport path from \"node:path\";\nimport type { EvalGoldSpan, EvalQuery } from \"./eval.js\";\n\n/** Lines of context kept either side of a non-block anchor (an error literal,\n *  a single statement) — enough to be a real target, tight enough that a\n *  60-line chunk elsewhere in the file does not count as a hit. */\nconst LINE_ANCHOR_PAD = 2;\n/** Upper bound on a resolved block, so an anchor that fails to find its\n *  closing brace cannot silently swallow an entire file. */\nconst MAX_BLOCK_LINES = 120;\n/** Fallback extent when a block anchor never finds its closing brace. */\nconst UNCLOSED_BLOCK_LINES = 15;\n\nexport interface GoldValidationIssue {\n\tqueryId: string;\n\tpath: string;\n\tproblem: string;\n}\n\nfunction readLines(corpusRoot: string, rel: string): string[] | undefined {\n\ttry {\n\t\treturn readFileSync(path.resolve(corpusRoot, rel), \"utf-8\").split(\"\\n\");\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/** Indices (0-based) of every line containing `anchor`. */\nfunction findAnchorLines(lines: readonly string[], anchor: string): number[] {\n\tconst found: number[] = [];\n\tfor (let i = 0; i < lines.length; i++) {\n\t\tif (lines[i].includes(anchor)) found.push(i);\n\t}\n\treturn found;\n}\n\n/**\n * Extent of the declaration an anchor names.\n *\n * A line ending in an opener (`{`, `(`, `[`) starts a block, which runs to the\n * first line closing at column 0 — the shape every top-level declaration in\n * this codebase has. Anything else is a statement, scored with a small pad.\n */\nfunction resolveExtent(lines: readonly string[], anchorIndex: number): { startLine: number; endLine: number } {\n\tconst head = lines[anchorIndex];\n\tconst opensBlock = /[{([]\\s*$/.test(head);\n\tif (!opensBlock) {\n\t\treturn {\n\t\t\tstartLine: Math.max(1, anchorIndex + 1 - LINE_ANCHOR_PAD),\n\t\t\tendLine: Math.min(lines.length, anchorIndex + 1 + LINE_ANCHOR_PAD),\n\t\t};\n\t}\n\tconst limit = Math.min(lines.length, anchorIndex + 1 + MAX_BLOCK_LINES);\n\tfor (let i = anchorIndex + 1; i < limit; i++) {\n\t\t// Column-0 close: `}`, `};`, `});`, `];` — the end of a top-level block.\n\t\tif (/^[}\\])]/.test(lines[i])) return { startLine: anchorIndex + 1, endLine: i + 1 };\n\t}\n\treturn { startLine: anchorIndex + 1, endLine: Math.min(lines.length, anchorIndex + 1 + UNCLOSED_BLOCK_LINES) };\n}\n\n/** Re-pin one gold span's line range from its anchor. Returns the span\n *  unchanged when it is file-scoped or has no anchor to resolve. */\nexport function resolveGoldSpan(corpusRoot: string, span: EvalGoldSpan): { span: EvalGoldSpan; problem?: string } {\n\tconst lines = readLines(corpusRoot, span.path);\n\tif (!lines) return { span, problem: `file not found: ${span.path}` };\n\n\tif (span.scope === \"file\") {\n\t\treturn { span: { ...span, startLine: 1, endLine: lines.length } };\n\t}\n\tif (!span.anchor) return { span, problem: `span-scoped gold needs an anchor: ${span.path}` };\n\n\tconst matches = findAnchorLines(lines, span.anchor);\n\tif (matches.length === 0) return { span, problem: `anchor not found in ${span.path}: ${span.anchor}` };\n\tif (matches.length > 1) {\n\t\treturn { span, problem: `anchor is ambiguous (${matches.length} matches) in ${span.path}: ${span.anchor}` };\n\t}\n\n\tconst { startLine, endLine } = resolveExtent(lines, matches[0]);\n\treturn { span: { ...span, startLine, endLine } };\n}\n\n/** Re-pin every gold span in the set. Problems are collected, not thrown, so\n *  a regeneration run reports all drift at once. */\nexport function resolveGoldSet(\n\tcorpusRoot: string,\n\tdataset: readonly EvalQuery[],\n): { dataset: EvalQuery[]; issues: GoldValidationIssue[] } {\n\tconst issues: GoldValidationIssue[] = [];\n\tconst resolved = dataset.map((query) => ({\n\t\t...query,\n\t\tgold: query.gold.map((span) => {\n\t\t\tconst result = resolveGoldSpan(corpusRoot, span);\n\t\t\tif (result.problem) issues.push({ queryId: query.id, path: span.path, problem: result.problem });\n\t\t\treturn result.span;\n\t\t}),\n\t}));\n\treturn { dataset: resolved, issues };\n}\n\n/**\n * Check the committed fixture against the corpus without rewriting it.\n *\n * Catches the two ways a gold set goes wrong: the anchor no longer exists (the\n * code was deleted or renamed), or it exists but has moved outside the\n * recorded range (the fixture is stale and every score computed from it is\n * wrong).\n */\nexport function validateGoldSet(corpusRoot: string, dataset: readonly EvalQuery[]): GoldValidationIssue[] {\n\tconst issues: GoldValidationIssue[] = [];\n\tconst seenIds = new Set<string>();\n\n\tfor (const query of dataset) {\n\t\tif (seenIds.has(query.id)) issues.push({ queryId: query.id, path: \"\", problem: \"duplicate query id\" });\n\t\tseenIds.add(query.id);\n\t\tif (query.gold.length === 0) issues.push({ queryId: query.id, path: \"\", problem: \"query has no gold spans\" });\n\n\t\tfor (const span of query.gold) {\n\t\t\tconst lines = readLines(corpusRoot, span.path);\n\t\t\tif (!lines) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"file not found\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.startLine === undefined || span.endLine === undefined) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"gold span is missing line numbers\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.startLine < 1 || span.endLine < span.startLine || span.endLine > lines.length) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `line range ${span.startLine}-${span.endLine} out of bounds (file has ${lines.length} lines)`,\n\t\t\t\t});\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.scope === \"file\") continue;\n\t\t\tif (!span.anchor) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"span-scoped gold needs an anchor\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tconst matches = findAnchorLines(lines, span.anchor);\n\t\t\tif (matches.length === 0) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: `anchor not found: ${span.anchor}` });\n\t\t\t} else if (matches.length > 1) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `anchor is ambiguous (${matches.length} matches): ${span.anchor}`,\n\t\t\t\t});\n\t\t\t} else if (matches[0] + 1 < span.startLine || matches[0] + 1 > span.endLine) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `anchor moved to line ${matches[0] + 1}, outside recorded range ${span.startLine}-${span.endLine}`,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n\treturn issues;\n}\n\n/** Load the gold set from a fixture file. */\nexport function loadGoldSet(fixturePath: string): EvalQuery[] {\n\treturn JSON.parse(readFileSync(fixturePath, \"utf-8\")) as EvalQuery[];\n}\n"]}
+{"version":3,"file":"eval-gold.d.ts","sourceRoot":"","sources":["../../../src/core/search/eval-gold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAYzD,MAAM,WAAW,mBAAmB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CAChB;AA8DD;oEACoE;AACpE,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,CAiBhH;AAED;oDACoD;AACpD,wBAAgB,cAAc,CAC7B,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,SAAS,SAAS,EAAE,GAC3B;IAAE,OAAO,EAAE,SAAS,EAAE,CAAC;IAAC,MAAM,EAAE,mBAAmB,EAAE,CAAA;CAAE,CAWzD;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,SAAS,EAAE,GAAG,mBAAmB,EAAE,CAmDxG;AAED,6CAA6C;AAC7C,wBAAgB,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,SAAS,EAAE,CAE5D","sourcesContent":["/**\n * Gold-set loading and anchor resolution.\n *\n * A gold span recorded as bare line numbers rots on the next refactor, and a\n * rotted gold set fails *silently* — it just scores lower. So each span also\n * carries an `anchor`: a literal snippet that must sit inside the range. The\n * anchor is the source of truth; the line numbers are a cache of where it was\n * when the fixture was last regenerated.\n *\n * That gives three properties the first gold set lacked:\n *  - `resolveGoldSet` recomputes ranges from anchors, so `--fix` re-pins the\n *    whole set after a refactor instead of someone hand-editing 60 numbers;\n *  - `validateGoldSet` fails loudly when an anchor has moved out of its\n *    recorded range, drifted, or become ambiguous;\n *  - anchors are never used for scoring, so this cannot flatter retrieval —\n *    `spanMatchesGold` still sees only a path and a line range.\n */\n\nimport { readFileSync } from \"node:fs\";\nimport path from \"node:path\";\nimport type { EvalGoldSpan, EvalQuery } from \"./eval.js\";\n\n/** Lines of context kept either side of a non-block anchor (an error literal,\n *  a single statement) — enough to be a real target, tight enough that a\n *  60-line chunk elsewhere in the file does not count as a hit. */\nconst LINE_ANCHOR_PAD = 2;\n/** Upper bound on a resolved block, so an anchor that fails to find its\n *  closing brace cannot silently swallow an entire file. */\nconst MAX_BLOCK_LINES = 120;\n/** Fallback extent when a block anchor never finds its closing brace. */\nconst UNCLOSED_BLOCK_LINES = 15;\n\nexport interface GoldValidationIssue {\n\tqueryId: string;\n\tpath: string;\n\tproblem: string;\n}\n\nfunction readLines(corpusRoot: string, rel: string): string[] | undefined {\n\ttry {\n\t\treturn readFileSync(path.resolve(corpusRoot, rel), \"utf-8\").split(\"\\n\");\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/** Indices (0-based) of every line containing `anchor`. */\nfunction findAnchorLines(lines: readonly string[], anchor: string): number[] {\n\tconst found: number[] = [];\n\tfor (let i = 0; i < lines.length; i++) {\n\t\tif (lines[i].includes(anchor)) found.push(i);\n\t}\n\treturn found;\n}\n\n/** Leading-whitespace width of a line, for matching a block's closer to its\n *  opener. Tabs count as one; this only ever compares lines in one file, which\n *  is consistently indented. */\nfunction indentWidth(line: string): number {\n\treturn /^[\\t ]*/.exec(line)?.[0].length ?? 0;\n}\n\n/**\n * Extent of the declaration an anchor names.\n *\n * A line ending in an opener (`{`, `(`, `[`) starts a block, which runs to the\n * first closing bracket indented no deeper than the opener. Anything else is a\n * statement, scored with a small pad.\n *\n * The indentation test is load-bearing. This used to close only on a bracket\n * at *column 0*, which is right for a top-level declaration and wrong for\n * every class member: a method's own `}` is indented, so the scan walked past\n * it to the end of the enclosing class. Three of the four boundary-class gold\n * spans were resolving to 80, 34 and 28 lines for methods that are 7, 4 and 4\n * lines long, all ending on the same line — the class's closing brace. Gold\n * that wide scores a hit for retrieving a neighbouring method, which is not\n * what the query asked for.\n */\nfunction resolveExtent(lines: readonly string[], anchorIndex: number): { startLine: number; endLine: number } {\n\tconst head = lines[anchorIndex];\n\tconst opensBlock = /[{([]\\s*$/.test(head);\n\tif (!opensBlock) {\n\t\treturn {\n\t\t\tstartLine: Math.max(1, anchorIndex + 1 - LINE_ANCHOR_PAD),\n\t\t\tendLine: Math.min(lines.length, anchorIndex + 1 + LINE_ANCHOR_PAD),\n\t\t};\n\t}\n\tconst anchorIndent = indentWidth(head);\n\tconst limit = Math.min(lines.length, anchorIndex + 1 + MAX_BLOCK_LINES);\n\tfor (let i = anchorIndex + 1; i < limit; i++) {\n\t\t// `}`, `};`, `});`, `];` at or outside the opener's indentation.\n\t\tif (/^[\\t ]*[}\\])]/.test(lines[i]) && indentWidth(lines[i]) <= anchorIndent) {\n\t\t\treturn { startLine: anchorIndex + 1, endLine: i + 1 };\n\t\t}\n\t}\n\treturn { startLine: anchorIndex + 1, endLine: Math.min(lines.length, anchorIndex + 1 + UNCLOSED_BLOCK_LINES) };\n}\n\n/** Re-pin one gold span's line range from its anchor. Returns the span\n *  unchanged when it is file-scoped or has no anchor to resolve. */\nexport function resolveGoldSpan(corpusRoot: string, span: EvalGoldSpan): { span: EvalGoldSpan; problem?: string } {\n\tconst lines = readLines(corpusRoot, span.path);\n\tif (!lines) return { span, problem: `file not found: ${span.path}` };\n\n\tif (span.scope === \"file\") {\n\t\treturn { span: { ...span, startLine: 1, endLine: lines.length } };\n\t}\n\tif (!span.anchor) return { span, problem: `span-scoped gold needs an anchor: ${span.path}` };\n\n\tconst matches = findAnchorLines(lines, span.anchor);\n\tif (matches.length === 0) return { span, problem: `anchor not found in ${span.path}: ${span.anchor}` };\n\tif (matches.length > 1) {\n\t\treturn { span, problem: `anchor is ambiguous (${matches.length} matches) in ${span.path}: ${span.anchor}` };\n\t}\n\n\tconst { startLine, endLine } = resolveExtent(lines, matches[0]);\n\treturn { span: { ...span, startLine, endLine } };\n}\n\n/** Re-pin every gold span in the set. Problems are collected, not thrown, so\n *  a regeneration run reports all drift at once. */\nexport function resolveGoldSet(\n\tcorpusRoot: string,\n\tdataset: readonly EvalQuery[],\n): { dataset: EvalQuery[]; issues: GoldValidationIssue[] } {\n\tconst issues: GoldValidationIssue[] = [];\n\tconst resolved = dataset.map((query) => ({\n\t\t...query,\n\t\tgold: query.gold.map((span) => {\n\t\t\tconst result = resolveGoldSpan(corpusRoot, span);\n\t\t\tif (result.problem) issues.push({ queryId: query.id, path: span.path, problem: result.problem });\n\t\t\treturn result.span;\n\t\t}),\n\t}));\n\treturn { dataset: resolved, issues };\n}\n\n/**\n * Check the committed fixture against the corpus without rewriting it.\n *\n * Catches the two ways a gold set goes wrong: the anchor no longer exists (the\n * code was deleted or renamed), or it exists but has moved outside the\n * recorded range (the fixture is stale and every score computed from it is\n * wrong).\n */\nexport function validateGoldSet(corpusRoot: string, dataset: readonly EvalQuery[]): GoldValidationIssue[] {\n\tconst issues: GoldValidationIssue[] = [];\n\tconst seenIds = new Set<string>();\n\n\tfor (const query of dataset) {\n\t\tif (seenIds.has(query.id)) issues.push({ queryId: query.id, path: \"\", problem: \"duplicate query id\" });\n\t\tseenIds.add(query.id);\n\t\tif (query.gold.length === 0) issues.push({ queryId: query.id, path: \"\", problem: \"query has no gold spans\" });\n\n\t\tfor (const span of query.gold) {\n\t\t\tconst lines = readLines(corpusRoot, span.path);\n\t\t\tif (!lines) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"file not found\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.startLine === undefined || span.endLine === undefined) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"gold span is missing line numbers\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.startLine < 1 || span.endLine < span.startLine || span.endLine > lines.length) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `line range ${span.startLine}-${span.endLine} out of bounds (file has ${lines.length} lines)`,\n\t\t\t\t});\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.scope === \"file\") continue;\n\t\t\tif (!span.anchor) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"span-scoped gold needs an anchor\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tconst matches = findAnchorLines(lines, span.anchor);\n\t\t\tif (matches.length === 0) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: `anchor not found: ${span.anchor}` });\n\t\t\t} else if (matches.length > 1) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `anchor is ambiguous (${matches.length} matches): ${span.anchor}`,\n\t\t\t\t});\n\t\t\t} else if (matches[0] + 1 < span.startLine || matches[0] + 1 > span.endLine) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `anchor moved to line ${matches[0] + 1}, outside recorded range ${span.startLine}-${span.endLine}`,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n\treturn issues;\n}\n\n/** Load the gold set from a fixture file. */\nexport function loadGoldSet(fixturePath: string): EvalQuery[] {\n\treturn JSON.parse(readFileSync(fixturePath, \"utf-8\")) as EvalQuery[];\n}\n"]}

package/dist/core/search/eval-gold.js

@@ -43,12 +43,27 @@ function findAnchorLines(lines, anchor) {
     }
     return found;
 }
+/** Leading-whitespace width of a line, for matching a block's closer to its
+ *  opener. Tabs count as one; this only ever compares lines in one file, which
+ *  is consistently indented. */
+function indentWidth(line) {
+    return /^[\t ]*/.exec(line)?.[0].length ?? 0;
+}
 /**
  * Extent of the declaration an anchor names.
  *
  * A line ending in an opener (`{`, `(`, `[`) starts a block, which runs to the
- * first line closing at column 0 — the shape every top-level declaration in
- * this codebase has. Anything else is a statement, scored with a small pad.
+ * first closing bracket indented no deeper than the opener. Anything else is a
+ * statement, scored with a small pad.
+ *
+ * The indentation test is load-bearing. This used to close only on a bracket
+ * at *column 0*, which is right for a top-level declaration and wrong for
+ * every class member: a method's own `}` is indented, so the scan walked past
+ * it to the end of the enclosing class. Three of the four boundary-class gold
+ * spans were resolving to 80, 34 and 28 lines for methods that are 7, 4 and 4
+ * lines long, all ending on the same line — the class's closing brace. Gold
+ * that wide scores a hit for retrieving a neighbouring method, which is not
+ * what the query asked for.
  */
 function resolveExtent(lines, anchorIndex) {
     const head = lines[anchorIndex];
@@ -59,11 +74,13 @@ function resolveExtent(lines, anchorIndex) {
             endLine: Math.min(lines.length, anchorIndex + 1 + LINE_ANCHOR_PAD),
         };
     }
+    const anchorIndent = indentWidth(head);
     const limit = Math.min(lines.length, anchorIndex + 1 + MAX_BLOCK_LINES);
     for (let i = anchorIndex + 1; i < limit; i++) {
-        // Column-0 close: `}`, `};`, `});`, `];` — the end of a top-level block.
-        if (/^[}\])]/.test(lines[i]))
+        // `}`, `};`, `});`, `];` at or outside the opener's indentation.
+        if (/^[\t ]*[}\])]/.test(lines[i]) && indentWidth(lines[i]) <= anchorIndent) {
             return { startLine: anchorIndex + 1, endLine: i + 1 };
+        }
     }
     return { startLine: anchorIndex + 1, endLine: Math.min(lines.length, anchorIndex + 1 + UNCLOSED_BLOCK_LINES) };
 }

package/dist/core/search/eval-gold.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-gold.js","sourceRoot":"","sources":["../../../src/core/search/eval-gold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B;;mEAEmE;AACnE,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B;4DAC4D;AAC5D,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,yEAAyE;AACzE,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAQhC,SAAS,SAAS,CAAC,UAAkB,EAAE,GAAW,EAAwB;IACzE,IAAI,CAAC;QACJ,OAAO,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED,2DAA2D;AAC3D,SAAS,eAAe,CAAC,KAAwB,EAAE,MAAc,EAAY;IAC5E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,KAAK,CAAC;AAAA,CACb;AAED;;;;;;GAMG;AACH,SAAS,aAAa,CAAC,KAAwB,EAAE,WAAmB,EAA0C;IAC7G,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;IAChC,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,OAAO;YACN,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,CAAC,GAAG,eAAe,CAAC;YACzD,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,eAAe,CAAC;SAClE,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;IACxE,KAAK,IAAI,CAAC,GAAG,WAAW,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,2EAAyE;QACzE,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;IACrF,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,oBAAoB,CAAC,EAAE,CAAC;AAAA,CAC/G;AAED;oEACoE;AACpE,MAAM,UAAU,eAAe,CAAC,UAAkB,EAAE,IAAkB,EAA4C;IACjH,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,mBAAmB,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;IAErE,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC3B,OAAO,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;IACnE,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,qCAAqC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;IAE7F,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,uBAAuB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;IACvG,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,wBAAwB,OAAO,CAAC,MAAM,gBAAgB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;IAC7G,CAAC;IAED,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;AAAA,CACjD;AAED;oDACoD;AACpD,MAAM,UAAU,cAAc,CAC7B,UAAkB,EAClB,OAA6B,EAC6B;IAC1D,MAAM,MAAM,GAA0B,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACxC,GAAG,KAAK;QACR,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YACjD,IAAI,MAAM,CAAC,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YACjG,OAAO,MAAM,CAAC,IAAI,CAAC;QAAA,CACnB,CAAC;KACF,CAAC,CAAC,CAAC;IACJ,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AAAA,CACrC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,UAAkB,EAAE,OAA6B,EAAyB;IACzG,MAAM,MAAM,GAA0B,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACvG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACtB,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,yBAAyB,EAAE,CAAC,CAAC;QAE9G,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/C,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;gBAC/E,SAAS;YACV,CAAC;YACD,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,mCAAmC,EAAE,CAAC,CAAC;gBAClG,SAAS;YACV,CAAC;YACD,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACxF,MAAM,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,cAAc,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,4BAA4B,KAAK,CAAC,MAAM,SAAS;iBACtG,CAAC,CAAC;gBACH,SAAS;YACV,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM;gBAAE,SAAS;YACpC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,kCAAkC,EAAE,CAAC,CAAC;gBACjG,SAAS;YACV,CAAC;YACD,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,qBAAqB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAClG,CAAC;iBAAM,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,wBAAwB,OAAO,CAAC,MAAM,cAAc,IAAI,CAAC,MAAM,EAAE;iBAC1E,CAAC,CAAC;YACJ,CAAC;iBAAM,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC7E,MAAM,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,wBAAwB,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,4BAA4B,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE;iBAC3G,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC;IACD,OAAO,MAAM,CAAC;AAAA,CACd;AAED,6CAA6C;AAC7C,MAAM,UAAU,WAAW,CAAC,WAAmB,EAAe;IAC7D,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAgB,CAAC;AAAA,CACrE","sourcesContent":["/**\n * Gold-set loading and anchor resolution.\n *\n * A gold span recorded as bare line numbers rots on the next refactor, and a\n * rotted gold set fails *silently* — it just scores lower. So each span also\n * carries an `anchor`: a literal snippet that must sit inside the range. The\n * anchor is the source of truth; the line numbers are a cache of where it was\n * when the fixture was last regenerated.\n *\n * That gives three properties the first gold set lacked:\n *  - `resolveGoldSet` recomputes ranges from anchors, so `--fix` re-pins the\n *    whole set after a refactor instead of someone hand-editing 60 numbers;\n *  - `validateGoldSet` fails loudly when an anchor has moved out of its\n *    recorded range, drifted, or become ambiguous;\n *  - anchors are never used for scoring, so this cannot flatter retrieval —\n *    `spanMatchesGold` still sees only a path and a line range.\n */\n\nimport { readFileSync } from \"node:fs\";\nimport path from \"node:path\";\nimport type { EvalGoldSpan, EvalQuery } from \"./eval.js\";\n\n/** Lines of context kept either side of a non-block anchor (an error literal,\n *  a single statement) — enough to be a real target, tight enough that a\n *  60-line chunk elsewhere in the file does not count as a hit. */\nconst LINE_ANCHOR_PAD = 2;\n/** Upper bound on a resolved block, so an anchor that fails to find its\n *  closing brace cannot silently swallow an entire file. */\nconst MAX_BLOCK_LINES = 120;\n/** Fallback extent when a block anchor never finds its closing brace. */\nconst UNCLOSED_BLOCK_LINES = 15;\n\nexport interface GoldValidationIssue {\n\tqueryId: string;\n\tpath: string;\n\tproblem: string;\n}\n\nfunction readLines(corpusRoot: string, rel: string): string[] | undefined {\n\ttry {\n\t\treturn readFileSync(path.resolve(corpusRoot, rel), \"utf-8\").split(\"\\n\");\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/** Indices (0-based) of every line containing `anchor`. */\nfunction findAnchorLines(lines: readonly string[], anchor: string): number[] {\n\tconst found: number[] = [];\n\tfor (let i = 0; i < lines.length; i++) {\n\t\tif (lines[i].includes(anchor)) found.push(i);\n\t}\n\treturn found;\n}\n\n/**\n * Extent of the declaration an anchor names.\n *\n * A line ending in an opener (`{`, `(`, `[`) starts a block, which runs to the\n * first line closing at column 0 — the shape every top-level declaration in\n * this codebase has. Anything else is a statement, scored with a small pad.\n */\nfunction resolveExtent(lines: readonly string[], anchorIndex: number): { startLine: number; endLine: number } {\n\tconst head = lines[anchorIndex];\n\tconst opensBlock = /[{([]\\s*$/.test(head);\n\tif (!opensBlock) {\n\t\treturn {\n\t\t\tstartLine: Math.max(1, anchorIndex + 1 - LINE_ANCHOR_PAD),\n\t\t\tendLine: Math.min(lines.length, anchorIndex + 1 + LINE_ANCHOR_PAD),\n\t\t};\n\t}\n\tconst limit = Math.min(lines.length, anchorIndex + 1 + MAX_BLOCK_LINES);\n\tfor (let i = anchorIndex + 1; i < limit; i++) {\n\t\t// Column-0 close: `}`, `};`, `});`, `];` — the end of a top-level block.\n\t\tif (/^[}\\])]/.test(lines[i])) return { startLine: anchorIndex + 1, endLine: i + 1 };\n\t}\n\treturn { startLine: anchorIndex + 1, endLine: Math.min(lines.length, anchorIndex + 1 + UNCLOSED_BLOCK_LINES) };\n}\n\n/** Re-pin one gold span's line range from its anchor. Returns the span\n *  unchanged when it is file-scoped or has no anchor to resolve. */\nexport function resolveGoldSpan(corpusRoot: string, span: EvalGoldSpan): { span: EvalGoldSpan; problem?: string } {\n\tconst lines = readLines(corpusRoot, span.path);\n\tif (!lines) return { span, problem: `file not found: ${span.path}` };\n\n\tif (span.scope === \"file\") {\n\t\treturn { span: { ...span, startLine: 1, endLine: lines.length } };\n\t}\n\tif (!span.anchor) return { span, problem: `span-scoped gold needs an anchor: ${span.path}` };\n\n\tconst matches = findAnchorLines(lines, span.anchor);\n\tif (matches.length === 0) return { span, problem: `anchor not found in ${span.path}: ${span.anchor}` };\n\tif (matches.length > 1) {\n\t\treturn { span, problem: `anchor is ambiguous (${matches.length} matches) in ${span.path}: ${span.anchor}` };\n\t}\n\n\tconst { startLine, endLine } = resolveExtent(lines, matches[0]);\n\treturn { span: { ...span, startLine, endLine } };\n}\n\n/** Re-pin every gold span in the set. Problems are collected, not thrown, so\n *  a regeneration run reports all drift at once. */\nexport function resolveGoldSet(\n\tcorpusRoot: string,\n\tdataset: readonly EvalQuery[],\n): { dataset: EvalQuery[]; issues: GoldValidationIssue[] } {\n\tconst issues: GoldValidationIssue[] = [];\n\tconst resolved = dataset.map((query) => ({\n\t\t...query,\n\t\tgold: query.gold.map((span) => {\n\t\t\tconst result = resolveGoldSpan(corpusRoot, span);\n\t\t\tif (result.problem) issues.push({ queryId: query.id, path: span.path, problem: result.problem });\n\t\t\treturn result.span;\n\t\t}),\n\t}));\n\treturn { dataset: resolved, issues };\n}\n\n/**\n * Check the committed fixture against the corpus without rewriting it.\n *\n * Catches the two ways a gold set goes wrong: the anchor no longer exists (the\n * code was deleted or renamed), or it exists but has moved outside the\n * recorded range (the fixture is stale and every score computed from it is\n * wrong).\n */\nexport function validateGoldSet(corpusRoot: string, dataset: readonly EvalQuery[]): GoldValidationIssue[] {\n\tconst issues: GoldValidationIssue[] = [];\n\tconst seenIds = new Set<string>();\n\n\tfor (const query of dataset) {\n\t\tif (seenIds.has(query.id)) issues.push({ queryId: query.id, path: \"\", problem: \"duplicate query id\" });\n\t\tseenIds.add(query.id);\n\t\tif (query.gold.length === 0) issues.push({ queryId: query.id, path: \"\", problem: \"query has no gold spans\" });\n\n\t\tfor (const span of query.gold) {\n\t\t\tconst lines = readLines(corpusRoot, span.path);\n\t\t\tif (!lines) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"file not found\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.startLine === undefined || span.endLine === undefined) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"gold span is missing line numbers\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.startLine < 1 || span.endLine < span.startLine || span.endLine > lines.length) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `line range ${span.startLine}-${span.endLine} out of bounds (file has ${lines.length} lines)`,\n\t\t\t\t});\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.scope === \"file\") continue;\n\t\t\tif (!span.anchor) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"span-scoped gold needs an anchor\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tconst matches = findAnchorLines(lines, span.anchor);\n\t\t\tif (matches.length === 0) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: `anchor not found: ${span.anchor}` });\n\t\t\t} else if (matches.length > 1) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `anchor is ambiguous (${matches.length} matches): ${span.anchor}`,\n\t\t\t\t});\n\t\t\t} else if (matches[0] + 1 < span.startLine || matches[0] + 1 > span.endLine) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `anchor moved to line ${matches[0] + 1}, outside recorded range ${span.startLine}-${span.endLine}`,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n\treturn issues;\n}\n\n/** Load the gold set from a fixture file. */\nexport function loadGoldSet(fixturePath: string): EvalQuery[] {\n\treturn JSON.parse(readFileSync(fixturePath, \"utf-8\")) as EvalQuery[];\n}\n"]}
+{"version":3,"file":"eval-gold.js","sourceRoot":"","sources":["../../../src/core/search/eval-gold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B;;mEAEmE;AACnE,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B;4DAC4D;AAC5D,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,yEAAyE;AACzE,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAQhC,SAAS,SAAS,CAAC,UAAkB,EAAE,GAAW,EAAwB;IACzE,IAAI,CAAC;QACJ,OAAO,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED,2DAA2D;AAC3D,SAAS,eAAe,CAAC,KAAwB,EAAE,MAAc,EAAY;IAC5E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,KAAK,CAAC;AAAA,CACb;AAED;;gCAEgC;AAChC,SAAS,WAAW,CAAC,IAAY,EAAU;IAC1C,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;AAAA,CAC7C;AAED;;;;;;;;;;;;;;;GAeG;AACH,SAAS,aAAa,CAAC,KAAwB,EAAE,WAAmB,EAA0C;IAC7G,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;IAChC,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,OAAO;YACN,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,CAAC,GAAG,eAAe,CAAC;YACzD,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,eAAe,CAAC;SAClE,CAAC;IACH,CAAC;IACD,MAAM,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;IACxE,KAAK,IAAI,CAAC,GAAG,WAAW,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,iEAAiE;QACjE,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,YAAY,EAAE,CAAC;YAC7E,OAAO,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;QACvD,CAAC;IACF,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,oBAAoB,CAAC,EAAE,CAAC;AAAA,CAC/G;AAED;oEACoE;AACpE,MAAM,UAAU,eAAe,CAAC,UAAkB,EAAE,IAAkB,EAA4C;IACjH,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,mBAAmB,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;IAErE,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC3B,OAAO,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;IACnE,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,qCAAqC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;IAE7F,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,uBAAuB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;IACvG,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,wBAAwB,OAAO,CAAC,MAAM,gBAAgB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;IAC7G,CAAC;IAED,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;AAAA,CACjD;AAED;oDACoD;AACpD,MAAM,UAAU,cAAc,CAC7B,UAAkB,EAClB,OAA6B,EAC6B;IAC1D,MAAM,MAAM,GAA0B,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACxC,GAAG,KAAK;QACR,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YACjD,IAAI,MAAM,CAAC,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YACjG,OAAO,MAAM,CAAC,IAAI,CAAC;QAAA,CACnB,CAAC;KACF,CAAC,CAAC,CAAC;IACJ,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AAAA,CACrC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,UAAkB,EAAE,OAA6B,EAAyB;IACzG,MAAM,MAAM,GAA0B,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACvG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACtB,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,yBAAyB,EAAE,CAAC,CAAC;QAE9G,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/C,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;gBAC/E,SAAS;YACV,CAAC;YACD,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,mCAAmC,EAAE,CAAC,CAAC;gBAClG,SAAS;YACV,CAAC;YACD,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACxF,MAAM,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,cAAc,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,4BAA4B,KAAK,CAAC,MAAM,SAAS;iBACtG,CAAC,CAAC;gBACH,SAAS;YACV,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM;gBAAE,SAAS;YACpC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,kCAAkC,EAAE,CAAC,CAAC;gBACjG,SAAS;YACV,CAAC;YACD,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,qBAAqB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAClG,CAAC;iBAAM,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,wBAAwB,OAAO,CAAC,MAAM,cAAc,IAAI,CAAC,MAAM,EAAE;iBAC1E,CAAC,CAAC;YACJ,CAAC;iBAAM,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC7E,MAAM,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,wBAAwB,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,4BAA4B,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE;iBAC3G,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC;IACD,OAAO,MAAM,CAAC;AAAA,CACd;AAED,6CAA6C;AAC7C,MAAM,UAAU,WAAW,CAAC,WAAmB,EAAe;IAC7D,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAgB,CAAC;AAAA,CACrE","sourcesContent":["/**\n * Gold-set loading and anchor resolution.\n *\n * A gold span recorded as bare line numbers rots on the next refactor, and a\n * rotted gold set fails *silently* — it just scores lower. So each span also\n * carries an `anchor`: a literal snippet that must sit inside the range. The\n * anchor is the source of truth; the line numbers are a cache of where it was\n * when the fixture was last regenerated.\n *\n * That gives three properties the first gold set lacked:\n *  - `resolveGoldSet` recomputes ranges from anchors, so `--fix` re-pins the\n *    whole set after a refactor instead of someone hand-editing 60 numbers;\n *  - `validateGoldSet` fails loudly when an anchor has moved out of its\n *    recorded range, drifted, or become ambiguous;\n *  - anchors are never used for scoring, so this cannot flatter retrieval —\n *    `spanMatchesGold` still sees only a path and a line range.\n */\n\nimport { readFileSync } from \"node:fs\";\nimport path from \"node:path\";\nimport type { EvalGoldSpan, EvalQuery } from \"./eval.js\";\n\n/** Lines of context kept either side of a non-block anchor (an error literal,\n *  a single statement) — enough to be a real target, tight enough that a\n *  60-line chunk elsewhere in the file does not count as a hit. */\nconst LINE_ANCHOR_PAD = 2;\n/** Upper bound on a resolved block, so an anchor that fails to find its\n *  closing brace cannot silently swallow an entire file. */\nconst MAX_BLOCK_LINES = 120;\n/** Fallback extent when a block anchor never finds its closing brace. */\nconst UNCLOSED_BLOCK_LINES = 15;\n\nexport interface GoldValidationIssue {\n\tqueryId: string;\n\tpath: string;\n\tproblem: string;\n}\n\nfunction readLines(corpusRoot: string, rel: string): string[] | undefined {\n\ttry {\n\t\treturn readFileSync(path.resolve(corpusRoot, rel), \"utf-8\").split(\"\\n\");\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/** Indices (0-based) of every line containing `anchor`. */\nfunction findAnchorLines(lines: readonly string[], anchor: string): number[] {\n\tconst found: number[] = [];\n\tfor (let i = 0; i < lines.length; i++) {\n\t\tif (lines[i].includes(anchor)) found.push(i);\n\t}\n\treturn found;\n}\n\n/** Leading-whitespace width of a line, for matching a block's closer to its\n *  opener. Tabs count as one; this only ever compares lines in one file, which\n *  is consistently indented. */\nfunction indentWidth(line: string): number {\n\treturn /^[\\t ]*/.exec(line)?.[0].length ?? 0;\n}\n\n/**\n * Extent of the declaration an anchor names.\n *\n * A line ending in an opener (`{`, `(`, `[`) starts a block, which runs to the\n * first closing bracket indented no deeper than the opener. Anything else is a\n * statement, scored with a small pad.\n *\n * The indentation test is load-bearing. This used to close only on a bracket\n * at *column 0*, which is right for a top-level declaration and wrong for\n * every class member: a method's own `}` is indented, so the scan walked past\n * it to the end of the enclosing class. Three of the four boundary-class gold\n * spans were resolving to 80, 34 and 28 lines for methods that are 7, 4 and 4\n * lines long, all ending on the same line — the class's closing brace. Gold\n * that wide scores a hit for retrieving a neighbouring method, which is not\n * what the query asked for.\n */\nfunction resolveExtent(lines: readonly string[], anchorIndex: number): { startLine: number; endLine: number } {\n\tconst head = lines[anchorIndex];\n\tconst opensBlock = /[{([]\\s*$/.test(head);\n\tif (!opensBlock) {\n\t\treturn {\n\t\t\tstartLine: Math.max(1, anchorIndex + 1 - LINE_ANCHOR_PAD),\n\t\t\tendLine: Math.min(lines.length, anchorIndex + 1 + LINE_ANCHOR_PAD),\n\t\t};\n\t}\n\tconst anchorIndent = indentWidth(head);\n\tconst limit = Math.min(lines.length, anchorIndex + 1 + MAX_BLOCK_LINES);\n\tfor (let i = anchorIndex + 1; i < limit; i++) {\n\t\t// `}`, `};`, `});`, `];` at or outside the opener's indentation.\n\t\tif (/^[\\t ]*[}\\])]/.test(lines[i]) && indentWidth(lines[i]) <= anchorIndent) {\n\t\t\treturn { startLine: anchorIndex + 1, endLine: i + 1 };\n\t\t}\n\t}\n\treturn { startLine: anchorIndex + 1, endLine: Math.min(lines.length, anchorIndex + 1 + UNCLOSED_BLOCK_LINES) };\n}\n\n/** Re-pin one gold span's line range from its anchor. Returns the span\n *  unchanged when it is file-scoped or has no anchor to resolve. */\nexport function resolveGoldSpan(corpusRoot: string, span: EvalGoldSpan): { span: EvalGoldSpan; problem?: string } {\n\tconst lines = readLines(corpusRoot, span.path);\n\tif (!lines) return { span, problem: `file not found: ${span.path}` };\n\n\tif (span.scope === \"file\") {\n\t\treturn { span: { ...span, startLine: 1, endLine: lines.length } };\n\t}\n\tif (!span.anchor) return { span, problem: `span-scoped gold needs an anchor: ${span.path}` };\n\n\tconst matches = findAnchorLines(lines, span.anchor);\n\tif (matches.length === 0) return { span, problem: `anchor not found in ${span.path}: ${span.anchor}` };\n\tif (matches.length > 1) {\n\t\treturn { span, problem: `anchor is ambiguous (${matches.length} matches) in ${span.path}: ${span.anchor}` };\n\t}\n\n\tconst { startLine, endLine } = resolveExtent(lines, matches[0]);\n\treturn { span: { ...span, startLine, endLine } };\n}\n\n/** Re-pin every gold span in the set. Problems are collected, not thrown, so\n *  a regeneration run reports all drift at once. */\nexport function resolveGoldSet(\n\tcorpusRoot: string,\n\tdataset: readonly EvalQuery[],\n): { dataset: EvalQuery[]; issues: GoldValidationIssue[] } {\n\tconst issues: GoldValidationIssue[] = [];\n\tconst resolved = dataset.map((query) => ({\n\t\t...query,\n\t\tgold: query.gold.map((span) => {\n\t\t\tconst result = resolveGoldSpan(corpusRoot, span);\n\t\t\tif (result.problem) issues.push({ queryId: query.id, path: span.path, problem: result.problem });\n\t\t\treturn result.span;\n\t\t}),\n\t}));\n\treturn { dataset: resolved, issues };\n}\n\n/**\n * Check the committed fixture against the corpus without rewriting it.\n *\n * Catches the two ways a gold set goes wrong: the anchor no longer exists (the\n * code was deleted or renamed), or it exists but has moved outside the\n * recorded range (the fixture is stale and every score computed from it is\n * wrong).\n */\nexport function validateGoldSet(corpusRoot: string, dataset: readonly EvalQuery[]): GoldValidationIssue[] {\n\tconst issues: GoldValidationIssue[] = [];\n\tconst seenIds = new Set<string>();\n\n\tfor (const query of dataset) {\n\t\tif (seenIds.has(query.id)) issues.push({ queryId: query.id, path: \"\", problem: \"duplicate query id\" });\n\t\tseenIds.add(query.id);\n\t\tif (query.gold.length === 0) issues.push({ queryId: query.id, path: \"\", problem: \"query has no gold spans\" });\n\n\t\tfor (const span of query.gold) {\n\t\t\tconst lines = readLines(corpusRoot, span.path);\n\t\t\tif (!lines) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"file not found\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.startLine === undefined || span.endLine === undefined) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"gold span is missing line numbers\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.startLine < 1 || span.endLine < span.startLine || span.endLine > lines.length) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `line range ${span.startLine}-${span.endLine} out of bounds (file has ${lines.length} lines)`,\n\t\t\t\t});\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.scope === \"file\") continue;\n\t\t\tif (!span.anchor) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"span-scoped gold needs an anchor\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tconst matches = findAnchorLines(lines, span.anchor);\n\t\t\tif (matches.length === 0) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: `anchor not found: ${span.anchor}` });\n\t\t\t} else if (matches.length > 1) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `anchor is ambiguous (${matches.length} matches): ${span.anchor}`,\n\t\t\t\t});\n\t\t\t} else if (matches[0] + 1 < span.startLine || matches[0] + 1 > span.endLine) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `anchor moved to line ${matches[0] + 1}, outside recorded range ${span.startLine}-${span.endLine}`,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n\treturn issues;\n}\n\n/** Load the gold set from a fixture file. */\nexport function loadGoldSet(fixturePath: string): EvalQuery[] {\n\treturn JSON.parse(readFileSync(fixturePath, \"utf-8\")) as EvalQuery[];\n}\n"]}

package/dist/core/search/eval-harness.d.ts

@@ -48,6 +48,10 @@ export interface EvalProvenance {
     /** Uncommitted changes present at run time. Only meaningful (and only
      *  possible) when `corpusFromWorkingTree` is true. */
     corpusDirty: boolean;
+    /** Files removed from the corpus before indexing — see
+     *  {@link CORPUS_EXCLUSIONS}. A score against a different exclusion list is
+     *  a score against a different corpus, so it is recorded, not assumed. */
+    corpusExcluded: string[];
     /** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,
      *  but differs when pinning an old corpus with today's code. */
     harnessSha: string;
@@ -109,9 +113,32 @@ export interface PinnedCorpus {
     sha: string;
     fromWorkingTree: boolean;
     dirty: boolean;
+    /** Files removed from the corpus before indexing. Empty when the corpus is
+     *  the live working tree, which is never mutated. */
+    excluded: string[];
     /** Removes the worktree, if one was created. */
     dispose: () => void;
 }
+/**
+ * Files that describe this eval rather than being searched by it.
+ *
+ * The fixtures hold all 62 query strings verbatim, so every query is a perfect
+ * lexical match against its own entry, and the design note quotes the same
+ * queries while discussing the classes they belong to. Measured before this
+ * exclusion existed: **56 of 62 queries had one of these files in the top 10,
+ * 27 of 62 had one as the #1 result, and they consumed 133 of the 620
+ * top-10 slots** — a fifth of the window, spent on the eval reading itself.
+ *
+ * That is not a ranking artifact a reranker can fix: it displaces real answers
+ * out of the window entirely, which is why two boundary-class queries were
+ * absent from the top *50* rather than merely buried. Retrieving your own
+ * question is not retrieval, so the corpus is scored without them.
+ *
+ * Removed from the pinned worktree before indexing, never from the repo — and
+ * recorded in the run's provenance so a score is never silently taken against
+ * a different corpus than it claims.
+ */
+export declare const CORPUS_EXCLUSIONS: readonly string[];
 /**
  * Materialize the corpus to evaluate.
  *

package/dist/core/search/eval-harness.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-harness.d.ts","sourceRoot":"","sources":["../../../src/core/search/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,KAAK,eAAe,EAAiB,MAAM,WAAW,CAAC;AAEjG,+DAA+D;AAC/D,MAAM,WAAW,aAAa;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,wCAAwC;IACxC,CAAC,EAAE,MAAM,CAAC;IACV,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED,2EAA2E;AAC3E,MAAM,WAAW,cAAc;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB;iEAC2D;IAC3D,qBAAqB,EAAE,OAAO,CAAC;IAC/B;0DACsD;IACtD,WAAW,EAAE,OAAO,CAAC;IACrB;oEACgE;IAChE,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;;OAMG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;yDACqD;IACrD,QAAQ,EAAE;QACT,SAAS,EAAE,OAAO,CAAC;QACnB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,0DAA0D;QAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;QACd;;8EAEsE;QACtE,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,aAAa,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IACF;kDAC8C;IAC9C,YAAY,CAAC,EAAE;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,WAAW,aAAa;IAC7B,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,OAAO,EAAE,SAAS,UAAU,EAAE,CAAC;IAC/B,UAAU,EAAE,aAAa,EAAE,CAAC;IAC5B,QAAQ,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,eAAe,EAAE,CAAA;KAAE,CAAC,CAAC;CAC3E;AAMD;0EAC0E;AAC1E,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CA+B5D;AAED,MAAM,WAAW,YAAY;IAC5B,qCAAqC;IACrC,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,OAAO,CAAC;IACzB,KAAK,EAAE,OAAO,CAAC;IACf,gDAAgD;IAChD,OAAO,EAAE,MAAM,IAAI,CAAC;CACpB;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,YAAY,CAyCjF;AAYD,wBAAgB,iBAAiB,CAChC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,YAAY,EACpB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,gBAAgB,GAAG,SAAS,EACrC,eAAe,CAAC,EAAE,MAAM,EACxB,aAAa,CAAC,EAAE,gBAAgB,GAC9B,cAAc,CA2BhB;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,SAAS,SAAS,EAAE,GAAG,aAAa,CAAC,SAAS,CAAC,CAQxF;AAED,MAAM,WAAW,mBAAmB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,SAAS,SAAS,EAAE,CAAC;IAC9B,OAAO,EAAE,SAAS,UAAU,EAAE,CAAC;IAC/B,OAAO,CAAC,EAAE,gBAAgB,CAAC;IAC3B;wEACoE;IACpE,aAAa,CAAC,EAAE,gBAAgB,CAAC;IACjC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,KAAK,IAAI,CAAC;CACpD;AAED,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC;IACzE,UAAU,EAAE,aAAa,EAAE,CAAC;IAC5B,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACpC,CAAC,CA4CD;AAED,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,GAAG,MAAM,CAcjF","sourcesContent":["/**\n * Eval harness: corpus pinning, provenance capture, and run records.\n *\n * The scoring math lives in `eval.ts`; this module is everything around it\n * that makes a number *comparable to a later number*. Three problems it\n * exists to solve, all of which bit the first eval round\n * (docs/hybrid-retrieval-design.md, \"Eval results\"):\n *\n *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,\n *     so every commit moves the thing being measured. A baseline taken today\n *     and a rerun taken after a retrieval change differ by both the change\n *     and the intervening commits, and nothing in the output says so. Fix:\n *     run against a detached git worktree pinned to an explicit SHA, and put\n *     that SHA in the record.\n *  2. **Nothing was recorded.** Results were printed to a terminal and\n *     hand-copied into a markdown table with no repo SHA, no embedder\n *     identity, and no index state. Fix: emit a machine-readable run record.\n *  3. **A degraded run looks like a real one.** With no embsearch binary the\n *     semantic and hybrid rows silently degrade to lexical, producing a table\n *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the\n *     record, plus a per-row degraded count that the writer refuses to hide.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readdirSync, readFileSync, statSync } from \"node:fs\";\nimport { execFileSync } from \"child_process\";\nimport { rmSync } from \"fs\";\nimport { tmpdir } from \"os\";\nimport path from \"path\";\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { type EvalConfig, type EvalQuery, type EvalQueryResult, evaluateQuery } from \"./eval.js\";\n\n/** Metrics aggregated per config across the whole gold set. */\nexport interface EvalAggregate {\n\tlabel: string;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n\t/** Queries scored under this config. */\n\tn: number;\n\t/** How many of them ran degraded (requested retriever unavailable). */\n\tdegraded: number;\n}\n\n/** Everything needed to decide whether two run records may be compared. */\nexport interface EvalProvenance {\n\ttimestampMs: number;\n\t/** SHA of the corpus actually indexed and searched. */\n\tcorpusSha: string;\n\t/** Ref the caller asked for, before resolution (e.g. \"HEAD\"). */\n\tcorpusRef: string;\n\t/** True when the corpus came from the live working tree rather than a\n\t *  pinned worktree — results are then not reproducible. */\n\tcorpusFromWorkingTree: boolean;\n\t/** Uncommitted changes present at run time. Only meaningful (and only\n\t *  possible) when `corpusFromWorkingTree` is true. */\n\tcorpusDirty: boolean;\n\t/** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,\n\t *  but differs when pinning an old corpus with today's code. */\n\tharnessSha: string;\n\t/**\n\t * Content hash of `src/core/search` + the chunker. Every tuning constant\n\t * that shapes a result — the fusion cap, top-k depths, rerank weights,\n\t * chunk sizing — lives in those files, so a changed hash means the\n\t * numbers are not comparable, without this module having to maintain a\n\t * hand-copied (and inevitably stale) list of constants.\n\t */\n\tretrievalSourceHash: string;\n\t/** Embedding backend state. `available: false` means every semantic and\n\t *  hybrid row in this record degraded to lexical. */\n\tembedder: {\n\t\tavailable: boolean;\n\t\treason?: string;\n\t\t/** Indexed chunk count when the index reached `ready`. */\n\t\tchunkCount?: number;\n\t\tphase: string;\n\t\t/** Binary that served the embeddings, and its self-reported version.\n\t\t *  The embedding model is baked into the binary at build time, so this\n\t\t *  is the only thing that identifies which model produced a score. */\n\t\tbinaryPath?: string;\n\t\tbinaryVersion?: string;\n\t};\n\t/** Daemon-side BM25 hybrid store, when the run included one. Absent means\n\t *  the record has no `daemon-hybrid` rows. */\n\tdaemonHybrid?: { available: boolean; phase: string };\n\truntime: { node: string; platform: string; arch: string };\n}\n\nexport interface EvalRunRecord {\n\tprovenance: EvalProvenance;\n\tgoldSet: { queryCount: number; byClass: Record<string, number>; goldSpanCount: number };\n\tconfigs: readonly EvalConfig[];\n\taggregates: EvalAggregate[];\n\tperQuery: Array<{ id: string; class: string; results: EvalQueryResult[] }>;\n}\n\nfunction git(repoRoot: string, args: string[]): string {\n\treturn execFileSync(\"git\", [\"-C\", repoRoot, ...args], { encoding: \"utf-8\" }).trim();\n}\n\n/** Hash every retrieval-shaping source file, so a tuning change is visible as\n *  a changed provenance field rather than an unexplained metric shift. */\nexport function hashRetrievalSource(repoRoot: string): string {\n\tconst roots = [\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/search\"),\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/embsearch/chunker.ts\"),\n\t];\n\tconst files: string[] = [];\n\tconst walk = (target: string): void => {\n\t\tlet stat: ReturnType<typeof statSync>;\n\t\ttry {\n\t\t\tstat = statSync(target);\n\t\t} catch {\n\t\t\treturn;\n\t\t}\n\t\tif (stat.isDirectory()) {\n\t\t\tfor (const entry of readdirSync(target).sort()) walk(path.join(target, entry));\n\t\t} else if (target.endsWith(\".ts\")) {\n\t\t\tfiles.push(target);\n\t\t}\n\t};\n\tfor (const root of roots) walk(root);\n\n\tconst hash = createHash(\"sha256\");\n\tfor (const file of files) {\n\t\t// Eval-only modules are excluded: changing how we measure must not look\n\t\t// like changing what we measure.\n\t\tconst base = path.basename(file);\n\t\tif (base.startsWith(\"eval\")) continue;\n\t\thash.update(path.relative(repoRoot, file).replace(/\\\\/g, \"/\"));\n\t\thash.update(readFileSync(file));\n\t}\n\treturn hash.digest(\"hex\").slice(0, 16);\n}\n\nexport interface PinnedCorpus {\n\t/** Directory to index and search. */\n\tcwd: string;\n\tsha: string;\n\tfromWorkingTree: boolean;\n\tdirty: boolean;\n\t/** Removes the worktree, if one was created. */\n\tdispose: () => void;\n}\n\n/**\n * Materialize the corpus to evaluate.\n *\n * With a `ref`, checks out a detached worktree at that commit so the corpus is\n * byte-identical on every rerun. Without one, falls back to the live working\n * tree and reports `dirty` so the record shows the run was not reproducible.\n */\nexport function pinCorpus(repoRoot: string, ref: string | undefined): PinnedCorpus {\n\tconst dirty = git(repoRoot, [\"status\", \"--porcelain\"]).length > 0;\n\tif (!ref) {\n\t\treturn {\n\t\t\tcwd: repoRoot,\n\t\t\tsha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\t\tfromWorkingTree: true,\n\t\t\tdirty,\n\t\t\tdispose: () => {},\n\t\t};\n\t}\n\n\tconst sha = git(repoRoot, [\"rev-parse\", ref]);\n\t// Deterministic path, not mkdtemp: the embedding store is keyed by a hash of\n\t// the corpus directory, so a fresh temp path every run would re-embed all\n\t// ~17k chunks (minutes) instead of reusing the store built for this exact\n\t// SHA. The worktree is still removed afterwards; only the store persists.\n\tconst dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}`);\n\tif (existsSync(dir)) {\n\t\t// Left behind by an interrupted run — drop it so `worktree add` succeeds.\n\t\ttry {\n\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t} catch {\n\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\tgit(repoRoot, [\"worktree\", \"prune\"]);\n\t\t}\n\t}\n\tgit(repoRoot, [\"worktree\", \"add\", \"--detach\", dir, sha]);\n\treturn {\n\t\tcwd: dir,\n\t\tsha,\n\t\tfromWorkingTree: false,\n\t\tdirty: false,\n\t\tdispose: () => {\n\t\t\ttry {\n\t\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t\t} catch {\n\t\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\t}\n\t\t},\n\t};\n}\n\n/** `<binary> --version`, or undefined when it cannot be run. */\nfunction probeBinaryVersion(binaryPath: string | undefined): string | undefined {\n\tif (!binaryPath) return undefined;\n\ttry {\n\t\treturn execFileSync(binaryPath, [\"--version\"], { encoding: \"utf-8\" }).trim();\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\nexport function collectProvenance(\n\trepoRoot: string,\n\tcorpus: PinnedCorpus,\n\tcorpusRef: string,\n\tservice: EmbsearchService | undefined,\n\tembsearchBinary?: string,\n\thybridService?: EmbsearchService,\n): EvalProvenance {\n\tconst state = service?.getState();\n\tconst phase = state?.phase ?? \"absent\";\n\treturn {\n\t\ttimestampMs: Date.now(),\n\t\tcorpusSha: corpus.sha,\n\t\tcorpusRef,\n\t\tcorpusFromWorkingTree: corpus.fromWorkingTree,\n\t\tcorpusDirty: corpus.dirty,\n\t\tharnessSha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\tretrievalSourceHash: hashRetrievalSource(repoRoot),\n\t\tembedder: {\n\t\t\t// `ready` is the only phase the service reaches with a real embedder:\n\t\t\t// it rejects the mock backend at startup, so availability here also\n\t\t\t// certifies the numbers came from a genuine ONNX build.\n\t\t\tavailable: service?.isAvailable() ?? false,\n\t\t\treason: state && \"reason\" in state ? state.reason : undefined,\n\t\t\tchunkCount: state?.phase === \"ready\" ? state.chunkCount : undefined,\n\t\t\tphase,\n\t\t\tbinaryPath: embsearchBinary,\n\t\t\tbinaryVersion: probeBinaryVersion(embsearchBinary),\n\t\t},\n\t\tdaemonHybrid: hybridService\n\t\t\t? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }\n\t\t\t: undefined,\n\t\truntime: { node: process.version, platform: process.platform, arch: process.arch },\n\t};\n}\n\nexport function summarizeGoldSet(dataset: readonly EvalQuery[]): EvalRunRecord[\"goldSet\"] {\n\tconst byClass: Record<string, number> = {};\n\tlet goldSpanCount = 0;\n\tfor (const query of dataset) {\n\t\tbyClass[query.class] = (byClass[query.class] ?? 0) + 1;\n\t\tgoldSpanCount += query.gold.length;\n\t}\n\treturn { queryCount: dataset.length, byClass, goldSpanCount };\n}\n\nexport interface RunEvalSuiteOptions {\n\tcwd: string;\n\tdataset: readonly EvalQuery[];\n\tconfigs: readonly EvalConfig[];\n\tservice?: EmbsearchService;\n\t/** Second service backed by a daemon-side BM25 hybrid store, for the\n\t *  `daemon-hybrid` configs. Absent means those rows are omitted. */\n\thybridService?: EmbsearchService;\n\tonQuery?: (index: number, query: EvalQuery) => void;\n}\n\nexport async function runEvalSuite(options: RunEvalSuiteOptions): Promise<{\n\taggregates: EvalAggregate[];\n\tperQuery: EvalRunRecord[\"perQuery\"];\n}> {\n\tconst { cwd, dataset, configs, service } = options;\n\tconst totals = new Map<string, EvalAggregate>();\n\tconst perQuery: EvalRunRecord[\"perQuery\"] = [];\n\n\tfor (const [index, evalQuery] of dataset.entries()) {\n\t\toptions.onQuery?.(index, evalQuery);\n\t\tconst results = await evaluateQuery(cwd, evalQuery, configs, service, options.hybridService);\n\t\tperQuery.push({ id: evalQuery.id, class: evalQuery.class, results });\n\t\tfor (const result of results) {\n\t\t\tconst total = totals.get(result.label) ?? {\n\t\t\t\tlabel: result.label,\n\t\t\t\trecallAt1: 0,\n\t\t\t\trecallAt5: 0,\n\t\t\t\trecallAt10: 0,\n\t\t\t\trecallAt50: 0,\n\t\t\t\tmrr: 0,\n\t\t\t\tn: 0,\n\t\t\t\tdegraded: 0,\n\t\t\t};\n\t\t\ttotal.recallAt1 += result.recallAt1;\n\t\t\ttotal.recallAt5 += result.recallAt5;\n\t\t\ttotal.recallAt10 += result.recallAt10;\n\t\t\ttotal.recallAt50 += result.recallAt50;\n\t\t\ttotal.mrr += result.mrr;\n\t\t\ttotal.n++;\n\t\t\tif (result.degraded) total.degraded++;\n\t\t\ttotals.set(result.label, total);\n\t\t}\n\t}\n\n\tconst aggregates = configs\n\t\t.map((config) => totals.get(config.label))\n\t\t.filter((total): total is EvalAggregate => total !== undefined)\n\t\t.map((total) => ({\n\t\t\t...total,\n\t\t\trecallAt1: total.recallAt1 / total.n,\n\t\t\trecallAt5: total.recallAt5 / total.n,\n\t\t\trecallAt10: total.recallAt10 / total.n,\n\t\t\trecallAt50: total.recallAt50 / total.n,\n\t\t\tmrr: total.mrr / total.n,\n\t\t}));\n\n\treturn { aggregates, perQuery };\n}\n\nexport function formatAggregateTable(aggregates: readonly EvalAggregate[]): string {\n\tconst pct = (x: number) => `${Math.round(x * 100)}%`.padStart(5);\n\tconst lines = [\n\t\t\"config           |  R@1  |  R@5  | R@10  | R@50  |  MRR  | notes\",\n\t\t\"-----------------|-------|-------|-------|-------|-------|------\",\n\t];\n\tfor (const a of aggregates) {\n\t\tconst notes = a.degraded === a.n ? \"degraded to lexical\" : a.degraded > 0 ? `${a.degraded}/${a.n} degraded` : \"\";\n\t\tlines.push(\n\t\t\t`${a.label.padEnd(16)} | ${pct(a.recallAt1)} | ${pct(a.recallAt5)} | ${pct(a.recallAt10)} | ` +\n\t\t\t\t`${pct(a.recallAt50)} | ${a.mrr.toFixed(3)} | ${notes}`,\n\t\t);\n\t}\n\treturn lines.join(\"\\n\");\n}\n"]}
+{"version":3,"file":"eval-harness.d.ts","sourceRoot":"","sources":["../../../src/core/search/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,KAAK,eAAe,EAAiB,MAAM,WAAW,CAAC;AAEjG,+DAA+D;AAC/D,MAAM,WAAW,aAAa;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,wCAAwC;IACxC,CAAC,EAAE,MAAM,CAAC;IACV,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED,2EAA2E;AAC3E,MAAM,WAAW,cAAc;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB;iEAC2D;IAC3D,qBAAqB,EAAE,OAAO,CAAC;IAC/B;0DACsD;IACtD,WAAW,EAAE,OAAO,CAAC;IACrB;;8EAE0E;IAC1E,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB;oEACgE;IAChE,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;;OAMG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;yDACqD;IACrD,QAAQ,EAAE;QACT,SAAS,EAAE,OAAO,CAAC;QACnB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,0DAA0D;QAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;QACd;;8EAEsE;QACtE,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,aAAa,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IACF;kDAC8C;IAC9C,YAAY,CAAC,EAAE;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,WAAW,aAAa;IAC7B,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,OAAO,EAAE,SAAS,UAAU,EAAE,CAAC;IAC/B,UAAU,EAAE,aAAa,EAAE,CAAC;IAC5B,QAAQ,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,eAAe,EAAE,CAAA;KAAE,CAAC,CAAC;CAC3E;AAMD;0EAC0E;AAC1E,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CA+B5D;AAED,MAAM,WAAW,YAAY;IAC5B,qCAAqC;IACrC,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,OAAO,CAAC;IACzB,KAAK,EAAE,OAAO,CAAC;IACf;yDACqD;IACrD,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,gDAAgD;IAChD,OAAO,EAAE,MAAM,IAAI,CAAC;CACpB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,iBAAiB,EAAE,SAAS,MAAM,EAK9C,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,YAAY,CAwDjF;AAYD,wBAAgB,iBAAiB,CAChC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,YAAY,EACpB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,gBAAgB,GAAG,SAAS,EACrC,eAAe,CAAC,EAAE,MAAM,EACxB,aAAa,CAAC,EAAE,gBAAgB,GAC9B,cAAc,CA4BhB;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,SAAS,SAAS,EAAE,GAAG,aAAa,CAAC,SAAS,CAAC,CAQxF;AAED,MAAM,WAAW,mBAAmB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,SAAS,SAAS,EAAE,CAAC;IAC9B,OAAO,EAAE,SAAS,UAAU,EAAE,CAAC;IAC/B,OAAO,CAAC,EAAE,gBAAgB,CAAC;IAC3B;wEACoE;IACpE,aAAa,CAAC,EAAE,gBAAgB,CAAC;IACjC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,KAAK,IAAI,CAAC;CACpD;AAED,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC;IACzE,UAAU,EAAE,aAAa,EAAE,CAAC;IAC5B,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACpC,CAAC,CA4CD;AAED,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,GAAG,MAAM,CAcjF","sourcesContent":["/**\n * Eval harness: corpus pinning, provenance capture, and run records.\n *\n * The scoring math lives in `eval.ts`; this module is everything around it\n * that makes a number *comparable to a later number*. Three problems it\n * exists to solve, all of which bit the first eval round\n * (docs/hybrid-retrieval-design.md, \"Eval results\"):\n *\n *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,\n *     so every commit moves the thing being measured. A baseline taken today\n *     and a rerun taken after a retrieval change differ by both the change\n *     and the intervening commits, and nothing in the output says so. Fix:\n *     run against a detached git worktree pinned to an explicit SHA, and put\n *     that SHA in the record.\n *  2. **Nothing was recorded.** Results were printed to a terminal and\n *     hand-copied into a markdown table with no repo SHA, no embedder\n *     identity, and no index state. Fix: emit a machine-readable run record.\n *  3. **A degraded run looks like a real one.** With no embsearch binary the\n *     semantic and hybrid rows silently degrade to lexical, producing a table\n *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the\n *     record, plus a per-row degraded count that the writer refuses to hide.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readdirSync, readFileSync, statSync } from \"node:fs\";\nimport { execFileSync } from \"child_process\";\nimport { rmSync } from \"fs\";\nimport { tmpdir } from \"os\";\nimport path from \"path\";\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { type EvalConfig, type EvalQuery, type EvalQueryResult, evaluateQuery } from \"./eval.js\";\n\n/** Metrics aggregated per config across the whole gold set. */\nexport interface EvalAggregate {\n\tlabel: string;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n\t/** Queries scored under this config. */\n\tn: number;\n\t/** How many of them ran degraded (requested retriever unavailable). */\n\tdegraded: number;\n}\n\n/** Everything needed to decide whether two run records may be compared. */\nexport interface EvalProvenance {\n\ttimestampMs: number;\n\t/** SHA of the corpus actually indexed and searched. */\n\tcorpusSha: string;\n\t/** Ref the caller asked for, before resolution (e.g. \"HEAD\"). */\n\tcorpusRef: string;\n\t/** True when the corpus came from the live working tree rather than a\n\t *  pinned worktree — results are then not reproducible. */\n\tcorpusFromWorkingTree: boolean;\n\t/** Uncommitted changes present at run time. Only meaningful (and only\n\t *  possible) when `corpusFromWorkingTree` is true. */\n\tcorpusDirty: boolean;\n\t/** Files removed from the corpus before indexing — see\n\t *  {@link CORPUS_EXCLUSIONS}. A score against a different exclusion list is\n\t *  a score against a different corpus, so it is recorded, not assumed. */\n\tcorpusExcluded: string[];\n\t/** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,\n\t *  but differs when pinning an old corpus with today's code. */\n\tharnessSha: string;\n\t/**\n\t * Content hash of `src/core/search` + the chunker. Every tuning constant\n\t * that shapes a result — the fusion cap, top-k depths, rerank weights,\n\t * chunk sizing — lives in those files, so a changed hash means the\n\t * numbers are not comparable, without this module having to maintain a\n\t * hand-copied (and inevitably stale) list of constants.\n\t */\n\tretrievalSourceHash: string;\n\t/** Embedding backend state. `available: false` means every semantic and\n\t *  hybrid row in this record degraded to lexical. */\n\tembedder: {\n\t\tavailable: boolean;\n\t\treason?: string;\n\t\t/** Indexed chunk count when the index reached `ready`. */\n\t\tchunkCount?: number;\n\t\tphase: string;\n\t\t/** Binary that served the embeddings, and its self-reported version.\n\t\t *  The embedding model is baked into the binary at build time, so this\n\t\t *  is the only thing that identifies which model produced a score. */\n\t\tbinaryPath?: string;\n\t\tbinaryVersion?: string;\n\t};\n\t/** Daemon-side BM25 hybrid store, when the run included one. Absent means\n\t *  the record has no `daemon-hybrid` rows. */\n\tdaemonHybrid?: { available: boolean; phase: string };\n\truntime: { node: string; platform: string; arch: string };\n}\n\nexport interface EvalRunRecord {\n\tprovenance: EvalProvenance;\n\tgoldSet: { queryCount: number; byClass: Record<string, number>; goldSpanCount: number };\n\tconfigs: readonly EvalConfig[];\n\taggregates: EvalAggregate[];\n\tperQuery: Array<{ id: string; class: string; results: EvalQueryResult[] }>;\n}\n\nfunction git(repoRoot: string, args: string[]): string {\n\treturn execFileSync(\"git\", [\"-C\", repoRoot, ...args], { encoding: \"utf-8\" }).trim();\n}\n\n/** Hash every retrieval-shaping source file, so a tuning change is visible as\n *  a changed provenance field rather than an unexplained metric shift. */\nexport function hashRetrievalSource(repoRoot: string): string {\n\tconst roots = [\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/search\"),\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/embsearch/chunker.ts\"),\n\t];\n\tconst files: string[] = [];\n\tconst walk = (target: string): void => {\n\t\tlet stat: ReturnType<typeof statSync>;\n\t\ttry {\n\t\t\tstat = statSync(target);\n\t\t} catch {\n\t\t\treturn;\n\t\t}\n\t\tif (stat.isDirectory()) {\n\t\t\tfor (const entry of readdirSync(target).sort()) walk(path.join(target, entry));\n\t\t} else if (target.endsWith(\".ts\")) {\n\t\t\tfiles.push(target);\n\t\t}\n\t};\n\tfor (const root of roots) walk(root);\n\n\tconst hash = createHash(\"sha256\");\n\tfor (const file of files) {\n\t\t// Eval-only modules are excluded: changing how we measure must not look\n\t\t// like changing what we measure.\n\t\tconst base = path.basename(file);\n\t\tif (base.startsWith(\"eval\")) continue;\n\t\thash.update(path.relative(repoRoot, file).replace(/\\\\/g, \"/\"));\n\t\thash.update(readFileSync(file));\n\t}\n\treturn hash.digest(\"hex\").slice(0, 16);\n}\n\nexport interface PinnedCorpus {\n\t/** Directory to index and search. */\n\tcwd: string;\n\tsha: string;\n\tfromWorkingTree: boolean;\n\tdirty: boolean;\n\t/** Files removed from the corpus before indexing. Empty when the corpus is\n\t *  the live working tree, which is never mutated. */\n\texcluded: string[];\n\t/** Removes the worktree, if one was created. */\n\tdispose: () => void;\n}\n\n/**\n * Files that describe this eval rather than being searched by it.\n *\n * The fixtures hold all 62 query strings verbatim, so every query is a perfect\n * lexical match against its own entry, and the design note quotes the same\n * queries while discussing the classes they belong to. Measured before this\n * exclusion existed: **56 of 62 queries had one of these files in the top 10,\n * 27 of 62 had one as the #1 result, and they consumed 133 of the 620\n * top-10 slots** — a fifth of the window, spent on the eval reading itself.\n *\n * That is not a ranking artifact a reranker can fix: it displaces real answers\n * out of the window entirely, which is why two boundary-class queries were\n * absent from the top *50* rather than merely buried. Retrieving your own\n * question is not retrieval, so the corpus is scored without them.\n *\n * Removed from the pinned worktree before indexing, never from the repo — and\n * recorded in the run's provenance so a score is never silently taken against\n * a different corpus than it claims.\n */\nexport const CORPUS_EXCLUSIONS: readonly string[] = [\n\t\"packages/coding-agent/test/fixtures/search-eval.json\",\n\t\"packages/coding-agent/test/fixtures/search-eval-live.json\",\n\t\"packages/coding-agent/test/fixtures/search-eval-baseline.json\",\n\t\"docs/hybrid-retrieval-design.md\",\n];\n\n/**\n * Materialize the corpus to evaluate.\n *\n * With a `ref`, checks out a detached worktree at that commit so the corpus is\n * byte-identical on every rerun. Without one, falls back to the live working\n * tree and reports `dirty` so the record shows the run was not reproducible.\n */\nexport function pinCorpus(repoRoot: string, ref: string | undefined): PinnedCorpus {\n\tconst dirty = git(repoRoot, [\"status\", \"--porcelain\"]).length > 0;\n\tif (!ref) {\n\t\t// The live working tree is the user's checkout; deleting files from it to\n\t\t// tidy a measurement would be an unforgivable trade. Working-tree runs\n\t\t// are already stamped non-reproducible, so they carry the contamination.\n\t\treturn {\n\t\t\tcwd: repoRoot,\n\t\t\tsha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\t\tfromWorkingTree: true,\n\t\t\tdirty,\n\t\t\texcluded: [],\n\t\t\tdispose: () => {},\n\t\t};\n\t}\n\n\tconst sha = git(repoRoot, [\"rev-parse\", ref]);\n\t// Deterministic path, not mkdtemp: the embedding store is keyed by a hash of\n\t// the corpus directory, so a fresh temp path every run would re-embed all\n\t// ~17k chunks (minutes) instead of reusing the store built for this exact\n\t// SHA. The worktree is still removed afterwards; only the store persists.\n\tconst dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}`);\n\tif (existsSync(dir)) {\n\t\t// Left behind by an interrupted run — drop it so `worktree add` succeeds.\n\t\ttry {\n\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t} catch {\n\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\tgit(repoRoot, [\"worktree\", \"prune\"]);\n\t\t}\n\t}\n\tgit(repoRoot, [\"worktree\", \"add\", \"--detach\", dir, sha]);\n\n\tconst excluded: string[] = [];\n\tfor (const rel of CORPUS_EXCLUSIONS) {\n\t\tconst target = path.join(dir, rel);\n\t\tif (existsSync(target)) {\n\t\t\trmSync(target, { force: true });\n\t\t\texcluded.push(rel);\n\t\t}\n\t}\n\n\treturn {\n\t\tcwd: dir,\n\t\tsha,\n\t\tfromWorkingTree: false,\n\t\tdirty: false,\n\t\texcluded,\n\t\tdispose: () => {\n\t\t\ttry {\n\t\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t\t} catch {\n\t\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\t}\n\t\t},\n\t};\n}\n\n/** `<binary> --version`, or undefined when it cannot be run. */\nfunction probeBinaryVersion(binaryPath: string | undefined): string | undefined {\n\tif (!binaryPath) return undefined;\n\ttry {\n\t\treturn execFileSync(binaryPath, [\"--version\"], { encoding: \"utf-8\" }).trim();\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\nexport function collectProvenance(\n\trepoRoot: string,\n\tcorpus: PinnedCorpus,\n\tcorpusRef: string,\n\tservice: EmbsearchService | undefined,\n\tembsearchBinary?: string,\n\thybridService?: EmbsearchService,\n): EvalProvenance {\n\tconst state = service?.getState();\n\tconst phase = state?.phase ?? \"absent\";\n\treturn {\n\t\ttimestampMs: Date.now(),\n\t\tcorpusSha: corpus.sha,\n\t\tcorpusRef,\n\t\tcorpusFromWorkingTree: corpus.fromWorkingTree,\n\t\tcorpusDirty: corpus.dirty,\n\t\tcorpusExcluded: corpus.excluded,\n\t\tharnessSha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\tretrievalSourceHash: hashRetrievalSource(repoRoot),\n\t\tembedder: {\n\t\t\t// `ready` is the only phase the service reaches with a real embedder:\n\t\t\t// it rejects the mock backend at startup, so availability here also\n\t\t\t// certifies the numbers came from a genuine ONNX build.\n\t\t\tavailable: service?.isAvailable() ?? false,\n\t\t\treason: state && \"reason\" in state ? state.reason : undefined,\n\t\t\tchunkCount: state?.phase === \"ready\" ? state.chunkCount : undefined,\n\t\t\tphase,\n\t\t\tbinaryPath: embsearchBinary,\n\t\t\tbinaryVersion: probeBinaryVersion(embsearchBinary),\n\t\t},\n\t\tdaemonHybrid: hybridService\n\t\t\t? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }\n\t\t\t: undefined,\n\t\truntime: { node: process.version, platform: process.platform, arch: process.arch },\n\t};\n}\n\nexport function summarizeGoldSet(dataset: readonly EvalQuery[]): EvalRunRecord[\"goldSet\"] {\n\tconst byClass: Record<string, number> = {};\n\tlet goldSpanCount = 0;\n\tfor (const query of dataset) {\n\t\tbyClass[query.class] = (byClass[query.class] ?? 0) + 1;\n\t\tgoldSpanCount += query.gold.length;\n\t}\n\treturn { queryCount: dataset.length, byClass, goldSpanCount };\n}\n\nexport interface RunEvalSuiteOptions {\n\tcwd: string;\n\tdataset: readonly EvalQuery[];\n\tconfigs: readonly EvalConfig[];\n\tservice?: EmbsearchService;\n\t/** Second service backed by a daemon-side BM25 hybrid store, for the\n\t *  `daemon-hybrid` configs. Absent means those rows are omitted. */\n\thybridService?: EmbsearchService;\n\tonQuery?: (index: number, query: EvalQuery) => void;\n}\n\nexport async function runEvalSuite(options: RunEvalSuiteOptions): Promise<{\n\taggregates: EvalAggregate[];\n\tperQuery: EvalRunRecord[\"perQuery\"];\n}> {\n\tconst { cwd, dataset, configs, service } = options;\n\tconst totals = new Map<string, EvalAggregate>();\n\tconst perQuery: EvalRunRecord[\"perQuery\"] = [];\n\n\tfor (const [index, evalQuery] of dataset.entries()) {\n\t\toptions.onQuery?.(index, evalQuery);\n\t\tconst results = await evaluateQuery(cwd, evalQuery, configs, service, options.hybridService);\n\t\tperQuery.push({ id: evalQuery.id, class: evalQuery.class, results });\n\t\tfor (const result of results) {\n\t\t\tconst total = totals.get(result.label) ?? {\n\t\t\t\tlabel: result.label,\n\t\t\t\trecallAt1: 0,\n\t\t\t\trecallAt5: 0,\n\t\t\t\trecallAt10: 0,\n\t\t\t\trecallAt50: 0,\n\t\t\t\tmrr: 0,\n\t\t\t\tn: 0,\n\t\t\t\tdegraded: 0,\n\t\t\t};\n\t\t\ttotal.recallAt1 += result.recallAt1;\n\t\t\ttotal.recallAt5 += result.recallAt5;\n\t\t\ttotal.recallAt10 += result.recallAt10;\n\t\t\ttotal.recallAt50 += result.recallAt50;\n\t\t\ttotal.mrr += result.mrr;\n\t\t\ttotal.n++;\n\t\t\tif (result.degraded) total.degraded++;\n\t\t\ttotals.set(result.label, total);\n\t\t}\n\t}\n\n\tconst aggregates = configs\n\t\t.map((config) => totals.get(config.label))\n\t\t.filter((total): total is EvalAggregate => total !== undefined)\n\t\t.map((total) => ({\n\t\t\t...total,\n\t\t\trecallAt1: total.recallAt1 / total.n,\n\t\t\trecallAt5: total.recallAt5 / total.n,\n\t\t\trecallAt10: total.recallAt10 / total.n,\n\t\t\trecallAt50: total.recallAt50 / total.n,\n\t\t\tmrr: total.mrr / total.n,\n\t\t}));\n\n\treturn { aggregates, perQuery };\n}\n\nexport function formatAggregateTable(aggregates: readonly EvalAggregate[]): string {\n\tconst pct = (x: number) => `${Math.round(x * 100)}%`.padStart(5);\n\tconst lines = [\n\t\t\"config           |  R@1  |  R@5  | R@10  | R@50  |  MRR  | notes\",\n\t\t\"-----------------|-------|-------|-------|-------|-------|------\",\n\t];\n\tfor (const a of aggregates) {\n\t\tconst notes = a.degraded === a.n ? \"degraded to lexical\" : a.degraded > 0 ? `${a.degraded}/${a.n} degraded` : \"\";\n\t\tlines.push(\n\t\t\t`${a.label.padEnd(16)} | ${pct(a.recallAt1)} | ${pct(a.recallAt5)} | ${pct(a.recallAt10)} | ` +\n\t\t\t\t`${pct(a.recallAt50)} | ${a.mrr.toFixed(3)} | ${notes}`,\n\t\t);\n\t}\n\treturn lines.join(\"\\n\");\n}\n"]}

package/dist/core/search/eval-harness.js

@@ -68,6 +68,31 @@ export function hashRetrievalSource(repoRoot) {
     }
     return hash.digest("hex").slice(0, 16);
 }
+/**
+ * Files that describe this eval rather than being searched by it.
+ *
+ * The fixtures hold all 62 query strings verbatim, so every query is a perfect
+ * lexical match against its own entry, and the design note quotes the same
+ * queries while discussing the classes they belong to. Measured before this
+ * exclusion existed: **56 of 62 queries had one of these files in the top 10,
+ * 27 of 62 had one as the #1 result, and they consumed 133 of the 620
+ * top-10 slots** — a fifth of the window, spent on the eval reading itself.
+ *
+ * That is not a ranking artifact a reranker can fix: it displaces real answers
+ * out of the window entirely, which is why two boundary-class queries were
+ * absent from the top *50* rather than merely buried. Retrieving your own
+ * question is not retrieval, so the corpus is scored without them.
+ *
+ * Removed from the pinned worktree before indexing, never from the repo — and
+ * recorded in the run's provenance so a score is never silently taken against
+ * a different corpus than it claims.
+ */
+export const CORPUS_EXCLUSIONS = [
+    "packages/coding-agent/test/fixtures/search-eval.json",
+    "packages/coding-agent/test/fixtures/search-eval-live.json",
+    "packages/coding-agent/test/fixtures/search-eval-baseline.json",
+    "docs/hybrid-retrieval-design.md",
+];
 /**
  * Materialize the corpus to evaluate.
  *
@@ -78,11 +103,15 @@ export function hashRetrievalSource(repoRoot) {
 export function pinCorpus(repoRoot, ref) {
     const dirty = git(repoRoot, ["status", "--porcelain"]).length > 0;
     if (!ref) {
+        // The live working tree is the user's checkout; deleting files from it to
+        // tidy a measurement would be an unforgivable trade. Working-tree runs
+        // are already stamped non-reproducible, so they carry the contamination.
         return {
             cwd: repoRoot,
             sha: git(repoRoot, ["rev-parse", "HEAD"]),
             fromWorkingTree: true,
             dirty,
+            excluded: [],
             dispose: () => { },
         };
     }
@@ -103,11 +132,20 @@ export function pinCorpus(repoRoot, ref) {
         }
     }
     git(repoRoot, ["worktree", "add", "--detach", dir, sha]);
+    const excluded = [];
+    for (const rel of CORPUS_EXCLUSIONS) {
+        const target = path.join(dir, rel);
+        if (existsSync(target)) {
+            rmSync(target, { force: true });
+            excluded.push(rel);
+        }
+    }
     return {
         cwd: dir,
         sha,
         fromWorkingTree: false,
         dirty: false,
+        excluded,
         dispose: () => {
             try {
                 git(repoRoot, ["worktree", "remove", "--force", dir]);
@@ -138,6 +176,7 @@ export function collectProvenance(repoRoot, corpus, corpusRef, service, embsearc
         corpusRef,
         corpusFromWorkingTree: corpus.fromWorkingTree,
         corpusDirty: corpus.dirty,
+        corpusExcluded: corpus.excluded,
         harnessSha: git(repoRoot, ["rev-parse", "HEAD"]),
         retrievalSourceHash: hashRetrievalSource(repoRoot),
         embedder: {

package/dist/core/search/eval-harness.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-harness.js","sourceRoot":"","sources":["../../../src/core/search/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAyD,aAAa,EAAE,MAAM,WAAW,CAAC;AAoEjG,SAAS,GAAG,CAAC,QAAgB,EAAE,IAAc,EAAU;IACtD,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAAA,CACpF;AAED;0EAC0E;AAC1E,MAAM,UAAU,mBAAmB,CAAC,QAAgB,EAAU;IAC7D,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,uCAAuC,CAAC;QAC5D,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,qDAAqD,CAAC;KAC1E,CAAC;IACF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,MAAc,EAAQ,EAAE,CAAC;QACtC,IAAI,IAAiC,CAAC;QACtC,IAAI,CAAC;YACJ,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACR,OAAO;QACR,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE;gBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;IAAA,CACD,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,wEAAwE;QACxE,iCAAiC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACtC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAAA,CACvC;AAYD;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,GAAuB,EAAgB;IAClF,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAClE,IAAI,CAAC,GAAG,EAAE,CAAC;QACV,OAAO;YACN,GAAG,EAAE,QAAQ;YACb,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACzC,eAAe,EAAE,IAAI;YACrB,KAAK;YACL,OAAO,EAAE,GAAG,EAAE,CAAC,EAAC,CAAC;SACjB,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9C,6EAA6E;IAC7E,0EAA0E;IAC1E,0EAA0E;IAC1E,0EAA0E;IAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,4EAA0E;QAC1E,IAAI,CAAC;YACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;QACtC,CAAC;IACF,CAAC;IACD,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IACzD,OAAO;QACN,GAAG,EAAE,GAAG;QACR,GAAG;QACH,eAAe,EAAE,KAAK;QACtB,KAAK,EAAE,KAAK;QACZ,OAAO,EAAE,GAAG,EAAE,CAAC;YACd,IAAI,CAAC;gBACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/C,CAAC;QAAA,CACD;KACD,CAAC;AAAA,CACF;AAED,gEAAgE;AAChE,SAAS,kBAAkB,CAAC,UAA8B,EAAsB;IAC/E,IAAI,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAClC,IAAI,CAAC;QACJ,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED,MAAM,UAAU,iBAAiB,CAChC,QAAgB,EAChB,MAAoB,EACpB,SAAiB,EACjB,OAAqC,EACrC,eAAwB,EACxB,aAAgC,EACf;IACjB,MAAM,KAAK,GAAG,OAAO,EAAE,QAAQ,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,KAAK,EAAE,KAAK,IAAI,QAAQ,CAAC;IACvC,OAAO;QACN,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;QACvB,SAAS,EAAE,MAAM,CAAC,GAAG;QACrB,SAAS;QACT,qBAAqB,EAAE,MAAM,CAAC,eAAe;QAC7C,WAAW,EAAE,MAAM,CAAC,KAAK;QACzB,UAAU,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAChD,mBAAmB,EAAE,mBAAmB,CAAC,QAAQ,CAAC;QAClD,QAAQ,EAAE;YACT,sEAAsE;YACtE,oEAAoE;YACpE,wDAAwD;YACxD,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,KAAK;YAC1C,MAAM,EAAE,KAAK,IAAI,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YAC7D,UAAU,EAAE,KAAK,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YACnE,KAAK;YACL,UAAU,EAAE,eAAe;YAC3B,aAAa,EAAE,kBAAkB,CAAC,eAAe,CAAC;SAClD;QACD,YAAY,EAAE,aAAa;YAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,aAAa,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,aAAa,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE;YACnF,CAAC,CAAC,SAAS;QACZ,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE;KAClF,CAAC;AAAA,CACF;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA6B,EAA4B;IACzF,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACvD,aAAa,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;IACpC,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AAAA,CAC9D;AAaD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B,EAG5D;IACF,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAChD,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAE/C,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC7F,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACrE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI;gBACzC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC;gBACb,UAAU,EAAE,CAAC;gBACb,GAAG,EAAE,CAAC;gBACN,CAAC,EAAE,CAAC;gBACJ,QAAQ,EAAE,CAAC;aACX,CAAC;YACF,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC;YACxB,KAAK,CAAC,CAAC,EAAE,CAAC;YACV,IAAI,MAAM,CAAC,QAAQ;gBAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACjC,CAAC;IACF,CAAC;IAED,MAAM,UAAU,GAAG,OAAO;SACxB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;SACzC,MAAM,CAAC,CAAC,KAAK,EAA0B,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC;SAC9D,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAChB,GAAG,KAAK;QACR,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;KACxB,CAAC,CAAC,CAAC;IAEL,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAAA,CAChC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAAoC,EAAU;IAClF,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG;QACb,kEAAkE;QAClE,kEAAkE;KAClE,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;QACjH,KAAK,CAAC,IAAI,CACT,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK;YAC5F,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,EAAE,CACxD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAAA,CACxB","sourcesContent":["/**\n * Eval harness: corpus pinning, provenance capture, and run records.\n *\n * The scoring math lives in `eval.ts`; this module is everything around it\n * that makes a number *comparable to a later number*. Three problems it\n * exists to solve, all of which bit the first eval round\n * (docs/hybrid-retrieval-design.md, \"Eval results\"):\n *\n *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,\n *     so every commit moves the thing being measured. A baseline taken today\n *     and a rerun taken after a retrieval change differ by both the change\n *     and the intervening commits, and nothing in the output says so. Fix:\n *     run against a detached git worktree pinned to an explicit SHA, and put\n *     that SHA in the record.\n *  2. **Nothing was recorded.** Results were printed to a terminal and\n *     hand-copied into a markdown table with no repo SHA, no embedder\n *     identity, and no index state. Fix: emit a machine-readable run record.\n *  3. **A degraded run looks like a real one.** With no embsearch binary the\n *     semantic and hybrid rows silently degrade to lexical, producing a table\n *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the\n *     record, plus a per-row degraded count that the writer refuses to hide.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readdirSync, readFileSync, statSync } from \"node:fs\";\nimport { execFileSync } from \"child_process\";\nimport { rmSync } from \"fs\";\nimport { tmpdir } from \"os\";\nimport path from \"path\";\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { type EvalConfig, type EvalQuery, type EvalQueryResult, evaluateQuery } from \"./eval.js\";\n\n/** Metrics aggregated per config across the whole gold set. */\nexport interface EvalAggregate {\n\tlabel: string;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n\t/** Queries scored under this config. */\n\tn: number;\n\t/** How many of them ran degraded (requested retriever unavailable). */\n\tdegraded: number;\n}\n\n/** Everything needed to decide whether two run records may be compared. */\nexport interface EvalProvenance {\n\ttimestampMs: number;\n\t/** SHA of the corpus actually indexed and searched. */\n\tcorpusSha: string;\n\t/** Ref the caller asked for, before resolution (e.g. \"HEAD\"). */\n\tcorpusRef: string;\n\t/** True when the corpus came from the live working tree rather than a\n\t *  pinned worktree — results are then not reproducible. */\n\tcorpusFromWorkingTree: boolean;\n\t/** Uncommitted changes present at run time. Only meaningful (and only\n\t *  possible) when `corpusFromWorkingTree` is true. */\n\tcorpusDirty: boolean;\n\t/** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,\n\t *  but differs when pinning an old corpus with today's code. */\n\tharnessSha: string;\n\t/**\n\t * Content hash of `src/core/search` + the chunker. Every tuning constant\n\t * that shapes a result — the fusion cap, top-k depths, rerank weights,\n\t * chunk sizing — lives in those files, so a changed hash means the\n\t * numbers are not comparable, without this module having to maintain a\n\t * hand-copied (and inevitably stale) list of constants.\n\t */\n\tretrievalSourceHash: string;\n\t/** Embedding backend state. `available: false` means every semantic and\n\t *  hybrid row in this record degraded to lexical. */\n\tembedder: {\n\t\tavailable: boolean;\n\t\treason?: string;\n\t\t/** Indexed chunk count when the index reached `ready`. */\n\t\tchunkCount?: number;\n\t\tphase: string;\n\t\t/** Binary that served the embeddings, and its self-reported version.\n\t\t *  The embedding model is baked into the binary at build time, so this\n\t\t *  is the only thing that identifies which model produced a score. */\n\t\tbinaryPath?: string;\n\t\tbinaryVersion?: string;\n\t};\n\t/** Daemon-side BM25 hybrid store, when the run included one. Absent means\n\t *  the record has no `daemon-hybrid` rows. */\n\tdaemonHybrid?: { available: boolean; phase: string };\n\truntime: { node: string; platform: string; arch: string };\n}\n\nexport interface EvalRunRecord {\n\tprovenance: EvalProvenance;\n\tgoldSet: { queryCount: number; byClass: Record<string, number>; goldSpanCount: number };\n\tconfigs: readonly EvalConfig[];\n\taggregates: EvalAggregate[];\n\tperQuery: Array<{ id: string; class: string; results: EvalQueryResult[] }>;\n}\n\nfunction git(repoRoot: string, args: string[]): string {\n\treturn execFileSync(\"git\", [\"-C\", repoRoot, ...args], { encoding: \"utf-8\" }).trim();\n}\n\n/** Hash every retrieval-shaping source file, so a tuning change is visible as\n *  a changed provenance field rather than an unexplained metric shift. */\nexport function hashRetrievalSource(repoRoot: string): string {\n\tconst roots = [\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/search\"),\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/embsearch/chunker.ts\"),\n\t];\n\tconst files: string[] = [];\n\tconst walk = (target: string): void => {\n\t\tlet stat: ReturnType<typeof statSync>;\n\t\ttry {\n\t\t\tstat = statSync(target);\n\t\t} catch {\n\t\t\treturn;\n\t\t}\n\t\tif (stat.isDirectory()) {\n\t\t\tfor (const entry of readdirSync(target).sort()) walk(path.join(target, entry));\n\t\t} else if (target.endsWith(\".ts\")) {\n\t\t\tfiles.push(target);\n\t\t}\n\t};\n\tfor (const root of roots) walk(root);\n\n\tconst hash = createHash(\"sha256\");\n\tfor (const file of files) {\n\t\t// Eval-only modules are excluded: changing how we measure must not look\n\t\t// like changing what we measure.\n\t\tconst base = path.basename(file);\n\t\tif (base.startsWith(\"eval\")) continue;\n\t\thash.update(path.relative(repoRoot, file).replace(/\\\\/g, \"/\"));\n\t\thash.update(readFileSync(file));\n\t}\n\treturn hash.digest(\"hex\").slice(0, 16);\n}\n\nexport interface PinnedCorpus {\n\t/** Directory to index and search. */\n\tcwd: string;\n\tsha: string;\n\tfromWorkingTree: boolean;\n\tdirty: boolean;\n\t/** Removes the worktree, if one was created. */\n\tdispose: () => void;\n}\n\n/**\n * Materialize the corpus to evaluate.\n *\n * With a `ref`, checks out a detached worktree at that commit so the corpus is\n * byte-identical on every rerun. Without one, falls back to the live working\n * tree and reports `dirty` so the record shows the run was not reproducible.\n */\nexport function pinCorpus(repoRoot: string, ref: string | undefined): PinnedCorpus {\n\tconst dirty = git(repoRoot, [\"status\", \"--porcelain\"]).length > 0;\n\tif (!ref) {\n\t\treturn {\n\t\t\tcwd: repoRoot,\n\t\t\tsha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\t\tfromWorkingTree: true,\n\t\t\tdirty,\n\t\t\tdispose: () => {},\n\t\t};\n\t}\n\n\tconst sha = git(repoRoot, [\"rev-parse\", ref]);\n\t// Deterministic path, not mkdtemp: the embedding store is keyed by a hash of\n\t// the corpus directory, so a fresh temp path every run would re-embed all\n\t// ~17k chunks (minutes) instead of reusing the store built for this exact\n\t// SHA. The worktree is still removed afterwards; only the store persists.\n\tconst dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}`);\n\tif (existsSync(dir)) {\n\t\t// Left behind by an interrupted run — drop it so `worktree add` succeeds.\n\t\ttry {\n\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t} catch {\n\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\tgit(repoRoot, [\"worktree\", \"prune\"]);\n\t\t}\n\t}\n\tgit(repoRoot, [\"worktree\", \"add\", \"--detach\", dir, sha]);\n\treturn {\n\t\tcwd: dir,\n\t\tsha,\n\t\tfromWorkingTree: false,\n\t\tdirty: false,\n\t\tdispose: () => {\n\t\t\ttry {\n\t\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t\t} catch {\n\t\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\t}\n\t\t},\n\t};\n}\n\n/** `<binary> --version`, or undefined when it cannot be run. */\nfunction probeBinaryVersion(binaryPath: string | undefined): string | undefined {\n\tif (!binaryPath) return undefined;\n\ttry {\n\t\treturn execFileSync(binaryPath, [\"--version\"], { encoding: \"utf-8\" }).trim();\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\nexport function collectProvenance(\n\trepoRoot: string,\n\tcorpus: PinnedCorpus,\n\tcorpusRef: string,\n\tservice: EmbsearchService | undefined,\n\tembsearchBinary?: string,\n\thybridService?: EmbsearchService,\n): EvalProvenance {\n\tconst state = service?.getState();\n\tconst phase = state?.phase ?? \"absent\";\n\treturn {\n\t\ttimestampMs: Date.now(),\n\t\tcorpusSha: corpus.sha,\n\t\tcorpusRef,\n\t\tcorpusFromWorkingTree: corpus.fromWorkingTree,\n\t\tcorpusDirty: corpus.dirty,\n\t\tharnessSha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\tretrievalSourceHash: hashRetrievalSource(repoRoot),\n\t\tembedder: {\n\t\t\t// `ready` is the only phase the service reaches with a real embedder:\n\t\t\t// it rejects the mock backend at startup, so availability here also\n\t\t\t// certifies the numbers came from a genuine ONNX build.\n\t\t\tavailable: service?.isAvailable() ?? false,\n\t\t\treason: state && \"reason\" in state ? state.reason : undefined,\n\t\t\tchunkCount: state?.phase === \"ready\" ? state.chunkCount : undefined,\n\t\t\tphase,\n\t\t\tbinaryPath: embsearchBinary,\n\t\t\tbinaryVersion: probeBinaryVersion(embsearchBinary),\n\t\t},\n\t\tdaemonHybrid: hybridService\n\t\t\t? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }\n\t\t\t: undefined,\n\t\truntime: { node: process.version, platform: process.platform, arch: process.arch },\n\t};\n}\n\nexport function summarizeGoldSet(dataset: readonly EvalQuery[]): EvalRunRecord[\"goldSet\"] {\n\tconst byClass: Record<string, number> = {};\n\tlet goldSpanCount = 0;\n\tfor (const query of dataset) {\n\t\tbyClass[query.class] = (byClass[query.class] ?? 0) + 1;\n\t\tgoldSpanCount += query.gold.length;\n\t}\n\treturn { queryCount: dataset.length, byClass, goldSpanCount };\n}\n\nexport interface RunEvalSuiteOptions {\n\tcwd: string;\n\tdataset: readonly EvalQuery[];\n\tconfigs: readonly EvalConfig[];\n\tservice?: EmbsearchService;\n\t/** Second service backed by a daemon-side BM25 hybrid store, for the\n\t *  `daemon-hybrid` configs. Absent means those rows are omitted. */\n\thybridService?: EmbsearchService;\n\tonQuery?: (index: number, query: EvalQuery) => void;\n}\n\nexport async function runEvalSuite(options: RunEvalSuiteOptions): Promise<{\n\taggregates: EvalAggregate[];\n\tperQuery: EvalRunRecord[\"perQuery\"];\n}> {\n\tconst { cwd, dataset, configs, service } = options;\n\tconst totals = new Map<string, EvalAggregate>();\n\tconst perQuery: EvalRunRecord[\"perQuery\"] = [];\n\n\tfor (const [index, evalQuery] of dataset.entries()) {\n\t\toptions.onQuery?.(index, evalQuery);\n\t\tconst results = await evaluateQuery(cwd, evalQuery, configs, service, options.hybridService);\n\t\tperQuery.push({ id: evalQuery.id, class: evalQuery.class, results });\n\t\tfor (const result of results) {\n\t\t\tconst total = totals.get(result.label) ?? {\n\t\t\t\tlabel: result.label,\n\t\t\t\trecallAt1: 0,\n\t\t\t\trecallAt5: 0,\n\t\t\t\trecallAt10: 0,\n\t\t\t\trecallAt50: 0,\n\t\t\t\tmrr: 0,\n\t\t\t\tn: 0,\n\t\t\t\tdegraded: 0,\n\t\t\t};\n\t\t\ttotal.recallAt1 += result.recallAt1;\n\t\t\ttotal.recallAt5 += result.recallAt5;\n\t\t\ttotal.recallAt10 += result.recallAt10;\n\t\t\ttotal.recallAt50 += result.recallAt50;\n\t\t\ttotal.mrr += result.mrr;\n\t\t\ttotal.n++;\n\t\t\tif (result.degraded) total.degraded++;\n\t\t\ttotals.set(result.label, total);\n\t\t}\n\t}\n\n\tconst aggregates = configs\n\t\t.map((config) => totals.get(config.label))\n\t\t.filter((total): total is EvalAggregate => total !== undefined)\n\t\t.map((total) => ({\n\t\t\t...total,\n\t\t\trecallAt1: total.recallAt1 / total.n,\n\t\t\trecallAt5: total.recallAt5 / total.n,\n\t\t\trecallAt10: total.recallAt10 / total.n,\n\t\t\trecallAt50: total.recallAt50 / total.n,\n\t\t\tmrr: total.mrr / total.n,\n\t\t}));\n\n\treturn { aggregates, perQuery };\n}\n\nexport function formatAggregateTable(aggregates: readonly EvalAggregate[]): string {\n\tconst pct = (x: number) => `${Math.round(x * 100)}%`.padStart(5);\n\tconst lines = [\n\t\t\"config           |  R@1  |  R@5  | R@10  | R@50  |  MRR  | notes\",\n\t\t\"-----------------|-------|-------|-------|-------|-------|------\",\n\t];\n\tfor (const a of aggregates) {\n\t\tconst notes = a.degraded === a.n ? \"degraded to lexical\" : a.degraded > 0 ? `${a.degraded}/${a.n} degraded` : \"\";\n\t\tlines.push(\n\t\t\t`${a.label.padEnd(16)} | ${pct(a.recallAt1)} | ${pct(a.recallAt5)} | ${pct(a.recallAt10)} | ` +\n\t\t\t\t`${pct(a.recallAt50)} | ${a.mrr.toFixed(3)} | ${notes}`,\n\t\t);\n\t}\n\treturn lines.join(\"\\n\");\n}\n"]}
+{"version":3,"file":"eval-harness.js","sourceRoot":"","sources":["../../../src/core/search/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAyD,aAAa,EAAE,MAAM,WAAW,CAAC;AAwEjG,SAAS,GAAG,CAAC,QAAgB,EAAE,IAAc,EAAU;IACtD,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAAA,CACpF;AAED;0EAC0E;AAC1E,MAAM,UAAU,mBAAmB,CAAC,QAAgB,EAAU;IAC7D,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,uCAAuC,CAAC;QAC5D,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,qDAAqD,CAAC;KAC1E,CAAC;IACF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,MAAc,EAAQ,EAAE,CAAC;QACtC,IAAI,IAAiC,CAAC;QACtC,IAAI,CAAC;YACJ,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACR,OAAO;QACR,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE;gBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;IAAA,CACD,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,wEAAwE;QACxE,iCAAiC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACtC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAAA,CACvC;AAeD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAsB;IACnD,sDAAsD;IACtD,2DAA2D;IAC3D,+DAA+D;IAC/D,iCAAiC;CACjC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,GAAuB,EAAgB;IAClF,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAClE,IAAI,CAAC,GAAG,EAAE,CAAC;QACV,0EAA0E;QAC1E,uEAAuE;QACvE,yEAAyE;QACzE,OAAO;YACN,GAAG,EAAE,QAAQ;YACb,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACzC,eAAe,EAAE,IAAI;YACrB,KAAK;YACL,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,GAAG,EAAE,CAAC,EAAC,CAAC;SACjB,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9C,6EAA6E;IAC7E,0EAA0E;IAC1E,0EAA0E;IAC1E,0EAA0E;IAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,4EAA0E;QAC1E,IAAI,CAAC;YACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;QACtC,CAAC;IACF,CAAC;IACD,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IAEzD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACnC,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAChC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACF,CAAC;IAED,OAAO;QACN,GAAG,EAAE,GAAG;QACR,GAAG;QACH,eAAe,EAAE,KAAK;QACtB,KAAK,EAAE,KAAK;QACZ,QAAQ;QACR,OAAO,EAAE,GAAG,EAAE,CAAC;YACd,IAAI,CAAC;gBACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/C,CAAC;QAAA,CACD;KACD,CAAC;AAAA,CACF;AAED,gEAAgE;AAChE,SAAS,kBAAkB,CAAC,UAA8B,EAAsB;IAC/E,IAAI,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAClC,IAAI,CAAC;QACJ,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED,MAAM,UAAU,iBAAiB,CAChC,QAAgB,EAChB,MAAoB,EACpB,SAAiB,EACjB,OAAqC,EACrC,eAAwB,EACxB,aAAgC,EACf;IACjB,MAAM,KAAK,GAAG,OAAO,EAAE,QAAQ,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,KAAK,EAAE,KAAK,IAAI,QAAQ,CAAC;IACvC,OAAO;QACN,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;QACvB,SAAS,EAAE,MAAM,CAAC,GAAG;QACrB,SAAS;QACT,qBAAqB,EAAE,MAAM,CAAC,eAAe;QAC7C,WAAW,EAAE,MAAM,CAAC,KAAK;QACzB,cAAc,EAAE,MAAM,CAAC,QAAQ;QAC/B,UAAU,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAChD,mBAAmB,EAAE,mBAAmB,CAAC,QAAQ,CAAC;QAClD,QAAQ,EAAE;YACT,sEAAsE;YACtE,oEAAoE;YACpE,wDAAwD;YACxD,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,KAAK;YAC1C,MAAM,EAAE,KAAK,IAAI,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YAC7D,UAAU,EAAE,KAAK,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YACnE,KAAK;YACL,UAAU,EAAE,eAAe;YAC3B,aAAa,EAAE,kBAAkB,CAAC,eAAe,CAAC;SAClD;QACD,YAAY,EAAE,aAAa;YAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,aAAa,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,aAAa,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE;YACnF,CAAC,CAAC,SAAS;QACZ,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE;KAClF,CAAC;AAAA,CACF;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA6B,EAA4B;IACzF,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACvD,aAAa,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;IACpC,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AAAA,CAC9D;AAaD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B,EAG5D;IACF,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAChD,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAE/C,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC7F,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACrE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI;gBACzC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC;gBACb,UAAU,EAAE,CAAC;gBACb,GAAG,EAAE,CAAC;gBACN,CAAC,EAAE,CAAC;gBACJ,QAAQ,EAAE,CAAC;aACX,CAAC;YACF,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC;YACxB,KAAK,CAAC,CAAC,EAAE,CAAC;YACV,IAAI,MAAM,CAAC,QAAQ;gBAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACjC,CAAC;IACF,CAAC;IAED,MAAM,UAAU,GAAG,OAAO;SACxB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;SACzC,MAAM,CAAC,CAAC,KAAK,EAA0B,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC;SAC9D,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAChB,GAAG,KAAK;QACR,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;KACxB,CAAC,CAAC,CAAC;IAEL,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAAA,CAChC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAAoC,EAAU;IAClF,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG;QACb,kEAAkE;QAClE,kEAAkE;KAClE,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;QACjH,KAAK,CAAC,IAAI,CACT,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK;YAC5F,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,EAAE,CACxD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAAA,CACxB","sourcesContent":["/**\n * Eval harness: corpus pinning, provenance capture, and run records.\n *\n * The scoring math lives in `eval.ts`; this module is everything around it\n * that makes a number *comparable to a later number*. Three problems it\n * exists to solve, all of which bit the first eval round\n * (docs/hybrid-retrieval-design.md, \"Eval results\"):\n *\n *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,\n *     so every commit moves the thing being measured. A baseline taken today\n *     and a rerun taken after a retrieval change differ by both the change\n *     and the intervening commits, and nothing in the output says so. Fix:\n *     run against a detached git worktree pinned to an explicit SHA, and put\n *     that SHA in the record.\n *  2. **Nothing was recorded.** Results were printed to a terminal and\n *     hand-copied into a markdown table with no repo SHA, no embedder\n *     identity, and no index state. Fix: emit a machine-readable run record.\n *  3. **A degraded run looks like a real one.** With no embsearch binary the\n *     semantic and hybrid rows silently degrade to lexical, producing a table\n *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the\n *     record, plus a per-row degraded count that the writer refuses to hide.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readdirSync, readFileSync, statSync } from \"node:fs\";\nimport { execFileSync } from \"child_process\";\nimport { rmSync } from \"fs\";\nimport { tmpdir } from \"os\";\nimport path from \"path\";\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { type EvalConfig, type EvalQuery, type EvalQueryResult, evaluateQuery } from \"./eval.js\";\n\n/** Metrics aggregated per config across the whole gold set. */\nexport interface EvalAggregate {\n\tlabel: string;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n\t/** Queries scored under this config. */\n\tn: number;\n\t/** How many of them ran degraded (requested retriever unavailable). */\n\tdegraded: number;\n}\n\n/** Everything needed to decide whether two run records may be compared. */\nexport interface EvalProvenance {\n\ttimestampMs: number;\n\t/** SHA of the corpus actually indexed and searched. */\n\tcorpusSha: string;\n\t/** Ref the caller asked for, before resolution (e.g. \"HEAD\"). */\n\tcorpusRef: string;\n\t/** True when the corpus came from the live working tree rather than a\n\t *  pinned worktree — results are then not reproducible. */\n\tcorpusFromWorkingTree: boolean;\n\t/** Uncommitted changes present at run time. Only meaningful (and only\n\t *  possible) when `corpusFromWorkingTree` is true. */\n\tcorpusDirty: boolean;\n\t/** Files removed from the corpus before indexing — see\n\t *  {@link CORPUS_EXCLUSIONS}. A score against a different exclusion list is\n\t *  a score against a different corpus, so it is recorded, not assumed. */\n\tcorpusExcluded: string[];\n\t/** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,\n\t *  but differs when pinning an old corpus with today's code. */\n\tharnessSha: string;\n\t/**\n\t * Content hash of `src/core/search` + the chunker. Every tuning constant\n\t * that shapes a result — the fusion cap, top-k depths, rerank weights,\n\t * chunk sizing — lives in those files, so a changed hash means the\n\t * numbers are not comparable, without this module having to maintain a\n\t * hand-copied (and inevitably stale) list of constants.\n\t */\n\tretrievalSourceHash: string;\n\t/** Embedding backend state. `available: false` means every semantic and\n\t *  hybrid row in this record degraded to lexical. */\n\tembedder: {\n\t\tavailable: boolean;\n\t\treason?: string;\n\t\t/** Indexed chunk count when the index reached `ready`. */\n\t\tchunkCount?: number;\n\t\tphase: string;\n\t\t/** Binary that served the embeddings, and its self-reported version.\n\t\t *  The embedding model is baked into the binary at build time, so this\n\t\t *  is the only thing that identifies which model produced a score. */\n\t\tbinaryPath?: string;\n\t\tbinaryVersion?: string;\n\t};\n\t/** Daemon-side BM25 hybrid store, when the run included one. Absent means\n\t *  the record has no `daemon-hybrid` rows. */\n\tdaemonHybrid?: { available: boolean; phase: string };\n\truntime: { node: string; platform: string; arch: string };\n}\n\nexport interface EvalRunRecord {\n\tprovenance: EvalProvenance;\n\tgoldSet: { queryCount: number; byClass: Record<string, number>; goldSpanCount: number };\n\tconfigs: readonly EvalConfig[];\n\taggregates: EvalAggregate[];\n\tperQuery: Array<{ id: string; class: string; results: EvalQueryResult[] }>;\n}\n\nfunction git(repoRoot: string, args: string[]): string {\n\treturn execFileSync(\"git\", [\"-C\", repoRoot, ...args], { encoding: \"utf-8\" }).trim();\n}\n\n/** Hash every retrieval-shaping source file, so a tuning change is visible as\n *  a changed provenance field rather than an unexplained metric shift. */\nexport function hashRetrievalSource(repoRoot: string): string {\n\tconst roots = [\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/search\"),\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/embsearch/chunker.ts\"),\n\t];\n\tconst files: string[] = [];\n\tconst walk = (target: string): void => {\n\t\tlet stat: ReturnType<typeof statSync>;\n\t\ttry {\n\t\t\tstat = statSync(target);\n\t\t} catch {\n\t\t\treturn;\n\t\t}\n\t\tif (stat.isDirectory()) {\n\t\t\tfor (const entry of readdirSync(target).sort()) walk(path.join(target, entry));\n\t\t} else if (target.endsWith(\".ts\")) {\n\t\t\tfiles.push(target);\n\t\t}\n\t};\n\tfor (const root of roots) walk(root);\n\n\tconst hash = createHash(\"sha256\");\n\tfor (const file of files) {\n\t\t// Eval-only modules are excluded: changing how we measure must not look\n\t\t// like changing what we measure.\n\t\tconst base = path.basename(file);\n\t\tif (base.startsWith(\"eval\")) continue;\n\t\thash.update(path.relative(repoRoot, file).replace(/\\\\/g, \"/\"));\n\t\thash.update(readFileSync(file));\n\t}\n\treturn hash.digest(\"hex\").slice(0, 16);\n}\n\nexport interface PinnedCorpus {\n\t/** Directory to index and search. */\n\tcwd: string;\n\tsha: string;\n\tfromWorkingTree: boolean;\n\tdirty: boolean;\n\t/** Files removed from the corpus before indexing. Empty when the corpus is\n\t *  the live working tree, which is never mutated. */\n\texcluded: string[];\n\t/** Removes the worktree, if one was created. */\n\tdispose: () => void;\n}\n\n/**\n * Files that describe this eval rather than being searched by it.\n *\n * The fixtures hold all 62 query strings verbatim, so every query is a perfect\n * lexical match against its own entry, and the design note quotes the same\n * queries while discussing the classes they belong to. Measured before this\n * exclusion existed: **56 of 62 queries had one of these files in the top 10,\n * 27 of 62 had one as the #1 result, and they consumed 133 of the 620\n * top-10 slots** — a fifth of the window, spent on the eval reading itself.\n *\n * That is not a ranking artifact a reranker can fix: it displaces real answers\n * out of the window entirely, which is why two boundary-class queries were\n * absent from the top *50* rather than merely buried. Retrieving your own\n * question is not retrieval, so the corpus is scored without them.\n *\n * Removed from the pinned worktree before indexing, never from the repo — and\n * recorded in the run's provenance so a score is never silently taken against\n * a different corpus than it claims.\n */\nexport const CORPUS_EXCLUSIONS: readonly string[] = [\n\t\"packages/coding-agent/test/fixtures/search-eval.json\",\n\t\"packages/coding-agent/test/fixtures/search-eval-live.json\",\n\t\"packages/coding-agent/test/fixtures/search-eval-baseline.json\",\n\t\"docs/hybrid-retrieval-design.md\",\n];\n\n/**\n * Materialize the corpus to evaluate.\n *\n * With a `ref`, checks out a detached worktree at that commit so the corpus is\n * byte-identical on every rerun. Without one, falls back to the live working\n * tree and reports `dirty` so the record shows the run was not reproducible.\n */\nexport function pinCorpus(repoRoot: string, ref: string | undefined): PinnedCorpus {\n\tconst dirty = git(repoRoot, [\"status\", \"--porcelain\"]).length > 0;\n\tif (!ref) {\n\t\t// The live working tree is the user's checkout; deleting files from it to\n\t\t// tidy a measurement would be an unforgivable trade. Working-tree runs\n\t\t// are already stamped non-reproducible, so they carry the contamination.\n\t\treturn {\n\t\t\tcwd: repoRoot,\n\t\t\tsha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\t\tfromWorkingTree: true,\n\t\t\tdirty,\n\t\t\texcluded: [],\n\t\t\tdispose: () => {},\n\t\t};\n\t}\n\n\tconst sha = git(repoRoot, [\"rev-parse\", ref]);\n\t// Deterministic path, not mkdtemp: the embedding store is keyed by a hash of\n\t// the corpus directory, so a fresh temp path every run would re-embed all\n\t// ~17k chunks (minutes) instead of reusing the store built for this exact\n\t// SHA. The worktree is still removed afterwards; only the store persists.\n\tconst dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}`);\n\tif (existsSync(dir)) {\n\t\t// Left behind by an interrupted run — drop it so `worktree add` succeeds.\n\t\ttry {\n\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t} catch {\n\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\tgit(repoRoot, [\"worktree\", \"prune\"]);\n\t\t}\n\t}\n\tgit(repoRoot, [\"worktree\", \"add\", \"--detach\", dir, sha]);\n\n\tconst excluded: string[] = [];\n\tfor (const rel of CORPUS_EXCLUSIONS) {\n\t\tconst target = path.join(dir, rel);\n\t\tif (existsSync(target)) {\n\t\t\trmSync(target, { force: true });\n\t\t\texcluded.push(rel);\n\t\t}\n\t}\n\n\treturn {\n\t\tcwd: dir,\n\t\tsha,\n\t\tfromWorkingTree: false,\n\t\tdirty: false,\n\t\texcluded,\n\t\tdispose: () => {\n\t\t\ttry {\n\t\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t\t} catch {\n\t\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\t}\n\t\t},\n\t};\n}\n\n/** `<binary> --version`, or undefined when it cannot be run. */\nfunction probeBinaryVersion(binaryPath: string | undefined): string | undefined {\n\tif (!binaryPath) return undefined;\n\ttry {\n\t\treturn execFileSync(binaryPath, [\"--version\"], { encoding: \"utf-8\" }).trim();\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\nexport function collectProvenance(\n\trepoRoot: string,\n\tcorpus: PinnedCorpus,\n\tcorpusRef: string,\n\tservice: EmbsearchService | undefined,\n\tembsearchBinary?: string,\n\thybridService?: EmbsearchService,\n): EvalProvenance {\n\tconst state = service?.getState();\n\tconst phase = state?.phase ?? \"absent\";\n\treturn {\n\t\ttimestampMs: Date.now(),\n\t\tcorpusSha: corpus.sha,\n\t\tcorpusRef,\n\t\tcorpusFromWorkingTree: corpus.fromWorkingTree,\n\t\tcorpusDirty: corpus.dirty,\n\t\tcorpusExcluded: corpus.excluded,\n\t\tharnessSha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\tretrievalSourceHash: hashRetrievalSource(repoRoot),\n\t\tembedder: {\n\t\t\t// `ready` is the only phase the service reaches with a real embedder:\n\t\t\t// it rejects the mock backend at startup, so availability here also\n\t\t\t// certifies the numbers came from a genuine ONNX build.\n\t\t\tavailable: service?.isAvailable() ?? false,\n\t\t\treason: state && \"reason\" in state ? state.reason : undefined,\n\t\t\tchunkCount: state?.phase === \"ready\" ? state.chunkCount : undefined,\n\t\t\tphase,\n\t\t\tbinaryPath: embsearchBinary,\n\t\t\tbinaryVersion: probeBinaryVersion(embsearchBinary),\n\t\t},\n\t\tdaemonHybrid: hybridService\n\t\t\t? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }\n\t\t\t: undefined,\n\t\truntime: { node: process.version, platform: process.platform, arch: process.arch },\n\t};\n}\n\nexport function summarizeGoldSet(dataset: readonly EvalQuery[]): EvalRunRecord[\"goldSet\"] {\n\tconst byClass: Record<string, number> = {};\n\tlet goldSpanCount = 0;\n\tfor (const query of dataset) {\n\t\tbyClass[query.class] = (byClass[query.class] ?? 0) + 1;\n\t\tgoldSpanCount += query.gold.length;\n\t}\n\treturn { queryCount: dataset.length, byClass, goldSpanCount };\n}\n\nexport interface RunEvalSuiteOptions {\n\tcwd: string;\n\tdataset: readonly EvalQuery[];\n\tconfigs: readonly EvalConfig[];\n\tservice?: EmbsearchService;\n\t/** Second service backed by a daemon-side BM25 hybrid store, for the\n\t *  `daemon-hybrid` configs. Absent means those rows are omitted. */\n\thybridService?: EmbsearchService;\n\tonQuery?: (index: number, query: EvalQuery) => void;\n}\n\nexport async function runEvalSuite(options: RunEvalSuiteOptions): Promise<{\n\taggregates: EvalAggregate[];\n\tperQuery: EvalRunRecord[\"perQuery\"];\n}> {\n\tconst { cwd, dataset, configs, service } = options;\n\tconst totals = new Map<string, EvalAggregate>();\n\tconst perQuery: EvalRunRecord[\"perQuery\"] = [];\n\n\tfor (const [index, evalQuery] of dataset.entries()) {\n\t\toptions.onQuery?.(index, evalQuery);\n\t\tconst results = await evaluateQuery(cwd, evalQuery, configs, service, options.hybridService);\n\t\tperQuery.push({ id: evalQuery.id, class: evalQuery.class, results });\n\t\tfor (const result of results) {\n\t\t\tconst total = totals.get(result.label) ?? {\n\t\t\t\tlabel: result.label,\n\t\t\t\trecallAt1: 0,\n\t\t\t\trecallAt5: 0,\n\t\t\t\trecallAt10: 0,\n\t\t\t\trecallAt50: 0,\n\t\t\t\tmrr: 0,\n\t\t\t\tn: 0,\n\t\t\t\tdegraded: 0,\n\t\t\t};\n\t\t\ttotal.recallAt1 += result.recallAt1;\n\t\t\ttotal.recallAt5 += result.recallAt5;\n\t\t\ttotal.recallAt10 += result.recallAt10;\n\t\t\ttotal.recallAt50 += result.recallAt50;\n\t\t\ttotal.mrr += result.mrr;\n\t\t\ttotal.n++;\n\t\t\tif (result.degraded) total.degraded++;\n\t\t\ttotals.set(result.label, total);\n\t\t}\n\t}\n\n\tconst aggregates = configs\n\t\t.map((config) => totals.get(config.label))\n\t\t.filter((total): total is EvalAggregate => total !== undefined)\n\t\t.map((total) => ({\n\t\t\t...total,\n\t\t\trecallAt1: total.recallAt1 / total.n,\n\t\t\trecallAt5: total.recallAt5 / total.n,\n\t\t\trecallAt10: total.recallAt10 / total.n,\n\t\t\trecallAt50: total.recallAt50 / total.n,\n\t\t\tmrr: total.mrr / total.n,\n\t\t}));\n\n\treturn { aggregates, perQuery };\n}\n\nexport function formatAggregateTable(aggregates: readonly EvalAggregate[]): string {\n\tconst pct = (x: number) => `${Math.round(x * 100)}%`.padStart(5);\n\tconst lines = [\n\t\t\"config           |  R@1  |  R@5  | R@10  | R@50  |  MRR  | notes\",\n\t\t\"-----------------|-------|-------|-------|-------|-------|------\",\n\t];\n\tfor (const a of aggregates) {\n\t\tconst notes = a.degraded === a.n ? \"degraded to lexical\" : a.degraded > 0 ? `${a.degraded}/${a.n} degraded` : \"\";\n\t\tlines.push(\n\t\t\t`${a.label.padEnd(16)} | ${pct(a.recallAt1)} | ${pct(a.recallAt5)} | ${pct(a.recallAt10)} | ` +\n\t\t\t\t`${pct(a.recallAt50)} | ${a.mrr.toFixed(3)} | ${notes}`,\n\t\t);\n\t}\n\treturn lines.join(\"\\n\");\n}\n"]}

package/examples/extensions/custom-provider-anthropic/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-custom-provider-anthropic",
 	"private": true,
-	"version": "0.2.163",
+	"version": "0.2.164",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/examples/extensions/custom-provider-gitlab-duo/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-custom-provider-gitlab-duo",
 	"private": true,
-	"version": "0.2.163",
+	"version": "0.2.164",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/examples/extensions/sandbox/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-sandbox",
 	"private": true,
-	"version": "0.2.163",
+	"version": "0.2.164",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/examples/extensions/with-deps/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-with-deps",
 	"private": true,
-	"version": "0.2.163",
+	"version": "0.2.164",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@kolisachint/hoocode-agent",
-	"version": "0.4.166",
+	"version": "0.4.167",
 	"description": "Coding agent CLI with read, bash, edit, write tools and session management",
 	"type": "module",
 	"hoocodeConfig": {
@@ -48,9 +48,9 @@
 		"prepublishOnly": "npm run clean && npm run build"
 	},
 	"dependencies": {
-		"@kolisachint/hoocode-agent-core": "^0.4.166",
-		"@kolisachint/hoocode-ai": "^0.4.166",
-		"@kolisachint/hoocode-tui": "^0.4.166",
+		"@kolisachint/hoocode-agent-core": "^0.4.167",
+		"@kolisachint/hoocode-ai": "^0.4.167",
+		"@kolisachint/hoocode-tui": "^0.4.167",
 		"@silvia-odwyer/photon-node": "^0.3.4",
 		"chalk": "^5.5.0",
 		"cli-highlight": "^2.1.11",