@kolisachint/hoocode-agent 0.4.164 → 0.4.166

package/dist/core/search/eval-gold.js

@@ -0,0 +1,172 @@
+/**
+ * Gold-set loading and anchor resolution.
+ *
+ * A gold span recorded as bare line numbers rots on the next refactor, and a
+ * rotted gold set fails *silently* — it just scores lower. So each span also
+ * carries an `anchor`: a literal snippet that must sit inside the range. The
+ * anchor is the source of truth; the line numbers are a cache of where it was
+ * when the fixture was last regenerated.
+ *
+ * That gives three properties the first gold set lacked:
+ *  - `resolveGoldSet` recomputes ranges from anchors, so `--fix` re-pins the
+ *    whole set after a refactor instead of someone hand-editing 60 numbers;
+ *  - `validateGoldSet` fails loudly when an anchor has moved out of its
+ *    recorded range, drifted, or become ambiguous;
+ *  - anchors are never used for scoring, so this cannot flatter retrieval —
+ *    `spanMatchesGold` still sees only a path and a line range.
+ */
+import { readFileSync } from "node:fs";
+import path from "node:path";
+/** Lines of context kept either side of a non-block anchor (an error literal,
+ *  a single statement) — enough to be a real target, tight enough that a
+ *  60-line chunk elsewhere in the file does not count as a hit. */
+const LINE_ANCHOR_PAD = 2;
+/** Upper bound on a resolved block, so an anchor that fails to find its
+ *  closing brace cannot silently swallow an entire file. */
+const MAX_BLOCK_LINES = 120;
+/** Fallback extent when a block anchor never finds its closing brace. */
+const UNCLOSED_BLOCK_LINES = 15;
+function readLines(corpusRoot, rel) {
+    try {
+        return readFileSync(path.resolve(corpusRoot, rel), "utf-8").split("\n");
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Indices (0-based) of every line containing `anchor`. */
+function findAnchorLines(lines, anchor) {
+    const found = [];
+    for (let i = 0; i < lines.length; i++) {
+        if (lines[i].includes(anchor))
+            found.push(i);
+    }
+    return found;
+}
+/**
+ * Extent of the declaration an anchor names.
+ *
+ * A line ending in an opener (`{`, `(`, `[`) starts a block, which runs to the
+ * first line closing at column 0 — the shape every top-level declaration in
+ * this codebase has. Anything else is a statement, scored with a small pad.
+ */
+function resolveExtent(lines, anchorIndex) {
+    const head = lines[anchorIndex];
+    const opensBlock = /[{([]\s*$/.test(head);
+    if (!opensBlock) {
+        return {
+            startLine: Math.max(1, anchorIndex + 1 - LINE_ANCHOR_PAD),
+            endLine: Math.min(lines.length, anchorIndex + 1 + LINE_ANCHOR_PAD),
+        };
+    }
+    const limit = Math.min(lines.length, anchorIndex + 1 + MAX_BLOCK_LINES);
+    for (let i = anchorIndex + 1; i < limit; i++) {
+        // Column-0 close: `}`, `};`, `});`, `];` — the end of a top-level block.
+        if (/^[}\])]/.test(lines[i]))
+            return { startLine: anchorIndex + 1, endLine: i + 1 };
+    }
+    return { startLine: anchorIndex + 1, endLine: Math.min(lines.length, anchorIndex + 1 + UNCLOSED_BLOCK_LINES) };
+}
+/** Re-pin one gold span's line range from its anchor. Returns the span
+ *  unchanged when it is file-scoped or has no anchor to resolve. */
+export function resolveGoldSpan(corpusRoot, span) {
+    const lines = readLines(corpusRoot, span.path);
+    if (!lines)
+        return { span, problem: `file not found: ${span.path}` };
+    if (span.scope === "file") {
+        return { span: { ...span, startLine: 1, endLine: lines.length } };
+    }
+    if (!span.anchor)
+        return { span, problem: `span-scoped gold needs an anchor: ${span.path}` };
+    const matches = findAnchorLines(lines, span.anchor);
+    if (matches.length === 0)
+        return { span, problem: `anchor not found in ${span.path}: ${span.anchor}` };
+    if (matches.length > 1) {
+        return { span, problem: `anchor is ambiguous (${matches.length} matches) in ${span.path}: ${span.anchor}` };
+    }
+    const { startLine, endLine } = resolveExtent(lines, matches[0]);
+    return { span: { ...span, startLine, endLine } };
+}
+/** Re-pin every gold span in the set. Problems are collected, not thrown, so
+ *  a regeneration run reports all drift at once. */
+export function resolveGoldSet(corpusRoot, dataset) {
+    const issues = [];
+    const resolved = dataset.map((query) => ({
+        ...query,
+        gold: query.gold.map((span) => {
+            const result = resolveGoldSpan(corpusRoot, span);
+            if (result.problem)
+                issues.push({ queryId: query.id, path: span.path, problem: result.problem });
+            return result.span;
+        }),
+    }));
+    return { dataset: resolved, issues };
+}
+/**
+ * Check the committed fixture against the corpus without rewriting it.
+ *
+ * Catches the two ways a gold set goes wrong: the anchor no longer exists (the
+ * code was deleted or renamed), or it exists but has moved outside the
+ * recorded range (the fixture is stale and every score computed from it is
+ * wrong).
+ */
+export function validateGoldSet(corpusRoot, dataset) {
+    const issues = [];
+    const seenIds = new Set();
+    for (const query of dataset) {
+        if (seenIds.has(query.id))
+            issues.push({ queryId: query.id, path: "", problem: "duplicate query id" });
+        seenIds.add(query.id);
+        if (query.gold.length === 0)
+            issues.push({ queryId: query.id, path: "", problem: "query has no gold spans" });
+        for (const span of query.gold) {
+            const lines = readLines(corpusRoot, span.path);
+            if (!lines) {
+                issues.push({ queryId: query.id, path: span.path, problem: "file not found" });
+                continue;
+            }
+            if (span.startLine === undefined || span.endLine === undefined) {
+                issues.push({ queryId: query.id, path: span.path, problem: "gold span is missing line numbers" });
+                continue;
+            }
+            if (span.startLine < 1 || span.endLine < span.startLine || span.endLine > lines.length) {
+                issues.push({
+                    queryId: query.id,
+                    path: span.path,
+                    problem: `line range ${span.startLine}-${span.endLine} out of bounds (file has ${lines.length} lines)`,
+                });
+                continue;
+            }
+            if (span.scope === "file")
+                continue;
+            if (!span.anchor) {
+                issues.push({ queryId: query.id, path: span.path, problem: "span-scoped gold needs an anchor" });
+                continue;
+            }
+            const matches = findAnchorLines(lines, span.anchor);
+            if (matches.length === 0) {
+                issues.push({ queryId: query.id, path: span.path, problem: `anchor not found: ${span.anchor}` });
+            }
+            else if (matches.length > 1) {
+                issues.push({
+                    queryId: query.id,
+                    path: span.path,
+                    problem: `anchor is ambiguous (${matches.length} matches): ${span.anchor}`,
+                });
+            }
+            else if (matches[0] + 1 < span.startLine || matches[0] + 1 > span.endLine) {
+                issues.push({
+                    queryId: query.id,
+                    path: span.path,
+                    problem: `anchor moved to line ${matches[0] + 1}, outside recorded range ${span.startLine}-${span.endLine}`,
+                });
+            }
+        }
+    }
+    return issues;
+}
+/** Load the gold set from a fixture file. */
+export function loadGoldSet(fixturePath) {
+    return JSON.parse(readFileSync(fixturePath, "utf-8"));
+}
+//# sourceMappingURL=eval-gold.js.map

package/dist/core/search/eval-gold.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-gold.js","sourceRoot":"","sources":["../../../src/core/search/eval-gold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B;;mEAEmE;AACnE,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B;4DAC4D;AAC5D,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,yEAAyE;AACzE,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAQhC,SAAS,SAAS,CAAC,UAAkB,EAAE,GAAW,EAAwB;IACzE,IAAI,CAAC;QACJ,OAAO,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED,2DAA2D;AAC3D,SAAS,eAAe,CAAC,KAAwB,EAAE,MAAc,EAAY;IAC5E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,KAAK,CAAC;AAAA,CACb;AAED;;;;;;GAMG;AACH,SAAS,aAAa,CAAC,KAAwB,EAAE,WAAmB,EAA0C;IAC7G,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;IAChC,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,OAAO;YACN,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,CAAC,GAAG,eAAe,CAAC;YACzD,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,eAAe,CAAC;SAClE,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;IACxE,KAAK,IAAI,CAAC,GAAG,WAAW,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,2EAAyE;QACzE,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;IACrF,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,WAAW,GAAG,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC,GAAG,oBAAoB,CAAC,EAAE,CAAC;AAAA,CAC/G;AAED;oEACoE;AACpE,MAAM,UAAU,eAAe,CAAC,UAAkB,EAAE,IAAkB,EAA4C;IACjH,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,mBAAmB,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;IAErE,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC3B,OAAO,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;IACnE,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,qCAAqC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;IAE7F,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,uBAAuB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;IACvG,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,wBAAwB,OAAO,CAAC,MAAM,gBAAgB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;IAC7G,CAAC;IAED,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC;AAAA,CACjD;AAED;oDACoD;AACpD,MAAM,UAAU,cAAc,CAC7B,UAAkB,EAClB,OAA6B,EAC6B;IAC1D,MAAM,MAAM,GAA0B,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACxC,GAAG,KAAK;QACR,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YACjD,IAAI,MAAM,CAAC,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YACjG,OAAO,MAAM,CAAC,IAAI,CAAC;QAAA,CACnB,CAAC;KACF,CAAC,CAAC,CAAC;IACJ,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AAAA,CACrC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,UAAkB,EAAE,OAA6B,EAAyB;IACzG,MAAM,MAAM,GAA0B,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACvG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACtB,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,yBAAyB,EAAE,CAAC,CAAC;QAE9G,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/C,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;gBAC/E,SAAS;YACV,CAAC;YACD,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,mCAAmC,EAAE,CAAC,CAAC;gBAClG,SAAS;YACV,CAAC;YACD,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACxF,MAAM,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,cAAc,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,4BAA4B,KAAK,CAAC,MAAM,SAAS;iBACtG,CAAC,CAAC;gBACH,SAAS;YACV,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM;gBAAE,SAAS;YACpC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,kCAAkC,EAAE,CAAC,CAAC;gBACjG,SAAS;YACV,CAAC;YACD,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACpD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,qBAAqB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAClG,CAAC;iBAAM,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,wBAAwB,OAAO,CAAC,MAAM,cAAc,IAAI,CAAC,MAAM,EAAE;iBAC1E,CAAC,CAAC;YACJ,CAAC;iBAAM,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC7E,MAAM,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,OAAO,EAAE,wBAAwB,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,4BAA4B,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE;iBAC3G,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC;IACD,OAAO,MAAM,CAAC;AAAA,CACd;AAED,6CAA6C;AAC7C,MAAM,UAAU,WAAW,CAAC,WAAmB,EAAe;IAC7D,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAgB,CAAC;AAAA,CACrE","sourcesContent":["/**\n * Gold-set loading and anchor resolution.\n *\n * A gold span recorded as bare line numbers rots on the next refactor, and a\n * rotted gold set fails *silently* — it just scores lower. So each span also\n * carries an `anchor`: a literal snippet that must sit inside the range. The\n * anchor is the source of truth; the line numbers are a cache of where it was\n * when the fixture was last regenerated.\n *\n * That gives three properties the first gold set lacked:\n *  - `resolveGoldSet` recomputes ranges from anchors, so `--fix` re-pins the\n *    whole set after a refactor instead of someone hand-editing 60 numbers;\n *  - `validateGoldSet` fails loudly when an anchor has moved out of its\n *    recorded range, drifted, or become ambiguous;\n *  - anchors are never used for scoring, so this cannot flatter retrieval —\n *    `spanMatchesGold` still sees only a path and a line range.\n */\n\nimport { readFileSync } from \"node:fs\";\nimport path from \"node:path\";\nimport type { EvalGoldSpan, EvalQuery } from \"./eval.js\";\n\n/** Lines of context kept either side of a non-block anchor (an error literal,\n *  a single statement) — enough to be a real target, tight enough that a\n *  60-line chunk elsewhere in the file does not count as a hit. */\nconst LINE_ANCHOR_PAD = 2;\n/** Upper bound on a resolved block, so an anchor that fails to find its\n *  closing brace cannot silently swallow an entire file. */\nconst MAX_BLOCK_LINES = 120;\n/** Fallback extent when a block anchor never finds its closing brace. */\nconst UNCLOSED_BLOCK_LINES = 15;\n\nexport interface GoldValidationIssue {\n\tqueryId: string;\n\tpath: string;\n\tproblem: string;\n}\n\nfunction readLines(corpusRoot: string, rel: string): string[] | undefined {\n\ttry {\n\t\treturn readFileSync(path.resolve(corpusRoot, rel), \"utf-8\").split(\"\\n\");\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/** Indices (0-based) of every line containing `anchor`. */\nfunction findAnchorLines(lines: readonly string[], anchor: string): number[] {\n\tconst found: number[] = [];\n\tfor (let i = 0; i < lines.length; i++) {\n\t\tif (lines[i].includes(anchor)) found.push(i);\n\t}\n\treturn found;\n}\n\n/**\n * Extent of the declaration an anchor names.\n *\n * A line ending in an opener (`{`, `(`, `[`) starts a block, which runs to the\n * first line closing at column 0 — the shape every top-level declaration in\n * this codebase has. Anything else is a statement, scored with a small pad.\n */\nfunction resolveExtent(lines: readonly string[], anchorIndex: number): { startLine: number; endLine: number } {\n\tconst head = lines[anchorIndex];\n\tconst opensBlock = /[{([]\\s*$/.test(head);\n\tif (!opensBlock) {\n\t\treturn {\n\t\t\tstartLine: Math.max(1, anchorIndex + 1 - LINE_ANCHOR_PAD),\n\t\t\tendLine: Math.min(lines.length, anchorIndex + 1 + LINE_ANCHOR_PAD),\n\t\t};\n\t}\n\tconst limit = Math.min(lines.length, anchorIndex + 1 + MAX_BLOCK_LINES);\n\tfor (let i = anchorIndex + 1; i < limit; i++) {\n\t\t// Column-0 close: `}`, `};`, `});`, `];` — the end of a top-level block.\n\t\tif (/^[}\\])]/.test(lines[i])) return { startLine: anchorIndex + 1, endLine: i + 1 };\n\t}\n\treturn { startLine: anchorIndex + 1, endLine: Math.min(lines.length, anchorIndex + 1 + UNCLOSED_BLOCK_LINES) };\n}\n\n/** Re-pin one gold span's line range from its anchor. Returns the span\n *  unchanged when it is file-scoped or has no anchor to resolve. */\nexport function resolveGoldSpan(corpusRoot: string, span: EvalGoldSpan): { span: EvalGoldSpan; problem?: string } {\n\tconst lines = readLines(corpusRoot, span.path);\n\tif (!lines) return { span, problem: `file not found: ${span.path}` };\n\n\tif (span.scope === \"file\") {\n\t\treturn { span: { ...span, startLine: 1, endLine: lines.length } };\n\t}\n\tif (!span.anchor) return { span, problem: `span-scoped gold needs an anchor: ${span.path}` };\n\n\tconst matches = findAnchorLines(lines, span.anchor);\n\tif (matches.length === 0) return { span, problem: `anchor not found in ${span.path}: ${span.anchor}` };\n\tif (matches.length > 1) {\n\t\treturn { span, problem: `anchor is ambiguous (${matches.length} matches) in ${span.path}: ${span.anchor}` };\n\t}\n\n\tconst { startLine, endLine } = resolveExtent(lines, matches[0]);\n\treturn { span: { ...span, startLine, endLine } };\n}\n\n/** Re-pin every gold span in the set. Problems are collected, not thrown, so\n *  a regeneration run reports all drift at once. */\nexport function resolveGoldSet(\n\tcorpusRoot: string,\n\tdataset: readonly EvalQuery[],\n): { dataset: EvalQuery[]; issues: GoldValidationIssue[] } {\n\tconst issues: GoldValidationIssue[] = [];\n\tconst resolved = dataset.map((query) => ({\n\t\t...query,\n\t\tgold: query.gold.map((span) => {\n\t\t\tconst result = resolveGoldSpan(corpusRoot, span);\n\t\t\tif (result.problem) issues.push({ queryId: query.id, path: span.path, problem: result.problem });\n\t\t\treturn result.span;\n\t\t}),\n\t}));\n\treturn { dataset: resolved, issues };\n}\n\n/**\n * Check the committed fixture against the corpus without rewriting it.\n *\n * Catches the two ways a gold set goes wrong: the anchor no longer exists (the\n * code was deleted or renamed), or it exists but has moved outside the\n * recorded range (the fixture is stale and every score computed from it is\n * wrong).\n */\nexport function validateGoldSet(corpusRoot: string, dataset: readonly EvalQuery[]): GoldValidationIssue[] {\n\tconst issues: GoldValidationIssue[] = [];\n\tconst seenIds = new Set<string>();\n\n\tfor (const query of dataset) {\n\t\tif (seenIds.has(query.id)) issues.push({ queryId: query.id, path: \"\", problem: \"duplicate query id\" });\n\t\tseenIds.add(query.id);\n\t\tif (query.gold.length === 0) issues.push({ queryId: query.id, path: \"\", problem: \"query has no gold spans\" });\n\n\t\tfor (const span of query.gold) {\n\t\t\tconst lines = readLines(corpusRoot, span.path);\n\t\t\tif (!lines) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"file not found\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.startLine === undefined || span.endLine === undefined) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"gold span is missing line numbers\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.startLine < 1 || span.endLine < span.startLine || span.endLine > lines.length) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `line range ${span.startLine}-${span.endLine} out of bounds (file has ${lines.length} lines)`,\n\t\t\t\t});\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (span.scope === \"file\") continue;\n\t\t\tif (!span.anchor) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: \"span-scoped gold needs an anchor\" });\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tconst matches = findAnchorLines(lines, span.anchor);\n\t\t\tif (matches.length === 0) {\n\t\t\t\tissues.push({ queryId: query.id, path: span.path, problem: `anchor not found: ${span.anchor}` });\n\t\t\t} else if (matches.length > 1) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `anchor is ambiguous (${matches.length} matches): ${span.anchor}`,\n\t\t\t\t});\n\t\t\t} else if (matches[0] + 1 < span.startLine || matches[0] + 1 > span.endLine) {\n\t\t\t\tissues.push({\n\t\t\t\t\tqueryId: query.id,\n\t\t\t\t\tpath: span.path,\n\t\t\t\t\tproblem: `anchor moved to line ${matches[0] + 1}, outside recorded range ${span.startLine}-${span.endLine}`,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n\treturn issues;\n}\n\n/** Load the gold set from a fixture file. */\nexport function loadGoldSet(fixturePath: string): EvalQuery[] {\n\treturn JSON.parse(readFileSync(fixturePath, \"utf-8\")) as EvalQuery[];\n}\n"]}

package/dist/core/search/eval-harness.d.ts

@@ -0,0 +1,140 @@
+/**
+ * Eval harness: corpus pinning, provenance capture, and run records.
+ *
+ * The scoring math lives in `eval.ts`; this module is everything around it
+ * that makes a number *comparable to a later number*. Three problems it
+ * exists to solve, all of which bit the first eval round
+ * (docs/hybrid-retrieval-design.md, "Eval results"):
+ *
+ *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,
+ *     so every commit moves the thing being measured. A baseline taken today
+ *     and a rerun taken after a retrieval change differ by both the change
+ *     and the intervening commits, and nothing in the output says so. Fix:
+ *     run against a detached git worktree pinned to an explicit SHA, and put
+ *     that SHA in the record.
+ *  2. **Nothing was recorded.** Results were printed to a terminal and
+ *     hand-copied into a markdown table with no repo SHA, no embedder
+ *     identity, and no index state. Fix: emit a machine-readable run record.
+ *  3. **A degraded run looks like a real one.** With no embsearch binary the
+ *     semantic and hybrid rows silently degrade to lexical, producing a table
+ *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the
+ *     record, plus a per-row degraded count that the writer refuses to hide.
+ */
+import type { EmbsearchService } from "../embsearch/embsearch-service.js";
+import { type EvalConfig, type EvalQuery, type EvalQueryResult } from "./eval.js";
+/** Metrics aggregated per config across the whole gold set. */
+export interface EvalAggregate {
+    label: string;
+    recallAt1: number;
+    recallAt5: number;
+    recallAt10: number;
+    recallAt50: number;
+    mrr: number;
+    /** Queries scored under this config. */
+    n: number;
+    /** How many of them ran degraded (requested retriever unavailable). */
+    degraded: number;
+}
+/** Everything needed to decide whether two run records may be compared. */
+export interface EvalProvenance {
+    timestampMs: number;
+    /** SHA of the corpus actually indexed and searched. */
+    corpusSha: string;
+    /** Ref the caller asked for, before resolution (e.g. "HEAD"). */
+    corpusRef: string;
+    /** True when the corpus came from the live working tree rather than a
+     *  pinned worktree — results are then not reproducible. */
+    corpusFromWorkingTree: boolean;
+    /** Uncommitted changes present at run time. Only meaningful (and only
+     *  possible) when `corpusFromWorkingTree` is true. */
+    corpusDirty: boolean;
+    /** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,
+     *  but differs when pinning an old corpus with today's code. */
+    harnessSha: string;
+    /**
+     * Content hash of `src/core/search` + the chunker. Every tuning constant
+     * that shapes a result — the fusion cap, top-k depths, rerank weights,
+     * chunk sizing — lives in those files, so a changed hash means the
+     * numbers are not comparable, without this module having to maintain a
+     * hand-copied (and inevitably stale) list of constants.
+     */
+    retrievalSourceHash: string;
+    /** Embedding backend state. `available: false` means every semantic and
+     *  hybrid row in this record degraded to lexical. */
+    embedder: {
+        available: boolean;
+        reason?: string;
+        /** Indexed chunk count when the index reached `ready`. */
+        chunkCount?: number;
+        phase: string;
+        /** Binary that served the embeddings, and its self-reported version.
+         *  The embedding model is baked into the binary at build time, so this
+         *  is the only thing that identifies which model produced a score. */
+        binaryPath?: string;
+        binaryVersion?: string;
+    };
+    /** Daemon-side BM25 hybrid store, when the run included one. Absent means
+     *  the record has no `daemon-hybrid` rows. */
+    daemonHybrid?: {
+        available: boolean;
+        phase: string;
+    };
+    runtime: {
+        node: string;
+        platform: string;
+        arch: string;
+    };
+}
+export interface EvalRunRecord {
+    provenance: EvalProvenance;
+    goldSet: {
+        queryCount: number;
+        byClass: Record<string, number>;
+        goldSpanCount: number;
+    };
+    configs: readonly EvalConfig[];
+    aggregates: EvalAggregate[];
+    perQuery: Array<{
+        id: string;
+        class: string;
+        results: EvalQueryResult[];
+    }>;
+}
+/** Hash every retrieval-shaping source file, so a tuning change is visible as
+ *  a changed provenance field rather than an unexplained metric shift. */
+export declare function hashRetrievalSource(repoRoot: string): string;
+export interface PinnedCorpus {
+    /** Directory to index and search. */
+    cwd: string;
+    sha: string;
+    fromWorkingTree: boolean;
+    dirty: boolean;
+    /** Removes the worktree, if one was created. */
+    dispose: () => void;
+}
+/**
+ * Materialize the corpus to evaluate.
+ *
+ * With a `ref`, checks out a detached worktree at that commit so the corpus is
+ * byte-identical on every rerun. Without one, falls back to the live working
+ * tree and reports `dirty` so the record shows the run was not reproducible.
+ */
+export declare function pinCorpus(repoRoot: string, ref: string | undefined): PinnedCorpus;
+export declare function collectProvenance(repoRoot: string, corpus: PinnedCorpus, corpusRef: string, service: EmbsearchService | undefined, embsearchBinary?: string, hybridService?: EmbsearchService): EvalProvenance;
+export declare function summarizeGoldSet(dataset: readonly EvalQuery[]): EvalRunRecord["goldSet"];
+export interface RunEvalSuiteOptions {
+    cwd: string;
+    dataset: readonly EvalQuery[];
+    configs: readonly EvalConfig[];
+    service?: EmbsearchService;
+    /** Second service backed by a daemon-side BM25 hybrid store, for the
+     *  `daemon-hybrid` configs. Absent means those rows are omitted. */
+    hybridService?: EmbsearchService;
+    onQuery?: (index: number, query: EvalQuery) => void;
+}
+export declare function runEvalSuite(options: RunEvalSuiteOptions): Promise<{
+    aggregates: EvalAggregate[];
+    perQuery: EvalRunRecord["perQuery"];
+}>;
+export declare function formatAggregateTable(aggregates: readonly EvalAggregate[]): string;
+//# sourceMappingURL=eval-harness.d.ts.map

package/dist/core/search/eval-harness.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-harness.d.ts","sourceRoot":"","sources":["../../../src/core/search/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,KAAK,eAAe,EAAiB,MAAM,WAAW,CAAC;AAEjG,+DAA+D;AAC/D,MAAM,WAAW,aAAa;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,wCAAwC;IACxC,CAAC,EAAE,MAAM,CAAC;IACV,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED,2EAA2E;AAC3E,MAAM,WAAW,cAAc;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB;iEAC2D;IAC3D,qBAAqB,EAAE,OAAO,CAAC;IAC/B;0DACsD;IACtD,WAAW,EAAE,OAAO,CAAC;IACrB;oEACgE;IAChE,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;;OAMG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;yDACqD;IACrD,QAAQ,EAAE;QACT,SAAS,EAAE,OAAO,CAAC;QACnB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,0DAA0D;QAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;QACd;;8EAEsE;QACtE,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,aAAa,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IACF;kDAC8C;IAC9C,YAAY,CAAC,EAAE;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,WAAW,aAAa;IAC7B,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,OAAO,EAAE,SAAS,UAAU,EAAE,CAAC;IAC/B,UAAU,EAAE,aAAa,EAAE,CAAC;IAC5B,QAAQ,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,eAAe,EAAE,CAAA;KAAE,CAAC,CAAC;CAC3E;AAMD;0EAC0E;AAC1E,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CA+B5D;AAED,MAAM,WAAW,YAAY;IAC5B,qCAAqC;IACrC,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,OAAO,CAAC;IACzB,KAAK,EAAE,OAAO,CAAC;IACf,gDAAgD;IAChD,OAAO,EAAE,MAAM,IAAI,CAAC;CACpB;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,YAAY,CAyCjF;AAYD,wBAAgB,iBAAiB,CAChC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,YAAY,EACpB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,gBAAgB,GAAG,SAAS,EACrC,eAAe,CAAC,EAAE,MAAM,EACxB,aAAa,CAAC,EAAE,gBAAgB,GAC9B,cAAc,CA2BhB;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,SAAS,SAAS,EAAE,GAAG,aAAa,CAAC,SAAS,CAAC,CAQxF;AAED,MAAM,WAAW,mBAAmB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,SAAS,SAAS,EAAE,CAAC;IAC9B,OAAO,EAAE,SAAS,UAAU,EAAE,CAAC;IAC/B,OAAO,CAAC,EAAE,gBAAgB,CAAC;IAC3B;wEACoE;IACpE,aAAa,CAAC,EAAE,gBAAgB,CAAC;IACjC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,KAAK,IAAI,CAAC;CACpD;AAED,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC;IACzE,UAAU,EAAE,aAAa,EAAE,CAAC;IAC5B,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACpC,CAAC,CA4CD;AAED,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,GAAG,MAAM,CAcjF","sourcesContent":["/**\n * Eval harness: corpus pinning, provenance capture, and run records.\n *\n * The scoring math lives in `eval.ts`; this module is everything around it\n * that makes a number *comparable to a later number*. Three problems it\n * exists to solve, all of which bit the first eval round\n * (docs/hybrid-retrieval-design.md, \"Eval results\"):\n *\n *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,\n *     so every commit moves the thing being measured. A baseline taken today\n *     and a rerun taken after a retrieval change differ by both the change\n *     and the intervening commits, and nothing in the output says so. Fix:\n *     run against a detached git worktree pinned to an explicit SHA, and put\n *     that SHA in the record.\n *  2. **Nothing was recorded.** Results were printed to a terminal and\n *     hand-copied into a markdown table with no repo SHA, no embedder\n *     identity, and no index state. Fix: emit a machine-readable run record.\n *  3. **A degraded run looks like a real one.** With no embsearch binary the\n *     semantic and hybrid rows silently degrade to lexical, producing a table\n *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the\n *     record, plus a per-row degraded count that the writer refuses to hide.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readdirSync, readFileSync, statSync } from \"node:fs\";\nimport { execFileSync } from \"child_process\";\nimport { rmSync } from \"fs\";\nimport { tmpdir } from \"os\";\nimport path from \"path\";\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { type EvalConfig, type EvalQuery, type EvalQueryResult, evaluateQuery } from \"./eval.js\";\n\n/** Metrics aggregated per config across the whole gold set. */\nexport interface EvalAggregate {\n\tlabel: string;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n\t/** Queries scored under this config. */\n\tn: number;\n\t/** How many of them ran degraded (requested retriever unavailable). */\n\tdegraded: number;\n}\n\n/** Everything needed to decide whether two run records may be compared. */\nexport interface EvalProvenance {\n\ttimestampMs: number;\n\t/** SHA of the corpus actually indexed and searched. */\n\tcorpusSha: string;\n\t/** Ref the caller asked for, before resolution (e.g. \"HEAD\"). */\n\tcorpusRef: string;\n\t/** True when the corpus came from the live working tree rather than a\n\t *  pinned worktree — results are then not reproducible. */\n\tcorpusFromWorkingTree: boolean;\n\t/** Uncommitted changes present at run time. Only meaningful (and only\n\t *  possible) when `corpusFromWorkingTree` is true. */\n\tcorpusDirty: boolean;\n\t/** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,\n\t *  but differs when pinning an old corpus with today's code. */\n\tharnessSha: string;\n\t/**\n\t * Content hash of `src/core/search` + the chunker. Every tuning constant\n\t * that shapes a result — the fusion cap, top-k depths, rerank weights,\n\t * chunk sizing — lives in those files, so a changed hash means the\n\t * numbers are not comparable, without this module having to maintain a\n\t * hand-copied (and inevitably stale) list of constants.\n\t */\n\tretrievalSourceHash: string;\n\t/** Embedding backend state. `available: false` means every semantic and\n\t *  hybrid row in this record degraded to lexical. */\n\tembedder: {\n\t\tavailable: boolean;\n\t\treason?: string;\n\t\t/** Indexed chunk count when the index reached `ready`. */\n\t\tchunkCount?: number;\n\t\tphase: string;\n\t\t/** Binary that served the embeddings, and its self-reported version.\n\t\t *  The embedding model is baked into the binary at build time, so this\n\t\t *  is the only thing that identifies which model produced a score. */\n\t\tbinaryPath?: string;\n\t\tbinaryVersion?: string;\n\t};\n\t/** Daemon-side BM25 hybrid store, when the run included one. Absent means\n\t *  the record has no `daemon-hybrid` rows. */\n\tdaemonHybrid?: { available: boolean; phase: string };\n\truntime: { node: string; platform: string; arch: string };\n}\n\nexport interface EvalRunRecord {\n\tprovenance: EvalProvenance;\n\tgoldSet: { queryCount: number; byClass: Record<string, number>; goldSpanCount: number };\n\tconfigs: readonly EvalConfig[];\n\taggregates: EvalAggregate[];\n\tperQuery: Array<{ id: string; class: string; results: EvalQueryResult[] }>;\n}\n\nfunction git(repoRoot: string, args: string[]): string {\n\treturn execFileSync(\"git\", [\"-C\", repoRoot, ...args], { encoding: \"utf-8\" }).trim();\n}\n\n/** Hash every retrieval-shaping source file, so a tuning change is visible as\n *  a changed provenance field rather than an unexplained metric shift. */\nexport function hashRetrievalSource(repoRoot: string): string {\n\tconst roots = [\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/search\"),\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/embsearch/chunker.ts\"),\n\t];\n\tconst files: string[] = [];\n\tconst walk = (target: string): void => {\n\t\tlet stat: ReturnType<typeof statSync>;\n\t\ttry {\n\t\t\tstat = statSync(target);\n\t\t} catch {\n\t\t\treturn;\n\t\t}\n\t\tif (stat.isDirectory()) {\n\t\t\tfor (const entry of readdirSync(target).sort()) walk(path.join(target, entry));\n\t\t} else if (target.endsWith(\".ts\")) {\n\t\t\tfiles.push(target);\n\t\t}\n\t};\n\tfor (const root of roots) walk(root);\n\n\tconst hash = createHash(\"sha256\");\n\tfor (const file of files) {\n\t\t// Eval-only modules are excluded: changing how we measure must not look\n\t\t// like changing what we measure.\n\t\tconst base = path.basename(file);\n\t\tif (base.startsWith(\"eval\")) continue;\n\t\thash.update(path.relative(repoRoot, file).replace(/\\\\/g, \"/\"));\n\t\thash.update(readFileSync(file));\n\t}\n\treturn hash.digest(\"hex\").slice(0, 16);\n}\n\nexport interface PinnedCorpus {\n\t/** Directory to index and search. */\n\tcwd: string;\n\tsha: string;\n\tfromWorkingTree: boolean;\n\tdirty: boolean;\n\t/** Removes the worktree, if one was created. */\n\tdispose: () => void;\n}\n\n/**\n * Materialize the corpus to evaluate.\n *\n * With a `ref`, checks out a detached worktree at that commit so the corpus is\n * byte-identical on every rerun. Without one, falls back to the live working\n * tree and reports `dirty` so the record shows the run was not reproducible.\n */\nexport function pinCorpus(repoRoot: string, ref: string | undefined): PinnedCorpus {\n\tconst dirty = git(repoRoot, [\"status\", \"--porcelain\"]).length > 0;\n\tif (!ref) {\n\t\treturn {\n\t\t\tcwd: repoRoot,\n\t\t\tsha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\t\tfromWorkingTree: true,\n\t\t\tdirty,\n\t\t\tdispose: () => {},\n\t\t};\n\t}\n\n\tconst sha = git(repoRoot, [\"rev-parse\", ref]);\n\t// Deterministic path, not mkdtemp: the embedding store is keyed by a hash of\n\t// the corpus directory, so a fresh temp path every run would re-embed all\n\t// ~17k chunks (minutes) instead of reusing the store built for this exact\n\t// SHA. The worktree is still removed afterwards; only the store persists.\n\tconst dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}`);\n\tif (existsSync(dir)) {\n\t\t// Left behind by an interrupted run — drop it so `worktree add` succeeds.\n\t\ttry {\n\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t} catch {\n\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\tgit(repoRoot, [\"worktree\", \"prune\"]);\n\t\t}\n\t}\n\tgit(repoRoot, [\"worktree\", \"add\", \"--detach\", dir, sha]);\n\treturn {\n\t\tcwd: dir,\n\t\tsha,\n\t\tfromWorkingTree: false,\n\t\tdirty: false,\n\t\tdispose: () => {\n\t\t\ttry {\n\t\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t\t} catch {\n\t\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\t}\n\t\t},\n\t};\n}\n\n/** `<binary> --version`, or undefined when it cannot be run. */\nfunction probeBinaryVersion(binaryPath: string | undefined): string | undefined {\n\tif (!binaryPath) return undefined;\n\ttry {\n\t\treturn execFileSync(binaryPath, [\"--version\"], { encoding: \"utf-8\" }).trim();\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\nexport function collectProvenance(\n\trepoRoot: string,\n\tcorpus: PinnedCorpus,\n\tcorpusRef: string,\n\tservice: EmbsearchService | undefined,\n\tembsearchBinary?: string,\n\thybridService?: EmbsearchService,\n): EvalProvenance {\n\tconst state = service?.getState();\n\tconst phase = state?.phase ?? \"absent\";\n\treturn {\n\t\ttimestampMs: Date.now(),\n\t\tcorpusSha: corpus.sha,\n\t\tcorpusRef,\n\t\tcorpusFromWorkingTree: corpus.fromWorkingTree,\n\t\tcorpusDirty: corpus.dirty,\n\t\tharnessSha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\tretrievalSourceHash: hashRetrievalSource(repoRoot),\n\t\tembedder: {\n\t\t\t// `ready` is the only phase the service reaches with a real embedder:\n\t\t\t// it rejects the mock backend at startup, so availability here also\n\t\t\t// certifies the numbers came from a genuine ONNX build.\n\t\t\tavailable: service?.isAvailable() ?? false,\n\t\t\treason: state && \"reason\" in state ? state.reason : undefined,\n\t\t\tchunkCount: state?.phase === \"ready\" ? state.chunkCount : undefined,\n\t\t\tphase,\n\t\t\tbinaryPath: embsearchBinary,\n\t\t\tbinaryVersion: probeBinaryVersion(embsearchBinary),\n\t\t},\n\t\tdaemonHybrid: hybridService\n\t\t\t? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }\n\t\t\t: undefined,\n\t\truntime: { node: process.version, platform: process.platform, arch: process.arch },\n\t};\n}\n\nexport function summarizeGoldSet(dataset: readonly EvalQuery[]): EvalRunRecord[\"goldSet\"] {\n\tconst byClass: Record<string, number> = {};\n\tlet goldSpanCount = 0;\n\tfor (const query of dataset) {\n\t\tbyClass[query.class] = (byClass[query.class] ?? 0) + 1;\n\t\tgoldSpanCount += query.gold.length;\n\t}\n\treturn { queryCount: dataset.length, byClass, goldSpanCount };\n}\n\nexport interface RunEvalSuiteOptions {\n\tcwd: string;\n\tdataset: readonly EvalQuery[];\n\tconfigs: readonly EvalConfig[];\n\tservice?: EmbsearchService;\n\t/** Second service backed by a daemon-side BM25 hybrid store, for the\n\t *  `daemon-hybrid` configs. Absent means those rows are omitted. */\n\thybridService?: EmbsearchService;\n\tonQuery?: (index: number, query: EvalQuery) => void;\n}\n\nexport async function runEvalSuite(options: RunEvalSuiteOptions): Promise<{\n\taggregates: EvalAggregate[];\n\tperQuery: EvalRunRecord[\"perQuery\"];\n}> {\n\tconst { cwd, dataset, configs, service } = options;\n\tconst totals = new Map<string, EvalAggregate>();\n\tconst perQuery: EvalRunRecord[\"perQuery\"] = [];\n\n\tfor (const [index, evalQuery] of dataset.entries()) {\n\t\toptions.onQuery?.(index, evalQuery);\n\t\tconst results = await evaluateQuery(cwd, evalQuery, configs, service, options.hybridService);\n\t\tperQuery.push({ id: evalQuery.id, class: evalQuery.class, results });\n\t\tfor (const result of results) {\n\t\t\tconst total = totals.get(result.label) ?? {\n\t\t\t\tlabel: result.label,\n\t\t\t\trecallAt1: 0,\n\t\t\t\trecallAt5: 0,\n\t\t\t\trecallAt10: 0,\n\t\t\t\trecallAt50: 0,\n\t\t\t\tmrr: 0,\n\t\t\t\tn: 0,\n\t\t\t\tdegraded: 0,\n\t\t\t};\n\t\t\ttotal.recallAt1 += result.recallAt1;\n\t\t\ttotal.recallAt5 += result.recallAt5;\n\t\t\ttotal.recallAt10 += result.recallAt10;\n\t\t\ttotal.recallAt50 += result.recallAt50;\n\t\t\ttotal.mrr += result.mrr;\n\t\t\ttotal.n++;\n\t\t\tif (result.degraded) total.degraded++;\n\t\t\ttotals.set(result.label, total);\n\t\t}\n\t}\n\n\tconst aggregates = configs\n\t\t.map((config) => totals.get(config.label))\n\t\t.filter((total): total is EvalAggregate => total !== undefined)\n\t\t.map((total) => ({\n\t\t\t...total,\n\t\t\trecallAt1: total.recallAt1 / total.n,\n\t\t\trecallAt5: total.recallAt5 / total.n,\n\t\t\trecallAt10: total.recallAt10 / total.n,\n\t\t\trecallAt50: total.recallAt50 / total.n,\n\t\t\tmrr: total.mrr / total.n,\n\t\t}));\n\n\treturn { aggregates, perQuery };\n}\n\nexport function formatAggregateTable(aggregates: readonly EvalAggregate[]): string {\n\tconst pct = (x: number) => `${Math.round(x * 100)}%`.padStart(5);\n\tconst lines = [\n\t\t\"config           |  R@1  |  R@5  | R@10  | R@50  |  MRR  | notes\",\n\t\t\"-----------------|-------|-------|-------|-------|-------|------\",\n\t];\n\tfor (const a of aggregates) {\n\t\tconst notes = a.degraded === a.n ? \"degraded to lexical\" : a.degraded > 0 ? `${a.degraded}/${a.n} degraded` : \"\";\n\t\tlines.push(\n\t\t\t`${a.label.padEnd(16)} | ${pct(a.recallAt1)} | ${pct(a.recallAt5)} | ${pct(a.recallAt10)} | ` +\n\t\t\t\t`${pct(a.recallAt50)} | ${a.mrr.toFixed(3)} | ${notes}`,\n\t\t);\n\t}\n\treturn lines.join(\"\\n\");\n}\n"]}

package/dist/core/search/eval-harness.js

@@ -0,0 +1,225 @@
+/**
+ * Eval harness: corpus pinning, provenance capture, and run records.
+ *
+ * The scoring math lives in `eval.ts`; this module is everything around it
+ * that makes a number *comparable to a later number*. Three problems it
+ * exists to solve, all of which bit the first eval round
+ * (docs/hybrid-retrieval-design.md, "Eval results"):
+ *
+ *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,
+ *     so every commit moves the thing being measured. A baseline taken today
+ *     and a rerun taken after a retrieval change differ by both the change
+ *     and the intervening commits, and nothing in the output says so. Fix:
+ *     run against a detached git worktree pinned to an explicit SHA, and put
+ *     that SHA in the record.
+ *  2. **Nothing was recorded.** Results were printed to a terminal and
+ *     hand-copied into a markdown table with no repo SHA, no embedder
+ *     identity, and no index state. Fix: emit a machine-readable run record.
+ *  3. **A degraded run looks like a real one.** With no embsearch binary the
+ *     semantic and hybrid rows silently degrade to lexical, producing a table
+ *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the
+ *     record, plus a per-row degraded count that the writer refuses to hide.
+ */
+import { createHash } from "node:crypto";
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { execFileSync } from "child_process";
+import { rmSync } from "fs";
+import { tmpdir } from "os";
+import path from "path";
+import { evaluateQuery } from "./eval.js";
+function git(repoRoot, args) {
+    return execFileSync("git", ["-C", repoRoot, ...args], { encoding: "utf-8" }).trim();
+}
+/** Hash every retrieval-shaping source file, so a tuning change is visible as
+ *  a changed provenance field rather than an unexplained metric shift. */
+export function hashRetrievalSource(repoRoot) {
+    const roots = [
+        path.join(repoRoot, "packages/coding-agent/src/core/search"),
+        path.join(repoRoot, "packages/coding-agent/src/core/embsearch/chunker.ts"),
+    ];
+    const files = [];
+    const walk = (target) => {
+        let stat;
+        try {
+            stat = statSync(target);
+        }
+        catch {
+            return;
+        }
+        if (stat.isDirectory()) {
+            for (const entry of readdirSync(target).sort())
+                walk(path.join(target, entry));
+        }
+        else if (target.endsWith(".ts")) {
+            files.push(target);
+        }
+    };
+    for (const root of roots)
+        walk(root);
+    const hash = createHash("sha256");
+    for (const file of files) {
+        // Eval-only modules are excluded: changing how we measure must not look
+        // like changing what we measure.
+        const base = path.basename(file);
+        if (base.startsWith("eval"))
+            continue;
+        hash.update(path.relative(repoRoot, file).replace(/\\/g, "/"));
+        hash.update(readFileSync(file));
+    }
+    return hash.digest("hex").slice(0, 16);
+}
+/**
+ * Materialize the corpus to evaluate.
+ *
+ * With a `ref`, checks out a detached worktree at that commit so the corpus is
+ * byte-identical on every rerun. Without one, falls back to the live working
+ * tree and reports `dirty` so the record shows the run was not reproducible.
+ */
+export function pinCorpus(repoRoot, ref) {
+    const dirty = git(repoRoot, ["status", "--porcelain"]).length > 0;
+    if (!ref) {
+        return {
+            cwd: repoRoot,
+            sha: git(repoRoot, ["rev-parse", "HEAD"]),
+            fromWorkingTree: true,
+            dirty,
+            dispose: () => { },
+        };
+    }
+    const sha = git(repoRoot, ["rev-parse", ref]);
+    // Deterministic path, not mkdtemp: the embedding store is keyed by a hash of
+    // the corpus directory, so a fresh temp path every run would re-embed all
+    // ~17k chunks (minutes) instead of reusing the store built for this exact
+    // SHA. The worktree is still removed afterwards; only the store persists.
+    const dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}`);
+    if (existsSync(dir)) {
+        // Left behind by an interrupted run — drop it so `worktree add` succeeds.
+        try {
+            git(repoRoot, ["worktree", "remove", "--force", dir]);
+        }
+        catch {
+            rmSync(dir, { recursive: true, force: true });
+            git(repoRoot, ["worktree", "prune"]);
+        }
+    }
+    git(repoRoot, ["worktree", "add", "--detach", dir, sha]);
+    return {
+        cwd: dir,
+        sha,
+        fromWorkingTree: false,
+        dirty: false,
+        dispose: () => {
+            try {
+                git(repoRoot, ["worktree", "remove", "--force", dir]);
+            }
+            catch {
+                rmSync(dir, { recursive: true, force: true });
+            }
+        },
+    };
+}
+/** `<binary> --version`, or undefined when it cannot be run. */
+function probeBinaryVersion(binaryPath) {
+    if (!binaryPath)
+        return undefined;
+    try {
+        return execFileSync(binaryPath, ["--version"], { encoding: "utf-8" }).trim();
+    }
+    catch {
+        return undefined;
+    }
+}
+export function collectProvenance(repoRoot, corpus, corpusRef, service, embsearchBinary, hybridService) {
+    const state = service?.getState();
+    const phase = state?.phase ?? "absent";
+    return {
+        timestampMs: Date.now(),
+        corpusSha: corpus.sha,
+        corpusRef,
+        corpusFromWorkingTree: corpus.fromWorkingTree,
+        corpusDirty: corpus.dirty,
+        harnessSha: git(repoRoot, ["rev-parse", "HEAD"]),
+        retrievalSourceHash: hashRetrievalSource(repoRoot),
+        embedder: {
+            // `ready` is the only phase the service reaches with a real embedder:
+            // it rejects the mock backend at startup, so availability here also
+            // certifies the numbers came from a genuine ONNX build.
+            available: service?.isAvailable() ?? false,
+            reason: state && "reason" in state ? state.reason : undefined,
+            chunkCount: state?.phase === "ready" ? state.chunkCount : undefined,
+            phase,
+            binaryPath: embsearchBinary,
+            binaryVersion: probeBinaryVersion(embsearchBinary),
+        },
+        daemonHybrid: hybridService
+            ? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }
+            : undefined,
+        runtime: { node: process.version, platform: process.platform, arch: process.arch },
+    };
+}
+export function summarizeGoldSet(dataset) {
+    const byClass = {};
+    let goldSpanCount = 0;
+    for (const query of dataset) {
+        byClass[query.class] = (byClass[query.class] ?? 0) + 1;
+        goldSpanCount += query.gold.length;
+    }
+    return { queryCount: dataset.length, byClass, goldSpanCount };
+}
+export async function runEvalSuite(options) {
+    const { cwd, dataset, configs, service } = options;
+    const totals = new Map();
+    const perQuery = [];
+    for (const [index, evalQuery] of dataset.entries()) {
+        options.onQuery?.(index, evalQuery);
+        const results = await evaluateQuery(cwd, evalQuery, configs, service, options.hybridService);
+        perQuery.push({ id: evalQuery.id, class: evalQuery.class, results });
+        for (const result of results) {
+            const total = totals.get(result.label) ?? {
+                label: result.label,
+                recallAt1: 0,
+                recallAt5: 0,
+                recallAt10: 0,
+                recallAt50: 0,
+                mrr: 0,
+                n: 0,
+                degraded: 0,
+            };
+            total.recallAt1 += result.recallAt1;
+            total.recallAt5 += result.recallAt5;
+            total.recallAt10 += result.recallAt10;
+            total.recallAt50 += result.recallAt50;
+            total.mrr += result.mrr;
+            total.n++;
+            if (result.degraded)
+                total.degraded++;
+            totals.set(result.label, total);
+        }
+    }
+    const aggregates = configs
+        .map((config) => totals.get(config.label))
+        .filter((total) => total !== undefined)
+        .map((total) => ({
+        ...total,
+        recallAt1: total.recallAt1 / total.n,
+        recallAt5: total.recallAt5 / total.n,
+        recallAt10: total.recallAt10 / total.n,
+        recallAt50: total.recallAt50 / total.n,
+        mrr: total.mrr / total.n,
+    }));
+    return { aggregates, perQuery };
+}
+export function formatAggregateTable(aggregates) {
+    const pct = (x) => `${Math.round(x * 100)}%`.padStart(5);
+    const lines = [
+        "config           |  R@1  |  R@5  | R@10  | R@50  |  MRR  | notes",
+        "-----------------|-------|-------|-------|-------|-------|------",
+    ];
+    for (const a of aggregates) {
+        const notes = a.degraded === a.n ? "degraded to lexical" : a.degraded > 0 ? `${a.degraded}/${a.n} degraded` : "";
+        lines.push(`${a.label.padEnd(16)} | ${pct(a.recallAt1)} | ${pct(a.recallAt5)} | ${pct(a.recallAt10)} | ` +
+            `${pct(a.recallAt50)} | ${a.mrr.toFixed(3)} | ${notes}`);
+    }
+    return lines.join("\n");
+}
+//# sourceMappingURL=eval-harness.js.map

package/dist/core/search/eval-harness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-harness.js","sourceRoot":"","sources":["../../../src/core/search/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAyD,aAAa,EAAE,MAAM,WAAW,CAAC;AAoEjG,SAAS,GAAG,CAAC,QAAgB,EAAE,IAAc,EAAU;IACtD,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAAA,CACpF;AAED;0EAC0E;AAC1E,MAAM,UAAU,mBAAmB,CAAC,QAAgB,EAAU;IAC7D,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,uCAAuC,CAAC;QAC5D,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,qDAAqD,CAAC;KAC1E,CAAC;IACF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,MAAc,EAAQ,EAAE,CAAC;QACtC,IAAI,IAAiC,CAAC;QACtC,IAAI,CAAC;YACJ,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACR,OAAO;QACR,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE;gBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;IAAA,CACD,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,wEAAwE;QACxE,iCAAiC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACtC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAAA,CACvC;AAYD;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,GAAuB,EAAgB;IAClF,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAClE,IAAI,CAAC,GAAG,EAAE,CAAC;QACV,OAAO;YACN,GAAG,EAAE,QAAQ;YACb,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACzC,eAAe,EAAE,IAAI;YACrB,KAAK;YACL,OAAO,EAAE,GAAG,EAAE,CAAC,EAAC,CAAC;SACjB,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9C,6EAA6E;IAC7E,0EAA0E;IAC1E,0EAA0E;IAC1E,0EAA0E;IAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,4EAA0E;QAC1E,IAAI,CAAC;YACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;QACtC,CAAC;IACF,CAAC;IACD,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IACzD,OAAO;QACN,GAAG,EAAE,GAAG;QACR,GAAG;QACH,eAAe,EAAE,KAAK;QACtB,KAAK,EAAE,KAAK;QACZ,OAAO,EAAE,GAAG,EAAE,CAAC;YACd,IAAI,CAAC;gBACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/C,CAAC;QAAA,CACD;KACD,CAAC;AAAA,CACF;AAED,gEAAgE;AAChE,SAAS,kBAAkB,CAAC,UAA8B,EAAsB;IAC/E,IAAI,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAClC,IAAI,CAAC;QACJ,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED,MAAM,UAAU,iBAAiB,CAChC,QAAgB,EAChB,MAAoB,EACpB,SAAiB,EACjB,OAAqC,EACrC,eAAwB,EACxB,aAAgC,EACf;IACjB,MAAM,KAAK,GAAG,OAAO,EAAE,QAAQ,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,KAAK,EAAE,KAAK,IAAI,QAAQ,CAAC;IACvC,OAAO;QACN,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;QACvB,SAAS,EAAE,MAAM,CAAC,GAAG;QACrB,SAAS;QACT,qBAAqB,EAAE,MAAM,CAAC,eAAe;QAC7C,WAAW,EAAE,MAAM,CAAC,KAAK;QACzB,UAAU,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAChD,mBAAmB,EAAE,mBAAmB,CAAC,QAAQ,CAAC;QAClD,QAAQ,EAAE;YACT,sEAAsE;YACtE,oEAAoE;YACpE,wDAAwD;YACxD,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,KAAK;YAC1C,MAAM,EAAE,KAAK,IAAI,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YAC7D,UAAU,EAAE,KAAK,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YACnE,KAAK;YACL,UAAU,EAAE,eAAe;YAC3B,aAAa,EAAE,kBAAkB,CAAC,eAAe,CAAC;SAClD;QACD,YAAY,EAAE,aAAa;YAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,aAAa,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,aAAa,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE;YACnF,CAAC,CAAC,SAAS;QACZ,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE;KAClF,CAAC;AAAA,CACF;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA6B,EAA4B;IACzF,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACvD,aAAa,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;IACpC,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AAAA,CAC9D;AAaD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B,EAG5D;IACF,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAChD,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAE/C,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC7F,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACrE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI;gBACzC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC;gBACb,UAAU,EAAE,CAAC;gBACb,GAAG,EAAE,CAAC;gBACN,CAAC,EAAE,CAAC;gBACJ,QAAQ,EAAE,CAAC;aACX,CAAC;YACF,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC;YACxB,KAAK,CAAC,CAAC,EAAE,CAAC;YACV,IAAI,MAAM,CAAC,QAAQ;gBAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACjC,CAAC;IACF,CAAC;IAED,MAAM,UAAU,GAAG,OAAO;SACxB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;SACzC,MAAM,CAAC,CAAC,KAAK,EAA0B,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC;SAC9D,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAChB,GAAG,KAAK;QACR,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;KACxB,CAAC,CAAC,CAAC;IAEL,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAAA,CAChC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAAoC,EAAU;IAClF,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG;QACb,kEAAkE;QAClE,kEAAkE;KAClE,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;QACjH,KAAK,CAAC,IAAI,CACT,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK;YAC5F,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,EAAE,CACxD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAAA,CACxB","sourcesContent":["/**\n * Eval harness: corpus pinning, provenance capture, and run records.\n *\n * The scoring math lives in `eval.ts`; this module is everything around it\n * that makes a number *comparable to a later number*. Three problems it\n * exists to solve, all of which bit the first eval round\n * (docs/hybrid-retrieval-design.md, \"Eval results\"):\n *\n *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,\n *     so every commit moves the thing being measured. A baseline taken today\n *     and a rerun taken after a retrieval change differ by both the change\n *     and the intervening commits, and nothing in the output says so. Fix:\n *     run against a detached git worktree pinned to an explicit SHA, and put\n *     that SHA in the record.\n *  2. **Nothing was recorded.** Results were printed to a terminal and\n *     hand-copied into a markdown table with no repo SHA, no embedder\n *     identity, and no index state. Fix: emit a machine-readable run record.\n *  3. **A degraded run looks like a real one.** With no embsearch binary the\n *     semantic and hybrid rows silently degrade to lexical, producing a table\n *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the\n *     record, plus a per-row degraded count that the writer refuses to hide.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readdirSync, readFileSync, statSync } from \"node:fs\";\nimport { execFileSync } from \"child_process\";\nimport { rmSync } from \"fs\";\nimport { tmpdir } from \"os\";\nimport path from \"path\";\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { type EvalConfig, type EvalQuery, type EvalQueryResult, evaluateQuery } from \"./eval.js\";\n\n/** Metrics aggregated per config across the whole gold set. */\nexport interface EvalAggregate {\n\tlabel: string;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n\t/** Queries scored under this config. */\n\tn: number;\n\t/** How many of them ran degraded (requested retriever unavailable). */\n\tdegraded: number;\n}\n\n/** Everything needed to decide whether two run records may be compared. */\nexport interface EvalProvenance {\n\ttimestampMs: number;\n\t/** SHA of the corpus actually indexed and searched. */\n\tcorpusSha: string;\n\t/** Ref the caller asked for, before resolution (e.g. \"HEAD\"). */\n\tcorpusRef: string;\n\t/** True when the corpus came from the live working tree rather than a\n\t *  pinned worktree — results are then not reproducible. */\n\tcorpusFromWorkingTree: boolean;\n\t/** Uncommitted changes present at run time. Only meaningful (and only\n\t *  possible) when `corpusFromWorkingTree` is true. */\n\tcorpusDirty: boolean;\n\t/** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,\n\t *  but differs when pinning an old corpus with today's code. */\n\tharnessSha: string;\n\t/**\n\t * Content hash of `src/core/search` + the chunker. Every tuning constant\n\t * that shapes a result — the fusion cap, top-k depths, rerank weights,\n\t * chunk sizing — lives in those files, so a changed hash means the\n\t * numbers are not comparable, without this module having to maintain a\n\t * hand-copied (and inevitably stale) list of constants.\n\t */\n\tretrievalSourceHash: string;\n\t/** Embedding backend state. `available: false` means every semantic and\n\t *  hybrid row in this record degraded to lexical. */\n\tembedder: {\n\t\tavailable: boolean;\n\t\treason?: string;\n\t\t/** Indexed chunk count when the index reached `ready`. */\n\t\tchunkCount?: number;\n\t\tphase: string;\n\t\t/** Binary that served the embeddings, and its self-reported version.\n\t\t *  The embedding model is baked into the binary at build time, so this\n\t\t *  is the only thing that identifies which model produced a score. */\n\t\tbinaryPath?: string;\n\t\tbinaryVersion?: string;\n\t};\n\t/** Daemon-side BM25 hybrid store, when the run included one. Absent means\n\t *  the record has no `daemon-hybrid` rows. */\n\tdaemonHybrid?: { available: boolean; phase: string };\n\truntime: { node: string; platform: string; arch: string };\n}\n\nexport interface EvalRunRecord {\n\tprovenance: EvalProvenance;\n\tgoldSet: { queryCount: number; byClass: Record<string, number>; goldSpanCount: number };\n\tconfigs: readonly EvalConfig[];\n\taggregates: EvalAggregate[];\n\tperQuery: Array<{ id: string; class: string; results: EvalQueryResult[] }>;\n}\n\nfunction git(repoRoot: string, args: string[]): string {\n\treturn execFileSync(\"git\", [\"-C\", repoRoot, ...args], { encoding: \"utf-8\" }).trim();\n}\n\n/** Hash every retrieval-shaping source file, so a tuning change is visible as\n *  a changed provenance field rather than an unexplained metric shift. */\nexport function hashRetrievalSource(repoRoot: string): string {\n\tconst roots = [\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/search\"),\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/embsearch/chunker.ts\"),\n\t];\n\tconst files: string[] = [];\n\tconst walk = (target: string): void => {\n\t\tlet stat: ReturnType<typeof statSync>;\n\t\ttry {\n\t\t\tstat = statSync(target);\n\t\t} catch {\n\t\t\treturn;\n\t\t}\n\t\tif (stat.isDirectory()) {\n\t\t\tfor (const entry of readdirSync(target).sort()) walk(path.join(target, entry));\n\t\t} else if (target.endsWith(\".ts\")) {\n\t\t\tfiles.push(target);\n\t\t}\n\t};\n\tfor (const root of roots) walk(root);\n\n\tconst hash = createHash(\"sha256\");\n\tfor (const file of files) {\n\t\t// Eval-only modules are excluded: changing how we measure must not look\n\t\t// like changing what we measure.\n\t\tconst base = path.basename(file);\n\t\tif (base.startsWith(\"eval\")) continue;\n\t\thash.update(path.relative(repoRoot, file).replace(/\\\\/g, \"/\"));\n\t\thash.update(readFileSync(file));\n\t}\n\treturn hash.digest(\"hex\").slice(0, 16);\n}\n\nexport interface PinnedCorpus {\n\t/** Directory to index and search. */\n\tcwd: string;\n\tsha: string;\n\tfromWorkingTree: boolean;\n\tdirty: boolean;\n\t/** Removes the worktree, if one was created. */\n\tdispose: () => void;\n}\n\n/**\n * Materialize the corpus to evaluate.\n *\n * With a `ref`, checks out a detached worktree at that commit so the corpus is\n * byte-identical on every rerun. Without one, falls back to the live working\n * tree and reports `dirty` so the record shows the run was not reproducible.\n */\nexport function pinCorpus(repoRoot: string, ref: string | undefined): PinnedCorpus {\n\tconst dirty = git(repoRoot, [\"status\", \"--porcelain\"]).length > 0;\n\tif (!ref) {\n\t\treturn {\n\t\t\tcwd: repoRoot,\n\t\t\tsha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\t\tfromWorkingTree: true,\n\t\t\tdirty,\n\t\t\tdispose: () => {},\n\t\t};\n\t}\n\n\tconst sha = git(repoRoot, [\"rev-parse\", ref]);\n\t// Deterministic path, not mkdtemp: the embedding store is keyed by a hash of\n\t// the corpus directory, so a fresh temp path every run would re-embed all\n\t// ~17k chunks (minutes) instead of reusing the store built for this exact\n\t// SHA. The worktree is still removed afterwards; only the store persists.\n\tconst dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}`);\n\tif (existsSync(dir)) {\n\t\t// Left behind by an interrupted run — drop it so `worktree add` succeeds.\n\t\ttry {\n\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t} catch {\n\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\tgit(repoRoot, [\"worktree\", \"prune\"]);\n\t\t}\n\t}\n\tgit(repoRoot, [\"worktree\", \"add\", \"--detach\", dir, sha]);\n\treturn {\n\t\tcwd: dir,\n\t\tsha,\n\t\tfromWorkingTree: false,\n\t\tdirty: false,\n\t\tdispose: () => {\n\t\t\ttry {\n\t\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t\t} catch {\n\t\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\t}\n\t\t},\n\t};\n}\n\n/** `<binary> --version`, or undefined when it cannot be run. */\nfunction probeBinaryVersion(binaryPath: string | undefined): string | undefined {\n\tif (!binaryPath) return undefined;\n\ttry {\n\t\treturn execFileSync(binaryPath, [\"--version\"], { encoding: \"utf-8\" }).trim();\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\nexport function collectProvenance(\n\trepoRoot: string,\n\tcorpus: PinnedCorpus,\n\tcorpusRef: string,\n\tservice: EmbsearchService | undefined,\n\tembsearchBinary?: string,\n\thybridService?: EmbsearchService,\n): EvalProvenance {\n\tconst state = service?.getState();\n\tconst phase = state?.phase ?? \"absent\";\n\treturn {\n\t\ttimestampMs: Date.now(),\n\t\tcorpusSha: corpus.sha,\n\t\tcorpusRef,\n\t\tcorpusFromWorkingTree: corpus.fromWorkingTree,\n\t\tcorpusDirty: corpus.dirty,\n\t\tharnessSha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\tretrievalSourceHash: hashRetrievalSource(repoRoot),\n\t\tembedder: {\n\t\t\t// `ready` is the only phase the service reaches with a real embedder:\n\t\t\t// it rejects the mock backend at startup, so availability here also\n\t\t\t// certifies the numbers came from a genuine ONNX build.\n\t\t\tavailable: service?.isAvailable() ?? false,\n\t\t\treason: state && \"reason\" in state ? state.reason : undefined,\n\t\t\tchunkCount: state?.phase === \"ready\" ? state.chunkCount : undefined,\n\t\t\tphase,\n\t\t\tbinaryPath: embsearchBinary,\n\t\t\tbinaryVersion: probeBinaryVersion(embsearchBinary),\n\t\t},\n\t\tdaemonHybrid: hybridService\n\t\t\t? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }\n\t\t\t: undefined,\n\t\truntime: { node: process.version, platform: process.platform, arch: process.arch },\n\t};\n}\n\nexport function summarizeGoldSet(dataset: readonly EvalQuery[]): EvalRunRecord[\"goldSet\"] {\n\tconst byClass: Record<string, number> = {};\n\tlet goldSpanCount = 0;\n\tfor (const query of dataset) {\n\t\tbyClass[query.class] = (byClass[query.class] ?? 0) + 1;\n\t\tgoldSpanCount += query.gold.length;\n\t}\n\treturn { queryCount: dataset.length, byClass, goldSpanCount };\n}\n\nexport interface RunEvalSuiteOptions {\n\tcwd: string;\n\tdataset: readonly EvalQuery[];\n\tconfigs: readonly EvalConfig[];\n\tservice?: EmbsearchService;\n\t/** Second service backed by a daemon-side BM25 hybrid store, for the\n\t *  `daemon-hybrid` configs. Absent means those rows are omitted. */\n\thybridService?: EmbsearchService;\n\tonQuery?: (index: number, query: EvalQuery) => void;\n}\n\nexport async function runEvalSuite(options: RunEvalSuiteOptions): Promise<{\n\taggregates: EvalAggregate[];\n\tperQuery: EvalRunRecord[\"perQuery\"];\n}> {\n\tconst { cwd, dataset, configs, service } = options;\n\tconst totals = new Map<string, EvalAggregate>();\n\tconst perQuery: EvalRunRecord[\"perQuery\"] = [];\n\n\tfor (const [index, evalQuery] of dataset.entries()) {\n\t\toptions.onQuery?.(index, evalQuery);\n\t\tconst results = await evaluateQuery(cwd, evalQuery, configs, service, options.hybridService);\n\t\tperQuery.push({ id: evalQuery.id, class: evalQuery.class, results });\n\t\tfor (const result of results) {\n\t\t\tconst total = totals.get(result.label) ?? {\n\t\t\t\tlabel: result.label,\n\t\t\t\trecallAt1: 0,\n\t\t\t\trecallAt5: 0,\n\t\t\t\trecallAt10: 0,\n\t\t\t\trecallAt50: 0,\n\t\t\t\tmrr: 0,\n\t\t\t\tn: 0,\n\t\t\t\tdegraded: 0,\n\t\t\t};\n\t\t\ttotal.recallAt1 += result.recallAt1;\n\t\t\ttotal.recallAt5 += result.recallAt5;\n\t\t\ttotal.recallAt10 += result.recallAt10;\n\t\t\ttotal.recallAt50 += result.recallAt50;\n\t\t\ttotal.mrr += result.mrr;\n\t\t\ttotal.n++;\n\t\t\tif (result.degraded) total.degraded++;\n\t\t\ttotals.set(result.label, total);\n\t\t}\n\t}\n\n\tconst aggregates = configs\n\t\t.map((config) => totals.get(config.label))\n\t\t.filter((total): total is EvalAggregate => total !== undefined)\n\t\t.map((total) => ({\n\t\t\t...total,\n\t\t\trecallAt1: total.recallAt1 / total.n,\n\t\t\trecallAt5: total.recallAt5 / total.n,\n\t\t\trecallAt10: total.recallAt10 / total.n,\n\t\t\trecallAt50: total.recallAt50 / total.n,\n\t\t\tmrr: total.mrr / total.n,\n\t\t}));\n\n\treturn { aggregates, perQuery };\n}\n\nexport function formatAggregateTable(aggregates: readonly EvalAggregate[]): string {\n\tconst pct = (x: number) => `${Math.round(x * 100)}%`.padStart(5);\n\tconst lines = [\n\t\t\"config           |  R@1  |  R@5  | R@10  | R@50  |  MRR  | notes\",\n\t\t\"-----------------|-------|-------|-------|-------|-------|------\",\n\t];\n\tfor (const a of aggregates) {\n\t\tconst notes = a.degraded === a.n ? \"degraded to lexical\" : a.degraded > 0 ? `${a.degraded}/${a.n} degraded` : \"\";\n\t\tlines.push(\n\t\t\t`${a.label.padEnd(16)} | ${pct(a.recallAt1)} | ${pct(a.recallAt5)} | ${pct(a.recallAt10)} | ` +\n\t\t\t\t`${pct(a.recallAt50)} | ${a.mrr.toFixed(3)} | ${notes}`,\n\t\t);\n\t}\n\treturn lines.join(\"\\n\");\n}\n"]}