@kolisachint/hoocode-agent 0.4.164 → 0.4.165

package/dist/core/search/eval-harness.js

@@ -0,0 +1,225 @@
+/**
+ * Eval harness: corpus pinning, provenance capture, and run records.
+ *
+ * The scoring math lives in `eval.ts`; this module is everything around it
+ * that makes a number *comparable to a later number*. Three problems it
+ * exists to solve, all of which bit the first eval round
+ * (docs/hybrid-retrieval-design.md, "Eval results"):
+ *
+ *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,
+ *     so every commit moves the thing being measured. A baseline taken today
+ *     and a rerun taken after a retrieval change differ by both the change
+ *     and the intervening commits, and nothing in the output says so. Fix:
+ *     run against a detached git worktree pinned to an explicit SHA, and put
+ *     that SHA in the record.
+ *  2. **Nothing was recorded.** Results were printed to a terminal and
+ *     hand-copied into a markdown table with no repo SHA, no embedder
+ *     identity, and no index state. Fix: emit a machine-readable run record.
+ *  3. **A degraded run looks like a real one.** With no embsearch binary the
+ *     semantic and hybrid rows silently degrade to lexical, producing a table
+ *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the
+ *     record, plus a per-row degraded count that the writer refuses to hide.
+ */
+import { createHash } from "node:crypto";
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { execFileSync } from "child_process";
+import { rmSync } from "fs";
+import { tmpdir } from "os";
+import path from "path";
+import { evaluateQuery } from "./eval.js";
+function git(repoRoot, args) {
+    return execFileSync("git", ["-C", repoRoot, ...args], { encoding: "utf-8" }).trim();
+}
+/** Hash every retrieval-shaping source file, so a tuning change is visible as
+ *  a changed provenance field rather than an unexplained metric shift. */
+export function hashRetrievalSource(repoRoot) {
+    const roots = [
+        path.join(repoRoot, "packages/coding-agent/src/core/search"),
+        path.join(repoRoot, "packages/coding-agent/src/core/embsearch/chunker.ts"),
+    ];
+    const files = [];
+    const walk = (target) => {
+        let stat;
+        try {
+            stat = statSync(target);
+        }
+        catch {
+            return;
+        }
+        if (stat.isDirectory()) {
+            for (const entry of readdirSync(target).sort())
+                walk(path.join(target, entry));
+        }
+        else if (target.endsWith(".ts")) {
+            files.push(target);
+        }
+    };
+    for (const root of roots)
+        walk(root);
+    const hash = createHash("sha256");
+    for (const file of files) {
+        // Eval-only modules are excluded: changing how we measure must not look
+        // like changing what we measure.
+        const base = path.basename(file);
+        if (base.startsWith("eval"))
+            continue;
+        hash.update(path.relative(repoRoot, file).replace(/\\/g, "/"));
+        hash.update(readFileSync(file));
+    }
+    return hash.digest("hex").slice(0, 16);
+}
+/**
+ * Materialize the corpus to evaluate.
+ *
+ * With a `ref`, checks out a detached worktree at that commit so the corpus is
+ * byte-identical on every rerun. Without one, falls back to the live working
+ * tree and reports `dirty` so the record shows the run was not reproducible.
+ */
+export function pinCorpus(repoRoot, ref) {
+    const dirty = git(repoRoot, ["status", "--porcelain"]).length > 0;
+    if (!ref) {
+        return {
+            cwd: repoRoot,
+            sha: git(repoRoot, ["rev-parse", "HEAD"]),
+            fromWorkingTree: true,
+            dirty,
+            dispose: () => { },
+        };
+    }
+    const sha = git(repoRoot, ["rev-parse", ref]);
+    // Deterministic path, not mkdtemp: the embedding store is keyed by a hash of
+    // the corpus directory, so a fresh temp path every run would re-embed all
+    // ~17k chunks (minutes) instead of reusing the store built for this exact
+    // SHA. The worktree is still removed afterwards; only the store persists.
+    const dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}`);
+    if (existsSync(dir)) {
+        // Left behind by an interrupted run — drop it so `worktree add` succeeds.
+        try {
+            git(repoRoot, ["worktree", "remove", "--force", dir]);
+        }
+        catch {
+            rmSync(dir, { recursive: true, force: true });
+            git(repoRoot, ["worktree", "prune"]);
+        }
+    }
+    git(repoRoot, ["worktree", "add", "--detach", dir, sha]);
+    return {
+        cwd: dir,
+        sha,
+        fromWorkingTree: false,
+        dirty: false,
+        dispose: () => {
+            try {
+                git(repoRoot, ["worktree", "remove", "--force", dir]);
+            }
+            catch {
+                rmSync(dir, { recursive: true, force: true });
+            }
+        },
+    };
+}
+/** `<binary> --version`, or undefined when it cannot be run. */
+function probeBinaryVersion(binaryPath) {
+    if (!binaryPath)
+        return undefined;
+    try {
+        return execFileSync(binaryPath, ["--version"], { encoding: "utf-8" }).trim();
+    }
+    catch {
+        return undefined;
+    }
+}
+export function collectProvenance(repoRoot, corpus, corpusRef, service, embsearchBinary, hybridService) {
+    const state = service?.getState();
+    const phase = state?.phase ?? "absent";
+    return {
+        timestampMs: Date.now(),
+        corpusSha: corpus.sha,
+        corpusRef,
+        corpusFromWorkingTree: corpus.fromWorkingTree,
+        corpusDirty: corpus.dirty,
+        harnessSha: git(repoRoot, ["rev-parse", "HEAD"]),
+        retrievalSourceHash: hashRetrievalSource(repoRoot),
+        embedder: {
+            // `ready` is the only phase the service reaches with a real embedder:
+            // it rejects the mock backend at startup, so availability here also
+            // certifies the numbers came from a genuine ONNX build.
+            available: service?.isAvailable() ?? false,
+            reason: state && "reason" in state ? state.reason : undefined,
+            chunkCount: state?.phase === "ready" ? state.chunkCount : undefined,
+            phase,
+            binaryPath: embsearchBinary,
+            binaryVersion: probeBinaryVersion(embsearchBinary),
+        },
+        daemonHybrid: hybridService
+            ? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }
+            : undefined,
+        runtime: { node: process.version, platform: process.platform, arch: process.arch },
+    };
+}
+export function summarizeGoldSet(dataset) {
+    const byClass = {};
+    let goldSpanCount = 0;
+    for (const query of dataset) {
+        byClass[query.class] = (byClass[query.class] ?? 0) + 1;
+        goldSpanCount += query.gold.length;
+    }
+    return { queryCount: dataset.length, byClass, goldSpanCount };
+}
+export async function runEvalSuite(options) {
+    const { cwd, dataset, configs, service } = options;
+    const totals = new Map();
+    const perQuery = [];
+    for (const [index, evalQuery] of dataset.entries()) {
+        options.onQuery?.(index, evalQuery);
+        const results = await evaluateQuery(cwd, evalQuery, configs, service, options.hybridService);
+        perQuery.push({ id: evalQuery.id, class: evalQuery.class, results });
+        for (const result of results) {
+            const total = totals.get(result.label) ?? {
+                label: result.label,
+                recallAt1: 0,
+                recallAt5: 0,
+                recallAt10: 0,
+                recallAt50: 0,
+                mrr: 0,
+                n: 0,
+                degraded: 0,
+            };
+            total.recallAt1 += result.recallAt1;
+            total.recallAt5 += result.recallAt5;
+            total.recallAt10 += result.recallAt10;
+            total.recallAt50 += result.recallAt50;
+            total.mrr += result.mrr;
+            total.n++;
+            if (result.degraded)
+                total.degraded++;
+            totals.set(result.label, total);
+        }
+    }
+    const aggregates = configs
+        .map((config) => totals.get(config.label))
+        .filter((total) => total !== undefined)
+        .map((total) => ({
+        ...total,
+        recallAt1: total.recallAt1 / total.n,
+        recallAt5: total.recallAt5 / total.n,
+        recallAt10: total.recallAt10 / total.n,
+        recallAt50: total.recallAt50 / total.n,
+        mrr: total.mrr / total.n,
+    }));
+    return { aggregates, perQuery };
+}
+export function formatAggregateTable(aggregates) {
+    const pct = (x) => `${Math.round(x * 100)}%`.padStart(5);
+    const lines = [
+        "config           |  R@1  |  R@5  | R@10  | R@50  |  MRR  | notes",
+        "-----------------|-------|-------|-------|-------|-------|------",
+    ];
+    for (const a of aggregates) {
+        const notes = a.degraded === a.n ? "degraded to lexical" : a.degraded > 0 ? `${a.degraded}/${a.n} degraded` : "";
+        lines.push(`${a.label.padEnd(16)} | ${pct(a.recallAt1)} | ${pct(a.recallAt5)} | ${pct(a.recallAt10)} | ` +
+            `${pct(a.recallAt50)} | ${a.mrr.toFixed(3)} | ${notes}`);
+    }
+    return lines.join("\n");
+}
+//# sourceMappingURL=eval-harness.js.map

package/dist/core/search/eval-harness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-harness.js","sourceRoot":"","sources":["../../../src/core/search/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAyD,aAAa,EAAE,MAAM,WAAW,CAAC;AAoEjG,SAAS,GAAG,CAAC,QAAgB,EAAE,IAAc,EAAU;IACtD,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAAA,CACpF;AAED;0EAC0E;AAC1E,MAAM,UAAU,mBAAmB,CAAC,QAAgB,EAAU;IAC7D,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,uCAAuC,CAAC;QAC5D,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,qDAAqD,CAAC;KAC1E,CAAC;IACF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,MAAc,EAAQ,EAAE,CAAC;QACtC,IAAI,IAAiC,CAAC;QACtC,IAAI,CAAC;YACJ,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACR,OAAO;QACR,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE;gBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;IAAA,CACD,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,wEAAwE;QACxE,iCAAiC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACtC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAAA,CACvC;AAYD;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,GAAuB,EAAgB;IAClF,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAClE,IAAI,CAAC,GAAG,EAAE,CAAC;QACV,OAAO;YACN,GAAG,EAAE,QAAQ;YACb,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACzC,eAAe,EAAE,IAAI;YACrB,KAAK;YACL,OAAO,EAAE,GAAG,EAAE,CAAC,EAAC,CAAC;SACjB,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9C,6EAA6E;IAC7E,0EAA0E;IAC1E,0EAA0E;IAC1E,0EAA0E;IAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,4EAA0E;QAC1E,IAAI,CAAC;YACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;QACtC,CAAC;IACF,CAAC;IACD,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IACzD,OAAO;QACN,GAAG,EAAE,GAAG;QACR,GAAG;QACH,eAAe,EAAE,KAAK;QACtB,KAAK,EAAE,KAAK;QACZ,OAAO,EAAE,GAAG,EAAE,CAAC;YACd,IAAI,CAAC;gBACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/C,CAAC;QAAA,CACD;KACD,CAAC;AAAA,CACF;AAED,gEAAgE;AAChE,SAAS,kBAAkB,CAAC,UAA8B,EAAsB;IAC/E,IAAI,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAClC,IAAI,CAAC;QACJ,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED,MAAM,UAAU,iBAAiB,CAChC,QAAgB,EAChB,MAAoB,EACpB,SAAiB,EACjB,OAAqC,EACrC,eAAwB,EACxB,aAAgC,EACf;IACjB,MAAM,KAAK,GAAG,OAAO,EAAE,QAAQ,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,KAAK,EAAE,KAAK,IAAI,QAAQ,CAAC;IACvC,OAAO;QACN,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;QACvB,SAAS,EAAE,MAAM,CAAC,GAAG;QACrB,SAAS;QACT,qBAAqB,EAAE,MAAM,CAAC,eAAe;QAC7C,WAAW,EAAE,MAAM,CAAC,KAAK;QACzB,UAAU,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAChD,mBAAmB,EAAE,mBAAmB,CAAC,QAAQ,CAAC;QAClD,QAAQ,EAAE;YACT,sEAAsE;YACtE,oEAAoE;YACpE,wDAAwD;YACxD,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,KAAK;YAC1C,MAAM,EAAE,KAAK,IAAI,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YAC7D,UAAU,EAAE,KAAK,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YACnE,KAAK;YACL,UAAU,EAAE,eAAe;YAC3B,aAAa,EAAE,kBAAkB,CAAC,eAAe,CAAC;SAClD;QACD,YAAY,EAAE,aAAa;YAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,aAAa,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,aAAa,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE;YACnF,CAAC,CAAC,SAAS;QACZ,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE;KAClF,CAAC;AAAA,CACF;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA6B,EAA4B;IACzF,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACvD,aAAa,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;IACpC,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AAAA,CAC9D;AAaD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B,EAG5D;IACF,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAChD,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAE/C,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC7F,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACrE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI;gBACzC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC;gBACb,UAAU,EAAE,CAAC;gBACb,GAAG,EAAE,CAAC;gBACN,CAAC,EAAE,CAAC;gBACJ,QAAQ,EAAE,CAAC;aACX,CAAC;YACF,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC;YACxB,KAAK,CAAC,CAAC,EAAE,CAAC;YACV,IAAI,MAAM,CAAC,QAAQ;gBAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACjC,CAAC;IACF,CAAC;IAED,MAAM,UAAU,GAAG,OAAO;SACxB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;SACzC,MAAM,CAAC,CAAC,KAAK,EAA0B,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC;SAC9D,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAChB,GAAG,KAAK;QACR,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;KACxB,CAAC,CAAC,CAAC;IAEL,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAAA,CAChC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAAoC,EAAU;IAClF,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG;QACb,kEAAkE;QAClE,kEAAkE;KAClE,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;QACjH,KAAK,CAAC,IAAI,CACT,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK;YAC5F,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,EAAE,CACxD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAAA,CACxB","sourcesContent":["/**\n * Eval harness: corpus pinning, provenance capture, and run records.\n *\n * The scoring math lives in `eval.ts`; this module is everything around it\n * that makes a number *comparable to a later number*. Three problems it\n * exists to solve, all of which bit the first eval round\n * (docs/hybrid-retrieval-design.md, \"Eval results\"):\n *\n *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,\n *     so every commit moves the thing being measured. A baseline taken today\n *     and a rerun taken after a retrieval change differ by both the change\n *     and the intervening commits, and nothing in the output says so. Fix:\n *     run against a detached git worktree pinned to an explicit SHA, and put\n *     that SHA in the record.\n *  2. **Nothing was recorded.** Results were printed to a terminal and\n *     hand-copied into a markdown table with no repo SHA, no embedder\n *     identity, and no index state. Fix: emit a machine-readable run record.\n *  3. **A degraded run looks like a real one.** With no embsearch binary the\n *     semantic and hybrid rows silently degrade to lexical, producing a table\n *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the\n *     record, plus a per-row degraded count that the writer refuses to hide.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readdirSync, readFileSync, statSync } from \"node:fs\";\nimport { execFileSync } from \"child_process\";\nimport { rmSync } from \"fs\";\nimport { tmpdir } from \"os\";\nimport path from \"path\";\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { type EvalConfig, type EvalQuery, type EvalQueryResult, evaluateQuery } from \"./eval.js\";\n\n/** Metrics aggregated per config across the whole gold set. */\nexport interface EvalAggregate {\n\tlabel: string;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n\t/** Queries scored under this config. */\n\tn: number;\n\t/** How many of them ran degraded (requested retriever unavailable). */\n\tdegraded: number;\n}\n\n/** Everything needed to decide whether two run records may be compared. */\nexport interface EvalProvenance {\n\ttimestampMs: number;\n\t/** SHA of the corpus actually indexed and searched. */\n\tcorpusSha: string;\n\t/** Ref the caller asked for, before resolution (e.g. \"HEAD\"). */\n\tcorpusRef: string;\n\t/** True when the corpus came from the live working tree rather than a\n\t *  pinned worktree — results are then not reproducible. */\n\tcorpusFromWorkingTree: boolean;\n\t/** Uncommitted changes present at run time. Only meaningful (and only\n\t *  possible) when `corpusFromWorkingTree` is true. */\n\tcorpusDirty: boolean;\n\t/** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,\n\t *  but differs when pinning an old corpus with today's code. */\n\tharnessSha: string;\n\t/**\n\t * Content hash of `src/core/search` + the chunker. Every tuning constant\n\t * that shapes a result — the fusion cap, top-k depths, rerank weights,\n\t * chunk sizing — lives in those files, so a changed hash means the\n\t * numbers are not comparable, without this module having to maintain a\n\t * hand-copied (and inevitably stale) list of constants.\n\t */\n\tretrievalSourceHash: string;\n\t/** Embedding backend state. `available: false` means every semantic and\n\t *  hybrid row in this record degraded to lexical. */\n\tembedder: {\n\t\tavailable: boolean;\n\t\treason?: string;\n\t\t/** Indexed chunk count when the index reached `ready`. */\n\t\tchunkCount?: number;\n\t\tphase: string;\n\t\t/** Binary that served the embeddings, and its self-reported version.\n\t\t *  The embedding model is baked into the binary at build time, so this\n\t\t *  is the only thing that identifies which model produced a score. */\n\t\tbinaryPath?: string;\n\t\tbinaryVersion?: string;\n\t};\n\t/** Daemon-side BM25 hybrid store, when the run included one. Absent means\n\t *  the record has no `daemon-hybrid` rows. */\n\tdaemonHybrid?: { available: boolean; phase: string };\n\truntime: { node: string; platform: string; arch: string };\n}\n\nexport interface EvalRunRecord {\n\tprovenance: EvalProvenance;\n\tgoldSet: { queryCount: number; byClass: Record<string, number>; goldSpanCount: number };\n\tconfigs: readonly EvalConfig[];\n\taggregates: EvalAggregate[];\n\tperQuery: Array<{ id: string; class: string; results: EvalQueryResult[] }>;\n}\n\nfunction git(repoRoot: string, args: string[]): string {\n\treturn execFileSync(\"git\", [\"-C\", repoRoot, ...args], { encoding: \"utf-8\" }).trim();\n}\n\n/** Hash every retrieval-shaping source file, so a tuning change is visible as\n *  a changed provenance field rather than an unexplained metric shift. */\nexport function hashRetrievalSource(repoRoot: string): string {\n\tconst roots = [\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/search\"),\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/embsearch/chunker.ts\"),\n\t];\n\tconst files: string[] = [];\n\tconst walk = (target: string): void => {\n\t\tlet stat: ReturnType<typeof statSync>;\n\t\ttry {\n\t\t\tstat = statSync(target);\n\t\t} catch {\n\t\t\treturn;\n\t\t}\n\t\tif (stat.isDirectory()) {\n\t\t\tfor (const entry of readdirSync(target).sort()) walk(path.join(target, entry));\n\t\t} else if (target.endsWith(\".ts\")) {\n\t\t\tfiles.push(target);\n\t\t}\n\t};\n\tfor (const root of roots) walk(root);\n\n\tconst hash = createHash(\"sha256\");\n\tfor (const file of files) {\n\t\t// Eval-only modules are excluded: changing how we measure must not look\n\t\t// like changing what we measure.\n\t\tconst base = path.basename(file);\n\t\tif (base.startsWith(\"eval\")) continue;\n\t\thash.update(path.relative(repoRoot, file).replace(/\\\\/g, \"/\"));\n\t\thash.update(readFileSync(file));\n\t}\n\treturn hash.digest(\"hex\").slice(0, 16);\n}\n\nexport interface PinnedCorpus {\n\t/** Directory to index and search. */\n\tcwd: string;\n\tsha: string;\n\tfromWorkingTree: boolean;\n\tdirty: boolean;\n\t/** Removes the worktree, if one was created. */\n\tdispose: () => void;\n}\n\n/**\n * Materialize the corpus to evaluate.\n *\n * With a `ref`, checks out a detached worktree at that commit so the corpus is\n * byte-identical on every rerun. Without one, falls back to the live working\n * tree and reports `dirty` so the record shows the run was not reproducible.\n */\nexport function pinCorpus(repoRoot: string, ref: string | undefined): PinnedCorpus {\n\tconst dirty = git(repoRoot, [\"status\", \"--porcelain\"]).length > 0;\n\tif (!ref) {\n\t\treturn {\n\t\t\tcwd: repoRoot,\n\t\t\tsha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\t\tfromWorkingTree: true,\n\t\t\tdirty,\n\t\t\tdispose: () => {},\n\t\t};\n\t}\n\n\tconst sha = git(repoRoot, [\"rev-parse\", ref]);\n\t// Deterministic path, not mkdtemp: the embedding store is keyed by a hash of\n\t// the corpus directory, so a fresh temp path every run would re-embed all\n\t// ~17k chunks (minutes) instead of reusing the store built for this exact\n\t// SHA. The worktree is still removed afterwards; only the store persists.\n\tconst dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}`);\n\tif (existsSync(dir)) {\n\t\t// Left behind by an interrupted run — drop it so `worktree add` succeeds.\n\t\ttry {\n\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t} catch {\n\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\tgit(repoRoot, [\"worktree\", \"prune\"]);\n\t\t}\n\t}\n\tgit(repoRoot, [\"worktree\", \"add\", \"--detach\", dir, sha]);\n\treturn {\n\t\tcwd: dir,\n\t\tsha,\n\t\tfromWorkingTree: false,\n\t\tdirty: false,\n\t\tdispose: () => {\n\t\t\ttry {\n\t\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t\t} catch {\n\t\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\t}\n\t\t},\n\t};\n}\n\n/** `<binary> --version`, or undefined when it cannot be run. */\nfunction probeBinaryVersion(binaryPath: string | undefined): string | undefined {\n\tif (!binaryPath) return undefined;\n\ttry {\n\t\treturn execFileSync(binaryPath, [\"--version\"], { encoding: \"utf-8\" }).trim();\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\nexport function collectProvenance(\n\trepoRoot: string,\n\tcorpus: PinnedCorpus,\n\tcorpusRef: string,\n\tservice: EmbsearchService | undefined,\n\tembsearchBinary?: string,\n\thybridService?: EmbsearchService,\n): EvalProvenance {\n\tconst state = service?.getState();\n\tconst phase = state?.phase ?? \"absent\";\n\treturn {\n\t\ttimestampMs: Date.now(),\n\t\tcorpusSha: corpus.sha,\n\t\tcorpusRef,\n\t\tcorpusFromWorkingTree: corpus.fromWorkingTree,\n\t\tcorpusDirty: corpus.dirty,\n\t\tharnessSha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\tretrievalSourceHash: hashRetrievalSource(repoRoot),\n\t\tembedder: {\n\t\t\t// `ready` is the only phase the service reaches with a real embedder:\n\t\t\t// it rejects the mock backend at startup, so availability here also\n\t\t\t// certifies the numbers came from a genuine ONNX build.\n\t\t\tavailable: service?.isAvailable() ?? false,\n\t\t\treason: state && \"reason\" in state ? state.reason : undefined,\n\t\t\tchunkCount: state?.phase === \"ready\" ? state.chunkCount : undefined,\n\t\t\tphase,\n\t\t\tbinaryPath: embsearchBinary,\n\t\t\tbinaryVersion: probeBinaryVersion(embsearchBinary),\n\t\t},\n\t\tdaemonHybrid: hybridService\n\t\t\t? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }\n\t\t\t: undefined,\n\t\truntime: { node: process.version, platform: process.platform, arch: process.arch },\n\t};\n}\n\nexport function summarizeGoldSet(dataset: readonly EvalQuery[]): EvalRunRecord[\"goldSet\"] {\n\tconst byClass: Record<string, number> = {};\n\tlet goldSpanCount = 0;\n\tfor (const query of dataset) {\n\t\tbyClass[query.class] = (byClass[query.class] ?? 0) + 1;\n\t\tgoldSpanCount += query.gold.length;\n\t}\n\treturn { queryCount: dataset.length, byClass, goldSpanCount };\n}\n\nexport interface RunEvalSuiteOptions {\n\tcwd: string;\n\tdataset: readonly EvalQuery[];\n\tconfigs: readonly EvalConfig[];\n\tservice?: EmbsearchService;\n\t/** Second service backed by a daemon-side BM25 hybrid store, for the\n\t *  `daemon-hybrid` configs. Absent means those rows are omitted. */\n\thybridService?: EmbsearchService;\n\tonQuery?: (index: number, query: EvalQuery) => void;\n}\n\nexport async function runEvalSuite(options: RunEvalSuiteOptions): Promise<{\n\taggregates: EvalAggregate[];\n\tperQuery: EvalRunRecord[\"perQuery\"];\n}> {\n\tconst { cwd, dataset, configs, service } = options;\n\tconst totals = new Map<string, EvalAggregate>();\n\tconst perQuery: EvalRunRecord[\"perQuery\"] = [];\n\n\tfor (const [index, evalQuery] of dataset.entries()) {\n\t\toptions.onQuery?.(index, evalQuery);\n\t\tconst results = await evaluateQuery(cwd, evalQuery, configs, service, options.hybridService);\n\t\tperQuery.push({ id: evalQuery.id, class: evalQuery.class, results });\n\t\tfor (const result of results) {\n\t\t\tconst total = totals.get(result.label) ?? {\n\t\t\t\tlabel: result.label,\n\t\t\t\trecallAt1: 0,\n\t\t\t\trecallAt5: 0,\n\t\t\t\trecallAt10: 0,\n\t\t\t\trecallAt50: 0,\n\t\t\t\tmrr: 0,\n\t\t\t\tn: 0,\n\t\t\t\tdegraded: 0,\n\t\t\t};\n\t\t\ttotal.recallAt1 += result.recallAt1;\n\t\t\ttotal.recallAt5 += result.recallAt5;\n\t\t\ttotal.recallAt10 += result.recallAt10;\n\t\t\ttotal.recallAt50 += result.recallAt50;\n\t\t\ttotal.mrr += result.mrr;\n\t\t\ttotal.n++;\n\t\t\tif (result.degraded) total.degraded++;\n\t\t\ttotals.set(result.label, total);\n\t\t}\n\t}\n\n\tconst aggregates = configs\n\t\t.map((config) => totals.get(config.label))\n\t\t.filter((total): total is EvalAggregate => total !== undefined)\n\t\t.map((total) => ({\n\t\t\t...total,\n\t\t\trecallAt1: total.recallAt1 / total.n,\n\t\t\trecallAt5: total.recallAt5 / total.n,\n\t\t\trecallAt10: total.recallAt10 / total.n,\n\t\t\trecallAt50: total.recallAt50 / total.n,\n\t\t\tmrr: total.mrr / total.n,\n\t\t}));\n\n\treturn { aggregates, perQuery };\n}\n\nexport function formatAggregateTable(aggregates: readonly EvalAggregate[]): string {\n\tconst pct = (x: number) => `${Math.round(x * 100)}%`.padStart(5);\n\tconst lines = [\n\t\t\"config           |  R@1  |  R@5  | R@10  | R@50  |  MRR  | notes\",\n\t\t\"-----------------|-------|-------|-------|-------|-------|------\",\n\t];\n\tfor (const a of aggregates) {\n\t\tconst notes = a.degraded === a.n ? \"degraded to lexical\" : a.degraded > 0 ? `${a.degraded}/${a.n} degraded` : \"\";\n\t\tlines.push(\n\t\t\t`${a.label.padEnd(16)} | ${pct(a.recallAt1)} | ${pct(a.recallAt5)} | ${pct(a.recallAt10)} | ` +\n\t\t\t\t`${pct(a.recallAt50)} | ${a.mrr.toFixed(3)} | ${notes}`,\n\t\t);\n\t}\n\treturn lines.join(\"\\n\");\n}\n"]}

package/dist/core/search/eval.d.ts

@@ -2,14 +2,19 @@
  * Retrieval evaluation gate (docs/hybrid-retrieval-design.md, step 6 of the
  * shipping order).
  *
- * Measures Recall@K for lexical, semantic, and hybrid retrieval across an
- * RRF `k` sweep, against a gold set keyed by **path + optional line range
- * matched by span overlap** — never by chunkId, which is only stable per
- * index build. Recall@50 doubles as the reranker gate: a gold span that
- * never reaches the fused top-50 cannot be rescued by any reranker.
+ * Measures Recall@K and MRR for lexical, semantic, and hybrid retrieval across
+ * an RRF `k` sweep, against a gold set keyed by **path + line range matched by
+ * span overlap** — never by chunkId, which is only stable per index build.
+ * Recall@50 doubles as the reranker gate: a gold span that never reaches the
+ * fused top-50 cannot be rescued by any reranker.
  *
- * Driven by `scripts/search-eval.mjs`; the scoring logic lives here so it is
- * unit-testable without an embedding index.
+ * Rank-sensitive metrics are the point of Recall@1 and MRR: an agent reads the
+ * top result or two, so "the gold span is somewhere in the top 5" understates
+ * how much ordering matters. Recall@5/10/50 are kept for continuity with the
+ * numbers already published in the design note.
+ *
+ * Driven by `scripts/search-eval.ts` via `eval-harness.ts`; the scoring logic
+ * lives here so it is unit-testable without an embedding index.
  */
 import type { EmbsearchService } from "../embsearch/embsearch-service.js";
 import type { CandidateSpan, ResolvedSearchMode, SearchMode } from "./types.js";
@@ -19,6 +24,22 @@ export interface EvalGoldSpan {
     /** 1-based inclusive; omit both to accept any span in the file. */
     startLine?: number;
     endLine?: number;
+    /**
+     * Literal source text that must occur inside `[startLine, endLine]`. Not
+     * used for scoring — it is how the gold set survives the corpus moving
+     * underneath it: `scripts/search-eval-gold.ts` re-resolves the line range
+     * from this anchor, and the fixture test fails when an anchor no longer
+     * sits inside its recorded range. Omitted for `scope: "file"` entries.
+     */
+    anchor?: string;
+    /**
+     * `"span"` (default) scores by overlap with the recorded line range;
+     * `"file"` accepts any span in the file. File scope is deliberate for
+     * path-class queries, where the whole file *is* the answer — it is
+     * recorded explicitly so a file-level score is never mistaken for a
+     * span-level one.
+     */
+    scope?: "span" | "file";
 }
 export interface EvalQuery {
     id: string;
@@ -33,17 +54,54 @@ export interface EvalConfig {
     mode: SearchMode;
     rrfK?: number;
     rerank?: boolean;
+    /**
+     * Score this config against a daemon-side hybrid store (BM25 fused with
+     * vectors inside the Rust daemon) instead of the dense-only index. Skipped
+     * when the harness has no hybrid service, so records without one simply
+     * omit the row rather than silently scoring it as plain semantic.
+     */
+    daemonHybrid?: boolean;
+    /**
+     * Fetch the daemon's BM25 index as a separate leg and fuse it here with
+     * dense and grep. Like `daemonHybrid` it needs a hybrid store, so it is
+     * skipped when the harness has no hybrid service.
+     */
+    bm25Leg?: boolean;
 }
+/**
+ * The sweep from the design doc — single retrievers, hybrid across k, the
+ * routed auto mode — plus reranked (`+rr`) variants for the step 7 gate.
+ *
+ * Exported because `scripts/search-eval.ts` and `test/search-eval.test.ts`
+ * both consume it. It previously lost its `export` to a knip-driven
+ * dead-export sweep (02efaab): the only importer was a `.mjs` script reading
+ * from `dist/`, which static analysis cannot see, and the eval gate silently
+ * stopped running. The TypeScript importers are the fix — do not "clean up"
+ * this export without checking them.
+ */
+export declare const EVAL_CONFIGS: readonly EvalConfig[];
 export declare function spanMatchesGold(span: CandidateSpan, gold: EvalGoldSpan): boolean;
 /** Fraction of gold spans matched by at least one of the top-`k` candidates. */
 export declare function recallAtK(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[], k: number): number;
+/**
+ * Mean reciprocal rank, averaged over gold spans: for each gold span, `1/rank`
+ * of the first candidate matching it (0 when it never appears).
+ *
+ * Averaging per gold span rather than taking the single first hit keeps
+ * multi-span queries (the cross-file class) honest — finding one of two
+ * required sites should not score like finding both. For single-span queries
+ * this reduces to textbook MRR.
+ */
+export declare function mrr(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[]): number;
 export interface EvalQueryResult {
     label: string;
     resolvedMode: ResolvedSearchMode;
     degraded: boolean;
+    recallAt1: number;
     recallAt5: number;
     recallAt10: number;
     recallAt50: number;
+    mrr: number;
 }
-export declare function evaluateQuery(cwd: string, evalQuery: EvalQuery, configs?: readonly EvalConfig[], service?: EmbsearchService): Promise<EvalQueryResult[]>;
+export declare function evaluateQuery(cwd: string, evalQuery: EvalQuery, configs?: readonly EvalConfig[], service?: EmbsearchService, hybridService?: EmbsearchService): Promise<EvalQueryResult[]>;
 //# sourceMappingURL=eval.d.ts.map

package/dist/core/search/eval.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../../src/core/search/eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAEhF,MAAM,WAAW,YAAY;IAC5B,gCAAgC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,mEAAmE;IACnE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS;IACzB,EAAE,EAAE,MAAM,CAAC;IACX;6DACyD;IACzD,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,YAAY,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;CACjB;AAsBD,wBAAgB,eAAe,CAAC,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,YAAY,GAAG,OAAO,CAIhF;AAED,gFAAgF;AAChF,wBAAgB,SAAS,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,YAAY,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAQhH;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,kBAAkB,CAAC;IACjC,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACnB;AAED,wBAAsB,aAAa,CAClC,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,SAAS,EACpB,OAAO,GAAE,SAAS,UAAU,EAAiB,EAC7C,OAAO,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,eAAe,EAAE,CAAC,CAsB5B","sourcesContent":["/**\n * Retrieval evaluation gate (docs/hybrid-retrieval-design.md, step 6 of the\n * shipping order).\n *\n * Measures Recall@K for lexical, semantic, and hybrid retrieval across an\n * RRF `k` sweep, against a gold set keyed by **path + optional line range\n * matched by span overlap** — never by chunkId, which is only stable per\n * index build. Recall@50 doubles as the reranker gate: a gold span that\n * never reaches the fused top-50 cannot be rescued by any reranker.\n *\n * Driven by `scripts/search-eval.mjs`; the scoring logic lives here so it is\n * unit-testable without an embedding index.\n */\n\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { retrieveCandidates } from \"./hybrid-search.js\";\nimport type { CandidateSpan, ResolvedSearchMode, SearchMode } from \"./types.js\";\n\nexport interface EvalGoldSpan {\n\t/** Repo-relative POSIX path. */\n\tpath: string;\n\t/** 1-based inclusive; omit both to accept any span in the file. */\n\tstartLine?: number;\n\tendLine?: number;\n}\n\nexport interface EvalQuery {\n\tid: string;\n\t/** Query class from the design doc (exact-symbol, path, error-fragment,\n\t *  conceptual, cross-file, boundary). Reporting only. */\n\tclass: string;\n\tquery: string;\n\tgold: EvalGoldSpan[];\n}\n\nexport interface EvalConfig {\n\tlabel: string;\n\tmode: SearchMode;\n\trrfK?: number;\n\trerank?: boolean;\n}\n\n/** The sweep from the design doc — single retrievers, hybrid across k, the\n *  routed auto mode — plus reranked (`+rr`) variants for the step 7 gate. */\nconst EVAL_CONFIGS: readonly EvalConfig[] = [\n\t{ label: \"lexical\", mode: \"lexical\" },\n\t{ label: \"semantic\", mode: \"semantic\" },\n\t{ label: \"hybrid k=0\", mode: \"hybrid\", rrfK: 0 },\n\t{ label: \"hybrid k=2\", mode: \"hybrid\", rrfK: 2 },\n\t{ label: \"hybrid k=10\", mode: \"hybrid\", rrfK: 10 },\n\t{ label: \"hybrid k=60\", mode: \"hybrid\", rrfK: 60 },\n\t{ label: \"auto\", mode: \"auto\" },\n\t{ label: \"lexical +rr\", mode: \"lexical\", rerank: true },\n\t{ label: \"semantic +rr\", mode: \"semantic\", rerank: true },\n\t{ label: \"hybrid k=2 +rr\", mode: \"hybrid\", rrfK: 2, rerank: true },\n\t{ label: \"hybrid k=60 +rr\", mode: \"hybrid\", rrfK: 60, rerank: true },\n\t{ label: \"auto +rr\", mode: \"auto\", rerank: true },\n];\n\n/** Candidates fetched per eval query — deep enough for the reranker gate. */\nconst EVAL_FETCH_LIMIT = 50;\n\nexport function spanMatchesGold(span: CandidateSpan, gold: EvalGoldSpan): boolean {\n\tif (span.path !== gold.path) return false;\n\tif (gold.startLine === undefined || gold.endLine === undefined) return true;\n\treturn span.startLine <= gold.endLine && span.endLine >= gold.startLine;\n}\n\n/** Fraction of gold spans matched by at least one of the top-`k` candidates. */\nexport function recallAtK(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[], k: number): number {\n\tif (gold.length === 0) return 0;\n\tconst top = candidates.slice(0, k);\n\tlet matched = 0;\n\tfor (const g of gold) {\n\t\tif (top.some((c) => spanMatchesGold(c, g))) matched++;\n\t}\n\treturn matched / gold.length;\n}\n\nexport interface EvalQueryResult {\n\tlabel: string;\n\tresolvedMode: ResolvedSearchMode;\n\tdegraded: boolean;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n}\n\nexport async function evaluateQuery(\n\tcwd: string,\n\tevalQuery: EvalQuery,\n\tconfigs: readonly EvalConfig[] = EVAL_CONFIGS,\n\tservice?: EmbsearchService,\n): Promise<EvalQueryResult[]> {\n\tconst results: EvalQueryResult[] = [];\n\tfor (const config of configs) {\n\t\tconst retrieved = await retrieveCandidates({\n\t\t\tcwd,\n\t\t\tquery: evalQuery.query,\n\t\t\tmode: config.mode,\n\t\t\trrfK: config.rrfK,\n\t\t\trerank: config.rerank ?? false,\n\t\t\tlimit: EVAL_FETCH_LIMIT,\n\t\t\tservice,\n\t\t});\n\t\tresults.push({\n\t\t\tlabel: config.label,\n\t\t\tresolvedMode: retrieved.resolvedMode,\n\t\t\tdegraded: retrieved.degradedReason !== undefined,\n\t\t\trecallAt5: recallAtK(retrieved.candidates, evalQuery.gold, 5),\n\t\t\trecallAt10: recallAtK(retrieved.candidates, evalQuery.gold, 10),\n\t\t\trecallAt50: recallAtK(retrieved.candidates, evalQuery.gold, 50),\n\t\t});\n\t}\n\treturn results;\n}\n"]}
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../../src/core/search/eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAEhF,MAAM,WAAW,YAAY;IAC5B,gCAAgC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,mEAAmE;IACnE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,SAAS;IACzB,EAAE,EAAE,MAAM,CAAC;IACX;6DACyD;IACzD,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,YAAY,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,YAAY,EAAE,SAAS,UAAU,EAwB7C,CAAC;AAKF,wBAAgB,eAAe,CAAC,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,YAAY,GAAG,OAAO,CAKhF;AAED,gFAAgF;AAChF,wBAAgB,SAAS,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,YAAY,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAQhH;AAED;;;;;;;;GAQG;AACH,wBAAgB,GAAG,CAAC,UAAU,EAAE,SAAS,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,YAAY,EAAE,GAAG,MAAM,CAQ/F;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,kBAAkB,CAAC;IACjC,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;CACZ;AAED,wBAAsB,aAAa,CAClC,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,SAAS,EACpB,OAAO,GAAE,SAAS,UAAU,EAAiB,EAC7C,OAAO,CAAC,EAAE,gBAAgB,EAC1B,aAAa,CAAC,EAAE,gBAAgB,GAC9B,OAAO,CAAC,eAAe,EAAE,CAAC,CA6B5B","sourcesContent":["/**\n * Retrieval evaluation gate (docs/hybrid-retrieval-design.md, step 6 of the\n * shipping order).\n *\n * Measures Recall@K and MRR for lexical, semantic, and hybrid retrieval across\n * an RRF `k` sweep, against a gold set keyed by **path + line range matched by\n * span overlap** — never by chunkId, which is only stable per index build.\n * Recall@50 doubles as the reranker gate: a gold span that never reaches the\n * fused top-50 cannot be rescued by any reranker.\n *\n * Rank-sensitive metrics are the point of Recall@1 and MRR: an agent reads the\n * top result or two, so \"the gold span is somewhere in the top 5\" understates\n * how much ordering matters. Recall@5/10/50 are kept for continuity with the\n * numbers already published in the design note.\n *\n * Driven by `scripts/search-eval.ts` via `eval-harness.ts`; the scoring logic\n * lives here so it is unit-testable without an embedding index.\n */\n\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { retrieveCandidates } from \"./hybrid-search.js\";\nimport type { CandidateSpan, ResolvedSearchMode, SearchMode } from \"./types.js\";\n\nexport interface EvalGoldSpan {\n\t/** Repo-relative POSIX path. */\n\tpath: string;\n\t/** 1-based inclusive; omit both to accept any span in the file. */\n\tstartLine?: number;\n\tendLine?: number;\n\t/**\n\t * Literal source text that must occur inside `[startLine, endLine]`. Not\n\t * used for scoring — it is how the gold set survives the corpus moving\n\t * underneath it: `scripts/search-eval-gold.ts` re-resolves the line range\n\t * from this anchor, and the fixture test fails when an anchor no longer\n\t * sits inside its recorded range. Omitted for `scope: \"file\"` entries.\n\t */\n\tanchor?: string;\n\t/**\n\t * `\"span\"` (default) scores by overlap with the recorded line range;\n\t * `\"file\"` accepts any span in the file. File scope is deliberate for\n\t * path-class queries, where the whole file *is* the answer — it is\n\t * recorded explicitly so a file-level score is never mistaken for a\n\t * span-level one.\n\t */\n\tscope?: \"span\" | \"file\";\n}\n\nexport interface EvalQuery {\n\tid: string;\n\t/** Query class from the design doc (exact-symbol, path, error-fragment,\n\t *  conceptual, cross-file, boundary). Reporting only. */\n\tclass: string;\n\tquery: string;\n\tgold: EvalGoldSpan[];\n}\n\nexport interface EvalConfig {\n\tlabel: string;\n\tmode: SearchMode;\n\trrfK?: number;\n\trerank?: boolean;\n\t/**\n\t * Score this config against a daemon-side hybrid store (BM25 fused with\n\t * vectors inside the Rust daemon) instead of the dense-only index. Skipped\n\t * when the harness has no hybrid service, so records without one simply\n\t * omit the row rather than silently scoring it as plain semantic.\n\t */\n\tdaemonHybrid?: boolean;\n\t/**\n\t * Fetch the daemon's BM25 index as a separate leg and fuse it here with\n\t * dense and grep. Like `daemonHybrid` it needs a hybrid store, so it is\n\t * skipped when the harness has no hybrid service.\n\t */\n\tbm25Leg?: boolean;\n}\n\n/**\n * The sweep from the design doc — single retrievers, hybrid across k, the\n * routed auto mode — plus reranked (`+rr`) variants for the step 7 gate.\n *\n * Exported because `scripts/search-eval.ts` and `test/search-eval.test.ts`\n * both consume it. It previously lost its `export` to a knip-driven\n * dead-export sweep (02efaab): the only importer was a `.mjs` script reading\n * from `dist/`, which static analysis cannot see, and the eval gate silently\n * stopped running. The TypeScript importers are the fix — do not \"clean up\"\n * this export without checking them.\n */\nexport const EVAL_CONFIGS: readonly EvalConfig[] = [\n\t{ label: \"lexical\", mode: \"lexical\" },\n\t{ label: \"semantic\", mode: \"semantic\" },\n\t{ label: \"hybrid k=0\", mode: \"hybrid\", rrfK: 0 },\n\t{ label: \"hybrid k=2\", mode: \"hybrid\", rrfK: 2 },\n\t{ label: \"hybrid k=10\", mode: \"hybrid\", rrfK: 10 },\n\t{ label: \"hybrid k=60\", mode: \"hybrid\", rrfK: 60 },\n\t{ label: \"auto\", mode: \"auto\" },\n\t{ label: \"lexical +rr\", mode: \"lexical\", rerank: true },\n\t{ label: \"semantic +rr\", mode: \"semantic\", rerank: true },\n\t{ label: \"hybrid k=2 +rr\", mode: \"hybrid\", rrfK: 2, rerank: true },\n\t{ label: \"hybrid k=60 +rr\", mode: \"hybrid\", rrfK: 60, rerank: true },\n\t{ label: \"auto +rr\", mode: \"auto\", rerank: true },\n\t// Is BM25 a better lexical leg than ripgrep? These two run the daemon's own\n\t// vector+BM25 fusion and no ripgrep at all, so comparing them against\n\t// \"semantic\" isolates what BM25 adds, and against \"hybrid k=2\" compares the\n\t// two lexical legs at the system level.\n\t{ label: \"daemon-hybrid\", mode: \"semantic\", daemonHybrid: true },\n\t{ label: \"daemon-hybrid +rr\", mode: \"semantic\", rerank: true, daemonHybrid: true },\n\t// Three-way fusion: dense + BM25 + grep, all fused here so `k` is ours and\n\t// the per-leg ranks survive into the trace. `bm25+dense` isolates what the\n\t// grep leg still contributes once BM25 is present.\n\t{ label: \"bm25+dense +rr\", mode: \"semantic\", rerank: true, bm25Leg: true },\n\t{ label: \"3-way +rr\", mode: \"hybrid\", rerank: true, bm25Leg: true },\n];\n\n/** Candidates fetched per eval query — deep enough for the reranker gate. */\nconst EVAL_FETCH_LIMIT = 50;\n\nexport function spanMatchesGold(span: CandidateSpan, gold: EvalGoldSpan): boolean {\n\tif (span.path !== gold.path) return false;\n\tif (gold.scope === \"file\") return true;\n\tif (gold.startLine === undefined || gold.endLine === undefined) return true;\n\treturn span.startLine <= gold.endLine && span.endLine >= gold.startLine;\n}\n\n/** Fraction of gold spans matched by at least one of the top-`k` candidates. */\nexport function recallAtK(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[], k: number): number {\n\tif (gold.length === 0) return 0;\n\tconst top = candidates.slice(0, k);\n\tlet matched = 0;\n\tfor (const g of gold) {\n\t\tif (top.some((c) => spanMatchesGold(c, g))) matched++;\n\t}\n\treturn matched / gold.length;\n}\n\n/**\n * Mean reciprocal rank, averaged over gold spans: for each gold span, `1/rank`\n * of the first candidate matching it (0 when it never appears).\n *\n * Averaging per gold span rather than taking the single first hit keeps\n * multi-span queries (the cross-file class) honest — finding one of two\n * required sites should not score like finding both. For single-span queries\n * this reduces to textbook MRR.\n */\nexport function mrr(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[]): number {\n\tif (gold.length === 0) return 0;\n\tlet total = 0;\n\tfor (const g of gold) {\n\t\tconst index = candidates.findIndex((c) => spanMatchesGold(c, g));\n\t\tif (index >= 0) total += 1 / (index + 1);\n\t}\n\treturn total / gold.length;\n}\n\nexport interface EvalQueryResult {\n\tlabel: string;\n\tresolvedMode: ResolvedSearchMode;\n\tdegraded: boolean;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n}\n\nexport async function evaluateQuery(\n\tcwd: string,\n\tevalQuery: EvalQuery,\n\tconfigs: readonly EvalConfig[] = EVAL_CONFIGS,\n\tservice?: EmbsearchService,\n\thybridService?: EmbsearchService,\n): Promise<EvalQueryResult[]> {\n\tconst results: EvalQueryResult[] = [];\n\tfor (const config of configs) {\n\t\t// A daemon-hybrid config against the plain store would error in the\n\t\t// daemon (`query_hybrid requires a hybrid store`); omit the row instead.\n\t\tif ((config.daemonHybrid || config.bm25Leg) && !hybridService) continue;\n\t\tconst retrieved = await retrieveCandidates({\n\t\t\tcwd,\n\t\t\tquery: evalQuery.query,\n\t\t\tmode: config.mode,\n\t\t\trrfK: config.rrfK,\n\t\t\trerank: config.rerank ?? false,\n\t\t\tlimit: EVAL_FETCH_LIMIT,\n\t\t\tservice: config.daemonHybrid || config.bm25Leg ? hybridService : service,\n\t\t\tdaemonHybrid: config.daemonHybrid,\n\t\t\tbm25Leg: config.bm25Leg,\n\t\t});\n\t\tresults.push({\n\t\t\tlabel: config.label,\n\t\t\tresolvedMode: retrieved.resolvedMode,\n\t\t\tdegraded: retrieved.degradedReason !== undefined,\n\t\t\trecallAt1: recallAtK(retrieved.candidates, evalQuery.gold, 1),\n\t\t\trecallAt5: recallAtK(retrieved.candidates, evalQuery.gold, 5),\n\t\t\trecallAt10: recallAtK(retrieved.candidates, evalQuery.gold, 10),\n\t\t\trecallAt50: recallAtK(retrieved.candidates, evalQuery.gold, 50),\n\t\t\tmrr: mrr(retrieved.candidates, evalQuery.gold),\n\t\t});\n\t}\n\treturn results;\n}\n"]}

package/dist/core/search/eval.js

@@ -2,19 +2,33 @@
  * Retrieval evaluation gate (docs/hybrid-retrieval-design.md, step 6 of the
  * shipping order).
  *
- * Measures Recall@K for lexical, semantic, and hybrid retrieval across an
- * RRF `k` sweep, against a gold set keyed by **path + optional line range
- * matched by span overlap** — never by chunkId, which is only stable per
- * index build. Recall@50 doubles as the reranker gate: a gold span that
- * never reaches the fused top-50 cannot be rescued by any reranker.
+ * Measures Recall@K and MRR for lexical, semantic, and hybrid retrieval across
+ * an RRF `k` sweep, against a gold set keyed by **path + line range matched by
+ * span overlap** — never by chunkId, which is only stable per index build.
+ * Recall@50 doubles as the reranker gate: a gold span that never reaches the
+ * fused top-50 cannot be rescued by any reranker.
  *
- * Driven by `scripts/search-eval.mjs`; the scoring logic lives here so it is
- * unit-testable without an embedding index.
+ * Rank-sensitive metrics are the point of Recall@1 and MRR: an agent reads the
+ * top result or two, so "the gold span is somewhere in the top 5" understates
+ * how much ordering matters. Recall@5/10/50 are kept for continuity with the
+ * numbers already published in the design note.
+ *
+ * Driven by `scripts/search-eval.ts` via `eval-harness.ts`; the scoring logic
+ * lives here so it is unit-testable without an embedding index.
  */
 import { retrieveCandidates } from "./hybrid-search.js";
-/** The sweep from the design doc — single retrievers, hybrid across k, the
- *  routed auto mode — plus reranked (`+rr`) variants for the step 7 gate. */
-const EVAL_CONFIGS = [
+/**
+ * The sweep from the design doc — single retrievers, hybrid across k, the
+ * routed auto mode — plus reranked (`+rr`) variants for the step 7 gate.
+ *
+ * Exported because `scripts/search-eval.ts` and `test/search-eval.test.ts`
+ * both consume it. It previously lost its `export` to a knip-driven
+ * dead-export sweep (02efaab): the only importer was a `.mjs` script reading
+ * from `dist/`, which static analysis cannot see, and the eval gate silently
+ * stopped running. The TypeScript importers are the fix — do not "clean up"
+ * this export without checking them.
+ */
+export const EVAL_CONFIGS = [
     { label: "lexical", mode: "lexical" },
     { label: "semantic", mode: "semantic" },
     { label: "hybrid k=0", mode: "hybrid", rrfK: 0 },
@@ -27,12 +41,25 @@ const EVAL_CONFIGS = [
     { label: "hybrid k=2 +rr", mode: "hybrid", rrfK: 2, rerank: true },
     { label: "hybrid k=60 +rr", mode: "hybrid", rrfK: 60, rerank: true },
     { label: "auto +rr", mode: "auto", rerank: true },
+    // Is BM25 a better lexical leg than ripgrep? These two run the daemon's own
+    // vector+BM25 fusion and no ripgrep at all, so comparing them against
+    // "semantic" isolates what BM25 adds, and against "hybrid k=2" compares the
+    // two lexical legs at the system level.
+    { label: "daemon-hybrid", mode: "semantic", daemonHybrid: true },
+    { label: "daemon-hybrid +rr", mode: "semantic", rerank: true, daemonHybrid: true },
+    // Three-way fusion: dense + BM25 + grep, all fused here so `k` is ours and
+    // the per-leg ranks survive into the trace. `bm25+dense` isolates what the
+    // grep leg still contributes once BM25 is present.
+    { label: "bm25+dense +rr", mode: "semantic", rerank: true, bm25Leg: true },
+    { label: "3-way +rr", mode: "hybrid", rerank: true, bm25Leg: true },
 ];
 /** Candidates fetched per eval query — deep enough for the reranker gate. */
 const EVAL_FETCH_LIMIT = 50;
 export function spanMatchesGold(span, gold) {
     if (span.path !== gold.path)
         return false;
+    if (gold.scope === "file")
+        return true;
     if (gold.startLine === undefined || gold.endLine === undefined)
         return true;
     return span.startLine <= gold.endLine && span.endLine >= gold.startLine;
@@ -49,9 +76,33 @@ export function recallAtK(candidates, gold, k) {
     }
     return matched / gold.length;
 }
-export async function evaluateQuery(cwd, evalQuery, configs = EVAL_CONFIGS, service) {
+/**
+ * Mean reciprocal rank, averaged over gold spans: for each gold span, `1/rank`
+ * of the first candidate matching it (0 when it never appears).
+ *
+ * Averaging per gold span rather than taking the single first hit keeps
+ * multi-span queries (the cross-file class) honest — finding one of two
+ * required sites should not score like finding both. For single-span queries
+ * this reduces to textbook MRR.
+ */
+export function mrr(candidates, gold) {
+    if (gold.length === 0)
+        return 0;
+    let total = 0;
+    for (const g of gold) {
+        const index = candidates.findIndex((c) => spanMatchesGold(c, g));
+        if (index >= 0)
+            total += 1 / (index + 1);
+    }
+    return total / gold.length;
+}
+export async function evaluateQuery(cwd, evalQuery, configs = EVAL_CONFIGS, service, hybridService) {
     const results = [];
     for (const config of configs) {
+        // A daemon-hybrid config against the plain store would error in the
+        // daemon (`query_hybrid requires a hybrid store`); omit the row instead.
+        if ((config.daemonHybrid || config.bm25Leg) && !hybridService)
+            continue;
         const retrieved = await retrieveCandidates({
             cwd,
             query: evalQuery.query,
@@ -59,15 +110,19 @@ export async function evaluateQuery(cwd, evalQuery, configs = EVAL_CONFIGS, serv
             rrfK: config.rrfK,
             rerank: config.rerank ?? false,
             limit: EVAL_FETCH_LIMIT,
-            service,
+            service: config.daemonHybrid || config.bm25Leg ? hybridService : service,
+            daemonHybrid: config.daemonHybrid,
+            bm25Leg: config.bm25Leg,
         });
         results.push({
             label: config.label,
             resolvedMode: retrieved.resolvedMode,
             degraded: retrieved.degradedReason !== undefined,
+            recallAt1: recallAtK(retrieved.candidates, evalQuery.gold, 1),
             recallAt5: recallAtK(retrieved.candidates, evalQuery.gold, 5),
             recallAt10: recallAtK(retrieved.candidates, evalQuery.gold, 10),
             recallAt50: recallAtK(retrieved.candidates, evalQuery.gold, 50),
+            mrr: mrr(retrieved.candidates, evalQuery.gold),
         });
     }
     return results;

package/dist/core/search/eval.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../../src/core/search/eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AA2BxD;+EAC6E;AAC7E,MAAM,YAAY,GAA0B;IAC3C,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE;IACrC,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE;IACvC,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE;IAChD,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE;IAChD,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE;IAClD,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE;IAClD,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;IAC/B,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE;IACvD,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE;IACzD,EAAE,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;IAClE,EAAE,KAAK,EAAE,iBAAiB,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;IACpE,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE;CACjD,CAAC;AAEF,+EAA6E;AAC7E,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,MAAM,UAAU,eAAe,CAAC,IAAmB,EAAE,IAAkB,EAAW;IACjF,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5E,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC;AAAA,CACxE;AAED,gFAAgF;AAChF,MAAM,UAAU,SAAS,CAAC,UAAoC,EAAE,IAA6B,EAAE,CAAS,EAAU;IACjH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACtB,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;IACvD,CAAC;IACD,OAAO,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;AAAA,CAC7B;AAWD,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,GAAW,EACX,SAAoB,EACpB,OAAO,GAA0B,YAAY,EAC7C,OAA0B,EACG;IAC7B,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC;YAC1C,GAAG;YACH,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,KAAK;YAC9B,KAAK,EAAE,gBAAgB;YACvB,OAAO;SACP,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC;YACZ,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,YAAY,EAAE,SAAS,CAAC,YAAY;YACpC,QAAQ,EAAE,SAAS,CAAC,cAAc,KAAK,SAAS;YAChD,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7D,UAAU,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/D,UAAU,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;SAC/D,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AAAA,CACf","sourcesContent":["/**\n * Retrieval evaluation gate (docs/hybrid-retrieval-design.md, step 6 of the\n * shipping order).\n *\n * Measures Recall@K for lexical, semantic, and hybrid retrieval across an\n * RRF `k` sweep, against a gold set keyed by **path + optional line range\n * matched by span overlap** — never by chunkId, which is only stable per\n * index build. Recall@50 doubles as the reranker gate: a gold span that\n * never reaches the fused top-50 cannot be rescued by any reranker.\n *\n * Driven by `scripts/search-eval.mjs`; the scoring logic lives here so it is\n * unit-testable without an embedding index.\n */\n\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { retrieveCandidates } from \"./hybrid-search.js\";\nimport type { CandidateSpan, ResolvedSearchMode, SearchMode } from \"./types.js\";\n\nexport interface EvalGoldSpan {\n\t/** Repo-relative POSIX path. */\n\tpath: string;\n\t/** 1-based inclusive; omit both to accept any span in the file. */\n\tstartLine?: number;\n\tendLine?: number;\n}\n\nexport interface EvalQuery {\n\tid: string;\n\t/** Query class from the design doc (exact-symbol, path, error-fragment,\n\t *  conceptual, cross-file, boundary). Reporting only. */\n\tclass: string;\n\tquery: string;\n\tgold: EvalGoldSpan[];\n}\n\nexport interface EvalConfig {\n\tlabel: string;\n\tmode: SearchMode;\n\trrfK?: number;\n\trerank?: boolean;\n}\n\n/** The sweep from the design doc — single retrievers, hybrid across k, the\n *  routed auto mode — plus reranked (`+rr`) variants for the step 7 gate. */\nconst EVAL_CONFIGS: readonly EvalConfig[] = [\n\t{ label: \"lexical\", mode: \"lexical\" },\n\t{ label: \"semantic\", mode: \"semantic\" },\n\t{ label: \"hybrid k=0\", mode: \"hybrid\", rrfK: 0 },\n\t{ label: \"hybrid k=2\", mode: \"hybrid\", rrfK: 2 },\n\t{ label: \"hybrid k=10\", mode: \"hybrid\", rrfK: 10 },\n\t{ label: \"hybrid k=60\", mode: \"hybrid\", rrfK: 60 },\n\t{ label: \"auto\", mode: \"auto\" },\n\t{ label: \"lexical +rr\", mode: \"lexical\", rerank: true },\n\t{ label: \"semantic +rr\", mode: \"semantic\", rerank: true },\n\t{ label: \"hybrid k=2 +rr\", mode: \"hybrid\", rrfK: 2, rerank: true },\n\t{ label: \"hybrid k=60 +rr\", mode: \"hybrid\", rrfK: 60, rerank: true },\n\t{ label: \"auto +rr\", mode: \"auto\", rerank: true },\n];\n\n/** Candidates fetched per eval query — deep enough for the reranker gate. */\nconst EVAL_FETCH_LIMIT = 50;\n\nexport function spanMatchesGold(span: CandidateSpan, gold: EvalGoldSpan): boolean {\n\tif (span.path !== gold.path) return false;\n\tif (gold.startLine === undefined || gold.endLine === undefined) return true;\n\treturn span.startLine <= gold.endLine && span.endLine >= gold.startLine;\n}\n\n/** Fraction of gold spans matched by at least one of the top-`k` candidates. */\nexport function recallAtK(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[], k: number): number {\n\tif (gold.length === 0) return 0;\n\tconst top = candidates.slice(0, k);\n\tlet matched = 0;\n\tfor (const g of gold) {\n\t\tif (top.some((c) => spanMatchesGold(c, g))) matched++;\n\t}\n\treturn matched / gold.length;\n}\n\nexport interface EvalQueryResult {\n\tlabel: string;\n\tresolvedMode: ResolvedSearchMode;\n\tdegraded: boolean;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n}\n\nexport async function evaluateQuery(\n\tcwd: string,\n\tevalQuery: EvalQuery,\n\tconfigs: readonly EvalConfig[] = EVAL_CONFIGS,\n\tservice?: EmbsearchService,\n): Promise<EvalQueryResult[]> {\n\tconst results: EvalQueryResult[] = [];\n\tfor (const config of configs) {\n\t\tconst retrieved = await retrieveCandidates({\n\t\t\tcwd,\n\t\t\tquery: evalQuery.query,\n\t\t\tmode: config.mode,\n\t\t\trrfK: config.rrfK,\n\t\t\trerank: config.rerank ?? false,\n\t\t\tlimit: EVAL_FETCH_LIMIT,\n\t\t\tservice,\n\t\t});\n\t\tresults.push({\n\t\t\tlabel: config.label,\n\t\t\tresolvedMode: retrieved.resolvedMode,\n\t\t\tdegraded: retrieved.degradedReason !== undefined,\n\t\t\trecallAt5: recallAtK(retrieved.candidates, evalQuery.gold, 5),\n\t\t\trecallAt10: recallAtK(retrieved.candidates, evalQuery.gold, 10),\n\t\t\trecallAt50: recallAtK(retrieved.candidates, evalQuery.gold, 50),\n\t\t});\n\t}\n\treturn results;\n}\n"]}
+{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../../src/core/search/eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAwDxD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,YAAY,GAA0B;IAClD,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE;IACrC,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE;IACvC,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE;IAChD,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE;IAChD,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE;IAClD,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE;IAClD,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;IAC/B,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE;IACvD,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE;IACzD,EAAE,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;IAClE,EAAE,KAAK,EAAE,iBAAiB,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;IACpE,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE;IACjD,4EAA4E;IAC5E,sEAAsE;IACtE,4EAA4E;IAC5E,wCAAwC;IACxC,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,IAAI,EAAE;IAChE,EAAE,KAAK,EAAE,mBAAmB,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE;IAClF,2EAA2E;IAC3E,2EAA2E;IAC3E,mDAAmD;IACnD,EAAE,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE;IAC1E,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE;CACnE,CAAC;AAEF,+EAA6E;AAC7E,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,MAAM,UAAU,eAAe,CAAC,IAAmB,EAAE,IAAkB,EAAW;IACjF,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACvC,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5E,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC;AAAA,CACxE;AAED,gFAAgF;AAChF,MAAM,UAAU,SAAS,CAAC,UAAoC,EAAE,IAA6B,EAAE,CAAS,EAAU;IACjH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACtB,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;IACvD,CAAC;IACD,OAAO,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;AAAA,CAC7B;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,GAAG,CAAC,UAAoC,EAAE,IAA6B,EAAU;IAChG,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjE,IAAI,KAAK,IAAI,CAAC;YAAE,KAAK,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;AAAA,CAC3B;AAaD,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,GAAW,EACX,SAAoB,EACpB,OAAO,GAA0B,YAAY,EAC7C,OAA0B,EAC1B,aAAgC,EACH;IAC7B,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,oEAAoE;QACpE,yEAAyE;QACzE,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa;YAAE,SAAS;QACxE,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC;YAC1C,GAAG;YACH,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,KAAK;YAC9B,KAAK,EAAE,gBAAgB;YACvB,OAAO,EAAE,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO;YACxE,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,OAAO,EAAE,MAAM,CAAC,OAAO;SACvB,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC;YACZ,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,YAAY,EAAE,SAAS,CAAC,YAAY;YACpC,QAAQ,EAAE,SAAS,CAAC,cAAc,KAAK,SAAS;YAChD,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7D,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7D,UAAU,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/D,UAAU,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/D,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,CAAC;SAC9C,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AAAA,CACf","sourcesContent":["/**\n * Retrieval evaluation gate (docs/hybrid-retrieval-design.md, step 6 of the\n * shipping order).\n *\n * Measures Recall@K and MRR for lexical, semantic, and hybrid retrieval across\n * an RRF `k` sweep, against a gold set keyed by **path + line range matched by\n * span overlap** — never by chunkId, which is only stable per index build.\n * Recall@50 doubles as the reranker gate: a gold span that never reaches the\n * fused top-50 cannot be rescued by any reranker.\n *\n * Rank-sensitive metrics are the point of Recall@1 and MRR: an agent reads the\n * top result or two, so \"the gold span is somewhere in the top 5\" understates\n * how much ordering matters. Recall@5/10/50 are kept for continuity with the\n * numbers already published in the design note.\n *\n * Driven by `scripts/search-eval.ts` via `eval-harness.ts`; the scoring logic\n * lives here so it is unit-testable without an embedding index.\n */\n\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { retrieveCandidates } from \"./hybrid-search.js\";\nimport type { CandidateSpan, ResolvedSearchMode, SearchMode } from \"./types.js\";\n\nexport interface EvalGoldSpan {\n\t/** Repo-relative POSIX path. */\n\tpath: string;\n\t/** 1-based inclusive; omit both to accept any span in the file. */\n\tstartLine?: number;\n\tendLine?: number;\n\t/**\n\t * Literal source text that must occur inside `[startLine, endLine]`. Not\n\t * used for scoring — it is how the gold set survives the corpus moving\n\t * underneath it: `scripts/search-eval-gold.ts` re-resolves the line range\n\t * from this anchor, and the fixture test fails when an anchor no longer\n\t * sits inside its recorded range. Omitted for `scope: \"file\"` entries.\n\t */\n\tanchor?: string;\n\t/**\n\t * `\"span\"` (default) scores by overlap with the recorded line range;\n\t * `\"file\"` accepts any span in the file. File scope is deliberate for\n\t * path-class queries, where the whole file *is* the answer — it is\n\t * recorded explicitly so a file-level score is never mistaken for a\n\t * span-level one.\n\t */\n\tscope?: \"span\" | \"file\";\n}\n\nexport interface EvalQuery {\n\tid: string;\n\t/** Query class from the design doc (exact-symbol, path, error-fragment,\n\t *  conceptual, cross-file, boundary). Reporting only. */\n\tclass: string;\n\tquery: string;\n\tgold: EvalGoldSpan[];\n}\n\nexport interface EvalConfig {\n\tlabel: string;\n\tmode: SearchMode;\n\trrfK?: number;\n\trerank?: boolean;\n\t/**\n\t * Score this config against a daemon-side hybrid store (BM25 fused with\n\t * vectors inside the Rust daemon) instead of the dense-only index. Skipped\n\t * when the harness has no hybrid service, so records without one simply\n\t * omit the row rather than silently scoring it as plain semantic.\n\t */\n\tdaemonHybrid?: boolean;\n\t/**\n\t * Fetch the daemon's BM25 index as a separate leg and fuse it here with\n\t * dense and grep. Like `daemonHybrid` it needs a hybrid store, so it is\n\t * skipped when the harness has no hybrid service.\n\t */\n\tbm25Leg?: boolean;\n}\n\n/**\n * The sweep from the design doc — single retrievers, hybrid across k, the\n * routed auto mode — plus reranked (`+rr`) variants for the step 7 gate.\n *\n * Exported because `scripts/search-eval.ts` and `test/search-eval.test.ts`\n * both consume it. It previously lost its `export` to a knip-driven\n * dead-export sweep (02efaab): the only importer was a `.mjs` script reading\n * from `dist/`, which static analysis cannot see, and the eval gate silently\n * stopped running. The TypeScript importers are the fix — do not \"clean up\"\n * this export without checking them.\n */\nexport const EVAL_CONFIGS: readonly EvalConfig[] = [\n\t{ label: \"lexical\", mode: \"lexical\" },\n\t{ label: \"semantic\", mode: \"semantic\" },\n\t{ label: \"hybrid k=0\", mode: \"hybrid\", rrfK: 0 },\n\t{ label: \"hybrid k=2\", mode: \"hybrid\", rrfK: 2 },\n\t{ label: \"hybrid k=10\", mode: \"hybrid\", rrfK: 10 },\n\t{ label: \"hybrid k=60\", mode: \"hybrid\", rrfK: 60 },\n\t{ label: \"auto\", mode: \"auto\" },\n\t{ label: \"lexical +rr\", mode: \"lexical\", rerank: true },\n\t{ label: \"semantic +rr\", mode: \"semantic\", rerank: true },\n\t{ label: \"hybrid k=2 +rr\", mode: \"hybrid\", rrfK: 2, rerank: true },\n\t{ label: \"hybrid k=60 +rr\", mode: \"hybrid\", rrfK: 60, rerank: true },\n\t{ label: \"auto +rr\", mode: \"auto\", rerank: true },\n\t// Is BM25 a better lexical leg than ripgrep? These two run the daemon's own\n\t// vector+BM25 fusion and no ripgrep at all, so comparing them against\n\t// \"semantic\" isolates what BM25 adds, and against \"hybrid k=2\" compares the\n\t// two lexical legs at the system level.\n\t{ label: \"daemon-hybrid\", mode: \"semantic\", daemonHybrid: true },\n\t{ label: \"daemon-hybrid +rr\", mode: \"semantic\", rerank: true, daemonHybrid: true },\n\t// Three-way fusion: dense + BM25 + grep, all fused here so `k` is ours and\n\t// the per-leg ranks survive into the trace. `bm25+dense` isolates what the\n\t// grep leg still contributes once BM25 is present.\n\t{ label: \"bm25+dense +rr\", mode: \"semantic\", rerank: true, bm25Leg: true },\n\t{ label: \"3-way +rr\", mode: \"hybrid\", rerank: true, bm25Leg: true },\n];\n\n/** Candidates fetched per eval query — deep enough for the reranker gate. */\nconst EVAL_FETCH_LIMIT = 50;\n\nexport function spanMatchesGold(span: CandidateSpan, gold: EvalGoldSpan): boolean {\n\tif (span.path !== gold.path) return false;\n\tif (gold.scope === \"file\") return true;\n\tif (gold.startLine === undefined || gold.endLine === undefined) return true;\n\treturn span.startLine <= gold.endLine && span.endLine >= gold.startLine;\n}\n\n/** Fraction of gold spans matched by at least one of the top-`k` candidates. */\nexport function recallAtK(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[], k: number): number {\n\tif (gold.length === 0) return 0;\n\tconst top = candidates.slice(0, k);\n\tlet matched = 0;\n\tfor (const g of gold) {\n\t\tif (top.some((c) => spanMatchesGold(c, g))) matched++;\n\t}\n\treturn matched / gold.length;\n}\n\n/**\n * Mean reciprocal rank, averaged over gold spans: for each gold span, `1/rank`\n * of the first candidate matching it (0 when it never appears).\n *\n * Averaging per gold span rather than taking the single first hit keeps\n * multi-span queries (the cross-file class) honest — finding one of two\n * required sites should not score like finding both. For single-span queries\n * this reduces to textbook MRR.\n */\nexport function mrr(candidates: readonly CandidateSpan[], gold: readonly EvalGoldSpan[]): number {\n\tif (gold.length === 0) return 0;\n\tlet total = 0;\n\tfor (const g of gold) {\n\t\tconst index = candidates.findIndex((c) => spanMatchesGold(c, g));\n\t\tif (index >= 0) total += 1 / (index + 1);\n\t}\n\treturn total / gold.length;\n}\n\nexport interface EvalQueryResult {\n\tlabel: string;\n\tresolvedMode: ResolvedSearchMode;\n\tdegraded: boolean;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n}\n\nexport async function evaluateQuery(\n\tcwd: string,\n\tevalQuery: EvalQuery,\n\tconfigs: readonly EvalConfig[] = EVAL_CONFIGS,\n\tservice?: EmbsearchService,\n\thybridService?: EmbsearchService,\n): Promise<EvalQueryResult[]> {\n\tconst results: EvalQueryResult[] = [];\n\tfor (const config of configs) {\n\t\t// A daemon-hybrid config against the plain store would error in the\n\t\t// daemon (`query_hybrid requires a hybrid store`); omit the row instead.\n\t\tif ((config.daemonHybrid || config.bm25Leg) && !hybridService) continue;\n\t\tconst retrieved = await retrieveCandidates({\n\t\t\tcwd,\n\t\t\tquery: evalQuery.query,\n\t\t\tmode: config.mode,\n\t\t\trrfK: config.rrfK,\n\t\t\trerank: config.rerank ?? false,\n\t\t\tlimit: EVAL_FETCH_LIMIT,\n\t\t\tservice: config.daemonHybrid || config.bm25Leg ? hybridService : service,\n\t\t\tdaemonHybrid: config.daemonHybrid,\n\t\t\tbm25Leg: config.bm25Leg,\n\t\t});\n\t\tresults.push({\n\t\t\tlabel: config.label,\n\t\t\tresolvedMode: retrieved.resolvedMode,\n\t\t\tdegraded: retrieved.degradedReason !== undefined,\n\t\t\trecallAt1: recallAtK(retrieved.candidates, evalQuery.gold, 1),\n\t\t\trecallAt5: recallAtK(retrieved.candidates, evalQuery.gold, 5),\n\t\t\trecallAt10: recallAtK(retrieved.candidates, evalQuery.gold, 10),\n\t\t\trecallAt50: recallAtK(retrieved.candidates, evalQuery.gold, 50),\n\t\t\tmrr: mrr(retrieved.candidates, evalQuery.gold),\n\t\t});\n\t}\n\treturn results;\n}\n"]}

package/dist/core/search/hybrid-search.d.ts

@@ -26,6 +26,22 @@ export interface RetrieveOptions {
     rrfK?: number;
     /** Rerank the fused top-50 before slicing to `limit`. Default: true. */
     rerank?: boolean;
+    /**
+     * Ask the daemon to fuse its own BM25 index with the vectors and return one
+     * already-fused ranking, instead of taking a dense-only list. Needs a store
+     * built with `--hybrid`. The fused list arrives as a single "embed" leg,
+     * because a pre-fused ranking has no per-retriever structure left to record.
+     *
+     * Prefer {@link bm25Leg}: fusing here keeps the legs separable in the trace
+     * and lets the grep leg participate.
+     */
+    daemonHybrid?: boolean;
+    /**
+     * Fetch the daemon's BM25 index as its own ranked list and fuse it here,
+     * alongside dense and grep. Needs a store built with `--hybrid` and an
+     * embsearch new enough to serve `retriever: "lexical"`.
+     */
+    bm25Leg?: boolean;
     service?: EmbsearchService;
     signal?: AbortSignal;
 }