@kolisachint/hoocode-agent 0.5.59 → 0.5.60

package/dist/core/search/eval-harness.js

@@ -26,6 +26,8 @@ import { execFileSync } from "child_process";
 import { rmSync } from "fs";
 import { tmpdir } from "os";
 import path from "path";
+import { chunkFile } from "../embsearch/chunker.js";
+import { scanRepo } from "../embsearch/repo-scan.js";
 import { evaluateQuery } from "./eval.js";
 function git(repoRoot, args) {
     return execFileSync("git", ["-C", repoRoot, ...args], { encoding: "utf-8" }).trim();
@@ -68,6 +70,116 @@ export function hashRetrievalSource(repoRoot) {
     }
     return hash.digest("hex").slice(0, 16);
 }
+/**
+ * Cut `dir` down to a chunk budget, in place.
+ *
+ * Counts chunks with the indexer's own chunker rather than estimating from
+ * file size, because the budget is meant to predict indexing time and
+ * indexing time is per chunk. Gold-bearing files are never candidates for
+ * removal: dropping one would make its queries unanswerable and score the
+ * arm on a corpus that cannot contain the answer.
+ *
+ * Deletion is what makes this apply to every leg at once. Filtering the
+ * indexer's file list instead would shrink the dense and BM25 legs while grep
+ * still walked the full tree, and the legs would then be answering about
+ * different corpora.
+ */
+function applySubsample(dir, request) {
+    const gold = new Set(request.keepRelPaths);
+    const counts = new Map();
+    for (const file of scanRepo(dir).files) {
+        // The scanner skips `.git` as a *directory*, but a linked worktree's
+        // `.git` is a file holding the path to the real gitdir — so it comes back
+        // as an ordinary indexable file. Deleting it detaches the worktree from
+        // the repo, and `worktree remove` then fails on a tree git can no longer
+        // validate. Never a candidate.
+        if (file.rel === ".git" || file.rel.startsWith(`.git${path.sep}`) || file.rel.startsWith(".git/")) {
+            continue;
+        }
+        let content;
+        try {
+            content = readFileSync(path.join(dir, file.rel), "utf8");
+        }
+        catch {
+            continue;
+        }
+        // Deliberately the *default* cap, never the arm's.
+        //
+        // The budget picks which files survive, and a chunk-cap sweep must
+        // compare caps over identical source text. Counting at the arm's cap
+        // would let a cap-2000 arm — whose chunks are bigger, so fewer fit the
+        // budget — keep far more of the repo than a cap-1000 arm, and the two
+        // would then differ by corpus as well as by cap. Sizing is a secondary
+        // concern to that: bigger caps produce fewer chunks and index faster
+        // anyway, so the budget only ever overestimates their cost.
+        const n = chunkFile(file.rel, content).length;
+        if (n > 0)
+            counts.set(file.rel, n);
+    }
+    const keep = new Set();
+    let chunks = 0;
+    let goldFilesKept = 0;
+    for (const rel of counts.keys()) {
+        if (!gold.has(rel))
+            continue;
+        keep.add(rel);
+        chunks += counts.get(rel) ?? 0;
+        goldFilesKept++;
+    }
+    // Shuffle the distractors rather than taking the scan's order, which is
+    // directory order — that would keep a few whole subtrees and drop the rest,
+    // making the sample a slice of the repo instead of a sample of it.
+    const rng = seededRandom(request.seed);
+    const others = [...counts.keys()].filter((rel) => !gold.has(rel));
+    for (let i = others.length - 1; i > 0; i--) {
+        const j = Math.floor(rng() * (i + 1));
+        [others[i], others[j]] = [others[j], others[i]];
+    }
+    for (const rel of others) {
+        if (chunks >= request.targetChunks)
+            break;
+        keep.add(rel);
+        chunks += counts.get(rel) ?? 0;
+    }
+    let filesDropped = 0;
+    for (const rel of counts.keys()) {
+        if (keep.has(rel))
+            continue;
+        try {
+            rmSync(path.join(dir, rel), { force: true });
+            filesDropped++;
+        }
+        catch {
+            // A file the scanner listed but cannot be removed stays in the
+            // corpus; it inflates the sample slightly and is not worth failing
+            // the run over.
+        }
+    }
+    return {
+        targetChunks: request.targetChunks,
+        chunkCount: chunks,
+        filesKept: keep.size,
+        filesDropped,
+        goldFilesKept,
+        seed: request.seed,
+    };
+}
+/**
+ * Deterministic PRNG (mulberry32).
+ *
+ * `Math.random()` would make a "reproducible" subsample a different corpus on
+ * every run, which is the one property this must not have.
+ */
+function seededRandom(seed) {
+    let a = seed >>> 0;
+    return () => {
+        a = (a + 0x6d2b79f5) >>> 0;
+        let t = a;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
 /**
  * Files that describe this eval rather than being searched by it.
  *
@@ -99,10 +211,22 @@ export const CORPUS_EXCLUSIONS = [
  * With a `ref`, checks out a detached worktree at that commit so the corpus is
  * byte-identical on every rerun. Without one, falls back to the live working
  * tree and reports `dirty` so the record shows the run was not reproducible.
+ *
+ * `subsample` shrinks the corpus to a chunk budget, for screening runs where a
+ * full arm costs too much to iterate on. It keeps every gold-bearing file and
+ * draws distractors deterministically. This is a real change to what is being
+ * measured — a smaller distractor pool makes retrieval easier and inflates
+ * every metric — so it is recorded in the record and folded into the worktree
+ * path, and it needs a `ref`.
  */
-export function pinCorpus(repoRoot, ref) {
+export function pinCorpus(repoRoot, ref, subsample) {
     const dirty = git(repoRoot, ["status", "--porcelain"]).length > 0;
     if (!ref) {
+        if (subsample) {
+            // Subsampling deletes files. Against the live checkout that is the
+            // user's source tree, so this refuses rather than asks.
+            throw new Error("subsampling requires --corpus-ref: it deletes files, and the working tree is not ours to cut");
+        }
         // The live working tree is the user's checkout; deleting files from it to
         // tidy a measurement would be an unforgivable trade. Working-tree runs
         // are already stamped non-reproducible, so they carry the contamination.
@@ -120,7 +244,16 @@ export function pinCorpus(repoRoot, ref) {
     // the corpus directory, so a fresh temp path every run would re-embed all
     // ~17k chunks (minutes) instead of reusing the store built for this exact
     // SHA. The worktree is still removed afterwards; only the store persists.
-    const dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}`);
+    //
+    // The subsample is part of the key. Without it a screening run and a full
+    // run at the same SHA would share this path *and* the store derived from it,
+    // so the second would silently score the first's index — a wrong number that
+    // looks entirely normal.
+    // No chunk cap in the key: the file set is cap-independent by construction
+    // (see `applySubsample`), so cap arms share one worktree. The *store* key
+    // does carry the cap, because the vectors differ.
+    const subsampleKey = subsample ? `-fast${subsample.targetChunks}s${subsample.seed}` : "";
+    const dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}${subsampleKey}`);
     if (existsSync(dir)) {
         // Left behind by an interrupted run — drop it so `worktree add` succeeds.
         try {
@@ -140,12 +273,14 @@ export function pinCorpus(repoRoot, ref) {
             excluded.push(rel);
         }
     }
+    const subsampleInfo = subsample ? applySubsample(dir, subsample) : undefined;
     return {
         cwd: dir,
         sha,
         fromWorkingTree: false,
         dirty: false,
         excluded,
+        subsample: subsampleInfo,
         dispose: () => {
             try {
                 git(repoRoot, ["worktree", "remove", "--force", dir]);
@@ -167,7 +302,7 @@ function probeBinaryVersion(binaryPath) {
         return undefined;
     }
 }
-export function collectProvenance(repoRoot, corpus, corpusRef, service, embsearchBinary, hybridService) {
+export function collectProvenance(repoRoot, corpus, corpusRef, service, embsearchBinary, hybridService, modelDir, timing, chunkMaxChars) {
     const state = service?.getState();
     const phase = state?.phase ?? "absent";
     return {
@@ -177,6 +312,7 @@ export function collectProvenance(repoRoot, corpus, corpusRef, service, embsearc
         corpusFromWorkingTree: corpus.fromWorkingTree,
         corpusDirty: corpus.dirty,
         corpusExcluded: corpus.excluded,
+        corpusSubsample: corpus.subsample,
         harnessSha: git(repoRoot, ["rev-parse", "HEAD"]),
         retrievalSourceHash: hashRetrievalSource(repoRoot),
         embedder: {
@@ -189,10 +325,14 @@ export function collectProvenance(repoRoot, corpus, corpusRef, service, embsearc
             phase,
             binaryPath: embsearchBinary,
             binaryVersion: probeBinaryVersion(embsearchBinary),
+            modelId: service?.modelId(),
+            modelDir,
         },
         daemonHybrid: hybridService
             ? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }
             : undefined,
+        timing,
+        chunkMaxChars,
         runtime: { node: process.version, platform: process.platform, arch: process.arch },
     };
 }

package/dist/core/search/eval-harness.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-harness.js","sourceRoot":"","sources":["../../../src/core/search/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAyD,aAAa,EAAE,MAAM,WAAW,CAAC;AAwEjG,SAAS,GAAG,CAAC,QAAgB,EAAE,IAAc,EAAU;IACtD,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAAA,CACpF;AAED;0EAC0E;AAC1E,MAAM,UAAU,mBAAmB,CAAC,QAAgB,EAAU;IAC7D,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,uCAAuC,CAAC;QAC5D,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,qDAAqD,CAAC;KAC1E,CAAC;IACF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,MAAc,EAAQ,EAAE,CAAC;QACtC,IAAI,IAAiC,CAAC;QACtC,IAAI,CAAC;YACJ,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACR,OAAO;QACR,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE;gBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;IAAA,CACD,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,wEAAwE;QACxE,iCAAiC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACtC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAAA,CACvC;AAeD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAsB;IACnD,sDAAsD;IACtD,2DAA2D;IAC3D,+DAA+D;IAC/D,iCAAiC;CACjC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,GAAuB,EAAgB;IAClF,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAClE,IAAI,CAAC,GAAG,EAAE,CAAC;QACV,0EAA0E;QAC1E,uEAAuE;QACvE,yEAAyE;QACzE,OAAO;YACN,GAAG,EAAE,QAAQ;YACb,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACzC,eAAe,EAAE,IAAI;YACrB,KAAK;YACL,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,GAAG,EAAE,CAAC,EAAC,CAAC;SACjB,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9C,6EAA6E;IAC7E,0EAA0E;IAC1E,0EAA0E;IAC1E,0EAA0E;IAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,4EAA0E;QAC1E,IAAI,CAAC;YACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;QACtC,CAAC;IACF,CAAC;IACD,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IAEzD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACnC,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAChC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACF,CAAC;IAED,OAAO;QACN,GAAG,EAAE,GAAG;QACR,GAAG;QACH,eAAe,EAAE,KAAK;QACtB,KAAK,EAAE,KAAK;QACZ,QAAQ;QACR,OAAO,EAAE,GAAG,EAAE,CAAC;YACd,IAAI,CAAC;gBACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/C,CAAC;QAAA,CACD;KACD,CAAC;AAAA,CACF;AAED,gEAAgE;AAChE,SAAS,kBAAkB,CAAC,UAA8B,EAAsB;IAC/E,IAAI,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAClC,IAAI,CAAC;QACJ,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED,MAAM,UAAU,iBAAiB,CAChC,QAAgB,EAChB,MAAoB,EACpB,SAAiB,EACjB,OAAqC,EACrC,eAAwB,EACxB,aAAgC,EACf;IACjB,MAAM,KAAK,GAAG,OAAO,EAAE,QAAQ,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,KAAK,EAAE,KAAK,IAAI,QAAQ,CAAC;IACvC,OAAO;QACN,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;QACvB,SAAS,EAAE,MAAM,CAAC,GAAG;QACrB,SAAS;QACT,qBAAqB,EAAE,MAAM,CAAC,eAAe;QAC7C,WAAW,EAAE,MAAM,CAAC,KAAK;QACzB,cAAc,EAAE,MAAM,CAAC,QAAQ;QAC/B,UAAU,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAChD,mBAAmB,EAAE,mBAAmB,CAAC,QAAQ,CAAC;QAClD,QAAQ,EAAE;YACT,sEAAsE;YACtE,oEAAoE;YACpE,wDAAwD;YACxD,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,KAAK;YAC1C,MAAM,EAAE,KAAK,IAAI,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YAC7D,UAAU,EAAE,KAAK,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YACnE,KAAK;YACL,UAAU,EAAE,eAAe;YAC3B,aAAa,EAAE,kBAAkB,CAAC,eAAe,CAAC;SAClD;QACD,YAAY,EAAE,aAAa;YAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,aAAa,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,aAAa,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE;YACnF,CAAC,CAAC,SAAS;QACZ,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE;KAClF,CAAC;AAAA,CACF;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA6B,EAA4B;IACzF,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACvD,aAAa,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;IACpC,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AAAA,CAC9D;AAaD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B,EAG5D;IACF,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAChD,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAE/C,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC7F,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACrE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI;gBACzC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC;gBACb,UAAU,EAAE,CAAC;gBACb,GAAG,EAAE,CAAC;gBACN,CAAC,EAAE,CAAC;gBACJ,QAAQ,EAAE,CAAC;aACX,CAAC;YACF,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC;YACxB,KAAK,CAAC,CAAC,EAAE,CAAC;YACV,IAAI,MAAM,CAAC,QAAQ;gBAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACjC,CAAC;IACF,CAAC;IAED,MAAM,UAAU,GAAG,OAAO;SACxB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;SACzC,MAAM,CAAC,CAAC,KAAK,EAA0B,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC;SAC9D,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAChB,GAAG,KAAK;QACR,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;KACxB,CAAC,CAAC,CAAC;IAEL,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAAA,CAChC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAAoC,EAAU;IAClF,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG;QACb,kEAAkE;QAClE,kEAAkE;KAClE,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;QACjH,KAAK,CAAC,IAAI,CACT,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK;YAC5F,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,EAAE,CACxD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAAA,CACxB","sourcesContent":["/**\n * Eval harness: corpus pinning, provenance capture, and run records.\n *\n * The scoring math lives in `eval.ts`; this module is everything around it\n * that makes a number *comparable to a later number*. Three problems it\n * exists to solve, all of which bit the first eval round\n * (docs/hybrid-retrieval-design.md, \"Eval results\"):\n *\n *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,\n *     so every commit moves the thing being measured. A baseline taken today\n *     and a rerun taken after a retrieval change differ by both the change\n *     and the intervening commits, and nothing in the output says so. Fix:\n *     run against a detached git worktree pinned to an explicit SHA, and put\n *     that SHA in the record.\n *  2. **Nothing was recorded.** Results were printed to a terminal and\n *     hand-copied into a markdown table with no repo SHA, no embedder\n *     identity, and no index state. Fix: emit a machine-readable run record.\n *  3. **A degraded run looks like a real one.** With no embsearch binary the\n *     semantic and hybrid rows silently degrade to lexical, producing a table\n *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the\n *     record, plus a per-row degraded count that the writer refuses to hide.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readdirSync, readFileSync, statSync } from \"node:fs\";\nimport { execFileSync } from \"child_process\";\nimport { rmSync } from \"fs\";\nimport { tmpdir } from \"os\";\nimport path from \"path\";\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { type EvalConfig, type EvalQuery, type EvalQueryResult, evaluateQuery } from \"./eval.js\";\n\n/** Metrics aggregated per config across the whole gold set. */\nexport interface EvalAggregate {\n\tlabel: string;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n\t/** Queries scored under this config. */\n\tn: number;\n\t/** How many of them ran degraded (requested retriever unavailable). */\n\tdegraded: number;\n}\n\n/** Everything needed to decide whether two run records may be compared. */\nexport interface EvalProvenance {\n\ttimestampMs: number;\n\t/** SHA of the corpus actually indexed and searched. */\n\tcorpusSha: string;\n\t/** Ref the caller asked for, before resolution (e.g. \"HEAD\"). */\n\tcorpusRef: string;\n\t/** True when the corpus came from the live working tree rather than a\n\t *  pinned worktree — results are then not reproducible. */\n\tcorpusFromWorkingTree: boolean;\n\t/** Uncommitted changes present at run time. Only meaningful (and only\n\t *  possible) when `corpusFromWorkingTree` is true. */\n\tcorpusDirty: boolean;\n\t/** Files removed from the corpus before indexing — see\n\t *  {@link CORPUS_EXCLUSIONS}. A score against a different exclusion list is\n\t *  a score against a different corpus, so it is recorded, not assumed. */\n\tcorpusExcluded: string[];\n\t/** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,\n\t *  but differs when pinning an old corpus with today's code. */\n\tharnessSha: string;\n\t/**\n\t * Content hash of `src/core/search` + the chunker. Every tuning constant\n\t * that shapes a result — the fusion cap, top-k depths, rerank weights,\n\t * chunk sizing — lives in those files, so a changed hash means the\n\t * numbers are not comparable, without this module having to maintain a\n\t * hand-copied (and inevitably stale) list of constants.\n\t */\n\tretrievalSourceHash: string;\n\t/** Embedding backend state. `available: false` means every semantic and\n\t *  hybrid row in this record degraded to lexical. */\n\tembedder: {\n\t\tavailable: boolean;\n\t\treason?: string;\n\t\t/** Indexed chunk count when the index reached `ready`. */\n\t\tchunkCount?: number;\n\t\tphase: string;\n\t\t/** Binary that served the embeddings, and its self-reported version.\n\t\t *  The embedding model is baked into the binary at build time, so this\n\t\t *  is the only thing that identifies which model produced a score. */\n\t\tbinaryPath?: string;\n\t\tbinaryVersion?: string;\n\t};\n\t/** Daemon-side BM25 hybrid store, when the run included one. Absent means\n\t *  the record has no `daemon-hybrid` rows. */\n\tdaemonHybrid?: { available: boolean; phase: string };\n\truntime: { node: string; platform: string; arch: string };\n}\n\nexport interface EvalRunRecord {\n\tprovenance: EvalProvenance;\n\tgoldSet: { queryCount: number; byClass: Record<string, number>; goldSpanCount: number };\n\tconfigs: readonly EvalConfig[];\n\taggregates: EvalAggregate[];\n\tperQuery: Array<{ id: string; class: string; results: EvalQueryResult[] }>;\n}\n\nfunction git(repoRoot: string, args: string[]): string {\n\treturn execFileSync(\"git\", [\"-C\", repoRoot, ...args], { encoding: \"utf-8\" }).trim();\n}\n\n/** Hash every retrieval-shaping source file, so a tuning change is visible as\n *  a changed provenance field rather than an unexplained metric shift. */\nexport function hashRetrievalSource(repoRoot: string): string {\n\tconst roots = [\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/search\"),\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/embsearch/chunker.ts\"),\n\t];\n\tconst files: string[] = [];\n\tconst walk = (target: string): void => {\n\t\tlet stat: ReturnType<typeof statSync>;\n\t\ttry {\n\t\t\tstat = statSync(target);\n\t\t} catch {\n\t\t\treturn;\n\t\t}\n\t\tif (stat.isDirectory()) {\n\t\t\tfor (const entry of readdirSync(target).sort()) walk(path.join(target, entry));\n\t\t} else if (target.endsWith(\".ts\")) {\n\t\t\tfiles.push(target);\n\t\t}\n\t};\n\tfor (const root of roots) walk(root);\n\n\tconst hash = createHash(\"sha256\");\n\tfor (const file of files) {\n\t\t// Eval-only modules are excluded: changing how we measure must not look\n\t\t// like changing what we measure.\n\t\tconst base = path.basename(file);\n\t\tif (base.startsWith(\"eval\")) continue;\n\t\thash.update(path.relative(repoRoot, file).replace(/\\\\/g, \"/\"));\n\t\thash.update(readFileSync(file));\n\t}\n\treturn hash.digest(\"hex\").slice(0, 16);\n}\n\nexport interface PinnedCorpus {\n\t/** Directory to index and search. */\n\tcwd: string;\n\tsha: string;\n\tfromWorkingTree: boolean;\n\tdirty: boolean;\n\t/** Files removed from the corpus before indexing. Empty when the corpus is\n\t *  the live working tree, which is never mutated. */\n\texcluded: string[];\n\t/** Removes the worktree, if one was created. */\n\tdispose: () => void;\n}\n\n/**\n * Files that describe this eval rather than being searched by it.\n *\n * The fixtures hold all 62 query strings verbatim, so every query is a perfect\n * lexical match against its own entry, and the design note quotes the same\n * queries while discussing the classes they belong to. Measured before this\n * exclusion existed: **56 of 62 queries had one of these files in the top 10,\n * 27 of 62 had one as the #1 result, and they consumed 133 of the 620\n * top-10 slots** — a fifth of the window, spent on the eval reading itself.\n *\n * That is not a ranking artifact a reranker can fix: it displaces real answers\n * out of the window entirely, which is why two boundary-class queries were\n * absent from the top *50* rather than merely buried. Retrieving your own\n * question is not retrieval, so the corpus is scored without them.\n *\n * Removed from the pinned worktree before indexing, never from the repo — and\n * recorded in the run's provenance so a score is never silently taken against\n * a different corpus than it claims.\n */\nexport const CORPUS_EXCLUSIONS: readonly string[] = [\n\t\"packages/coding-agent/test/fixtures/search-eval.json\",\n\t\"packages/coding-agent/test/fixtures/search-eval-live.json\",\n\t\"packages/coding-agent/test/fixtures/search-eval-baseline.json\",\n\t\"docs/hybrid-retrieval-design.md\",\n];\n\n/**\n * Materialize the corpus to evaluate.\n *\n * With a `ref`, checks out a detached worktree at that commit so the corpus is\n * byte-identical on every rerun. Without one, falls back to the live working\n * tree and reports `dirty` so the record shows the run was not reproducible.\n */\nexport function pinCorpus(repoRoot: string, ref: string | undefined): PinnedCorpus {\n\tconst dirty = git(repoRoot, [\"status\", \"--porcelain\"]).length > 0;\n\tif (!ref) {\n\t\t// The live working tree is the user's checkout; deleting files from it to\n\t\t// tidy a measurement would be an unforgivable trade. Working-tree runs\n\t\t// are already stamped non-reproducible, so they carry the contamination.\n\t\treturn {\n\t\t\tcwd: repoRoot,\n\t\t\tsha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\t\tfromWorkingTree: true,\n\t\t\tdirty,\n\t\t\texcluded: [],\n\t\t\tdispose: () => {},\n\t\t};\n\t}\n\n\tconst sha = git(repoRoot, [\"rev-parse\", ref]);\n\t// Deterministic path, not mkdtemp: the embedding store is keyed by a hash of\n\t// the corpus directory, so a fresh temp path every run would re-embed all\n\t// ~17k chunks (minutes) instead of reusing the store built for this exact\n\t// SHA. The worktree is still removed afterwards; only the store persists.\n\tconst dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}`);\n\tif (existsSync(dir)) {\n\t\t// Left behind by an interrupted run — drop it so `worktree add` succeeds.\n\t\ttry {\n\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t} catch {\n\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\tgit(repoRoot, [\"worktree\", \"prune\"]);\n\t\t}\n\t}\n\tgit(repoRoot, [\"worktree\", \"add\", \"--detach\", dir, sha]);\n\n\tconst excluded: string[] = [];\n\tfor (const rel of CORPUS_EXCLUSIONS) {\n\t\tconst target = path.join(dir, rel);\n\t\tif (existsSync(target)) {\n\t\t\trmSync(target, { force: true });\n\t\t\texcluded.push(rel);\n\t\t}\n\t}\n\n\treturn {\n\t\tcwd: dir,\n\t\tsha,\n\t\tfromWorkingTree: false,\n\t\tdirty: false,\n\t\texcluded,\n\t\tdispose: () => {\n\t\t\ttry {\n\t\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t\t} catch {\n\t\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\t}\n\t\t},\n\t};\n}\n\n/** `<binary> --version`, or undefined when it cannot be run. */\nfunction probeBinaryVersion(binaryPath: string | undefined): string | undefined {\n\tif (!binaryPath) return undefined;\n\ttry {\n\t\treturn execFileSync(binaryPath, [\"--version\"], { encoding: \"utf-8\" }).trim();\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\nexport function collectProvenance(\n\trepoRoot: string,\n\tcorpus: PinnedCorpus,\n\tcorpusRef: string,\n\tservice: EmbsearchService | undefined,\n\tembsearchBinary?: string,\n\thybridService?: EmbsearchService,\n): EvalProvenance {\n\tconst state = service?.getState();\n\tconst phase = state?.phase ?? \"absent\";\n\treturn {\n\t\ttimestampMs: Date.now(),\n\t\tcorpusSha: corpus.sha,\n\t\tcorpusRef,\n\t\tcorpusFromWorkingTree: corpus.fromWorkingTree,\n\t\tcorpusDirty: corpus.dirty,\n\t\tcorpusExcluded: corpus.excluded,\n\t\tharnessSha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\tretrievalSourceHash: hashRetrievalSource(repoRoot),\n\t\tembedder: {\n\t\t\t// `ready` is the only phase the service reaches with a real embedder:\n\t\t\t// it rejects the mock backend at startup, so availability here also\n\t\t\t// certifies the numbers came from a genuine ONNX build.\n\t\t\tavailable: service?.isAvailable() ?? false,\n\t\t\treason: state && \"reason\" in state ? state.reason : undefined,\n\t\t\tchunkCount: state?.phase === \"ready\" ? state.chunkCount : undefined,\n\t\t\tphase,\n\t\t\tbinaryPath: embsearchBinary,\n\t\t\tbinaryVersion: probeBinaryVersion(embsearchBinary),\n\t\t},\n\t\tdaemonHybrid: hybridService\n\t\t\t? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }\n\t\t\t: undefined,\n\t\truntime: { node: process.version, platform: process.platform, arch: process.arch },\n\t};\n}\n\nexport function summarizeGoldSet(dataset: readonly EvalQuery[]): EvalRunRecord[\"goldSet\"] {\n\tconst byClass: Record<string, number> = {};\n\tlet goldSpanCount = 0;\n\tfor (const query of dataset) {\n\t\tbyClass[query.class] = (byClass[query.class] ?? 0) + 1;\n\t\tgoldSpanCount += query.gold.length;\n\t}\n\treturn { queryCount: dataset.length, byClass, goldSpanCount };\n}\n\nexport interface RunEvalSuiteOptions {\n\tcwd: string;\n\tdataset: readonly EvalQuery[];\n\tconfigs: readonly EvalConfig[];\n\tservice?: EmbsearchService;\n\t/** Second service backed by a daemon-side BM25 hybrid store, for the\n\t *  `daemon-hybrid` configs. Absent means those rows are omitted. */\n\thybridService?: EmbsearchService;\n\tonQuery?: (index: number, query: EvalQuery) => void;\n}\n\nexport async function runEvalSuite(options: RunEvalSuiteOptions): Promise<{\n\taggregates: EvalAggregate[];\n\tperQuery: EvalRunRecord[\"perQuery\"];\n}> {\n\tconst { cwd, dataset, configs, service } = options;\n\tconst totals = new Map<string, EvalAggregate>();\n\tconst perQuery: EvalRunRecord[\"perQuery\"] = [];\n\n\tfor (const [index, evalQuery] of dataset.entries()) {\n\t\toptions.onQuery?.(index, evalQuery);\n\t\tconst results = await evaluateQuery(cwd, evalQuery, configs, service, options.hybridService);\n\t\tperQuery.push({ id: evalQuery.id, class: evalQuery.class, results });\n\t\tfor (const result of results) {\n\t\t\tconst total = totals.get(result.label) ?? {\n\t\t\t\tlabel: result.label,\n\t\t\t\trecallAt1: 0,\n\t\t\t\trecallAt5: 0,\n\t\t\t\trecallAt10: 0,\n\t\t\t\trecallAt50: 0,\n\t\t\t\tmrr: 0,\n\t\t\t\tn: 0,\n\t\t\t\tdegraded: 0,\n\t\t\t};\n\t\t\ttotal.recallAt1 += result.recallAt1;\n\t\t\ttotal.recallAt5 += result.recallAt5;\n\t\t\ttotal.recallAt10 += result.recallAt10;\n\t\t\ttotal.recallAt50 += result.recallAt50;\n\t\t\ttotal.mrr += result.mrr;\n\t\t\ttotal.n++;\n\t\t\tif (result.degraded) total.degraded++;\n\t\t\ttotals.set(result.label, total);\n\t\t}\n\t}\n\n\tconst aggregates = configs\n\t\t.map((config) => totals.get(config.label))\n\t\t.filter((total): total is EvalAggregate => total !== undefined)\n\t\t.map((total) => ({\n\t\t\t...total,\n\t\t\trecallAt1: total.recallAt1 / total.n,\n\t\t\trecallAt5: total.recallAt5 / total.n,\n\t\t\trecallAt10: total.recallAt10 / total.n,\n\t\t\trecallAt50: total.recallAt50 / total.n,\n\t\t\tmrr: total.mrr / total.n,\n\t\t}));\n\n\treturn { aggregates, perQuery };\n}\n\nexport function formatAggregateTable(aggregates: readonly EvalAggregate[]): string {\n\tconst pct = (x: number) => `${Math.round(x * 100)}%`.padStart(5);\n\tconst lines = [\n\t\t\"config           |  R@1  |  R@5  | R@10  | R@50  |  MRR  | notes\",\n\t\t\"-----------------|-------|-------|-------|-------|-------|------\",\n\t];\n\tfor (const a of aggregates) {\n\t\tconst notes = a.degraded === a.n ? \"degraded to lexical\" : a.degraded > 0 ? `${a.degraded}/${a.n} degraded` : \"\";\n\t\tlines.push(\n\t\t\t`${a.label.padEnd(16)} | ${pct(a.recallAt1)} | ${pct(a.recallAt5)} | ${pct(a.recallAt10)} | ` +\n\t\t\t\t`${pct(a.recallAt50)} | ${a.mrr.toFixed(3)} | ${notes}`,\n\t\t);\n\t}\n\treturn lines.join(\"\\n\");\n}\n"]}
+{"version":3,"file":"eval-harness.js","sourceRoot":"","sources":["../../../src/core/search/eval-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAEpD,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAyD,aAAa,EAAE,MAAM,WAAW,CAAC;AA2HjG,SAAS,GAAG,CAAC,QAAgB,EAAE,IAAc,EAAU;IACtD,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAAA,CACpF;AAED;0EAC0E;AAC1E,MAAM,UAAU,mBAAmB,CAAC,QAAgB,EAAU;IAC7D,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,uCAAuC,CAAC;QAC5D,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,qDAAqD,CAAC;KAC1E,CAAC;IACF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,MAAc,EAAQ,EAAE,CAAC;QACtC,IAAI,IAAiC,CAAC;QACtC,IAAI,CAAC;YACJ,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACR,OAAO;QACR,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE;gBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;IAAA,CACD,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,wEAAwE;QACxE,iCAAiC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACtC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAAA,CACvC;AAwCD;;;;;;;;;;;;;GAaG;AACH,SAAS,cAAc,CAAC,GAAW,EAAE,OAA+B,EAAuB;IAC1F,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;QACxC,qEAAqE;QACrE,4EAA0E;QAC1E,wEAAwE;QACxE,yEAAyE;QACzE,+BAA+B;QAC/B,IAAI,IAAI,CAAC,GAAG,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACnG,SAAS;QACV,CAAC;QACD,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACJ,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1D,CAAC;QAAC,MAAM,CAAC;YACR,SAAS;QACV,CAAC;QACD,mDAAmD;QACnD,EAAE;QACF,mEAAmE;QACnE,qEAAqE;QACrE,yEAAuE;QACvE,wEAAsE;QACtE,uEAAuE;QACvE,qEAAqE;QACrE,4DAA4D;QAC5D,MAAM,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,MAAM,CAAC;QAC9C,IAAI,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC7B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC/B,aAAa,EAAE,CAAC;IACjB,CAAC;IAED,wEAAwE;IACxE,8EAA4E;IAC5E,mEAAmE;IACnE,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAClE,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,MAAM,IAAI,OAAO,CAAC,YAAY;YAAE,MAAM;QAC1C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;QACjC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC;YACJ,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7C,YAAY,EAAE,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACR,+DAA+D;YAC/D,mEAAmE;YACnE,gBAAgB;QACjB,CAAC;IACF,CAAC;IAED,OAAO;QACN,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,UAAU,EAAE,MAAM;QAClB,SAAS,EAAE,IAAI,CAAC,IAAI;QACpB,YAAY;QACZ,aAAa;QACb,IAAI,EAAE,OAAO,CAAC,IAAI;KAClB,CAAC;AAAA,CACF;AAED;;;;;GAKG;AACH,SAAS,YAAY,CAAC,IAAY,EAAgB;IACjD,IAAI,CAAC,GAAG,IAAI,KAAK,CAAC,CAAC;IACnB,OAAO,GAAG,EAAE,CAAC;QACZ,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC3B,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,CAAC;IAAA,CAC7C,CAAC;AAAA,CACF;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAsB;IACnD,sDAAsD;IACtD,2DAA2D;IAC3D,+DAA+D;IAC/D,iCAAiC;CACjC,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB,EAAE,GAAuB,EAAE,SAAkC,EAAgB;IACtH,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAClE,IAAI,CAAC,GAAG,EAAE,CAAC;QACV,IAAI,SAAS,EAAE,CAAC;YACf,mEAAmE;YACnE,wDAAwD;YACxD,MAAM,IAAI,KAAK,CACd,8FAA8F,CAC9F,CAAC;QACH,CAAC;QACD,0EAA0E;QAC1E,uEAAuE;QACvE,yEAAyE;QACzE,OAAO;YACN,GAAG,EAAE,QAAQ;YACb,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACzC,eAAe,EAAE,IAAI;YACrB,KAAK;YACL,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,GAAG,EAAE,CAAC,EAAC,CAAC;SACjB,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9C,6EAA6E;IAC7E,0EAA0E;IAC1E,0EAA0E;IAC1E,0EAA0E;IAC1E,EAAE;IACF,0EAA0E;IAC1E,6EAA6E;IAC7E,+EAA6E;IAC7E,yBAAyB;IACzB,2EAA2E;IAC3E,0EAA0E;IAC1E,kDAAkD;IAClD,MAAM,YAAY,GAAG,SAAS,CAAC,CAAC,CAAC,QAAQ,SAAS,CAAC,YAAY,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACzF,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,CAAC;IAC1F,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,4EAA0E;QAC1E,IAAI,CAAC;YACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;QACtC,CAAC;IACF,CAAC;IACD,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IAEzD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACnC,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAChC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACF,CAAC;IAED,MAAM,aAAa,GAAG,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE7E,OAAO;QACN,GAAG,EAAE,GAAG;QACR,GAAG;QACH,eAAe,EAAE,KAAK;QACtB,KAAK,EAAE,KAAK;QACZ,QAAQ;QACR,SAAS,EAAE,aAAa;QACxB,OAAO,EAAE,GAAG,EAAE,CAAC;YACd,IAAI,CAAC;gBACJ,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACR,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/C,CAAC;QAAA,CACD;KACD,CAAC;AAAA,CACF;AAED,gEAAgE;AAChE,SAAS,kBAAkB,CAAC,UAA8B,EAAsB;IAC/E,IAAI,CAAC,UAAU;QAAE,OAAO,SAAS,CAAC;IAClC,IAAI,CAAC;QACJ,OAAO,YAAY,CAAC,UAAU,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED,MAAM,UAAU,iBAAiB,CAChC,QAAgB,EAChB,MAAoB,EACpB,SAAiB,EACjB,OAAqC,EACrC,eAAwB,EACxB,aAAgC,EAChC,QAAiB,EACjB,MAAiC,EACjC,aAAsB,EACL;IACjB,MAAM,KAAK,GAAG,OAAO,EAAE,QAAQ,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,KAAK,EAAE,KAAK,IAAI,QAAQ,CAAC;IACvC,OAAO;QACN,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;QACvB,SAAS,EAAE,MAAM,CAAC,GAAG;QACrB,SAAS;QACT,qBAAqB,EAAE,MAAM,CAAC,eAAe;QAC7C,WAAW,EAAE,MAAM,CAAC,KAAK;QACzB,cAAc,EAAE,MAAM,CAAC,QAAQ;QAC/B,eAAe,EAAE,MAAM,CAAC,SAAS;QACjC,UAAU,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAChD,mBAAmB,EAAE,mBAAmB,CAAC,QAAQ,CAAC;QAClD,QAAQ,EAAE;YACT,sEAAsE;YACtE,oEAAoE;YACpE,wDAAwD;YACxD,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,KAAK;YAC1C,MAAM,EAAE,KAAK,IAAI,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YAC7D,UAAU,EAAE,KAAK,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YACnE,KAAK;YACL,UAAU,EAAE,eAAe;YAC3B,aAAa,EAAE,kBAAkB,CAAC,eAAe,CAAC;YAClD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE;YAC3B,QAAQ;SACR;QACD,YAAY,EAAE,aAAa;YAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,aAAa,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,aAAa,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE;YACnF,CAAC,CAAC,SAAS;QACZ,MAAM;QACN,aAAa;QACb,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE;KAClF,CAAC;AAAA,CACF;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA6B,EAA4B;IACzF,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACvD,aAAa,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;IACpC,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AAAA,CAC9D;AAaD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B,EAG5D;IACF,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAChD,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAE/C,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC7F,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACrE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI;gBACzC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC;gBACb,UAAU,EAAE,CAAC;gBACb,GAAG,EAAE,CAAC;gBACN,CAAC,EAAE,CAAC;gBACJ,QAAQ,EAAE,CAAC;aACX,CAAC;YACF,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACpC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;YACtC,KAAK,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC;YACxB,KAAK,CAAC,CAAC,EAAE,CAAC;YACV,IAAI,MAAM,CAAC,QAAQ;gBAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACjC,CAAC;IACF,CAAC;IAED,MAAM,UAAU,GAAG,OAAO;SACxB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;SACzC,MAAM,CAAC,CAAC,KAAK,EAA0B,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC;SAC9D,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAChB,GAAG,KAAK;QACR,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC;QACpC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC;QACtC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;KACxB,CAAC,CAAC,CAAC;IAEL,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAAA,CAChC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAAoC,EAAU;IAClF,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG;QACb,kEAAkE;QAClE,kEAAkE;KAClE,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;QACjH,KAAK,CAAC,IAAI,CACT,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK;YAC5F,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,EAAE,CACxD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAAA,CACxB","sourcesContent":["/**\n * Eval harness: corpus pinning, provenance capture, and run records.\n *\n * The scoring math lives in `eval.ts`; this module is everything around it\n * that makes a number *comparable to a later number*. Three problems it\n * exists to solve, all of which bit the first eval round\n * (docs/hybrid-retrieval-design.md, \"Eval results\"):\n *\n *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,\n *     so every commit moves the thing being measured. A baseline taken today\n *     and a rerun taken after a retrieval change differ by both the change\n *     and the intervening commits, and nothing in the output says so. Fix:\n *     run against a detached git worktree pinned to an explicit SHA, and put\n *     that SHA in the record.\n *  2. **Nothing was recorded.** Results were printed to a terminal and\n *     hand-copied into a markdown table with no repo SHA, no embedder\n *     identity, and no index state. Fix: emit a machine-readable run record.\n *  3. **A degraded run looks like a real one.** With no embsearch binary the\n *     semantic and hybrid rows silently degrade to lexical, producing a table\n *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the\n *     record, plus a per-row degraded count that the writer refuses to hide.\n */\n\nimport { createHash } from \"node:crypto\";\nimport { existsSync, readdirSync, readFileSync, statSync } from \"node:fs\";\nimport { execFileSync } from \"child_process\";\nimport { rmSync } from \"fs\";\nimport { tmpdir } from \"os\";\nimport path from \"path\";\nimport { chunkFile } from \"../embsearch/chunker.js\";\nimport type { EmbsearchService } from \"../embsearch/embsearch-service.js\";\nimport { scanRepo } from \"../embsearch/repo-scan.js\";\nimport { type EvalConfig, type EvalQuery, type EvalQueryResult, evaluateQuery } from \"./eval.js\";\n\n/** Metrics aggregated per config across the whole gold set. */\nexport interface EvalAggregate {\n\tlabel: string;\n\trecallAt1: number;\n\trecallAt5: number;\n\trecallAt10: number;\n\trecallAt50: number;\n\tmrr: number;\n\t/** Queries scored under this config. */\n\tn: number;\n\t/** How many of them ran degraded (requested retriever unavailable). */\n\tdegraded: number;\n}\n\n/** Everything needed to decide whether two run records may be compared. */\nexport interface EvalProvenance {\n\ttimestampMs: number;\n\t/** SHA of the corpus actually indexed and searched. */\n\tcorpusSha: string;\n\t/** Ref the caller asked for, before resolution (e.g. \"HEAD\"). */\n\tcorpusRef: string;\n\t/** True when the corpus came from the live working tree rather than a\n\t *  pinned worktree — results are then not reproducible. */\n\tcorpusFromWorkingTree: boolean;\n\t/** Uncommitted changes present at run time. Only meaningful (and only\n\t *  possible) when `corpusFromWorkingTree` is true. */\n\tcorpusDirty: boolean;\n\t/** Files removed from the corpus before indexing — see\n\t *  {@link CORPUS_EXCLUSIONS}. A score against a different exclusion list is\n\t *  a score against a different corpus, so it is recorded, not assumed. */\n\tcorpusExcluded: string[];\n\t/**\n\t * Set only when the run scored a deliberately shrunk corpus.\n\t *\n\t * A smaller distractor pool makes every query easier, so these metrics are\n\t * higher than a full-corpus run's and are **not** comparable to one. They\n\t * are comparable to another subsampled run with the same target and seed,\n\t * which is what makes this useful for screening model arms.\n\t */\n\tcorpusSubsample?: CorpusSubsampleInfo;\n\t/** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,\n\t *  but differs when pinning an old corpus with today's code. */\n\tharnessSha: string;\n\t/**\n\t * Content hash of `src/core/search` + the chunker. Every tuning constant\n\t * that shapes a result — the fusion cap, top-k depths, rerank weights,\n\t * chunk sizing — lives in those files, so a changed hash means the\n\t * numbers are not comparable, without this module having to maintain a\n\t * hand-copied (and inevitably stale) list of constants.\n\t */\n\tretrievalSourceHash: string;\n\t/** Embedding backend state. `available: false` means every semantic and\n\t *  hybrid row in this record degraded to lexical. */\n\tembedder: {\n\t\tavailable: boolean;\n\t\treason?: string;\n\t\t/** Indexed chunk count when the index reached `ready`. */\n\t\tchunkCount?: number;\n\t\tphase: string;\n\t\t/** Binary that served the embeddings, and its self-reported version. */\n\t\tbinaryPath?: string;\n\t\tbinaryVersion?: string;\n\t\t/**\n\t\t * Model id the daemon reported — the thing that actually identifies\n\t\t * which model produced these scores.\n\t\t *\n\t\t * This used to be inferred from `binaryVersion`, on the reasoning that\n\t\t * the model was baked into the binary at build time. `--model <dir>`\n\t\t * ends that: one binary now serves any number of models, so two arms of\n\t\t * a model comparison would have carried identical provenance and been\n\t\t * indistinguishable in the record. The id is a hash over the model's\n\t\t * whole spec (pooling, token limit, prefixes), so a change to any of\n\t\t * them shows up here.\n\t\t */\n\t\tmodelId?: string;\n\t\t/** Model directory passed as `--model`, when the run overrode the\n\t\t *  bundled model. Absent means the binary's own model was used. */\n\t\tmodelDir?: string;\n\t};\n\t/** Daemon-side BM25 hybrid store, when the run included one. Absent means\n\t *  the record has no `daemon-hybrid` rows. */\n\tdaemonHybrid?: { available: boolean; phase: string };\n\t/**\n\t * Wall time, split at the seam between building the index and scoring the\n\t * gold set.\n\t *\n\t * Recorded because the cost side of a model comparison is almost entirely\n\t * indexing, and a single total cannot show it: the first such comparison\n\t * could only report \"17 -> 60 min\" for whole runs and had to note that the\n\t * figure was \"not isolated from query work\", which left the headline cost\n\t * of the change unmeasured. These two numbers are machine- and\n\t * load-dependent and say nothing about retrieval quality; they are a budget,\n\t * not a metric.\n\t */\n\t/**\n\t * Chunker character cap, when an arm overrode it. Absent means the shipped\n\t * `CHUNK_MAX_CHARS`. Records differing here are not comparable: the chunks\n\t * are different text, so every id, span and vector differs.\n\t */\n\tchunkMaxChars?: number;\n\ttiming?: {\n\t\t/** Seconds spent bringing the index(es) to `ready`, model load included. */\n\t\tindexSeconds: number;\n\t\t/** Seconds spent running every config over every gold query. */\n\t\tquerySeconds: number;\n\t\t/** True when one hybrid store served both the dense and BM25 roles\n\t\t *  rather than the corpus being embedded twice. Runs with this false\n\t\t *  paid roughly double the indexing time. */\n\t\tsharedStore: boolean;\n\t};\n\truntime: { node: string; platform: string; arch: string };\n}\n\nexport interface EvalRunRecord {\n\tprovenance: EvalProvenance;\n\tgoldSet: { queryCount: number; byClass: Record<string, number>; goldSpanCount: number };\n\tconfigs: readonly EvalConfig[];\n\taggregates: EvalAggregate[];\n\tperQuery: Array<{ id: string; class: string; results: EvalQueryResult[] }>;\n}\n\nfunction git(repoRoot: string, args: string[]): string {\n\treturn execFileSync(\"git\", [\"-C\", repoRoot, ...args], { encoding: \"utf-8\" }).trim();\n}\n\n/** Hash every retrieval-shaping source file, so a tuning change is visible as\n *  a changed provenance field rather than an unexplained metric shift. */\nexport function hashRetrievalSource(repoRoot: string): string {\n\tconst roots = [\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/search\"),\n\t\tpath.join(repoRoot, \"packages/coding-agent/src/core/embsearch/chunker.ts\"),\n\t];\n\tconst files: string[] = [];\n\tconst walk = (target: string): void => {\n\t\tlet stat: ReturnType<typeof statSync>;\n\t\ttry {\n\t\t\tstat = statSync(target);\n\t\t} catch {\n\t\t\treturn;\n\t\t}\n\t\tif (stat.isDirectory()) {\n\t\t\tfor (const entry of readdirSync(target).sort()) walk(path.join(target, entry));\n\t\t} else if (target.endsWith(\".ts\")) {\n\t\t\tfiles.push(target);\n\t\t}\n\t};\n\tfor (const root of roots) walk(root);\n\n\tconst hash = createHash(\"sha256\");\n\tfor (const file of files) {\n\t\t// Eval-only modules are excluded: changing how we measure must not look\n\t\t// like changing what we measure.\n\t\tconst base = path.basename(file);\n\t\tif (base.startsWith(\"eval\")) continue;\n\t\thash.update(path.relative(repoRoot, file).replace(/\\\\/g, \"/\"));\n\t\thash.update(readFileSync(file));\n\t}\n\treturn hash.digest(\"hex\").slice(0, 16);\n}\n\nexport interface PinnedCorpus {\n\t/** Directory to index and search. */\n\tcwd: string;\n\tsha: string;\n\tfromWorkingTree: boolean;\n\tdirty: boolean;\n\t/** Files removed from the corpus before indexing. Empty when the corpus is\n\t *  the live working tree, which is never mutated. */\n\texcluded: string[];\n\t/** Present only on a subsampled run. Its presence is what marks a record as\n\t *  incomparable to a full-corpus one. */\n\tsubsample?: CorpusSubsampleInfo;\n\t/** Removes the worktree, if one was created. */\n\tdispose: () => void;\n}\n\n/** Request to shrink the corpus to a chunk budget. See {@link pinCorpus}. */\nexport interface CorpusSubsampleRequest {\n\t/** Approximate chunk budget. Gold-bearing files are kept past it. */\n\ttargetChunks: number;\n\t/** Files that must survive regardless of budget — the gold-bearing ones. */\n\tkeepRelPaths: readonly string[];\n\t/** Seed for the distractor draw, so a budget reproduces exactly. */\n\tseed: number;\n}\n\n/** What a subsampled run did, recorded so it can never be read as a full one. */\nexport interface CorpusSubsampleInfo {\n\ttargetChunks: number;\n\t/** Chunks actually kept. Exceeds the target when gold files alone do. */\n\tchunkCount: number;\n\tfilesKept: number;\n\tfilesDropped: number;\n\t/** Gold-bearing files, all of which are kept unconditionally. */\n\tgoldFilesKept: number;\n\tseed: number;\n}\n\n/**\n * Cut `dir` down to a chunk budget, in place.\n *\n * Counts chunks with the indexer's own chunker rather than estimating from\n * file size, because the budget is meant to predict indexing time and\n * indexing time is per chunk. Gold-bearing files are never candidates for\n * removal: dropping one would make its queries unanswerable and score the\n * arm on a corpus that cannot contain the answer.\n *\n * Deletion is what makes this apply to every leg at once. Filtering the\n * indexer's file list instead would shrink the dense and BM25 legs while grep\n * still walked the full tree, and the legs would then be answering about\n * different corpora.\n */\nfunction applySubsample(dir: string, request: CorpusSubsampleRequest): CorpusSubsampleInfo {\n\tconst gold = new Set(request.keepRelPaths);\n\tconst counts = new Map<string, number>();\n\tfor (const file of scanRepo(dir).files) {\n\t\t// The scanner skips `.git` as a *directory*, but a linked worktree's\n\t\t// `.git` is a file holding the path to the real gitdir — so it comes back\n\t\t// as an ordinary indexable file. Deleting it detaches the worktree from\n\t\t// the repo, and `worktree remove` then fails on a tree git can no longer\n\t\t// validate. Never a candidate.\n\t\tif (file.rel === \".git\" || file.rel.startsWith(`.git${path.sep}`) || file.rel.startsWith(\".git/\")) {\n\t\t\tcontinue;\n\t\t}\n\t\tlet content: string;\n\t\ttry {\n\t\t\tcontent = readFileSync(path.join(dir, file.rel), \"utf8\");\n\t\t} catch {\n\t\t\tcontinue;\n\t\t}\n\t\t// Deliberately the *default* cap, never the arm's.\n\t\t//\n\t\t// The budget picks which files survive, and a chunk-cap sweep must\n\t\t// compare caps over identical source text. Counting at the arm's cap\n\t\t// would let a cap-2000 arm — whose chunks are bigger, so fewer fit the\n\t\t// budget — keep far more of the repo than a cap-1000 arm, and the two\n\t\t// would then differ by corpus as well as by cap. Sizing is a secondary\n\t\t// concern to that: bigger caps produce fewer chunks and index faster\n\t\t// anyway, so the budget only ever overestimates their cost.\n\t\tconst n = chunkFile(file.rel, content).length;\n\t\tif (n > 0) counts.set(file.rel, n);\n\t}\n\n\tconst keep = new Set<string>();\n\tlet chunks = 0;\n\tlet goldFilesKept = 0;\n\tfor (const rel of counts.keys()) {\n\t\tif (!gold.has(rel)) continue;\n\t\tkeep.add(rel);\n\t\tchunks += counts.get(rel) ?? 0;\n\t\tgoldFilesKept++;\n\t}\n\n\t// Shuffle the distractors rather than taking the scan's order, which is\n\t// directory order — that would keep a few whole subtrees and drop the rest,\n\t// making the sample a slice of the repo instead of a sample of it.\n\tconst rng = seededRandom(request.seed);\n\tconst others = [...counts.keys()].filter((rel) => !gold.has(rel));\n\tfor (let i = others.length - 1; i > 0; i--) {\n\t\tconst j = Math.floor(rng() * (i + 1));\n\t\t[others[i], others[j]] = [others[j], others[i]];\n\t}\n\tfor (const rel of others) {\n\t\tif (chunks >= request.targetChunks) break;\n\t\tkeep.add(rel);\n\t\tchunks += counts.get(rel) ?? 0;\n\t}\n\n\tlet filesDropped = 0;\n\tfor (const rel of counts.keys()) {\n\t\tif (keep.has(rel)) continue;\n\t\ttry {\n\t\t\trmSync(path.join(dir, rel), { force: true });\n\t\t\tfilesDropped++;\n\t\t} catch {\n\t\t\t// A file the scanner listed but cannot be removed stays in the\n\t\t\t// corpus; it inflates the sample slightly and is not worth failing\n\t\t\t// the run over.\n\t\t}\n\t}\n\n\treturn {\n\t\ttargetChunks: request.targetChunks,\n\t\tchunkCount: chunks,\n\t\tfilesKept: keep.size,\n\t\tfilesDropped,\n\t\tgoldFilesKept,\n\t\tseed: request.seed,\n\t};\n}\n\n/**\n * Deterministic PRNG (mulberry32).\n *\n * `Math.random()` would make a \"reproducible\" subsample a different corpus on\n * every run, which is the one property this must not have.\n */\nfunction seededRandom(seed: number): () => number {\n\tlet a = seed >>> 0;\n\treturn () => {\n\t\ta = (a + 0x6d2b79f5) >>> 0;\n\t\tlet t = a;\n\t\tt = Math.imul(t ^ (t >>> 15), t | 1);\n\t\tt ^= t + Math.imul(t ^ (t >>> 7), t | 61);\n\t\treturn ((t ^ (t >>> 14)) >>> 0) / 4294967296;\n\t};\n}\n\n/**\n * Files that describe this eval rather than being searched by it.\n *\n * The fixtures hold all 62 query strings verbatim, so every query is a perfect\n * lexical match against its own entry, and the design note quotes the same\n * queries while discussing the classes they belong to. Measured before this\n * exclusion existed: **56 of 62 queries had one of these files in the top 10,\n * 27 of 62 had one as the #1 result, and they consumed 133 of the 620\n * top-10 slots** — a fifth of the window, spent on the eval reading itself.\n *\n * That is not a ranking artifact a reranker can fix: it displaces real answers\n * out of the window entirely, which is why two boundary-class queries were\n * absent from the top *50* rather than merely buried. Retrieving your own\n * question is not retrieval, so the corpus is scored without them.\n *\n * Removed from the pinned worktree before indexing, never from the repo — and\n * recorded in the run's provenance so a score is never silently taken against\n * a different corpus than it claims.\n */\nexport const CORPUS_EXCLUSIONS: readonly string[] = [\n\t\"packages/coding-agent/test/fixtures/search-eval.json\",\n\t\"packages/coding-agent/test/fixtures/search-eval-live.json\",\n\t\"packages/coding-agent/test/fixtures/search-eval-baseline.json\",\n\t\"docs/hybrid-retrieval-design.md\",\n];\n\n/**\n * Materialize the corpus to evaluate.\n *\n * With a `ref`, checks out a detached worktree at that commit so the corpus is\n * byte-identical on every rerun. Without one, falls back to the live working\n * tree and reports `dirty` so the record shows the run was not reproducible.\n *\n * `subsample` shrinks the corpus to a chunk budget, for screening runs where a\n * full arm costs too much to iterate on. It keeps every gold-bearing file and\n * draws distractors deterministically. This is a real change to what is being\n * measured — a smaller distractor pool makes retrieval easier and inflates\n * every metric — so it is recorded in the record and folded into the worktree\n * path, and it needs a `ref`.\n */\nexport function pinCorpus(repoRoot: string, ref: string | undefined, subsample?: CorpusSubsampleRequest): PinnedCorpus {\n\tconst dirty = git(repoRoot, [\"status\", \"--porcelain\"]).length > 0;\n\tif (!ref) {\n\t\tif (subsample) {\n\t\t\t// Subsampling deletes files. Against the live checkout that is the\n\t\t\t// user's source tree, so this refuses rather than asks.\n\t\t\tthrow new Error(\n\t\t\t\t\"subsampling requires --corpus-ref: it deletes files, and the working tree is not ours to cut\",\n\t\t\t);\n\t\t}\n\t\t// The live working tree is the user's checkout; deleting files from it to\n\t\t// tidy a measurement would be an unforgivable trade. Working-tree runs\n\t\t// are already stamped non-reproducible, so they carry the contamination.\n\t\treturn {\n\t\t\tcwd: repoRoot,\n\t\t\tsha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\t\tfromWorkingTree: true,\n\t\t\tdirty,\n\t\t\texcluded: [],\n\t\t\tdispose: () => {},\n\t\t};\n\t}\n\n\tconst sha = git(repoRoot, [\"rev-parse\", ref]);\n\t// Deterministic path, not mkdtemp: the embedding store is keyed by a hash of\n\t// the corpus directory, so a fresh temp path every run would re-embed all\n\t// ~17k chunks (minutes) instead of reusing the store built for this exact\n\t// SHA. The worktree is still removed afterwards; only the store persists.\n\t//\n\t// The subsample is part of the key. Without it a screening run and a full\n\t// run at the same SHA would share this path *and* the store derived from it,\n\t// so the second would silently score the first's index — a wrong number that\n\t// looks entirely normal.\n\t// No chunk cap in the key: the file set is cap-independent by construction\n\t// (see `applySubsample`), so cap arms share one worktree. The *store* key\n\t// does carry the cap, because the vectors differ.\n\tconst subsampleKey = subsample ? `-fast${subsample.targetChunks}s${subsample.seed}` : \"\";\n\tconst dir = path.join(tmpdir(), `hoocode-search-eval-${sha.slice(0, 12)}${subsampleKey}`);\n\tif (existsSync(dir)) {\n\t\t// Left behind by an interrupted run — drop it so `worktree add` succeeds.\n\t\ttry {\n\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t} catch {\n\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\tgit(repoRoot, [\"worktree\", \"prune\"]);\n\t\t}\n\t}\n\tgit(repoRoot, [\"worktree\", \"add\", \"--detach\", dir, sha]);\n\n\tconst excluded: string[] = [];\n\tfor (const rel of CORPUS_EXCLUSIONS) {\n\t\tconst target = path.join(dir, rel);\n\t\tif (existsSync(target)) {\n\t\t\trmSync(target, { force: true });\n\t\t\texcluded.push(rel);\n\t\t}\n\t}\n\n\tconst subsampleInfo = subsample ? applySubsample(dir, subsample) : undefined;\n\n\treturn {\n\t\tcwd: dir,\n\t\tsha,\n\t\tfromWorkingTree: false,\n\t\tdirty: false,\n\t\texcluded,\n\t\tsubsample: subsampleInfo,\n\t\tdispose: () => {\n\t\t\ttry {\n\t\t\t\tgit(repoRoot, [\"worktree\", \"remove\", \"--force\", dir]);\n\t\t\t} catch {\n\t\t\t\trmSync(dir, { recursive: true, force: true });\n\t\t\t}\n\t\t},\n\t};\n}\n\n/** `<binary> --version`, or undefined when it cannot be run. */\nfunction probeBinaryVersion(binaryPath: string | undefined): string | undefined {\n\tif (!binaryPath) return undefined;\n\ttry {\n\t\treturn execFileSync(binaryPath, [\"--version\"], { encoding: \"utf-8\" }).trim();\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\nexport function collectProvenance(\n\trepoRoot: string,\n\tcorpus: PinnedCorpus,\n\tcorpusRef: string,\n\tservice: EmbsearchService | undefined,\n\tembsearchBinary?: string,\n\thybridService?: EmbsearchService,\n\tmodelDir?: string,\n\ttiming?: EvalProvenance[\"timing\"],\n\tchunkMaxChars?: number,\n): EvalProvenance {\n\tconst state = service?.getState();\n\tconst phase = state?.phase ?? \"absent\";\n\treturn {\n\t\ttimestampMs: Date.now(),\n\t\tcorpusSha: corpus.sha,\n\t\tcorpusRef,\n\t\tcorpusFromWorkingTree: corpus.fromWorkingTree,\n\t\tcorpusDirty: corpus.dirty,\n\t\tcorpusExcluded: corpus.excluded,\n\t\tcorpusSubsample: corpus.subsample,\n\t\tharnessSha: git(repoRoot, [\"rev-parse\", \"HEAD\"]),\n\t\tretrievalSourceHash: hashRetrievalSource(repoRoot),\n\t\tembedder: {\n\t\t\t// `ready` is the only phase the service reaches with a real embedder:\n\t\t\t// it rejects the mock backend at startup, so availability here also\n\t\t\t// certifies the numbers came from a genuine ONNX build.\n\t\t\tavailable: service?.isAvailable() ?? false,\n\t\t\treason: state && \"reason\" in state ? state.reason : undefined,\n\t\t\tchunkCount: state?.phase === \"ready\" ? state.chunkCount : undefined,\n\t\t\tphase,\n\t\t\tbinaryPath: embsearchBinary,\n\t\t\tbinaryVersion: probeBinaryVersion(embsearchBinary),\n\t\t\tmodelId: service?.modelId(),\n\t\t\tmodelDir,\n\t\t},\n\t\tdaemonHybrid: hybridService\n\t\t\t? { available: hybridService.isAvailable(), phase: hybridService.getState().phase }\n\t\t\t: undefined,\n\t\ttiming,\n\t\tchunkMaxChars,\n\t\truntime: { node: process.version, platform: process.platform, arch: process.arch },\n\t};\n}\n\nexport function summarizeGoldSet(dataset: readonly EvalQuery[]): EvalRunRecord[\"goldSet\"] {\n\tconst byClass: Record<string, number> = {};\n\tlet goldSpanCount = 0;\n\tfor (const query of dataset) {\n\t\tbyClass[query.class] = (byClass[query.class] ?? 0) + 1;\n\t\tgoldSpanCount += query.gold.length;\n\t}\n\treturn { queryCount: dataset.length, byClass, goldSpanCount };\n}\n\nexport interface RunEvalSuiteOptions {\n\tcwd: string;\n\tdataset: readonly EvalQuery[];\n\tconfigs: readonly EvalConfig[];\n\tservice?: EmbsearchService;\n\t/** Second service backed by a daemon-side BM25 hybrid store, for the\n\t *  `daemon-hybrid` configs. Absent means those rows are omitted. */\n\thybridService?: EmbsearchService;\n\tonQuery?: (index: number, query: EvalQuery) => void;\n}\n\nexport async function runEvalSuite(options: RunEvalSuiteOptions): Promise<{\n\taggregates: EvalAggregate[];\n\tperQuery: EvalRunRecord[\"perQuery\"];\n}> {\n\tconst { cwd, dataset, configs, service } = options;\n\tconst totals = new Map<string, EvalAggregate>();\n\tconst perQuery: EvalRunRecord[\"perQuery\"] = [];\n\n\tfor (const [index, evalQuery] of dataset.entries()) {\n\t\toptions.onQuery?.(index, evalQuery);\n\t\tconst results = await evaluateQuery(cwd, evalQuery, configs, service, options.hybridService);\n\t\tperQuery.push({ id: evalQuery.id, class: evalQuery.class, results });\n\t\tfor (const result of results) {\n\t\t\tconst total = totals.get(result.label) ?? {\n\t\t\t\tlabel: result.label,\n\t\t\t\trecallAt1: 0,\n\t\t\t\trecallAt5: 0,\n\t\t\t\trecallAt10: 0,\n\t\t\t\trecallAt50: 0,\n\t\t\t\tmrr: 0,\n\t\t\t\tn: 0,\n\t\t\t\tdegraded: 0,\n\t\t\t};\n\t\t\ttotal.recallAt1 += result.recallAt1;\n\t\t\ttotal.recallAt5 += result.recallAt5;\n\t\t\ttotal.recallAt10 += result.recallAt10;\n\t\t\ttotal.recallAt50 += result.recallAt50;\n\t\t\ttotal.mrr += result.mrr;\n\t\t\ttotal.n++;\n\t\t\tif (result.degraded) total.degraded++;\n\t\t\ttotals.set(result.label, total);\n\t\t}\n\t}\n\n\tconst aggregates = configs\n\t\t.map((config) => totals.get(config.label))\n\t\t.filter((total): total is EvalAggregate => total !== undefined)\n\t\t.map((total) => ({\n\t\t\t...total,\n\t\t\trecallAt1: total.recallAt1 / total.n,\n\t\t\trecallAt5: total.recallAt5 / total.n,\n\t\t\trecallAt10: total.recallAt10 / total.n,\n\t\t\trecallAt50: total.recallAt50 / total.n,\n\t\t\tmrr: total.mrr / total.n,\n\t\t}));\n\n\treturn { aggregates, perQuery };\n}\n\nexport function formatAggregateTable(aggregates: readonly EvalAggregate[]): string {\n\tconst pct = (x: number) => `${Math.round(x * 100)}%`.padStart(5);\n\tconst lines = [\n\t\t\"config           |  R@1  |  R@5  | R@10  | R@50  |  MRR  | notes\",\n\t\t\"-----------------|-------|-------|-------|-------|-------|------\",\n\t];\n\tfor (const a of aggregates) {\n\t\tconst notes = a.degraded === a.n ? \"degraded to lexical\" : a.degraded > 0 ? `${a.degraded}/${a.n} degraded` : \"\";\n\t\tlines.push(\n\t\t\t`${a.label.padEnd(16)} | ${pct(a.recallAt1)} | ${pct(a.recallAt5)} | ${pct(a.recallAt10)} | ` +\n\t\t\t\t`${pct(a.recallAt50)} | ${a.mrr.toFixed(3)} | ${notes}`,\n\t\t);\n\t}\n\treturn lines.join(\"\\n\");\n}\n"]}

package/examples/extensions/custom-provider-anthropic/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-custom-provider-anthropic",
 	"private": true,
-	"version": "0.3.59",
+	"version": "0.3.60",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/examples/extensions/custom-provider-gitlab-duo/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-custom-provider-gitlab-duo",
 	"private": true,
-	"version": "0.3.59",
+	"version": "0.3.60",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/examples/extensions/sandbox/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-sandbox",
 	"private": true,
-	"version": "0.3.59",
+	"version": "0.3.60",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/examples/extensions/with-deps/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-with-deps",
 	"private": true,
-	"version": "0.3.59",
+	"version": "0.3.60",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@kolisachint/hoocode-agent",
-	"version": "0.5.59",
+	"version": "0.5.60",
 	"description": "Coding agent CLI with read, bash, edit, write tools and session management",
 	"type": "module",
 	"hoocodeConfig": {
@@ -50,9 +50,9 @@
 		"prepublishOnly": "npm run clean && npm run build"
 	},
 	"dependencies": {
-		"@kolisachint/hoocode-agent-core": "^0.5.59",
-		"@kolisachint/hoocode-ai": "^0.5.59",
-		"@kolisachint/hoocode-tui": "^0.5.59",
+		"@kolisachint/hoocode-agent-core": "^0.5.60",
+		"@kolisachint/hoocode-ai": "^0.5.60",
+		"@kolisachint/hoocode-tui": "^0.5.60",
 		"@silvia-odwyer/photon-node": "^0.3.4",
 		"chalk": "^5.5.0",
 		"cli-highlight": "^2.1.11",