@gamaze/hicortex 0.22.0 → 0.22.2

package/dist/eval/relevance-eval.js

@@ -60,17 +60,33 @@
  * Run:
  *   npm run eval:relevance -- <snapshot.db> [prompts.json] [report.md] \
  *       [--judge-delay-ms=2000] [--max-calls=900] [--resume] \
- *       [--verdicts-json=path] [--verdicts-jsonl=path]
+ *       [--verdicts-json=path] [--verdicts-jsonl=path] \
+ *       [--now=<ISO>] [--runs=N]
+ *
+ * #458 clock pin + judge variance protocol:
+ *   - `--now=<ISO>` pins the clock the retrieve sweep scores against (the
+ *     retrieve() `now` seam) — before/after runs become wall-clock-independent
+ *     (default: live clock, the pre-#458 behavior).
+ *   - `--runs=N` (default 1) runs judge phases 1+2 N times over the SAME
+ *     selections (one retrieve sweep) and reports the run-to-run noise floor
+ *     (median + spread + selection-identical flip counts, §15b). Phases 3+4
+ *     stay single-shot; the shared --max-calls budget spans all runs and the
+ *     default scales ×N unless --max-calls was passed explicitly.
  */
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseArgs = parseArgs;
+exports.scaleMaxCalls = scaleMaxCalls;
+exports.median = median;
+exports.computeJudgeVariance = computeJudgeVariance;
 const node_fs_1 = require("node:fs");
 const node_path_1 = require("node:path");
 const node_os_1 = require("node:os");
 const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
 const eval_db_js_1 = require("./eval-db.js");
+const eval_clock_js_1 = require("./eval-clock.js");
 const embedder_js_1 = require("../embedder.js");
 const retrieval_js_1 = require("../retrieval.js");
 const recall_index_js_1 = require("../recall-index.js");
@@ -379,7 +395,7 @@ function computeSweepSubsetIndices(prompts) {
 // ---------------------------------------------------------------------------
 // Retrieve sweep (scope OFF vs ON) — LLM-free system-under-test
 // ---------------------------------------------------------------------------
-async function runRetrieveSweep(db, prompts) {
+async function runRetrieveSweep(db, prompts, opts) {
     const embedCache = new Map();
     for (const p of prompts) {
         if (!embedCache.has(p.prompt))
@@ -401,6 +417,9 @@ async function runRetrieveSweep(db, prompts) {
                 limit: RESULT_K,
                 queryEmbedding,
                 noStrengthen: true,
+                // #458: the pinned clock (undefined = live) — identical selections
+                // across before/after runs regardless of when each run executes.
+                ...(opts?.now ? { now: opts.now } : {}),
                 ...scopeOpts,
             });
             const topIds = retrieved.map((r) => r.id);
@@ -1063,7 +1082,7 @@ function similarityBucketLabel(sim) {
     }
     return ">=1.00";
 }
-function buildVerdictRows(prompts, results, sweepRows) {
+function buildVerdictRows(prompts, results, sweepRows, runsTotal = 1) {
     const byKey = new Map();
     for (const r of results) {
         const pIdx = prompts.findIndex((p) => p.prompt === r.prompt.prompt && p.agent === r.prompt.agent);
@@ -1104,6 +1123,7 @@ function buildVerdictRows(prompts, results, sweepRows) {
                     line_reason: lv ? lv.reason : lineOk ? "" : r.lineVerdicts.judgeError,
                     rendered_line: row ? (0, recall_index_js_1.formatIndexLine)(row, PROD_TITLE_CHARS) : "",
                     snippet_len_variant: String(PROD_TITLE_CHARS),
+                    ...(runsTotal > 1 ? { run: 1 } : {}),
                 });
             }
         }
@@ -1148,13 +1168,18 @@ function buildVerdictRows(prompts, results, sweepRows) {
     }
     return rows;
 }
+/** Parsed and exported for tests (#458). Throws on an invalid --runs value
+ *  (NaN or <1) — fail-explicit, never a silent default. */
 function parseArgs(argv) {
     const positionals = [];
     let judgeDelayMs = DEFAULT_JUDGE_DELAY_MS;
     let maxCalls = DEFAULT_MAX_CALLS;
+    let maxCallsExplicit = false;
     let resume = false;
     let verdictsJsonPath = VERDICTS_JSON_PATH;
     let verdictsJsonlPath = VERDICTS_JSONL_PATH;
+    let nowIso;
+    let runs = 1;
     for (const arg of argv) {
         if (arg === "--resume") {
             resume = true;
@@ -1164,17 +1189,132 @@ function parseArgs(argv) {
         if (m) {
             if (m[1] === "judge-delay-ms")
                 judgeDelayMs = parseInt(m[2], 10);
-            else if (m[1] === "max-calls")
+            else if (m[1] === "max-calls") {
                 maxCalls = parseInt(m[2], 10);
+                maxCallsExplicit = true;
+            }
             else if (m[1] === "verdicts-json")
                 verdictsJsonPath = m[2];
             else if (m[1] === "verdicts-jsonl")
                 verdictsJsonlPath = m[2];
+            else if (m[1] === "now")
+                nowIso = m[2];
+            else if (m[1] === "runs") {
+                const parsed = Number(m[2]);
+                if (!Number.isInteger(parsed) || parsed < 1) {
+                    throw new Error(`--runs must be an integer >= 1 (got "${m[2]}")`);
+                }
+                runs = parsed;
+            }
             continue;
         }
         positionals.push(arg);
     }
-    return { positionals, judgeDelayMs, maxCalls, resume, verdictsJsonPath, verdictsJsonlPath };
+    return {
+        positionals,
+        judgeDelayMs,
+        maxCalls,
+        maxCallsExplicit,
+        resume,
+        verdictsJsonPath,
+        verdictsJsonlPath,
+        nowIso,
+        runs,
+    };
+}
+/**
+ * #458 — resolve the effective --max-calls budget. The ONE shared budget
+ * spans ALL judge runs, so when --runs>1 and the caller did NOT pass
+ * --max-calls explicitly, the default scales ×N (each run re-judges every
+ * unit). An explicit budget always wins — the caller sized it for the whole
+ * invocation. Pure, exported for tests.
+ */
+function scaleMaxCalls(runs, explicitMaxCalls, defaultMaxCalls) {
+    if (explicitMaxCalls !== undefined)
+        return explicitMaxCalls;
+    return runs > 1 ? defaultMaxCalls * runs : defaultMaxCalls;
+}
+// ---------------------------------------------------------------------------
+// Judge variance (#458) — median, spread, and selection-identical flip
+// counts over N runs of the SAME selections. Pure + synthetic-verdict
+// testable; the orchestration (running the judge N times) lives in main.
+// ---------------------------------------------------------------------------
+/** Median of a numeric sample (even count → mean of the two middle values).
+ *  Empty input → NaN, the mean() convention above. Does not mutate input. */
+function median(xs) {
+    if (xs.length === 0)
+        return NaN;
+    const sorted = [...xs].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 1 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+}
+/**
+ * Compute the #458 judge-variance account over N runs of the same selections:
+ * per-metric median + spread (the noise floor), and pairwise verdict flip
+ * counts (rows = one Q-label judgment on one (prompt,mode) unit; units flip
+ * if any row does). `perRunVerdicts[r]` is run r+1's units in a stable
+ * order/identity; `metricSeries[s].values[r]` is run r+1's headline value.
+ */
+function computeJudgeVariance(perRunVerdicts, metricSeries) {
+    const runs = perRunVerdicts.length;
+    const flips = [];
+    let maxRows = 0;
+    let maxUnits = 0;
+    for (let i = 0; i < runs; i++) {
+        for (let j = i + 1; j < runs; j++) {
+            const byKeyI = new Map(perRunVerdicts[i].map((u) => [u.key, u.verdicts]));
+            let rowsCompared = 0;
+            let rowsFlipped = 0;
+            let unitsCompared = 0;
+            let unitsFlipped = 0;
+            for (const unitJ of perRunVerdicts[j]) {
+                const vi = byKeyI.get(unitJ.key);
+                if (!vi || !Array.isArray(vi) || !Array.isArray(unitJ.verdicts))
+                    continue; // judge_error/absent — not comparable
+                const vj = unitJ.verdicts;
+                unitsCompared++;
+                let unitFlipped = false;
+                const byLabelJ = new Map(vj.map((v) => [v.id, v.verdict]));
+                for (const rowI of vi) {
+                    const verdictJ = byLabelJ.get(rowI.id);
+                    if (verdictJ === undefined)
+                        continue; // label omitted in run j — not comparable
+                    rowsCompared++;
+                    if (verdictJ !== rowI.verdict) {
+                        rowsFlipped++;
+                        unitFlipped = true;
+                    }
+                }
+                if (unitFlipped)
+                    unitsFlipped++;
+            }
+            flips.push({ runA: i + 1, runB: j + 1, rowsCompared, rowsFlipped, unitsCompared, unitsFlipped });
+            maxRows = Math.max(maxRows, rowsFlipped);
+            maxUnits = Math.max(maxUnits, unitsFlipped);
+        }
+    }
+    const metrics = metricSeries.map((s) => {
+        const usable = s.values.filter((v) => v !== null && Number.isFinite(v));
+        if (usable.length === 0)
+            return { ...s, median: null, spreadPts: null };
+        return {
+            ...s,
+            median: median(usable),
+            spreadPts: (Math.max(...usable) - Math.min(...usable)) * 100,
+        };
+    });
+    const judgeErrorUnitsPerRun = perRunVerdicts.map((units) => units.filter((u) => !Array.isArray(u.verdicts)).length);
+    return {
+        runs,
+        metrics,
+        flips,
+        maxRowsFlipped: maxRows,
+        maxUnitsFlipped: maxUnits,
+        meanRowsFlipped: flips.length > 0 ? mean(flips.map((f) => f.rowsFlipped)) : 0,
+        meanUnitsFlipped: flips.length > 0 ? mean(flips.map((f) => f.unitsFlipped)) : 0,
+        totalComparableRows: flips.length > 0 ? flips[0].rowsCompared : 0,
+        judgeErrorUnitsPerRun,
+    };
 }
 // ---------------------------------------------------------------------------
 // Reporting
@@ -1207,7 +1347,8 @@ function renderReport(args) {
     L.push(`Snapshot: \`${args.snapshotPath}\`  \n` +
         `Prompts: \`${args.promptsPath}\` (${prompts.length} prompts; requested ${SOURCE_TAGS.length * N_PER_SOURCE} — ` +
         `see §0 for any per-source shortfall)  \n` +
-        `Generated: ${args.generatedAt}\n`);
+        `Generated: ${args.generatedAt}  \n` +
+        `Clock: ${args.clock} (the instant the retrieve sweep scored against, #458)${args.variance ? `  \nJudge runs: ${args.variance.runs} (phases 1+2 re-judged over fixed selections — see §15b)` : ""}\n`);
     L.push(`_Spec: \`specs/2026-08-02-relevance-eval.md\`. Real agent prompts replayed through \`retrieve()\` on a ` +
         `READONLY snapshot; GLM-5.2 (z.ai) is the JUDGE only — retrieve() is LLM-free. Two independent, BLIND judge ` +
         `calls per (prompt × mode): \`full_verdict\` (up to 2000 chars) and \`line_verdict\` (ONLY the rendered ` +
@@ -1608,6 +1749,38 @@ function renderReport(args) {
         L.push(`**Scope ON−OFF delta (precision@${PROD_MENU_K}):** ${deltaStr} — **${distinguishable}**\n`);
     }
     // =====================================================================
+    // §15b — judge variance (#458): the measured run-to-run noise floor.
+    // Rendered ONLY when --runs>1.
+    // =====================================================================
+    if (args.variance) {
+        const v = args.variance;
+        L.push(`## 15b. Judge variance — run-to-run noise floor (#458, ${v.runs} runs)\n`);
+        L.push(`_Judge phases 1+2 ran ${v.runs} times over the SAME selections (one retrieve sweep feeds every run — ` +
+            `the sweep precedes judging, so selections are fixed under a live clock too). ALL drift measured here is ` +
+            `judge-only instrument noise, the class PR C evidenced at 38/380 flipped verdicts. Read every body section ` +
+            `above as run 1; this section carries the median and the band._\n`);
+        L.push("| metric | " + v.metrics.map((_, i) => `run ${i + 1}`).join(" | ") + " | median | spread (max−min) |");
+        L.push("|---|" + v.metrics.map(() => "---").join("|") + "|---|---|");
+        for (const m of v.metrics) {
+            const cells = m.values.map((x) => (x === null || !Number.isFinite(x) ? "n/a" : pct(x))).join(" | ");
+            L.push(`| ${m.label} | ${cells} | **${m.median !== null && Number.isFinite(m.median) ? pct(m.median) : "n/a"}** | ${m.spreadPts !== null && Number.isFinite(m.spreadPts) ? `${m.spreadPts.toFixed(1)}pts` : "n/a"} |`);
+        }
+        L.push("");
+        L.push(`_Pairwise verdict flips on selection-identical units (full_verdict rows; judge_error units excluded from ` +
+            `the denominators — error units per run: ${v.judgeErrorUnitsPerRun.map((e, i) => `r${i + 1} ${e}`).join(", ")}). ` +
+            `A verdict ROW is one Q-label judgment on one (prompt, mode) unit; a UNIT flips if any of its rows differ._\n`);
+        L.push("| pair | rows flipped / compared | units flipped / compared |");
+        L.push("|---|---|---|");
+        for (const f of v.flips) {
+            L.push(`| r${f.runA}↔r${f.runB} | ${f.rowsFlipped} / ${f.rowsCompared} | ${f.unitsFlipped} / ${f.unitsCompared} |`);
+        }
+        L.push("");
+        L.push(`**Noise floor:** max pairwise flips ${v.maxRowsFlipped} rows / ${v.maxUnitsFlipped} units; mean ` +
+            `${v.meanRowsFlipped.toFixed(1)} rows / ${v.meanUnitsFlipped.toFixed(1)} units over ${v.flips.length} pair(s); ` +
+            `total comparable rows ${v.totalComparableRows}. A before/after headline delta smaller than the spread row ` +
+            `above is NOT resolvable by this instrument — report it as within judge noise._\n`);
+    }
+    // =====================================================================
     // §16 — DECISION THRESHOLDS (spec §9) — filled with measured values
     // =====================================================================
     L.push("## 16. Decision thresholds (spec §9) — measured values + triggered actions\n");
@@ -1743,7 +1916,8 @@ function renderReport(args) {
 function usage() {
     console.error("Usage: relevance-eval.js <snapshot.db> [prompts.json] [report.md] " +
         "[--judge-delay-ms=2000] [--max-calls=900] [--resume] " +
-        "[--verdicts-json=path] [--verdicts-jsonl=path]\n" +
+        "[--verdicts-json=path] [--verdicts-jsonl=path] " +
+        "[--now=<ISO>] [--runs=N]\n" +
         "  <snapshot.db>  — REQUIRED. Readonly bedrock snapshot (opened via openSnapshot).\n" +
         "  [prompts.json] — omitted: build the v2 corpus fresh (20/source × 5 sources, spec §2),\n" +
         "                   saved to data/prompts.json. A valid existing v2 set is reused verbatim.\n" +
@@ -1753,6 +1927,14 @@ function usage() {
         "                   JSONL paths (default data/relevance-verdicts.json[l]). ALWAYS override\n" +
         "                   these for a re-run against a new snapshot — otherwise a re-run silently\n" +
         "                   clobbers a prior baseline's raw verdicts.\n" +
+        "  --now=<ISO>    — #458: pin the clock the retrieve sweep scores against (full ISO 8601\n" +
+        "                   instant). Before/after runs become wall-clock-independent. Default: the\n" +
+        "                   live clock. Invalid input is an error, never a silent live fallback.\n" +
+        "  --runs=N       — #458: run judge phases 1+2 N times over the SAME selections (one\n" +
+        "                   retrieve sweep) and report the run-to-run noise floor (median + spread\n" +
+        "                   + selection-identical flip counts, §15b). Default 1 (single-run\n" +
+        "                   behavior). The shared --max-calls budget spans ALL runs; without an\n" +
+        "                   explicit --max-calls the default scales ×N. Phases 3+4 stay single-shot.\n" +
         "Env: ZAI_API_KEY (required — GLM-5.2 via z.ai is the judge)\n" +
         "Preconditions: /tmp/hermes-{lenny,raider,nano}-state.db (readonly Hermes state DBs) when\n" +
         "building the corpus fresh; ~/.claude/projects/.../infrastructure + .../aironic-marine dirs.");
@@ -1765,10 +1947,39 @@ async function main() {
     (0, retrieval_js_1.configureDecay)();
     (0, retrieval_js_1.configureRecall)();
     (0, retrieval_js_1.configureSessionIntent)();
-    const { positionals, judgeDelayMs, maxCalls, resume, verdictsJsonPath, verdictsJsonlPath } = parseArgs(process.argv.slice(2));
+    let parsed;
+    try {
+        parsed = parseArgs(process.argv.slice(2));
+    }
+    catch (err) {
+        console.error(`[relevance-eval] ${err instanceof Error ? err.message : err}`);
+        usage();
+    }
+    const { positionals, judgeDelayMs, maxCalls: maxCallsArg, maxCallsExplicit, resume, verdictsJsonPath, verdictsJsonlPath, nowIso, runs, } = parsed;
     const [snapshotArg, promptsSourceArg, reportArg] = positionals;
     if (!snapshotArg)
         usage();
+    // #458: pin the eval clock (invalid → usage-style error, never a silent
+    // live fallback — a run that believed it was pinned would silently drift).
+    let now;
+    try {
+        now = (0, eval_clock_js_1.parsePinnedNow)(nowIso);
+    }
+    catch (err) {
+        console.error(`[relevance-eval] ${err instanceof Error ? err.message : err}`);
+        usage();
+    }
+    console.log(`[relevance-eval] clock: ${(0, eval_clock_js_1.clockLabel)(now)}${now === null ? " (pass --now=<ISO> to pin before/after runs)" : ""}`);
+    // #458: the ONE shared call budget spans ALL judge runs. When --runs>1 and
+    // --max-calls was not passed explicitly, scale the default ×N — loudly.
+    const maxCalls = scaleMaxCalls(runs, maxCallsExplicit ? maxCallsArg : undefined, DEFAULT_MAX_CALLS);
+    if (runs > 1 && !maxCallsExplicit) {
+        console.log(`[relevance-eval] --runs=${runs} without an explicit --max-calls: scaling the default budget ` +
+            `${DEFAULT_MAX_CALLS} × ${runs} = ${maxCalls} (the budget spans ALL runs)`);
+    }
+    if (runs > 1) {
+        console.log(`[relevance-eval] judge variance protocol: phases 1+2 run ${runs}× over the fixed selections (see §15b)`);
+    }
     const apiKey = process.env.ZAI_API_KEY;
     if (!apiKey) {
         console.error("relevance-eval: ZAI_API_KEY env var is required (GLM-5.2 via z.ai is the judge)");
@@ -1809,8 +2020,8 @@ async function main() {
         console.log(`[relevance-eval] snapshot opened readonly: ${snapshotArg} (${prompts.length} prompts × 2 modes = ${prompts.length * 2} retrieve() calls)`);
         console.log("[relevance-eval] loading bge-small-en-v1.5 + replaying retrieve()...");
         const t0 = Date.now();
-        const results = await runRetrieveSweep(db, prompts);
-        console.log(`[relevance-eval] retrieve sweep done in ${Date.now() - t0}ms (${results.length} prompt×mode batches, LLM-free)`);
+        const results = await runRetrieveSweep(db, prompts, { now: now ?? undefined });
+        console.log(`[relevance-eval] retrieve sweep done in ${Date.now() - t0}ms (${results.length} prompt×mode batches, LLM-free, clock ${(0, eval_clock_js_1.clockLabel)(now)})`);
         // ---- judge run state (rate limiting + resumability, spec §6b) ----
         (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(verdictsJsonlPath), { recursive: true });
         if (!resume)
@@ -1838,25 +2049,53 @@ async function main() {
             console.log(`[relevance-eval] --resume: ${resumedMap.size} units already checkpointed in ${verdictsJsonlPath}`);
         }
         const sweepRows = [];
+        // #458: judge phases 1+2 run `runs` times over the SAME selections (the
+        // retrieve sweep above ran ONCE, before judging — so selections are fixed
+        // across runs under a live clock too). Per-run verdict arrays feed the
+        // §15b variance section; run 1's verdicts land in `results` so every body
+        // section renders exactly as a single-run report. Checkpoint keys carry a
+        // 1-based run prefix when runs>1 (`full|r1|…`) so sidecar entries never
+        // collide across runs; the legacy single-run keys are byte-unchanged when
+        // runs==1 (old single-run sidecars still resume — an N-run invocation
+        // starts a fresh sidecar by default).
+        const perRunFull = [];
+        const perRunLine = [];
+        const judgeRunKey = (phase, run, ...parts) => runs > 1 ? [phase, `r${run}`, ...parts].join("|") : [phase, ...parts].join("|");
         try {
-            // ---- Phase 1: full_verdict (all prompts × both modes) ----
-            console.log(`[relevance-eval] phase 1/4: full-content judging (${results.length} units)...`);
-            for (const r of results) {
-                const key = `full|${prompts.indexOf(r.prompt)}|${r.mode}`;
-                const expectedLabels = new Set(r.topIds.map((_, i) => `Q${i + 1}`));
-                const promptText = buildFullContentJudgePrompt(r.prompt.prompt, r.topIds, r.contents);
-                r.fullVerdicts = await judgeVerdictUnit(judgeState, key, promptText, expectedLabels);
+            // ---- Phase 1: full_verdict (all prompts × both modes), N runs ----
+            for (let run = 1; run <= runs; run++) {
+                console.log(`[relevance-eval] phase 1/4: full-content judging (${results.length} units, run ${run}/${runs})...`);
+                const units = [];
+                for (const r of results) {
+                    const pIdx = prompts.indexOf(r.prompt);
+                    const key = judgeRunKey("full", run, pIdx, r.mode);
+                    const expectedLabels = new Set(r.topIds.map((_, i) => `Q${i + 1}`));
+                    const promptText = buildFullContentJudgePrompt(r.prompt.prompt, r.topIds, r.contents);
+                    const verdicts = await judgeVerdictUnit(judgeState, key, promptText, expectedLabels);
+                    units.push({ key: `${pIdx}|${r.mode}`, verdicts });
+                    if (run === 1)
+                        r.fullVerdicts = verdicts;
+                }
+                perRunFull.push(units);
             }
-            // ---- Phase 2: line_verdict at maxLen=PROD_TITLE_CHARS (production render) ----
-            console.log(`[relevance-eval] phase 2/4: line-only judging at production maxLen=${PROD_TITLE_CHARS} (${results.length} units)...`);
-            for (const r of results) {
-                const key = `line|${prompts.indexOf(r.prompt)}|${r.mode}`;
-                const expectedLabels = new Set(r.topIds.map((_, i) => `Q${i + 1}`));
-                const lines = {};
-                for (const id of r.topIds)
-                    lines[id] = (0, recall_index_js_1.formatIndexLine)(r.rows[id], PROD_TITLE_CHARS);
-                const promptText = buildLineJudgePrompt(r.prompt.prompt, r.topIds, lines);
-                r.lineVerdicts = await judgeVerdictUnit(judgeState, key, promptText, expectedLabels);
+            // ---- Phase 2: line_verdict at maxLen=PROD_TITLE_CHARS (production render), N runs ----
+            for (let run = 1; run <= runs; run++) {
+                console.log(`[relevance-eval] phase 2/4: line-only judging at production maxLen=${PROD_TITLE_CHARS} (${results.length} units, run ${run}/${runs})...`);
+                const units = [];
+                for (const r of results) {
+                    const pIdx = prompts.indexOf(r.prompt);
+                    const key = judgeRunKey("line", run, pIdx, r.mode);
+                    const expectedLabels = new Set(r.topIds.map((_, i) => `Q${i + 1}`));
+                    const lines = {};
+                    for (const id of r.topIds)
+                        lines[id] = (0, recall_index_js_1.formatIndexLine)(r.rows[id], PROD_TITLE_CHARS);
+                    const promptText = buildLineJudgePrompt(r.prompt.prompt, r.topIds, lines);
+                    const verdicts = await judgeVerdictUnit(judgeState, key, promptText, expectedLabels);
+                    units.push({ key: `${pIdx}|${r.mode}`, verdicts });
+                    if (run === 1)
+                        r.lineVerdicts = verdicts;
+                }
+                perRunLine.push(units);
             }
             // ---- Phase 3: snippet-length sweep (40-subset, scope OFF, 4 variants) ----
             const offResultByPromptIdx = new Map();
@@ -1905,8 +2144,47 @@ async function main() {
                 throw err;
             }
         }
+        // ---- #458: per-run headline metrics + the judge-variance account ----
+        // Abort-tolerant like everything downstream of the judge loop: a run
+        // stopped by the budget / error-rate gate leaves partial per-run arrays,
+        // and the report still renders (run-1 verdicts + a null variance).
+        let variance = null;
+        const varianceComplete = perRunFull.length === runs &&
+            perRunLine.length === runs &&
+            perRunFull.every((u) => u.length === results.length) &&
+            perRunLine.every((u) => u.length === results.length);
+        if (runs > 1 && !varianceComplete) {
+            console.log(`[relevance-eval] judge runs incomplete (${perRunFull.length}/${runs} full-verdict runs filled) — §15b variance skipped`);
+        }
+        if (runs > 1 && varianceComplete) {
+            const runResults = (run) => results.map((r, i) => ({
+                ...r,
+                fullVerdicts: perRunFull[run - 1][i].verdicts,
+                lineVerdicts: perRunLine[run - 1][i].verdicts,
+            }));
+            const series = [
+                { label: `precision@${PROD_MENU_K} OFF`, values: [] },
+                { label: `precision@${PROD_MENU_K} ON`, values: [] },
+                { label: "snippet_failure_rate OFF", values: [] },
+            ];
+            for (let run = 1; run <= runs; run++) {
+                const rs = runResults(run);
+                const offRun = rs.filter((r) => r.mode === "off");
+                const onRun = rs.filter((r) => r.mode === "on");
+                const mOff = computeMetricsAtK(offRun, PROD_MENU_K);
+                const mOn = computeMetricsAtK(onRun, PROD_MENU_K);
+                const gap = computeSnippetGap(offRun);
+                series[0].values.push(mOff ? mOff.precision : null);
+                series[1].values.push(mOn ? mOn.precision : null);
+                series[2].values.push(gap.fullRelevantTotal > 0 ? gap.snippetFailure / gap.fullRelevantTotal : null);
+            }
+            variance = computeJudgeVariance(perRunFull, series);
+            console.log(`[relevance-eval] judge variance over ${runs} runs (selections fixed): ` +
+                variance.metrics.map((m) => `${m.label} median ${m.median !== null ? pct(m.median) : "n/a"}, spread ${m.spreadPts !== null ? m.spreadPts.toFixed(1) + "pts" : "n/a"}`).join("; ") +
+                `; max pairwise flips ${variance.maxRowsFlipped}/${variance.totalComparableRows} rows, ${variance.maxUnitsFlipped} units`);
+        }
         // ---- persist raw verdicts (source of truth, spec §6) ----
-        const verdictRows = buildVerdictRows(prompts, results, sweepRows);
+        const verdictRows = buildVerdictRows(prompts, results, sweepRows, runs);
         (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(verdictsJsonPath), { recursive: true });
         (0, node_fs_1.writeFileSync)(verdictsJsonPath, JSON.stringify(verdictRows, null, 2), "utf-8");
         console.log(`[relevance-eval] raw verdicts saved: ${verdictsJsonPath} (${verdictRows.length} rows), checkpoint sidecar: ${verdictsJsonlPath}`);
@@ -1922,6 +2200,8 @@ async function main() {
             judgeState,
             generatedAt: new Date().toISOString(),
             db,
+            clock: (0, eval_clock_js_1.clockLabel)(now),
+            variance,
         });
         const reportPath = reportArg ?? DEFAULT_REPORT_PATH;
         (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(reportPath), { recursive: true });
@@ -1948,7 +2228,13 @@ async function main() {
         }
     }
 }
-main().catch((err) => {
-    console.error("[relevance-eval] FAILED:", err instanceof Error ? err.stack : String(err));
-    process.exitCode = 1;
-});
+// The importance-eval.ts guard: run main() only when executed directly, so
+// the exported pure helpers (parseArgs, scaleMaxCalls, median,
+// computeJudgeVariance — #458) are importable from the vitest suite without
+// firing the CLI.
+if (process.argv[1] && process.argv[1].endsWith("relevance-eval.js")) {
+    main().catch((err) => {
+        console.error("[relevance-eval] FAILED:", err instanceof Error ? err.stack : String(err));
+        process.exitCode = 1;
+    });
+}

package/dist/eval/run-eval.d.ts

@@ -8,10 +8,13 @@
  * never `initDb()`), never writes to the DB. Not wired into `cli.ts` — this
  * is an internal measurement tool, run via the `eval` npm script:
  *
- *   npm run eval -- <snapshot.db> [report.md]
+ *   npm run eval -- <snapshot.db> [report.md] [--now=<ISO>]
  *
- * `state.json` is expected alongside the snapshot DB (same directory) for
- * the `domainCursor`/`relinkCursor` watermarks. Report path defaults to
- * `eval-report.md` next to the DB.
+ * `--now=<ISO>` (#458) pins the clock the age/decay-derived stats (structural
+ * strength, adoption age buckets) measure against, so before/after audits are
+ * wall-clock-independent. Default: the live clock; the report header records
+ * which clock produced it. `state.json` is expected alongside the snapshot DB
+ * (same directory) for the `domainCursor`/`relinkCursor` watermarks. Report
+ * path defaults to `eval-report.md` next to the DB.
  */
 export {};

package/dist/eval/run-eval.js

@@ -9,16 +9,20 @@
  * never `initDb()`), never writes to the DB. Not wired into `cli.ts` — this
  * is an internal measurement tool, run via the `eval` npm script:
  *
- *   npm run eval -- <snapshot.db> [report.md]
+ *   npm run eval -- <snapshot.db> [report.md] [--now=<ISO>]
  *
- * `state.json` is expected alongside the snapshot DB (same directory) for
- * the `domainCursor`/`relinkCursor` watermarks. Report path defaults to
- * `eval-report.md` next to the DB.
+ * `--now=<ISO>` (#458) pins the clock the age/decay-derived stats (structural
+ * strength, adoption age buckets) measure against, so before/after audits are
+ * wall-clock-independent. Default: the live clock; the report header records
+ * which clock produced it. `state.json` is expected alongside the snapshot DB
+ * (same directory) for the `domainCursor`/`relinkCursor` watermarks. Report
+ * path defaults to `eval-report.md` next to the DB.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 const node_path_1 = require("node:path");
 const node_fs_1 = require("node:fs");
 const eval_db_js_1 = require("./eval-db.js");
+const eval_clock_js_1 = require("./eval-clock.js");
 const dups_js_1 = require("./dups.js");
 const decay_eval_js_1 = require("./decay-eval.js");
 const graph_eval_js_1 = require("./graph-eval.js");
@@ -216,7 +220,7 @@ function renderPhaseB(dups, prune, structural, backlog, graph, adoption) {
 function renderReport(input) {
     const parts = [];
     parts.push("# Hicortex Mechanical Audit Baseline (#191)\n");
-    parts.push(`Snapshot: \`${input.dbPath}\`  \nGenerated: ${input.generatedAt}  \nRead-only, zero LLM, zero human grading — mechanical measurement only.\n`);
+    parts.push(`Snapshot: \`${input.dbPath}\`  \nGenerated: ${input.generatedAt}  \nClock: ${input.clock} (#458 — the instant the age/decay-derived stats measured against)  \nRead-only, zero LLM, zero human grading — mechanical measurement only.\n`);
     parts.push(renderDups(input.dups));
     parts.push(renderDecay(input.domainBacklog, input.pruneDryRun, input.structural, input.adoption));
     parts.push(renderGraph(input.graph));
@@ -228,12 +232,34 @@ function renderReport(input) {
 // CLI entry
 // ---------------------------------------------------------------------------
 function main() {
-    const [, , dbPathArg, reportPathArg] = process.argv;
+    // #458: --now=<ISO> pins the age/decay-derived stats' clock; positionals
+    // (snapshot, report path) keep working unchanged.
+    const positionals = [];
+    let nowIso;
+    for (const arg of process.argv.slice(2)) {
+        const m = arg.match(/^--now=(.+)$/);
+        if (m)
+            nowIso = m[1];
+        else
+            positionals.push(arg);
+    }
+    const [dbPathArg, reportPathArg] = positionals;
     if (!dbPathArg) {
-        console.error("Usage: run-eval.js <snapshot.db> [report.md]");
+        console.error("Usage: run-eval.js <snapshot.db> [report.md] [--now=<ISO>]");
+        process.exitCode = 1;
+        return;
+    }
+    let now;
+    try {
+        now = (0, eval_clock_js_1.parsePinnedNow)(nowIso);
+    }
+    catch (err) {
+        console.error(`[eval] ${err instanceof Error ? err.message : err}`);
         process.exitCode = 1;
         return;
     }
+    console.log(`[eval] clock: ${(0, eval_clock_js_1.clockLabel)(now)}`);
+    const nowArg = now ?? new Date(); // the live clock, made explicit for threading
     const dbPath = dbPathArg;
     const stateDir = (0, node_path_1.dirname)(dbPath);
     const reportPath = reportPathArg ?? (0, node_path_1.join)(stateDir, "eval-report.md");
@@ -246,9 +272,9 @@ function main() {
         console.log("[eval] D4 prune dry-run...");
         const pruneDryRun = (0, decay_eval_js_1.runPruneDryRun)(db, retrieval_js_1.DEFAULT_DECAY_HALF_LIFE_DAYS);
         console.log("[eval] D4 structural strength stats...");
-        const structural = (0, decay_eval_js_1.runStructuralStrengthStats)(db);
+        const structural = (0, decay_eval_js_1.runStructuralStrengthStats)(db, nowArg);
         console.log("[eval] D4 adoption stats...");
-        const adoption = (0, decay_eval_js_1.runAdoptionStats)(db);
+        const adoption = (0, decay_eval_js_1.runAdoptionStats)(db, nowArg);
         console.log("[eval] D6 link-graph audit...");
         const graph = (0, graph_eval_js_1.runGraphAudit)(db, stateDir);
         console.log("[eval] reflection census...");
@@ -256,6 +282,7 @@ function main() {
         const report = renderReport({
             dbPath,
             generatedAt: new Date().toISOString(),
+            clock: (0, eval_clock_js_1.clockLabel)(now),
             dups,
             domainBacklog,
             pruneDryRun,