@gamaze/hicortex 0.16.0 → 0.16.1

package/dist/eval/relevance-eval.js

@@ -0,0 +1,1954 @@
+#!/usr/bin/env node
+"use strict";
+/**
+ * Real-query relevance + SNIPPET eval — recall QUALITY on real agent prompts.
+ *
+ * v2 (spec `specs/2026-08-02-relevance-eval.md`) extends the v1 selection-only
+ * eval with the SNIPPET layer: v1 asked "did retrieve() surface the right
+ * memories?" (judge sees up to 2000 chars). Production shows the agent only a
+ * ~100-char one-liner (`recall-index.ts#memoryTitle`), so a memory can be
+ * genuinely relevant while its rendered line is useless — v1 scored that as a
+ * win. v2 grades BOTH: `full_verdict` (selection quality, judge sees full
+ * content) and `line_verdict` (snippet quality, judge sees ONLY the rendered
+ * production one-liner — imported from `recall-index.ts`, never reimplemented).
+ *
+ * v2 additions (spec §4, §5, §6, §6b) layered onto the v1 base (prompt
+ * sampling, readonly snapshot handling, embed-once + neverCalledEmbed,
+ * lenient JSON parse, distribution/CI reporting):
+ *   1. Dual verdict per surfaced memory — TWO separate, blind judge calls.
+ *   2. Snippet-length sweep (100/200/300/title+1st-sentence) on a fixed
+ *      40-prompt subset (8 per source).
+ *   3. Similarity-floor + retrieval-source analysis (near-free — logged, not
+ *      re-judged).
+ *   4. Token-cost estimate (char/4) per K and per snippet-length variant.
+ *   5. ~20 rendered ACTUAL production blocks dumped into the report.
+ *   6. Redundancy — one set-level judge call per (prompt × mode) over the
+ *      production 6.
+ *   7. Rate limiting + resumability (§6b, MANDATORY): serial calls,
+ *      `--judge-delay-ms` (default 2000), exponential backoff with jitter on
+ *      429/5xx/timeout (5→10→20→40→80s, max 5 retries, respects
+ *      `Retry-After`), checkpoint-per-call to a `.jsonl` sidecar, `--resume`,
+ *      progress logging, 10%-error-rate abort, `--max-calls` budget guard
+ *      (default 900).
+ *
+ * Prompt corpus (spec §2, owner decision §11.1): EVEN split, 20 prompts per
+ * source × 5 sources — Hermes (lenny, raider, nano) + CC (the DevOps
+ * `infrastructure` project, the `aironic-marine` project). Saved to
+ * `data/prompts.json`, stable/reused verbatim once a valid v2 set exists.
+ *
+ * Judge: GLM-5.2 via z.ai — the INSTRUMENT only. It never picks candidates;
+ * retrieve() (LLM-free) does. A dedicated raw HTTP caller (NOT `LlmClient`) is
+ * used here on purpose: `LlmClient.completeReflect` bakes in a
+ * nightly-tolerant retry policy (30s/60s/120s, unlimited rate-limit patience)
+ * that conflicts with §6b's specific real-time batch policy (5/10/20/40/80s +
+ * jitter, 5 retries, a hard call budget). Implemented directly here rather
+ * than adding a second retry mode to `llm.ts` (out of scope for this eval,
+ * and another agent is concurrently working elsewhere in this repo).
+ *
+ * Honesty invariants (non-negotiable — mirror recall-sweep.ts + spec §7):
+ *   - Snapshot opened READONLY via openSnapshot — never initDb.
+ *   - noStrengthen: true on every retrieve() call.
+ *   - Real bge-small-en-v1.5 embedder, embed-once + queryEmbedding reuse.
+ *   - neverCalledEmbed self-check ABORTS the run if retrieve() ignores
+ *     queryEmbedding (would invalidate every measured number).
+ *   - GLM-5.2 is the JUDGE only, real prompts, no synthetic queries.
+ *   - Production renderer (`formatIndexLine`/`memoryTitle`) imported from
+ *     `recall-index.ts`, never reimplemented.
+ *   - judge_error batches/units excluded from every denominator, reported
+ *     separately.
+ *
+ * Run:
+ *   npm run eval:relevance -- <snapshot.db> [prompts.json] [report.md] \
+ *       [--judge-delay-ms=2000] [--max-calls=900] [--resume] \
+ *       [--verdicts-json=path] [--verdicts-jsonl=path]
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const node_os_1 = require("node:os");
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+const eval_db_js_1 = require("./eval-db.js");
+const embedder_js_1 = require("../embedder.js");
+const retrieval_js_1 = require("../retrieval.js");
+const recall_index_js_1 = require("../recall-index.js");
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const RESULT_K = 8; // top-k replayed per prompt — the MEASUREMENT WINDOW. Two
+// slots PAST the production menu (PROD_MENU_K=6) on purpose: the bubble
+// (Q7-Q8) is cap-tuning data.
+// All three mirror the shipped defaults in recall-index.ts (kept in sync
+// manually — CLAUDE.md). Updated 2026-08-03 alongside the recall-quality PR.
+const PROD_MENU_K = 5; // the shipped `recallMaxItems` default (was 6).
+// K is a reporting/labelling param here (precision@4/@6/@8 all derive from
+// the same stored verdict rows), so prior runs stay fully comparable.
+const PROD_MIN_SIMILARITY = 0.62; // the shipped `recallMinSimilarity` floor (was 0.55).
+const PROD_TITLE_CHARS = 100; // the shipped `recallTitleChars` default (was 150).
+// Methodology note: this controls what the judge SEES in the baseline
+// line_verdict pass, so eval #4's snippet_failure_rate is NOT directly
+// comparable to #3's 24.3% (judged at 150). Cross-run comparability survives
+// because SWEEP_VARIANTS still measures 150 on the 40-prompt subset — do NOT
+// prune 150 from SWEEP_VARIANTS without recording that it breaks the #3↔#4
+// comparison (lower N, wider CI, but comparable).
+const MIN_PROMPT_CHARS = 20; // drop trivial/short prompts (<20 chars)
+const MAX_MEM_CHARS_IN_JUDGE = 2000; // full_verdict per-memory cap.
+const ZAI_BASE_URL = "https://api.z.ai/api/anthropic";
+const ZAI_MODEL = "glm-5.2";
+const N_PER_SOURCE = 20; // spec §2, §11.1: even 20/source.
+const SWEEP_N_PER_SOURCE = 8; // spec §4.2: fixed 40-prompt subset (8/source).
+const DEFAULT_JUDGE_DELAY_MS = 2000; // spec §6b: 1.5-2s between calls.
+const BACKOFF_SCHEDULE_MS = [5_000, 10_000, 20_000, 40_000, 80_000]; // spec §6b.
+const ERROR_RATE_ABORT_THRESHOLD = 0.10; // spec §6b.
+const ERROR_RATE_MIN_SAMPLE = 20; // don't abort on noise from a tiny sample.
+const DEFAULT_MAX_CALLS = 900; // spec §6b budget guard.
+const DEFAULT_CC_PROJECTS_DIR = (0, node_path_1.join)((0, node_os_1.homedir)(), ".claude", "projects");
+const PROMPTS_SAVE_PATH = (0, node_path_1.join)(process.cwd(), "data", "prompts.json");
+const DEFAULT_REPORT_PATH = (0, node_path_1.join)(process.cwd(), "data", "relevance-eval-report.md");
+const VERDICTS_JSON_PATH = (0, node_path_1.join)(process.cwd(), "data", "relevance-verdicts.json");
+const VERDICTS_JSONL_PATH = (0, node_path_1.join)(process.cwd(), "data", "relevance-verdicts.jsonl");
+/** Spec §2 — the 5 sources, in stable corpus order. */
+const SOURCE_TAGS = [
+    "hermes-lenny",
+    "hermes-raider",
+    "hermes-nano",
+    "cc-infrastructure",
+    "cc-marine",
+];
+/** Required Hermes profile state DBs (flat /tmp layout, read-only, owner drop). */
+const HERMES_PROFILE_DBS = {
+    lenny: "/tmp/hermes-lenny-state.db",
+    raider: "/tmp/hermes-raider-state.db",
+    nano: "/tmp/hermes-nano-state.db",
+};
+/** Profile → declared mission domains (corpus-derived, owner directive — see
+ *  v1 header history). Simulates the per-agent config not yet set on bedrock;
+ *  the scope-ON sweep measures what #203 domain affinity WOULD do. */
+const AGENT_MISSION_DOMAINS = {
+    lenny: ["Health", "Ventures"],
+    nano: ["Finances", "Ventures"],
+    raider: ["Ventures"],
+};
+/** CC project dirs named in spec §2. "aironic-marine" matches BOTH the
+ *  AironicVentures and GAV checkouts — collected together as ONE source
+ *  ("cc-marine"), since the spec says "the `*aironic-marine*` DIRS" (plural). */
+const CC_INFRA_DIR = (0, node_path_1.join)(DEFAULT_CC_PROJECTS_DIR, "-Users-mattias-Development-DevOps-infrastructure");
+const CC_MARINE_DIRS = [
+    (0, node_path_1.join)(DEFAULT_CC_PROJECTS_DIR, "-Users-mattias-Development-AironicVentures-aironic-marine"),
+    (0, node_path_1.join)(DEFAULT_CC_PROJECTS_DIR, "-Users-mattias-Development-GAV-aironic-marine"),
+];
+/** Hermes session `source` values that are NOT primary conversations (cron =
+ *  scheduled tasks, automated scans) — their "user" turns are boilerplate
+ *  ("Run daily scan") and would inject synthetic-feeling prompts, undermining
+ *  spec §1's "real prompts, not synthetic" requirement. Excluded, same as v1. */
+const NON_PRIMARY_HERMES_SOURCES = new Set(["cron"]);
+const SWEEP_VARIANTS = ["100", "150", "title1sent"];
+const SWEEP_KS = [4, 6, 8]; // K-sweep (supporting table, §5.1).
+// ---------------------------------------------------------------------------
+// Prompt sampling — spec §2: 20/source × 5 sources (3 Hermes + 2 CC)
+// ---------------------------------------------------------------------------
+function normalizeForDedupe(s) {
+    return s.toLowerCase().replace(/\s+/g, " ").trim();
+}
+/** Pull user prompts from ONE Hermes profile state.db (flat /tmp path).
+ *  Read-only; missing/locked/schema-drifted DB returns []. */
+function collectHermesPrompts(dbPath, profile, sourceTag) {
+    const out = [];
+    const missionDomains = AGENT_MISSION_DOMAINS[profile] ?? ["Work"];
+    let db;
+    try {
+        db = new better_sqlite3_1.default(dbPath, { readonly: true, fileMustExist: true });
+    }
+    catch {
+        return [];
+    }
+    try {
+        const sessions = db
+            .prepare("SELECT id FROM sessions WHERE ended_at IS NOT NULL AND (source IS NULL OR source NOT IN ('cron'))")
+            .all();
+        const stmt = db.prepare("SELECT content FROM messages WHERE session_id = ? AND role = 'user' AND content IS NOT NULL ORDER BY id");
+        for (const s of sessions) {
+            const rows = stmt.all(s.id);
+            for (const r of rows) {
+                const text = (r.content ?? "").trim();
+                if (text.length < MIN_PROMPT_CHARS)
+                    continue;
+                out.push({ prompt: text, agent: profile, sourceTag, missionDomains, project: null, source: dbPath });
+            }
+        }
+    }
+    catch {
+        return [];
+    }
+    finally {
+        try {
+            db.close();
+        }
+        catch {
+            /* already closed */
+        }
+    }
+    return out;
+}
+/** Decode a CC project directory name to a project name (last dash-split
+ *  token) — mirrors transcript-reader.ts's decodeProjectDirName. */
+function decodeProjectName(dirName) {
+    const parts = dirName.split("-").filter(Boolean);
+    return parts.length === 0 ? dirName : parts[parts.length - 1];
+}
+function extractUserText(content) {
+    if (typeof content === "string")
+        return content;
+    if (!Array.isArray(content))
+        return "";
+    const texts = [];
+    for (const block of content) {
+        if (typeof block !== "object" || block === null)
+            continue;
+        if (block.type === "text") {
+            texts.push(String(block.text ?? ""));
+        }
+    }
+    return texts.join("\n").trim();
+}
+/** Collect every user prompt from ONE CC project dir's .jsonl session files. */
+function collectCcPromptsFromDir(projectDir, sourceTag) {
+    const out = [];
+    let files;
+    try {
+        if (!(0, node_fs_1.statSync)(projectDir).isDirectory())
+            return [];
+        files = (0, node_fs_1.readdirSync)(projectDir);
+    }
+    catch {
+        return [];
+    }
+    const project = decodeProjectName(projectDir.split("/").filter(Boolean).pop() ?? projectDir);
+    for (const file of files) {
+        if (!file.endsWith(".jsonl"))
+            continue;
+        const filePath = (0, node_path_1.join)(projectDir, file);
+        let raw;
+        try {
+            raw = (0, node_fs_1.readFileSync)(filePath, "utf-8");
+        }
+        catch {
+            continue;
+        }
+        for (const line of raw.split("\n")) {
+            if (!line.trim())
+                continue;
+            let entry;
+            try {
+                entry = JSON.parse(line);
+            }
+            catch {
+                continue;
+            }
+            if (typeof entry !== "object" || entry === null)
+                continue;
+            const e = entry;
+            const nested = e.message;
+            const role = String(e.type ?? nested?.role ?? "");
+            if (role !== "user")
+                continue;
+            const text = extractUserText(e.content ?? nested?.content).trim();
+            if (text.length < MIN_PROMPT_CHARS)
+                continue;
+            out.push({
+                prompt: text,
+                agent: `cc-${project}`,
+                sourceTag,
+                missionDomains: [],
+                project,
+                source: filePath,
+            });
+        }
+    }
+    return out;
+}
+/** Dedupe (normalized text) + sort length DESC (prefer substantial prompts,
+ *  matching v1's heuristic) + take the first `n`. Logs a shortfall loudly —
+ *  never pads with synthetic content. */
+function pickTopN(all, n, sourceTag) {
+    const seen = new Set();
+    const deduped = [];
+    for (const p of all) {
+        const key = normalizeForDedupe(p.prompt);
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        deduped.push(p);
+    }
+    deduped.sort((a, b) => b.prompt.length - a.prompt.length);
+    const picked = deduped.slice(0, n);
+    if (picked.length < n) {
+        console.warn(`[relevance-eval] WARNING: source "${sourceTag}" yielded only ${picked.length}/${n} prompts ` +
+            `after dedupe — corpus will be short by ${n - picked.length}. Not padded with synthetic content.`);
+    }
+    return picked;
+}
+/** Fail loudly if a required Hermes profile DB is missing — this is a
+ *  precondition (task instructions), not a soft corpus shortfall. */
+function verifyHermesPreconditions() {
+    const missing = Object.keys(HERMES_PROFILE_DBS).filter((p) => !(0, node_fs_1.existsSync)(HERMES_PROFILE_DBS[p]));
+    if (missing.length > 0) {
+        const lines = missing.map((p) => `  ssh agents@bedrock 'sqlite3 ~/.hermes/profiles/${p}/state.db ".backup /tmp/hermes-${p}-state.db"' && ` +
+            `scp agents@bedrock:/tmp/hermes-${p}-state.db ${HERMES_PROFILE_DBS[p]}`);
+        throw new Error(`relevance-eval: missing required Hermes state DB(s) for: ${missing.join(", ")}.\n` +
+            `Create via a consistent SQLite backup:\n${lines.join("\n")}`);
+    }
+}
+/** Build the v2 corpus fresh: 20 prompts/source × 5 sources (spec §2). */
+function buildV2Corpus() {
+    verifyHermesPreconditions();
+    const buckets = [];
+    for (const [profile, dbPath] of Object.entries(HERMES_PROFILE_DBS)) {
+        const sourceTag = `hermes-${profile}`;
+        const all = collectHermesPrompts(dbPath, profile, sourceTag);
+        buckets.push(...pickTopN(all, N_PER_SOURCE, sourceTag));
+    }
+    const infraAll = collectCcPromptsFromDir(CC_INFRA_DIR, "cc-infrastructure");
+    buckets.push(...pickTopN(infraAll, N_PER_SOURCE, "cc-infrastructure"));
+    const marineAll = CC_MARINE_DIRS.flatMap((d) => collectCcPromptsFromDir(d, "cc-marine"));
+    buckets.push(...pickTopN(marineAll, N_PER_SOURCE, "cc-marine"));
+    if (buckets.length === 0) {
+        throw new Error("relevance-eval: corpus construction yielded 0 prompts across all 5 sources — aborting.");
+    }
+    return buckets;
+}
+/** A saved prompts.json satisfies the v2 corpus contract iff every one of the
+ *  5 sources is present (count may be < 20/source if a source was exhausted —
+ *  see pickTopN's logged shortfall — but ALL 5 sources must be represented,
+ *  never silently missing one). */
+function isValidV2PromptSet(prompts) {
+    if (prompts.length === 0)
+        return false;
+    const present = new Set(prompts.map((p) => p.sourceTag));
+    return SOURCE_TAGS.every((t) => present.has(t));
+}
+function loadPromptsJson(path) {
+    const raw = (0, node_fs_1.readFileSync)(path, "utf-8");
+    const parsed = JSON.parse(raw);
+    if (!Array.isArray(parsed))
+        throw new Error(`relevance-eval: ${path} is not a JSON array`);
+    return parsed.map((p, i) => {
+        if (typeof p !== "object" || p === null)
+            throw new Error(`relevance-eval: ${path}[${i}] is not an object`);
+        const o = p;
+        const prompt = o.prompt;
+        if (typeof prompt !== "string")
+            throw new Error(`relevance-eval: ${path}[${i}] missing string prompt`);
+        const project = typeof o.project === "string" ? o.project : null;
+        const missionDomainsRaw = o.missionDomains;
+        const missionDomains = Array.isArray(missionDomainsRaw)
+            ? missionDomainsRaw.filter((d) => typeof d === "string")
+            : [];
+        const agent = typeof o.agent === "string" ? o.agent : project !== null ? `cc-${project}` : "unknown";
+        const source = typeof o.source === "string" ? o.source : "(saved set)";
+        const sourceTag = SOURCE_TAGS.includes(String(o.sourceTag))
+            ? o.sourceTag
+            : project !== null
+                ? `cc-${project}` // best-effort backfill for a pre-v2 saved set
+                : `hermes-${agent}`;
+        return { prompt, agent, sourceTag, missionDomains, project, source };
+    });
+}
+function savePromptsJson(path, prompts) {
+    (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(path), { recursive: true });
+    (0, node_fs_1.writeFileSync)(path, JSON.stringify(prompts, null, 2), "utf-8");
+}
+/** For each source (in SOURCE_TAGS order), the first SWEEP_N_PER_SOURCE
+ *  prompts (by their position in the saved corpus) — the fixed 40-prompt
+ *  length-sweep subset (spec §4.2). Deterministic given the saved order. */
+function computeSweepSubsetIndices(prompts) {
+    const bySource = new Map();
+    prompts.forEach((p, idx) => {
+        const arr = bySource.get(p.sourceTag) ?? [];
+        arr.push(idx);
+        bySource.set(p.sourceTag, arr);
+    });
+    const out = [];
+    for (const tag of SOURCE_TAGS) {
+        out.push(...(bySource.get(tag) ?? []).slice(0, SWEEP_N_PER_SOURCE));
+    }
+    return out;
+}
+// ---------------------------------------------------------------------------
+// Retrieve sweep (scope OFF vs ON) — LLM-free system-under-test
+// ---------------------------------------------------------------------------
+async function runRetrieveSweep(db, prompts) {
+    const embedCache = new Map();
+    for (const p of prompts) {
+        if (!embedCache.has(p.prompt))
+            embedCache.set(p.prompt, await (0, embedder_js_1.embed)(p.prompt));
+    }
+    const neverCalledEmbed = async () => {
+        throw new Error("relevance-eval: retrieve() called the embedFn — queryEmbedding was not honored. Eval aborted (results would be invalid).");
+    };
+    const results = [];
+    for (const sp of prompts) {
+        const queryEmbedding = embedCache.get(sp.prompt);
+        for (const mode of ["off", "on"]) {
+            const scopeOpts = mode === "on"
+                ? sp.project !== null
+                    ? { project: sp.project }
+                    : { missionDomains: sp.missionDomains }
+                : {};
+            const retrieved = await (0, retrieval_js_1.retrieve)(db, neverCalledEmbed, sp.prompt, {
+                limit: RESULT_K,
+                queryEmbedding,
+                noStrengthen: true,
+                ...scopeOpts,
+            });
+            const topIds = retrieved.map((r) => r.id);
+            const contents = {};
+            const rows = {};
+            for (const r of retrieved) {
+                contents[r.id] = r.content;
+                rows[r.id] = r;
+            }
+            results.push({ prompt: sp, mode, topIds, contents, rows, fullVerdicts: [], lineVerdicts: [], redundancy: null });
+        }
+    }
+    return results;
+}
+// ---------------------------------------------------------------------------
+// Rendering — production surface (imported, never reimplemented) + one
+// exploratory variant (title + first sentence) that has no production
+// equivalent, so it lives here.
+// ---------------------------------------------------------------------------
+const RECALL_BLOCK_HEADER = [
+    "## Memory recall (auto)",
+    // Copied verbatim from recall-index.ts's handleRecallIndex block header —
+    // DISPLAY TEXT only, not rendering LOGIC (the logic under measurement,
+    // formatIndexLine/memoryTitle, IS imported). Keep in sync if it changes.
+    "Possibly relevant memories — dates matter, newer supersedes older. Fetch with `hicortex_get(id)` when an entry could change your action. Cite what you rely on by id + date, and mark it `FETCHED` if you read the full memory or `SNIPPET` if you're citing the one-line entry unread — don't pass a SNIPPET citation off as established fact.",
+];
+/** Not a production surface — spec §4.2's exploratory 4th variant. Title
+ *  (same memoryTitle() production helper) + one sentence of body, so the
+ *  sweep can measure whether ONE extra sentence (not just a longer title)
+ *  closes the snippet gap. Date formatting is a local dd.mm.yyyy helper (not
+ *  exported from recall-index.ts) since this whole render path is synthetic. */
+function ddmmyyyy(iso) {
+    const d = new Date(iso);
+    if (isNaN(d.getTime()))
+        return "";
+    const dd = String(d.getDate()).padStart(2, "0");
+    const mm = String(d.getMonth() + 1).padStart(2, "0");
+    return `${dd}.${mm}.${d.getFullYear()}`;
+}
+function firstSentence(text) {
+    const cleaned = text
+        .replace(/^#+\s*/, "")
+        .replace(/^Session Memory:\s*/i, "")
+        .replace(/^Lesson:\s*/i, "")
+        .trim();
+    const m = cleaned.match(/^(.*?[.!?])(\s|$)/);
+    return (m ? m[1] : cleaned).trim();
+}
+function renderTitleFirstSentenceLine(r) {
+    const title = (0, recall_index_js_1.memoryTitle)(r.content, 100);
+    const lines = r.content
+        .split("\n")
+        .map((l) => l.trim())
+        .filter((l) => l.length > 0);
+    const restAfterTitleLine = lines.slice(1).join(" ");
+    const sentence = firstSentence(restAfterTitleLine.length > 0 ? restAfterTitleLine : r.content);
+    const body = sentence && sentence.toLowerCase() !== title.toLowerCase() ? `${title} — ${sentence}` : title;
+    const meta = [ddmmyyyy(r.created_at), r.domain ?? r.project ?? undefined, r.source_agent ?? undefined, r.memory_type]
+        .filter(Boolean)
+        .join(", ");
+    return `- [${r.id}] ${body}${meta ? ` (${meta})` : ""}`;
+}
+function renderVariantLine(r, variant) {
+    if (variant === "title1sent")
+        return renderTitleFirstSentenceLine(r);
+    return (0, recall_index_js_1.formatIndexLine)(r, parseInt(variant, 10));
+}
+/** The ACTUAL production block: relevance-gated (passesRelevanceGate,
+ *  imported), capped at maxItems, rendered via formatIndexLine (imported). */
+function renderProductionBlock(rows, minSimilarity, maxItems, maxLen = PROD_TITLE_CHARS) {
+    const picked = rows.filter((r) => (0, recall_index_js_1.passesRelevanceGate)(r, minSimilarity)).slice(0, maxItems);
+    const lines = {};
+    for (const r of picked)
+        lines[r.id] = (0, recall_index_js_1.formatIndexLine)(r, maxLen);
+    const block = [...RECALL_BLOCK_HEADER, ...picked.map((r) => lines[r.id])].join("\n");
+    return { block, shownIds: picked.map((r) => r.id), lines };
+}
+/** char/4 token estimate — no tokenizer available locally; stated in the
+ *  report. Good enough for a relative (variant-to-variant, K-to-K) comparison. */
+function estimateTokens(s) {
+    return Math.ceil(s.length / 4);
+}
+// ---------------------------------------------------------------------------
+// Judge prompts
+// ---------------------------------------------------------------------------
+function buildFullContentJudgePrompt(userPrompt, topIds, contents) {
+    const memBlock = [];
+    for (let i = 0; i < topIds.length; i++) {
+        const id = topIds[i];
+        const label = `Q${i + 1}`;
+        const full = contents[id] ?? "";
+        const trimmed = full.length > MAX_MEM_CHARS_IN_JUDGE ? full.slice(0, MAX_MEM_CHARS_IN_JUDGE) + " […truncated]" : full;
+        memBlock.push(`[${label}] (memory id ${id})\n${trimmed}`);
+    }
+    return ("You are a STRICT relevance judge for an AI coding agent's memory system. " +
+        "A real user prompt and the memories the retrieval system surfaced are " +
+        "below. Grade EACH memory independently against the SPECIFIC prompt.\n\n" +
+        "Verdicts (pick exactly one per memory):\n" +
+        "- relevant — genuinely useful for handling the user's actual need. The memory " +
+        "carries information, context, or a lesson that would help answer or " +
+        "contextualize the prompt. A tangential keyword overlap is NOT relevant.\n" +
+        "- noise — irrelevant. Off-topic, wrong scope, or surfaced purely on a " +
+        "keyword / lexical collision with no genuine connection to what the user is " +
+        "asking.\n" +
+        "- stale — once relevant but now outdated or superseded. The decision was " +
+        "reversed, the value moved, the API was removed, the lesson no longer holds.\n\n" +
+        "Grade relevance to the SPECIFIC prompt, not how interesting the memory is " +
+        "in general. When in doubt between 'relevant' and 'noise', prefer 'noise' — " +
+        "relevance inflation defeats the eval.\n\n" +
+        `USER PROMPT:\n${userPrompt}\n\n` +
+        `MEMORIES (Q1 = top-ranked):\n${memBlock.join("\n\n")}\n\n` +
+        "Return ONLY a JSON array — one entry per memory, in this exact shape, no " +
+        "prose, no markdown fences:\n" +
+        '[{"id": "Q1", "verdict": "relevant|noise|stale", "reason": "<= 12 words"}, ...]');
+}
+/** Spec §4.1 — the line judge is BLIND to full content: a separate call that
+ *  sees ONLY the rendered production one-liner (or a sweep variant of it). */
+function buildLineJudgePrompt(userPrompt, topIds, lines) {
+    const memBlock = [];
+    for (let i = 0; i < topIds.length; i++) {
+        const id = topIds[i];
+        memBlock.push(`[Q${i + 1}] ${lines[id] ?? ""}`);
+    }
+    return ("You are a STRICT relevance judge for an AI coding agent's memory system. " +
+        "The agent is shown ONLY the one-line index entry below per memory — " +
+        "NEVER the full content. Judge whether the AGENT COULD TELL, FROM THE LINE " +
+        "ALONE, that the memory is relevant to its actual need. Do not assume " +
+        "information that isn't visible in the line.\n\n" +
+        "Verdicts (pick exactly one per line):\n" +
+        "- relevant — the line itself signals genuine relevance to the prompt.\n" +
+        "- noise — the line signals no genuine connection, or is too vague/generic " +
+        "to tell anything from.\n" +
+        "- stale — the line itself signals outdated/superseded content.\n\n" +
+        "When in doubt, prefer 'noise' — relevance inflation defeats this measurement.\n\n" +
+        `USER PROMPT:\n${userPrompt}\n\n` +
+        `INDEX LINES (Q1 = top-ranked):\n${memBlock.join("\n")}\n\n` +
+        "Return ONLY a JSON array, same shape, no prose, no markdown fences:\n" +
+        '[{"id": "Q1", "verdict": "relevant|noise|stale", "reason": "<= 12 words"}, ...]');
+}
+function buildRedundancyPrompt(userPrompt, shownLines) {
+    const block = shownLines.map((l, i) => `[R${i + 1}] ${l}`).join("\n");
+    return ("You are grading REDUNDANCY in a set of memory index lines an AI agent " +
+        "would see TOGETHER for one prompt. Two lines are redundant if reading one " +
+        "makes the other add NOTHING NEW (near-duplicates, restatements, or a " +
+        "supersede-pair where only the newer one matters).\n\n" +
+        `USER PROMPT:\n${userPrompt}\n\n` +
+        `INDEX LINES SHOWN TOGETHER:\n${block}\n\n` +
+        `Return ONLY JSON, no prose, no markdown fences: {"non_redundant_count": <integer 0-${shownLines.length}>, "reason": "<= 15 words"}\n` +
+        "non_redundant_count = how many DISTINCT pieces of information are represented " +
+        "(total lines minus duplicates counted beyond the first in each redundant group).");
+}
+// ---------------------------------------------------------------------------
+// Lenient JSON parse (v1, unchanged) — full/line/sweep share this parser.
+// ---------------------------------------------------------------------------
+const VALID_VERDICTS = new Set(["relevant", "noise", "stale"]);
+function normalizeVerdict(v) {
+    if (typeof v !== "string")
+        return null;
+    const lower = v.toLowerCase().trim();
+    if (lower.startsWith("rel"))
+        return "relevant";
+    if (lower.startsWith("noise") || lower === "irrelevant")
+        return "noise";
+    if (lower.startsWith("stale") || lower.startsWith("outdated") || lower.startsWith("superse"))
+        return "stale";
+    return VALID_VERDICTS.has(lower) ? lower : null;
+}
+function parseJudgeOutput(raw, expectedLabels) {
+    const cleaned = raw.replace(/```(?:json)?/gi, "").trim();
+    const start = cleaned.indexOf("[");
+    const end = cleaned.lastIndexOf("]");
+    if (start >= 0 && end > start) {
+        try {
+            const parsed = JSON.parse(cleaned.slice(start, end + 1));
+            if (Array.isArray(parsed)) {
+                const out = [];
+                for (const item of parsed) {
+                    if (typeof item !== "object" || item === null)
+                        continue;
+                    const id = String(item.id ?? "").toUpperCase();
+                    if (!expectedLabels.has(id))
+                        continue;
+                    const verdict = normalizeVerdict(item.verdict);
+                    if (!verdict)
+                        continue;
+                    const reason = String(item.reason ?? "").trim();
+                    out.push({ id, verdict, reason });
+                }
+                if (out.length > 0)
+                    return out;
+            }
+        }
+        catch {
+            // fall through to heuristic
+        }
+    }
+    const out = [];
+    for (const line of cleaned.split("\n")) {
+        const m = line.match(/\b(Q\d+)\b[\s\-:)]*([A-Za-z]+)/);
+        if (!m)
+            continue;
+        const id = m[1].toUpperCase();
+        if (!expectedLabels.has(id))
+            continue;
+        const verdict = normalizeVerdict(m[2]);
+        if (!verdict)
+            continue;
+        const tail = line.slice((m.index ?? 0) + m[0].length).replace(/^[\s\-:;.,)]+/, "");
+        const reason = tail.split(/[.\n]/)[0].trim().slice(0, 80);
+        out.push({ id, verdict, reason });
+    }
+    return out.length > 0 ? out : null;
+}
+function parseRedundancyOutput(raw, shown) {
+    const cleaned = raw.replace(/```(?:json)?/gi, "").trim();
+    const start = cleaned.indexOf("{");
+    const end = cleaned.lastIndexOf("}");
+    if (start >= 0 && end > start) {
+        try {
+            const obj = JSON.parse(cleaned.slice(start, end + 1));
+            const n = Number(obj.non_redundant_count);
+            if (Number.isFinite(n)) {
+                return {
+                    nonRedundantCount: Math.max(0, Math.min(shown, Math.round(n))),
+                    reason: String(obj.reason ?? "").trim(),
+                };
+            }
+        }
+        catch {
+            // fall through
+        }
+    }
+    const m = cleaned.match(/(\d+)/);
+    if (m)
+        return { nonRedundantCount: Math.max(0, Math.min(shown, parseInt(m[1], 10))), reason: "" };
+    return null;
+}
+// ---------------------------------------------------------------------------
+// Rate limiting + resumability (spec §6b, MANDATORY)
+// ---------------------------------------------------------------------------
+class JudgeHttpError extends Error {
+    status;
+    retryAfterMs;
+    constructor(status, message, retryAfterMs) {
+        super(message);
+        this.name = "JudgeHttpError";
+        this.status = status;
+        this.retryAfterMs = retryAfterMs;
+    }
+}
+/** Thrown when the run must stop (budget or error-rate abort). Caught once at
+ *  the top of the judging pipeline in main() — whatever completed so far is
+ *  still persisted and reported, clearly marked as an aborted/partial run. */
+class RunAbortedError extends Error {
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/** ONE raw HTTP attempt against z.ai's Anthropic-compatible endpoint. Mirrors
+ *  llm.ts#completeAnthropic's request shape exactly (model/messages/max_tokens,
+ *  x-api-key header) but with NO built-in retry — the backoff wrapper below
+ *  owns retries so the schedule matches spec §6b precisely. */
+async function callZaiOnce(apiKey, prompt, maxTokens, timeoutMs = 90_000) {
+    const url = `${ZAI_BASE_URL.replace(/\/$/, "")}/v1/messages`;
+    let resp;
+    try {
+        resp = await fetch(url, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "x-api-key": apiKey,
+                "anthropic-version": "2023-06-01",
+            },
+            body: JSON.stringify({ model: ZAI_MODEL, messages: [{ role: "user", content: prompt }], max_tokens: maxTokens }),
+            signal: AbortSignal.timeout(timeoutMs),
+        });
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        throw new JudgeHttpError(0, `network/timeout: ${msg}`); // status 0 = retryable (network/timeout)
+    }
+    if (resp.status === 429 || resp.status >= 500) {
+        const retryAfterHeader = resp.headers.get("retry-after");
+        const retryAfterMs = retryAfterHeader ? parseInt(retryAfterHeader, 10) * 1000 : undefined;
+        const text = await resp.text().catch(() => "");
+        throw new JudgeHttpError(resp.status, `HTTP ${resp.status}: ${text.slice(0, 200)}`, retryAfterMs);
+    }
+    if (!resp.ok) {
+        const text = await resp.text().catch(() => "");
+        throw new JudgeHttpError(resp.status, `HTTP ${resp.status} (non-retryable): ${text.slice(0, 200)}`);
+    }
+    const data = (await resp.json());
+    const block = data.content?.find((c) => c.type === "text");
+    return (block?.text ?? "").trim();
+}
+function jitter(ms) {
+    return Math.round(ms * (0.8 + Math.random() * 0.4)); // ±20%
+}
+/** Retryable = 429, 5xx, or network/timeout (status 0). Any other non-ok
+ *  status (400/401/403/404) fails immediately — retrying a malformed request
+ *  or an auth failure would waste the call budget on a guaranteed repeat. */
+function isRetryable(err) {
+    return err instanceof JudgeHttpError && (err.status === 429 || err.status >= 500 || err.status === 0);
+}
+async function callZaiWithBackoff(prompt, maxTokens, state) {
+    let lastErr;
+    for (let attempt = 0; attempt <= BACKOFF_SCHEDULE_MS.length; attempt++) {
+        try {
+            return await callZaiOnce(state.apiKey, prompt, maxTokens);
+        }
+        catch (err) {
+            lastErr = err;
+            if (!isRetryable(err) || attempt === BACKOFF_SCHEDULE_MS.length)
+                throw err;
+            const scheduled = BACKOFF_SCHEDULE_MS[attempt];
+            const waitMs = err.retryAfterMs ?? jitter(scheduled); // Retry-After wins over the schedule.
+            state.totalRetries++;
+            console.log(`[relevance-eval] judge call failed (${err.message.slice(0, 100)}), ` +
+                `retry ${attempt + 1}/${BACKOFF_SCHEDULE_MS.length} in ${Math.round(waitMs / 1000)}s...`);
+            await sleep(waitMs);
+        }
+    }
+    throw lastErr;
+}
+function loadResumeMap(sidecarPath) {
+    const map = new Map();
+    if (!(0, node_fs_1.existsSync)(sidecarPath))
+        return map;
+    const raw = (0, node_fs_1.readFileSync)(sidecarPath, "utf-8");
+    for (const line of raw.split("\n")) {
+        if (!line.trim())
+            continue;
+        try {
+            const obj = JSON.parse(line);
+            if (obj && typeof obj.key === "string")
+                map.set(obj.key, obj.result);
+        }
+        catch {
+            // corrupt checkpoint line — skip, don't crash the whole resume.
+        }
+    }
+    return map;
+}
+/** One judge UNIT: resume-cache check → budget check → delay → call w/
+ *  backoff → parse → checkpoint-append → progress log → error-rate check. */
+async function runJudgeUnit(state, key, buildPrompt, maxTokens, parse, parseErrorLabel) {
+    const cached = state.resumed.get(key);
+    if (cached !== undefined) {
+        console.log(`[relevance-eval] resumed: ${key}`);
+        return cached;
+    }
+    if (state.aborted)
+        throw new RunAbortedError(state.aborted.reason);
+    if (state.totalCalls >= state.maxCalls) {
+        state.aborted = { reason: `--max-calls budget (${state.maxCalls}) reached` };
+        throw new RunAbortedError(state.aborted.reason);
+    }
+    await sleep(state.judgeDelayMs);
+    state.totalCalls++;
+    let result;
+    try {
+        const raw = await callZaiWithBackoff(buildPrompt(), maxTokens, state);
+        const parsed = parse(raw);
+        result = parsed ?? { judgeError: `${parseErrorLabel}; raw head: ${raw.slice(0, 200).replace(/\s+/g, " ")}` };
+    }
+    catch (err) {
+        result = { judgeError: (err instanceof Error ? err.message : String(err)).slice(0, 300) };
+    }
+    state.completedUnits++;
+    if (typeof result === "object" && result !== null && "judgeError" in result) {
+        state.errorUnits++;
+    }
+    (0, node_fs_1.appendFileSync)(state.sidecarPath, JSON.stringify({ key, result }) + "\n", "utf-8");
+    const elapsedS = Math.round((Date.now() - state.startTime) / 1000);
+    console.log(`[relevance-eval] judged ${state.completedUnits} (calls ${state.totalCalls}/${state.maxCalls}, ` +
+        `${elapsedS}s elapsed, ${state.totalRetries} retries, ${state.errorUnits} judge_error) — ${key}`);
+    if (state.completedUnits >= ERROR_RATE_MIN_SAMPLE && state.errorUnits / state.completedUnits > ERROR_RATE_ABORT_THRESHOLD) {
+        state.aborted = {
+            reason: `judge-error rate ${((state.errorUnits / state.completedUnits) * 100).toFixed(1)}% ` +
+                `exceeds the 10% abort threshold after ${state.completedUnits} units`,
+        };
+        throw new RunAbortedError(state.aborted.reason);
+    }
+    return result;
+}
+async function judgeVerdictUnit(state, key, promptText, expectedLabels) {
+    if (expectedLabels.size === 0)
+        return { judgeError: "retrieve() surfaced 0 memories — nothing to grade" };
+    return runJudgeUnit(state, key, () => promptText, 2048, (raw) => parseJudgeOutput(raw, expectedLabels), "parse failed");
+}
+async function judgeRedundancyUnit(state, key, promptText, shown) {
+    return runJudgeUnit(state, key, () => promptText, 512, (raw) => parseRedundancyOutput(raw, shown), "parse failed");
+}
+function computeMetricsAtK(results, k) {
+    const judged = results.filter((r) => Array.isArray(r.fullVerdicts));
+    if (judged.length === 0)
+        return null;
+    const inK = (id) => {
+        const idx = parseInt(id.slice(1), 10);
+        return Number.isInteger(idx) && idx >= 1 && idx <= k;
+    };
+    let relTotal = 0, noiseTotal = 0, staleTotal = 0, verdictTotal = 0;
+    const perPrompt = [];
+    for (const r of judged) {
+        let promptRel = 0;
+        for (const v of r.fullVerdicts) {
+            if (!inK(v.id))
+                continue;
+            verdictTotal++;
+            if (v.verdict === "relevant") {
+                relTotal++;
+                promptRel++;
+            }
+            else if (v.verdict === "noise")
+                noiseTotal++;
+            else
+                staleTotal++;
+        }
+        perPrompt.push(promptRel / k);
+    }
+    return {
+        precision: perPrompt.reduce((a, b) => a + b, 0) / perPrompt.length,
+        noiseRate: verdictTotal > 0 ? noiseTotal / verdictTotal : 0,
+        staleRate: verdictTotal > 0 ? staleTotal / verdictTotal : 0,
+        n: judged.length,
+        perPrompt,
+    };
+}
+function computePromptHistogram(results, mode) {
+    const buckets = new Array(RESULT_K + 1).fill(0);
+    for (const r of results.filter((r) => r.mode === mode)) {
+        if (!Array.isArray(r.fullVerdicts))
+            continue;
+        const rel = r.fullVerdicts.filter((v) => v.verdict === "relevant").length;
+        if (rel >= 0 && rel <= RESULT_K)
+            buckets[rel]++;
+    }
+    return buckets;
+}
+function findWorstBestPrompts(results, mode, worstRelMax, bestRelMin) {
+    const named = [];
+    for (const r of results.filter((r) => r.mode === mode)) {
+        if (!Array.isArray(r.fullVerdicts))
+            continue;
+        let rel = 0, noise = 0, stale = 0;
+        const perRank = [];
+        for (const v of r.fullVerdicts) {
+            const rank = parseInt(v.id.slice(1), 10);
+            const memId = r.topIds[rank - 1] ?? "?";
+            if (v.verdict === "relevant")
+                rel++;
+            else if (v.verdict === "noise")
+                noise++;
+            else
+                stale++;
+            perRank.push({ rank, memory_id: memId, verdict: v.verdict, reason: v.reason, content: r.contents[memId] ?? "" });
+        }
+        named.push({ prompt: r.prompt.prompt, agent: r.prompt.agent, relevant: rel, noise, stale, perRank });
+    }
+    return {
+        worst: named.filter((p) => p.relevant <= worstRelMax),
+        best: named.filter((p) => p.relevant >= bestRelMin),
+    };
+}
+function rankChronicMemories(results, metaById) {
+    const byId = new Map();
+    for (const r of results) {
+        if (!Array.isArray(r.fullVerdicts))
+            continue;
+        const seenNoiseForPrompt = new Set();
+        const seenStaleForPrompt = new Set();
+        const seenSurfaceForPrompt = new Set();
+        for (const v of r.fullVerdicts) {
+            const rank = parseInt(v.id.slice(1), 10);
+            const memId = r.topIds[rank - 1];
+            if (!memId)
+                continue;
+            let entry = byId.get(memId);
+            if (!entry) {
+                const meta = metaById.get(memId) ?? { content: r.contents[memId] ?? "", memory_type: null, project: null };
+                entry = { memory_id: memId, noisePrompts: 0, stalePrompts: 0, totalSurfaces: 0, ...meta };
+                byId.set(memId, entry);
+            }
+            if (!seenSurfaceForPrompt.has(memId)) {
+                entry.totalSurfaces++;
+                seenSurfaceForPrompt.add(memId);
+            }
+            if (v.verdict === "noise" && !seenNoiseForPrompt.has(memId)) {
+                entry.noisePrompts++;
+                seenNoiseForPrompt.add(memId);
+            }
+            else if (v.verdict === "stale" && !seenStaleForPrompt.has(memId)) {
+                entry.stalePrompts++;
+                seenStaleForPrompt.add(memId);
+            }
+        }
+    }
+    return Array.from(byId.values());
+}
+function lookupMemoryMeta(db, results) {
+    const ids = new Set();
+    for (const r of results)
+        for (const id of r.topIds)
+            ids.add(id);
+    const out = new Map();
+    if (ids.size === 0)
+        return out;
+    const placeholders = Array.from(ids, () => "?").join(",");
+    try {
+        const rows = db
+            .prepare(`SELECT id, content, memory_type, project FROM memories WHERE id IN (${placeholders})`)
+            .all(...ids);
+        for (const row of rows) {
+            out.set(row.id, { content: row.content ?? "", memory_type: row.memory_type, project: row.project });
+        }
+    }
+    catch {
+        // Schema drift — empty map; callers fall back to cached content.
+    }
+    for (const r of results) {
+        for (const id of r.topIds) {
+            if (!out.has(id))
+                out.set(id, { content: r.contents[id] ?? "", memory_type: null, project: null });
+        }
+    }
+    return out;
+}
+function computeRankMetrics(results) {
+    const judged = results.filter((r) => Array.isArray(r.fullVerdicts));
+    const rel = new Array(RESULT_K).fill(0);
+    const noise = new Array(RESULT_K).fill(0);
+    const stale = new Array(RESULT_K).fill(0);
+    const n = new Array(RESULT_K).fill(0);
+    for (const r of judged) {
+        for (const v of r.fullVerdicts) {
+            const idx = parseInt(v.id.slice(1), 10) - 1;
+            if (!Number.isInteger(idx) || idx < 0 || idx >= RESULT_K)
+                continue;
+            n[idx]++;
+            if (v.verdict === "relevant")
+                rel[idx]++;
+            else if (v.verdict === "noise")
+                noise[idx]++;
+            else
+                stale[idx]++;
+        }
+    }
+    const rows = [];
+    for (let i = 0; i < RESULT_K; i++) {
+        rows.push({
+            rank: i + 1,
+            n: n[i],
+            relevantRate: n[i] > 0 ? rel[i] / n[i] : 0,
+            noiseRate: n[i] > 0 ? noise[i] / n[i] : 0,
+            staleRate: n[i] > 0 ? stale[i] / n[i] : 0,
+        });
+    }
+    return rows;
+}
+function aggregateRankBand(results, fromRank, toRank) {
+    const judged = results.filter((r) => Array.isArray(r.fullVerdicts));
+    const want = new Set();
+    for (let r = fromRank; r <= toRank; r++)
+        want.add(`Q${r}`);
+    let noise = 0, total = 0;
+    for (const r of judged) {
+        for (const v of r.fullVerdicts) {
+            if (!want.has(v.id))
+                continue;
+            total++;
+            if (v.verdict === "noise")
+                noise++;
+        }
+    }
+    return { noise, total, rate: total > 0 ? noise / total : 0 };
+}
+function computeSnippetGap(results) {
+    let actionable = 0, snippetFailure = 0, misleadingLine = 0, fullRelevantTotal = 0, n = 0;
+    for (const r of results) {
+        if (!Array.isArray(r.fullVerdicts) || !Array.isArray(r.lineVerdicts))
+            continue;
+        const lineById = new Map(r.lineVerdicts.map((v) => [v.id, v]));
+        for (const fv of r.fullVerdicts) {
+            const lv = lineById.get(fv.id);
+            if (!lv)
+                continue;
+            n++;
+            const fullRel = fv.verdict === "relevant";
+            const lineRel = lv.verdict === "relevant";
+            if (fullRel)
+                fullRelevantTotal++;
+            if (fullRel && lineRel)
+                actionable++;
+            else if (fullRel && !lineRel)
+                snippetFailure++;
+            else if (!fullRel && lineRel)
+                misleadingLine++;
+        }
+    }
+    return { actionable, snippetFailure, misleadingLine, fullRelevantTotal, n };
+}
+function findNamedSnippetFailures(results, limit) {
+    const out = [];
+    for (const r of results) {
+        if (!Array.isArray(r.fullVerdicts) || !Array.isArray(r.lineVerdicts))
+            continue;
+        const lineById = new Map(r.lineVerdicts.map((v) => [v.id, v]));
+        for (const fv of r.fullVerdicts) {
+            const lv = lineById.get(fv.id);
+            if (!lv)
+                continue;
+            if (fv.verdict === "relevant" && lv.verdict !== "relevant") {
+                const rank = parseInt(fv.id.slice(1), 10);
+                const memId = r.topIds[rank - 1] ?? "?";
+                const row = r.rows[memId];
+                out.push({
+                    prompt: r.prompt.prompt,
+                    agent: r.prompt.agent,
+                    mode: r.mode,
+                    memory_id: memId,
+                    rendered_line: row ? (0, recall_index_js_1.formatIndexLine)(row, PROD_TITLE_CHARS) : "(unavailable)",
+                    full_reason: fv.reason,
+                    line_reason: lv.reason,
+                    content: r.contents[memId] ?? "",
+                });
+            }
+        }
+    }
+    return out.slice(0, limit);
+}
+function mean(xs) {
+    return xs.length === 0 ? NaN : xs.reduce((a, b) => a + b, 0) / xs.length;
+}
+function sd(xs) {
+    if (xs.length < 2)
+        return 0;
+    const m = mean(xs);
+    return Math.sqrt(xs.reduce((a, b) => a + (b - m) ** 2, 0) / (xs.length - 1));
+}
+function ci95(xs) {
+    const m = mean(xs);
+    const s = sd(xs);
+    const se = xs.length > 0 ? s / Math.sqrt(xs.length) : NaN;
+    return { mean: m, sd: s, se, lo: m - 1.96 * se, hi: m + 1.96 * se, n: xs.length };
+}
+/** Similarity buckets for §5.9 — width-0.05 bands from 0.30 to 1.00, plus a
+ *  catch-all "<0.30" and a "n/a" bucket for FTS/graph hits (no measured
+ *  cosine — they bypass the floor entirely per passesRelevanceGate). */
+const SIM_BUCKET_EDGES = [0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1.001];
+function similarityBucketLabel(sim) {
+    if (sim === null || sim === undefined)
+        return "n/a (fts/graph, no cosine)";
+    if (sim < SIM_BUCKET_EDGES[0])
+        return "<0.30";
+    for (let i = 0; i < SIM_BUCKET_EDGES.length - 1; i++) {
+        if (sim >= SIM_BUCKET_EDGES[i] && sim < SIM_BUCKET_EDGES[i + 1]) {
+            const hi = Math.min(SIM_BUCKET_EDGES[i + 1], 1);
+            return `${SIM_BUCKET_EDGES[i].toFixed(2)}–${hi.toFixed(2)}`;
+        }
+    }
+    return ">=1.00";
+}
+function buildVerdictRows(prompts, results, sweepRows) {
+    const byKey = new Map();
+    for (const r of results) {
+        const pIdx = prompts.findIndex((p) => p.prompt === r.prompt.prompt && p.agent === r.prompt.agent);
+        byKey.set(`${pIdx}:${r.mode}`, r);
+    }
+    const rows = [];
+    // Baseline (variant PROD_TITLE_CHARS, production render) — one row per (prompt, mode, rank).
+    for (let i = 0; i < prompts.length; i++) {
+        for (const mode of ["off", "on"]) {
+            const r = byKey.get(`${i}:${mode}`);
+            if (!r)
+                continue;
+            const fullOk = Array.isArray(r.fullVerdicts);
+            const lineOk = Array.isArray(r.lineVerdicts);
+            const fullById = fullOk ? new Map(r.fullVerdicts.map((v) => [v.id, v])) : null;
+            const lineById = lineOk ? new Map(r.lineVerdicts.map((v) => [v.id, v])) : null;
+            const ranks = r.topIds.length > 0 ? r.topIds.map((_, idx) => idx + 1) : [0];
+            for (const rank of ranks) {
+                const label = `Q${rank}`;
+                const memId = rank > 0 ? r.topIds[rank - 1] : "";
+                const row = memId ? r.rows[memId] : undefined;
+                const fv = fullById?.get(label);
+                const lv = lineById?.get(label);
+                rows.push({
+                    prompt_idx: i,
+                    prompt: r.prompt.prompt,
+                    source: r.prompt.sourceTag,
+                    agent: r.prompt.agent,
+                    mode,
+                    rank,
+                    memory_id: memId,
+                    memory_type: row?.memory_type ?? null,
+                    similarity: row?.similarity ?? null,
+                    retrieval_source: row?.source ?? null,
+                    full_verdict: fv ? fv.verdict : fullOk ? null : "judge_error",
+                    full_reason: fv ? fv.reason : fullOk ? "" : r.fullVerdicts.judgeError,
+                    line_verdict: lv ? lv.verdict : lineOk ? null : "judge_error",
+                    line_reason: lv ? lv.reason : lineOk ? "" : r.lineVerdicts.judgeError,
+                    rendered_line: row ? (0, recall_index_js_1.formatIndexLine)(row, PROD_TITLE_CHARS) : "",
+                    snippet_len_variant: String(PROD_TITLE_CHARS),
+                });
+            }
+        }
+    }
+    // Sweep rows (variants 100/120/200/title1sent — PROD_TITLE_CHARS (150)
+    // already covered above by the baseline pass; mode is always "off" per the
+    // resource-bounded design, spec §4.2).
+    for (const sw of sweepRows) {
+        if (sw.variant === String(PROD_TITLE_CHARS))
+            continue; // covered by the baseline pass above.
+        const r = byKey.get(`${sw.promptIdx}:off`);
+        if (!r)
+            continue;
+        const fullById = Array.isArray(r.fullVerdicts) ? new Map(r.fullVerdicts.map((v) => [v.id, v])) : null;
+        const lineOk = Array.isArray(sw.verdicts);
+        const lineById = lineOk ? new Map(sw.verdicts.map((v) => [v.id, v])) : null;
+        for (let rank = 1; rank <= r.topIds.length; rank++) {
+            const label = `Q${rank}`;
+            const memId = r.topIds[rank - 1];
+            const row = r.rows[memId];
+            const fv = fullById?.get(label);
+            const lv = lineById?.get(label);
+            rows.push({
+                prompt_idx: sw.promptIdx,
+                prompt: r.prompt.prompt,
+                source: r.prompt.sourceTag,
+                agent: r.prompt.agent,
+                mode: "off",
+                rank,
+                memory_id: memId,
+                memory_type: row?.memory_type ?? null,
+                similarity: row?.similarity ?? null,
+                retrieval_source: row?.source ?? null,
+                full_verdict: fv ? fv.verdict : null, // carried over from the baseline full judging; not re-run per variant.
+                full_reason: fv ? fv.reason : "",
+                line_verdict: lv ? lv.verdict : lineOk ? null : "judge_error",
+                line_reason: lv ? lv.reason : lineOk ? "" : sw.verdicts.judgeError,
+                rendered_line: sw.lines[memId] ?? "",
+                snippet_len_variant: sw.variant,
+            });
+        }
+    }
+    return rows;
+}
+function parseArgs(argv) {
+    const positionals = [];
+    let judgeDelayMs = DEFAULT_JUDGE_DELAY_MS;
+    let maxCalls = DEFAULT_MAX_CALLS;
+    let resume = false;
+    let verdictsJsonPath = VERDICTS_JSON_PATH;
+    let verdictsJsonlPath = VERDICTS_JSONL_PATH;
+    for (const arg of argv) {
+        if (arg === "--resume") {
+            resume = true;
+            continue;
+        }
+        const m = arg.match(/^--([a-z-]+)=(.+)$/);
+        if (m) {
+            if (m[1] === "judge-delay-ms")
+                judgeDelayMs = parseInt(m[2], 10);
+            else if (m[1] === "max-calls")
+                maxCalls = parseInt(m[2], 10);
+            else if (m[1] === "verdicts-json")
+                verdictsJsonPath = m[2];
+            else if (m[1] === "verdicts-jsonl")
+                verdictsJsonlPath = m[2];
+            continue;
+        }
+        positionals.push(arg);
+    }
+    return { positionals, judgeDelayMs, maxCalls, resume, verdictsJsonPath, verdictsJsonlPath };
+}
+// ---------------------------------------------------------------------------
+// Reporting
+// ---------------------------------------------------------------------------
+function pct(x) {
+    return Number.isFinite(x) ? `${(x * 100).toFixed(1)}%` : "n/a";
+}
+function pts(x) {
+    return Number.isFinite(x) ? `${(x * 100).toFixed(1)}pts` : "n/a";
+}
+function fmtNum(x, decimals = 2) {
+    return Number.isFinite(x) ? x.toFixed(decimals) : "n/a";
+}
+function head(s, n) {
+    const flat = s.replace(/\s+/g, " ").trim();
+    return flat.length > n ? flat.slice(0, n) + "…" : flat;
+}
+function renderReport(args) {
+    const { prompts, results, sweepRows, sweepSubsetIdx, verdictRows, judgeState } = args;
+    const off = results.filter((r) => r.mode === "off");
+    const on = results.filter((r) => r.mode === "on");
+    const L = [];
+    if (judgeState.aborted) {
+        L.push(`# ⚠ ABORTED RUN — ${judgeState.aborted.reason}\n\n` +
+            `_The report below reflects only what completed before the abort. ${judgeState.completedUnits} judge ` +
+            `units completed, ${judgeState.errorUnits} judge_error, ${judgeState.totalCalls} fresh calls this ` +
+            `invocation. Re-run with \`--resume\` after addressing the cause._\n`);
+    }
+    L.push("# Recall Relevance + Snippet Eval — v2\n");
+    L.push(`Snapshot: \`${args.snapshotPath}\`  \n` +
+        `Prompts: \`${args.promptsPath}\` (${prompts.length} prompts; requested ${SOURCE_TAGS.length * N_PER_SOURCE} — ` +
+        `see §0 for any per-source shortfall)  \n` +
+        `Generated: ${args.generatedAt}\n`);
+    L.push(`_Spec: \`specs/2026-08-02-relevance-eval.md\`. Real agent prompts replayed through \`retrieve()\` on a ` +
+        `READONLY snapshot; GLM-5.2 (z.ai) is the JUDGE only — retrieve() is LLM-free. Two independent, BLIND judge ` +
+        `calls per (prompt × mode): \`full_verdict\` (up to 2000 chars) and \`line_verdict\` (ONLY the rendered ` +
+        `production one-liner, imported from \`recall-index.ts#formatIndexLine\`/\`memoryTitle\` — never ` +
+        `reimplemented). Static DB (\`noStrengthen: true\`); real bge-small-en-v1.5 embedder; embed-once via ` +
+        `\`queryEmbedding\` with a \`neverCalledEmbed\` self-check. Retrieve K=${RESULT_K}; precision@4/@${PROD_MENU_K}/@${RESULT_K} ` +
+        `all derived from the SAME verdicts. Token counts are a char/4 ESTIMATE (no tokenizer available locally — stated ` +
+        `explicitly wherever used). Raw per-verdict rows saved to \`${args.verdictsJsonPath}\` for ad-hoc tuning ` +
+        `analysis without re-judging._\n`);
+    // =====================================================================
+    // §0 — corpus construction (source counts, sweep subset)
+    // =====================================================================
+    L.push("## 0. Corpus\n");
+    const bySource = new Map();
+    for (const p of prompts)
+        bySource.set(p.sourceTag, (bySource.get(p.sourceTag) ?? 0) + 1);
+    L.push("| source | prompts | of requested |");
+    L.push("|---|---|---|");
+    for (const tag of SOURCE_TAGS) {
+        L.push(`| ${tag} | ${bySource.get(tag) ?? 0} | ${N_PER_SOURCE} |`);
+    }
+    L.push("");
+    L.push(`Length-sweep fixed subset (spec §4.2): ${sweepSubsetIdx.length} prompts (${SWEEP_N_PER_SOURCE}/source), ` +
+        `scope OFF only (the true baseline).\n`);
+    // =====================================================================
+    // §1 — per-prompt precision histogram (v1, unchanged shape)
+    // =====================================================================
+    L.push(`## 1. Per-prompt precision histogram (relevant-count in Q1-Q${RESULT_K}, full_verdict)\n`);
+    const offHist = computePromptHistogram(results, "off");
+    const onHist = computePromptHistogram(results, "on");
+    L.push("| relevant-count | OFF prompt-count | ON prompt-count |");
+    L.push("|---|---|---|");
+    for (let i = 0; i <= RESULT_K; i++) {
+        const marker = i <= 1 ? " ← worst" : i >= RESULT_K - 1 ? " ← best" : "";
+        L.push(`| ${i}${marker} | ${offHist[i]} | ${onHist[i]} |`);
+    }
+    L.push("");
+    // =====================================================================
+    // §2 — worst/best named prompts
+    // =====================================================================
+    L.push(`## 2. Worst (0-1 relevant) and best (${RESULT_K - 1}-${RESULT_K} relevant) prompts — scope OFF\n`);
+    const offWb = findWorstBestPrompts(results, "off", 1, RESULT_K - 1);
+    L.push(`### Worst — ${offWb.worst.length} prompt(s)\n`);
+    if (offWb.worst.length === 0) {
+        L.push("_(none)_\n");
+    }
+    else {
+        for (const p of offWb.worst.slice(0, 6)) {
+            L.push(`- **[${p.agent}]** \`${head(p.prompt, 160)}\``);
+            L.push(`  - tally: ${p.relevant} relevant / ${p.noise} noise / ${p.stale} stale`);
+            for (const v of p.perRank) {
+                L.push(`  - Q${v.rank} \`${v.memory_id.slice(0, 8)}\` **${v.verdict}** — ${v.reason || "_(no reason)_"} — \`${head(v.content, 100)}\``);
+            }
+        }
+        L.push("");
+    }
+    L.push(`### Best — ${offWb.best.length} prompt(s)\n`);
+    if (offWb.best.length === 0) {
+        L.push("_(none)_\n");
+    }
+    else {
+        for (const p of offWb.best.slice(0, 4)) {
+            L.push(`- **[${p.agent}]** \`${head(p.prompt, 160)}\` — ${p.relevant} relevant / ${p.noise} noise / ${p.stale} stale`);
+        }
+        L.push("");
+    }
+    // =====================================================================
+    // §3/§4 — chronic-noise / chronic-stale
+    // =====================================================================
+    const metaById = lookupMemoryMeta(args.db, results);
+    const chronic = rankChronicMemories(results, metaById);
+    const chronicNoise = [...chronic].sort((a, b) => b.noisePrompts - a.noisePrompts).filter((c) => c.noisePrompts > 0);
+    const chronicStale = [...chronic].sort((a, b) => b.stalePrompts - a.stalePrompts).filter((c) => c.stalePrompts > 0);
+    L.push(`## 3. Chronic-noise memories (ranked pollution sources) — HEADLINE\n`);
+    if (chronicNoise.length === 0) {
+        L.push("_(none)_\n");
+    }
+    else {
+        L.push("| # | noise-prompts | stale-prompts | surfaces | type | project | id | content snippet |");
+        L.push("|---|---|---|---|---|---|---|---|");
+        for (let i = 0; i < Math.min(15, chronicNoise.length); i++) {
+            const c = chronicNoise[i];
+            L.push(`| ${i + 1} | ${c.noisePrompts} | ${c.stalePrompts} | ${c.totalSurfaces} | ${c.memory_type ?? "-"} | ` +
+                `${c.project ?? "-"} | \`${c.memory_id.slice(0, 8)}\` | \`${head(c.content, 100)}\` |`);
+        }
+        L.push("");
+    }
+    L.push(`## 4. Chronic-stale memories (supersession gaps)\n`);
+    if (chronicStale.length === 0) {
+        L.push("_(none)_\n");
+    }
+    else {
+        L.push("| # | stale-prompts | noise-prompts | surfaces | type | project | id | content snippet |");
+        L.push("|---|---|---|---|---|---|---|---|");
+        for (let i = 0; i < Math.min(15, chronicStale.length); i++) {
+            const c = chronicStale[i];
+            L.push(`| ${i + 1} | ${c.stalePrompts} | ${c.noisePrompts} | ${c.totalSurfaces} | ${c.memory_type ?? "-"} | ` +
+                `${c.project ?? "-"} | \`${c.memory_id.slice(0, 8)}\` | \`${head(c.content, 100)}\` |`);
+        }
+        L.push("");
+    }
+    // =====================================================================
+    // §5 — K-sweep + per-rank + top/bottom + bubble (v1, supporting)
+    // =====================================================================
+    L.push(`## 5. K-sweep (supporting — precision@4 / @${PROD_MENU_K} / @${RESULT_K}, full_verdict)\n`);
+    L.push("| mode | precision@4 | precision@6 | precision@8 | noise@4 | noise@6 | noise@8 | stale@6 |");
+    L.push("|---|---|---|---|---|---|---|---|");
+    const m4off = computeMetricsAtK(off, 4), m6off = computeMetricsAtK(off, 6), m8off = computeMetricsAtK(off, 8);
+    const m4on = computeMetricsAtK(on, 4), m6on = computeMetricsAtK(on, 6), m8on = computeMetricsAtK(on, 8);
+    if (m4off && m6off && m8off) {
+        L.push(`| OFF | ${pct(m4off.precision)} | ${pct(m6off.precision)} | ${pct(m8off.precision)} | ${pct(m4off.noiseRate)} | ${pct(m6off.noiseRate)} | ${pct(m8off.noiseRate)} | ${pct(m6off.staleRate)} |`);
+    }
+    if (m4on && m6on && m8on) {
+        L.push(`| ON | ${pct(m4on.precision)} | ${pct(m6on.precision)} | ${pct(m8on.precision)} | ${pct(m4on.noiseRate)} | ${pct(m6on.noiseRate)} | ${pct(m8on.noiseRate)} | ${pct(m6on.staleRate)} |`);
+    }
+    L.push("");
+    L.push(`## 6. Per-rank verdict distribution (Q1..Q${RESULT_K}, full_verdict)\n`);
+    for (const [label, rs] of [["OFF", off], ["ON", on]]) {
+        const rows = computeRankMetrics(rs);
+        L.push(`### Scope ${label}\n`);
+        L.push("| rank | band | N | relevant | noise | stale |");
+        L.push("|---|---|---|---|---|---|");
+        for (const r of rows) {
+            const band = r.rank <= PROD_MENU_K ? "menu" : "bubble";
+            L.push(`| Q${r.rank} | ${band} | ${r.n} | ${Math.round(r.relevantRate * r.n)} | ${Math.round(r.noiseRate * r.n)} | ${Math.round(r.staleRate * r.n)} |`);
+        }
+        L.push("");
+    }
+    const offTop = aggregateRankBand(off, 1, 3);
+    const offBot = aggregateRankBand(off, 4, PROD_MENU_K);
+    const onTop = aggregateRankBand(on, 1, 3);
+    const onBot = aggregateRankBand(on, 4, PROD_MENU_K);
+    L.push(`## 7. Top-3 vs bottom-3 noise split + bubble (supporting)\n`);
+    L.push("| mode | band | noise / N | noise-rate |");
+    L.push("|---|---|---|---|");
+    L.push(`| OFF | top-3 | ${offTop.noise}/${offTop.total} | **${pct(offTop.rate)}** |`);
+    L.push(`| OFF | bottom-3 | ${offBot.noise}/${offBot.total} | ${pct(offBot.rate)} |`);
+    L.push(`| ON | top-3 | ${onTop.noise}/${onTop.total} | **${pct(onTop.rate)}** |`);
+    L.push(`| ON | bottom-3 | ${onBot.noise}/${onBot.total} | ${pct(onBot.rate)} |`);
+    L.push("");
+    const rateFromRows = (rows) => {
+        const rel = rows.reduce((s, r) => s + Math.round(r.relevantRate * r.n), 0);
+        const total = rows.reduce((s, r) => s + r.n, 0);
+        return { rel, total, rate: total > 0 ? rel / total : 0 };
+    };
+    const offMenuRel = rateFromRows(computeRankMetrics(off).slice(0, PROD_MENU_K));
+    const onMenuRel = rateFromRows(computeRankMetrics(on).slice(0, PROD_MENU_K));
+    const offBubRel = rateFromRows(computeRankMetrics(off).slice(PROD_MENU_K));
+    const onBubRel = rateFromRows(computeRankMetrics(on).slice(PROD_MENU_K));
+    L.push(`Bubble (Q${PROD_MENU_K + 1}-Q${RESULT_K}) relevant-rate vs the menu.\n`);
+    L.push("| mode | band | relevant / N | relevant-rate |");
+    L.push("|---|---|---|---|");
+    L.push(`| OFF | menu | ${offMenuRel.rel}/${offMenuRel.total} | ${pct(offMenuRel.rate)} |`);
+    L.push(`| OFF | bubble | ${offBubRel.rel}/${offBubRel.total} | **${pct(offBubRel.rate)}** |`);
+    L.push(`| ON | menu | ${onMenuRel.rel}/${onMenuRel.total} | ${pct(onMenuRel.rate)} |`);
+    L.push(`| ON | bubble | ${onBubRel.rel}/${onBubRel.total} | **${pct(onBubRel.rate)}** |`);
+    L.push("");
+    // =====================================================================
+    // §8 — per-source breakdown (spec §5.7)
+    // =====================================================================
+    L.push("## 8. Per-source breakdown (scope OFF, precision@6)\n");
+    L.push("| source | precision@6 | noise@6 | stale@6 | n prompts |");
+    L.push("|---|---|---|---|---|");
+    for (const tag of SOURCE_TAGS) {
+        const rs = off.filter((r) => r.prompt.sourceTag === tag);
+        const m = computeMetricsAtK(rs, PROD_MENU_K);
+        L.push(`| ${tag} | ${m ? pct(m.precision) : "n/a"} | ${m ? pct(m.noiseRate) : "n/a"} | ${m ? pct(m.staleRate) : "n/a"} | ${m ? m.n : 0} |`);
+    }
+    L.push("");
+    // =====================================================================
+    // §9 — snippet layer: dual verdict + named failures (spec §4.1, §5.8) — HEADLINE
+    // =====================================================================
+    L.push("## 9. Snippet layer — dual verdict (full_verdict vs line_verdict) — HEADLINE\n");
+    const gapOff = computeSnippetGap(off);
+    const gapOn = computeSnippetGap(on);
+    const gapAll = computeSnippetGap(results);
+    const snippetFailureRate = (g) => (g.fullRelevantTotal > 0 ? g.snippetFailure / g.fullRelevantTotal : NaN);
+    L.push("| scope | full-relevant (denominator) | actionable | snippet_failure | snippet_failure_rate | misleading_line |");
+    L.push("|---|---|---|---|---|---|");
+    L.push(`| OFF | ${gapOff.fullRelevantTotal} | ${gapOff.actionable} | ${gapOff.snippetFailure} | **${pct(snippetFailureRate(gapOff))}** | ${gapOff.misleadingLine} |`);
+    L.push(`| ON | ${gapOn.fullRelevantTotal} | ${gapOn.actionable} | ${gapOn.snippetFailure} | **${pct(snippetFailureRate(gapOn))}** | ${gapOn.misleadingLine} |`);
+    L.push(`| BOTH | ${gapAll.fullRelevantTotal} | ${gapAll.actionable} | ${gapAll.snippetFailure} | **${pct(snippetFailureRate(gapAll))}** | ${gapAll.misleadingLine} |`);
+    L.push("");
+    L.push(`_snippet_failure_rate = snippet_failure / full-relevant — "of the memories retrieve() got RIGHT, what fraction ` +
+        `does the ~100-char production line fail to communicate as relevant?" misleading_line = the agent is BAITED ` +
+        `by a line that reads relevant for a memory that (on full inspection) is not._\n`);
+    const namedFailures = findNamedSnippetFailures(results, 15);
+    L.push(`### Named worst snippet failures (up to 15)\n`);
+    if (namedFailures.length === 0) {
+        L.push("_(none — every full-relevant memory's line also read as relevant)_\n");
+    }
+    else {
+        for (const f of namedFailures) {
+            L.push(`- **[${f.mode}/${f.agent}]** prompt: \`${head(f.prompt, 120)}\``);
+            L.push(`  - rendered line: \`${f.rendered_line}\``);
+            L.push(`  - full_verdict=relevant (${f.full_reason || "no reason"}) but line_verdict≠relevant (${f.line_reason || "no reason"})`);
+            L.push(`  - full content: \`${head(f.content, 140)}\``);
+        }
+        L.push("");
+    }
+    // =====================================================================
+    // §10 — snippet-length sweep (spec §4.2, §5.8) — HEADLINE
+    // =====================================================================
+    L.push(`## 10. Snippet-length sweep (N=${sweepSubsetIdx.length}, scope OFF, Q1-Q${RESULT_K}) — HEADLINE\n`);
+    L.push(`_Same prompts, same memories — only the rendering changes per variant. CI is ~1.6× wider than the N=100 tables ` +
+        `(spec §8); only effects ≥ ~15pts are distinguishable here. ${PROD_TITLE_CHARS} is the PRODUCTION baseline ` +
+        `(shipped \`recallTitleChars\`) — deltas are reported against it, not against 100._\n`);
+    L.push(`| variant | line-relevant-rate | 95% CI | Δ vs ${PROD_TITLE_CHARS} (prod) | mean tokens/block (prod-6, char/4) |`);
+    L.push("|---|---|---|---|---|");
+    const sweepByVariant = new Map();
+    for (const v of SWEEP_VARIANTS)
+        sweepByVariant.set(v, sweepRows.filter((s) => s.variant === v));
+    const sweepVariantCi = new Map();
+    const sweepVariantTokens = new Map();
+    // Pass 1: compute CI + token estimate per variant (order-independent — the
+    // baseline lookup below must not depend on iterating variants in a
+    // particular order, since PROD_TITLE_CHARS (150) is no longer first in
+    // SWEEP_VARIANTS' display order).
+    for (const variant of SWEEP_VARIANTS) {
+        const rowsForVariant = sweepByVariant.get(variant) ?? [];
+        const perPrompt = [];
+        const tokenSamples = [];
+        for (const sw of rowsForVariant) {
+            if (!Array.isArray(sw.verdicts))
+                continue;
+            const relCount = sw.verdicts.filter((v) => v.verdict === "relevant").length;
+            const denom = sw.verdicts.length || 1;
+            perPrompt.push(relCount / denom);
+            const off6 = off.find((r) => prompts.indexOf(r.prompt) === sw.promptIdx);
+            if (off6) {
+                const rowsArr = off6.topIds.map((id) => off6.rows[id]).filter(Boolean);
+                const gated = rowsArr.filter((r) => (0, recall_index_js_1.passesRelevanceGate)(r, PROD_MIN_SIMILARITY)).slice(0, PROD_MENU_K);
+                const block = [...RECALL_BLOCK_HEADER, ...gated.map((r) => sw.lines[r.id] ?? "")].join("\n");
+                tokenSamples.push(estimateTokens(block));
+            }
+        }
+        sweepVariantCi.set(variant, ci95(perPrompt));
+        sweepVariantTokens.set(variant, tokenSamples.length ? mean(tokenSamples) : NaN);
+    }
+    // Pass 2: render rows in SWEEP_VARIANTS order, delta always vs the
+    // production baseline (PROD_TITLE_CHARS), looked up by key not by position.
+    const baselineCi = sweepVariantCi.get(String(PROD_TITLE_CHARS)) ?? null;
+    for (const variant of SWEEP_VARIANTS) {
+        const c = sweepVariantCi.get(variant);
+        const tokenSamples = sweepVariantTokens.get(variant);
+        const isBaseline = variant === String(PROD_TITLE_CHARS);
+        const delta = baselineCi && !isBaseline ? c.mean - baselineCi.mean : null;
+        const variantLabel = isBaseline ? `${variant} (prod)` : variant;
+        L.push(`| ${variantLabel} | ${pct(c.mean)} | [${pct(c.lo)}, ${pct(c.hi)}] | ${delta === null ? "—" : pts(delta)} | ${Number.isFinite(tokenSamples) ? tokenSamples.toFixed(0) : "n/a"} |`);
+    }
+    L.push("");
+    // =====================================================================
+    // §11 — similarity-floor + retrieval-source analysis (spec §5.9)
+    // =====================================================================
+    L.push(`## 11. Similarity-floor + retrieval-source analysis (full_verdict, variant=${PROD_TITLE_CHARS} baseline only)\n`);
+    const baseRows = verdictRows.filter((r) => r.snippet_len_variant === String(PROD_TITLE_CHARS) && r.full_verdict !== null);
+    const byBucket = new Map();
+    for (const r of baseRows) {
+        if (r.full_verdict === "judge_error")
+            continue;
+        const label = similarityBucketLabel(r.similarity);
+        const e = byBucket.get(label) ?? { relevant: 0, noise: 0, stale: 0, total: 0 };
+        e.total++;
+        if (r.full_verdict === "relevant")
+            e.relevant++;
+        else if (r.full_verdict === "noise")
+            e.noise++;
+        else if (r.full_verdict === "stale")
+            e.stale++;
+        byBucket.set(label, e);
+    }
+    L.push("### By similarity band\n");
+    L.push("| band | n | relevant | noise | stale | noise-rate |");
+    L.push("|---|---|---|---|---|---|");
+    const bucketOrder = ["n/a (fts/graph, no cosine)", "<0.30", ...SIM_BUCKET_EDGES.slice(0, -1).map((_, i) => `${SIM_BUCKET_EDGES[i].toFixed(2)}–${Math.min(SIM_BUCKET_EDGES[i + 1], 1).toFixed(2)}`)];
+    for (const label of bucketOrder) {
+        const e = byBucket.get(label);
+        if (!e)
+            continue;
+        L.push(`| ${label} | ${e.total} | ${e.relevant} | ${e.noise} | ${e.stale} | ${pct(e.total > 0 ? e.noise / e.total : 0)} |`);
+    }
+    L.push("");
+    L.push("### By retrieval source (channel)\n");
+    const bySrc = new Map();
+    for (const r of baseRows) {
+        if (r.full_verdict === "judge_error")
+            continue;
+        const label = r.retrieval_source ?? "(unknown)";
+        const e = bySrc.get(label) ?? { relevant: 0, noise: 0, stale: 0, total: 0 };
+        e.total++;
+        if (r.full_verdict === "relevant")
+            e.relevant++;
+        else if (r.full_verdict === "noise")
+            e.noise++;
+        else if (r.full_verdict === "stale")
+            e.stale++;
+        bySrc.set(label, e);
+    }
+    L.push("| retrieval_source | n | relevant | noise | stale | noise-rate |");
+    L.push("|---|---|---|---|---|---|");
+    for (const [label, e] of Array.from(bySrc.entries()).sort((a, b) => b[1].total - a[1].total)) {
+        L.push(`| ${label} | ${e.total} | ${e.relevant} | ${e.noise} | ${e.stale} | ${pct(e.total > 0 ? e.noise / e.total : 0)} |`);
+    }
+    L.push("");
+    // =====================================================================
+    // §12 — token cost per K (spec §5.10)
+    // =====================================================================
+    L.push(`## 12. Token cost per K (production render, maxLen=${PROD_TITLE_CHARS}, scope OFF, char/4 estimate)\n`);
+    L.push("| K | mean tokens/block | mean chars/block |");
+    L.push("|---|---|---|");
+    for (const k of SWEEP_KS) {
+        const tokenSamples = [];
+        for (const r of off) {
+            const rowsArr = r.topIds.map((id) => r.rows[id]).filter(Boolean);
+            const { block } = renderProductionBlock(rowsArr, PROD_MIN_SIMILARITY, k, PROD_TITLE_CHARS);
+            tokenSamples.push(estimateTokens(block));
+        }
+        L.push(`| ${k} | ${fmtNum(mean(tokenSamples), 0)} | ${fmtNum(mean(tokenSamples) * 4, 0)} |`);
+    }
+    L.push("\n_Per-variant token cost is in §10's sweep table (same char/4 estimate, prod-6-gated block)._\n");
+    // =====================================================================
+    // §13 — rendered production blocks (spec §5.11) — ~20 mixed best/worst
+    // =====================================================================
+    L.push(`## 13. Rendered ACTUAL production blocks (mixed best/worst, scope OFF, maxLen=${PROD_TITLE_CHARS})\n`);
+    L.push(`_Exactly what \`/recall-index\` would render: \`passesRelevanceGate\` (minSimilarity=${PROD_MIN_SIMILARITY}) then ` +
+        `capped at ${PROD_MENU_K}. NOTE: candidate pool here is Q1-Q${RESULT_K} (this eval's replay window); production ` +
+        `over-fetches \`maxItems×3\` candidates before gating, so a real production block could differ slightly on a ` +
+        `sparse query — a known, documented simplification, not a bug._\n`);
+    const worstBlocks = offWb.worst.slice(0, 10);
+    const bestBlocks = offWb.best.slice(0, 10);
+    let shown = 0;
+    for (const label of ["worst", "best"]) {
+        const set = label === "worst" ? worstBlocks : bestBlocks;
+        for (const p of set) {
+            if (shown >= 20)
+                break;
+            const r = off.find((x) => x.prompt.prompt === p.prompt && x.prompt.agent === p.agent);
+            if (!r)
+                continue;
+            const rowsArr = r.topIds.map((id) => r.rows[id]).filter(Boolean);
+            const { block } = renderProductionBlock(rowsArr, PROD_MIN_SIMILARITY, PROD_MENU_K, PROD_TITLE_CHARS);
+            L.push(`### [${label}] ${p.agent}: \`${head(p.prompt, 100)}\`\n`);
+            L.push("```");
+            L.push(block || "(null — no candidate passed the relevance gate)");
+            L.push("```\n");
+            shown++;
+        }
+    }
+    // =====================================================================
+    // §14 — redundancy (spec §5.12)
+    // =====================================================================
+    L.push("## 14. Redundancy over the production 6 (set-level judge call per prompt × mode)\n");
+    const redundancySamples = [];
+    for (const r of results) {
+        if (!r.redundancy)
+            continue;
+        if ("skipped" in r.redundancy)
+            redundancySamples.push({ mode: r.mode, shown: r.redundancy.shown, nonRedundant: r.redundancy.shown });
+        else if (!("judgeError" in r.redundancy))
+            redundancySamples.push({ mode: r.mode, shown: r.redundancy.shown, nonRedundant: r.redundancy.nonRedundantCount });
+    }
+    L.push("| mode | n prompts | mean shown | mean non-redundant | mean redundant slots |");
+    L.push("|---|---|---|---|---|");
+    for (const label of ["off", "on"]) {
+        const rs = redundancySamples.filter((s) => s.mode === label);
+        const meanShown = mean(rs.map((s) => s.shown));
+        const meanNr = mean(rs.map((s) => s.nonRedundant));
+        L.push(`| ${label.toUpperCase()} | ${rs.length} | ${fmtNum(meanShown)} | ${fmtNum(meanNr)} | ${fmtNum(meanShown - meanNr)} |`);
+    }
+    L.push("");
+    // =====================================================================
+    // §15 — uncertainty (spec §5.13, §8)
+    // =====================================================================
+    L.push("## 15. Uncertainty — SD / SE / 95% CI\n");
+    const p6off = computeMetricsAtK(off, PROD_MENU_K);
+    const p6on = computeMetricsAtK(on, PROD_MENU_K);
+    L.push("| metric | mode | mean | SD | SE | 95% CI |");
+    L.push("|---|---|---|---|---|---|");
+    if (p6off) {
+        const c = ci95(p6off.perPrompt);
+        L.push(`| precision@${PROD_MENU_K} | OFF | ${pct(c.mean)} | ${pct(c.sd)} | ${pct(c.se)} | [${pct(c.lo)}, ${pct(c.hi)}] |`);
+    }
+    if (p6on) {
+        const c = ci95(p6on.perPrompt);
+        L.push(`| precision@${PROD_MENU_K} | ON | ${pct(c.mean)} | ${pct(c.sd)} | ${pct(c.se)} | [${pct(c.lo)}, ${pct(c.hi)}] |`);
+    }
+    L.push("");
+    let distinguishable = "n/a";
+    let deltaStr = "n/a";
+    if (p6off && p6on) {
+        const cOff = ci95(p6off.perPrompt);
+        const cOn = ci95(p6on.perPrompt);
+        const delta = cOn.mean - cOff.mean;
+        const seDelta = Math.sqrt(cOff.se ** 2 + cOn.se ** 2);
+        const thresholdPts = 1.96 * seDelta;
+        distinguishable = Math.abs(delta) > thresholdPts ? "YES" : "NO — within noise";
+        deltaStr = `${pts(delta)} (SE_delta=${pts(seDelta)}, 95% threshold=±${pts(thresholdPts)})`;
+        L.push(`**Scope ON−OFF delta (precision@${PROD_MENU_K}):** ${deltaStr} — **${distinguishable}**\n`);
+    }
+    // =====================================================================
+    // §16 — DECISION THRESHOLDS (spec §9) — filled with measured values
+    // =====================================================================
+    L.push("## 16. Decision thresholds (spec §9) — measured values + triggered actions\n");
+    const noiseRate6off = m6off ? m6off.noiseRate : NaN;
+    const noiseHighTriggered = Number.isFinite(noiseRate6off) && noiseRate6off > 0.3;
+    // Worst similarity band with meaningful sample (n>=5), excluding n/a.
+    let worstBand = null;
+    for (const [label, e] of byBucket.entries()) {
+        if (label.startsWith("n/a"))
+            continue;
+        if (e.total < 5)
+            continue;
+        const rate = e.noise / e.total;
+        if (!worstBand || rate > worstBand.rate)
+            worstBand = { label, rate, n: e.total };
+    }
+    const bandTriggered = worstBand !== null && worstBand.rate >= 0.6;
+    const ftsE = bySrc.get("fts");
+    const vecE = bySrc.get("vector");
+    const ftsNoiseRate = ftsE && ftsE.total > 0 ? ftsE.noise / ftsE.total : NaN;
+    const vecNoiseRate = vecE && vecE.total > 0 ? vecE.noise / vecE.total : NaN;
+    const ftsRatio = Number.isFinite(ftsNoiseRate) && Number.isFinite(vecNoiseRate) && vecNoiseRate > 0 ? ftsNoiseRate / vecNoiseRate : NaN;
+    const ftsTriggered = Number.isFinite(ftsRatio) && ftsRatio > 1.5;
+    const snippetFailOffRate = snippetFailureRate(gapOff);
+    const snippetFailTriggered = Number.isFinite(snippetFailOffRate) && snippetFailOffRate > 0.2;
+    // Baseline is now PROD_TITLE_CHARS (150), not 100 — 300 was dropped from the
+    // sweep (replaced by 120) in the 2026-08-02 post-rewrite run config.
+    const ciBase = sweepVariantCi.get(String(PROD_TITLE_CHARS));
+    const ciLong = sweepVariantCi.get("200");
+    const deltaBaseToLong = ciBase && ciLong ? ciLong.mean - ciBase.mean : NaN;
+    const lengthHelpsTriggered = Number.isFinite(deltaBaseToLong) && deltaBaseToLong >= 0.05;
+    // Owner question (2026-08-02): can maxLen be LOWERED 150→120 without losing
+    // signal (token savings)? "No meaningful loss" = delta not more negative
+    // than -5pts.
+    const ciShort = sweepVariantCi.get("120");
+    const deltaBaseToShort = ciBase && ciShort ? ciShort.mean - ciBase.mean : NaN;
+    const canLowerTo120 = Number.isFinite(deltaBaseToShort) && deltaBaseToShort >= -0.05;
+    const bubbleGapPts = offMenuRel.rate - offBubRel.rate; // menu minus bubble
+    const bubbleWithin5 = Number.isFinite(bubbleGapPts) && Math.abs(bubbleGapPts) <= 0.05;
+    const bubbleMuchLower = Number.isFinite(bubbleGapPts) && bubbleGapPts > 0.15;
+    const topLoaded = offTop.total > 0 && offBot.total > 0 && offTop.rate > offBot.rate;
+    const topChronicNoise = chronicNoise[0] ?? null;
+    const chronicTriggered = topChronicNoise !== null && topChronicNoise.noisePrompts > 10;
+    const offRedundancy = redundancySamples.filter((s) => s.mode === "off");
+    const meanRedundantSlotsOff = offRedundancy.length > 0 ? mean(offRedundancy.map((s) => s.shown - s.nonRedundant)) : NaN;
+    const redundancyTriggered = Number.isFinite(meanRedundantSlotsOff) && meanRedundantSlotsOff > 1.5;
+    const scopeDeltaPts = p6off && p6on ? (ci95(p6on.perPrompt).mean - ci95(p6off.perPrompt).mean) * 100 : NaN;
+    L.push("| finding | measured | threshold | triggered? | action |");
+    L.push("|---|---|---|---|---|");
+    L.push(`| Overall noise-rate (prod K=6, OFF) | ${pct(noiseRate6off)} | >30% | ${noiseHighTriggered ? "**YES**" : "no"} | Raise \`recallMinSimilarity\` (0.55 → floor indicated by the band table) |`);
+    L.push(`| Noise concentrated below a band | worst band ${worstBand ? `${worstBand.label} (${pct(worstBand.rate)}, n=${worstBand.n})` : "n/a"} | band ≥60% noise | ${bandTriggered ? "**YES**" : "no"} | Set \`recallMinSimilarity\` at that band's lower edge |`);
+    L.push(`| FTS-matched hits disproportionately noise | fts ${pct(ftsNoiseRate)} vs vector ${pct(vecNoiseRate)} (ratio ${Number.isFinite(ftsRatio) ? ftsRatio.toFixed(2) : "n/a"}×) | >1.5× | ${ftsTriggered ? "**YES**" : "no"} | Stop letting FTS bypass the floor (\`passesRelevanceGate\`) |`);
+    L.push(`| Snippet failures | ${pct(snippetFailOffRate)} (OFF) | >20% | ${snippetFailTriggered ? "**YES**" : "no"} | Raise \`maxLen\` to the sweep's best variant, or switch to title+first-sentence |`);
+    L.push(`| Longer snippet doesn't improve line-relevance (${PROD_TITLE_CHARS}→200) | Δ ${Number.isFinite(deltaBaseToLong) ? pts(deltaBaseToLong) : "n/a"} | <5pts | ${!lengthHelpsTriggered ? "**YES (no improvement)**" : "no (improves ≥5pts)"} | Keep \`maxLen=${PROD_TITLE_CHARS}\` — the info problem is distillation quality, not length |`);
+    L.push(`| Can \`maxLen\` be LOWERED ${PROD_TITLE_CHARS}→120 without losing signal? | Δ ${Number.isFinite(deltaBaseToShort) ? pts(deltaBaseToShort) : "n/a"} | ≥-5pts (no meaningful loss) | ${canLowerTo120 ? "**YES**" : "no"} | ${canLowerTo120 ? "120 is a viable token-saving downgrade — no measurable quality loss on this evidence" : `Keep \`maxLen=${PROD_TITLE_CHARS}\` — 120 measurably loses signal`} |`);
+    L.push(`| Bubble (Q7-8) relevant-rate within ~5pts of menu | menu ${pct(offMenuRel.rate)} vs bubble ${pct(offBubRel.rate)} (gap ${pts(bubbleGapPts)}) | within ±5pts | ${bubbleWithin5 ? "**YES**" : "no"} | Raise \`recallMaxItems\` 6→8 if token cost is acceptable |`);
+    L.push(`| Bubble much lower than menu | gap ${pts(bubbleGapPts)} | >15pts | ${bubbleMuchLower ? "**YES**" : "no"} | Keep 6, or lower to 4 if Q5-6 are also weak |`);
+    L.push(`| Noise top-loaded (Q1-3 worse than Q4-6) | top-3 ${pct(offTop.rate)} vs bottom-3 ${pct(offBot.rate)} | top > bottom | ${topLoaded ? "**YES**" : "no"} | Ranking bug — investigate \`computeScore\`, don't tune the cap |`);
+    L.push(`| Chronic-noise memory | top offender pollutes ${topChronicNoise ? topChronicNoise.noisePrompts : 0} prompts${topChronicNoise ? ` (\`${topChronicNoise.memory_id.slice(0, 8)}\`)` : ""} | >10 prompts | ${chronicTriggered ? "**YES**" : "no"} | Named target: dedup / demote / re-distill |`);
+    L.push(`| Redundancy in the production 6 | mean ${Number.isFinite(meanRedundantSlotsOff) ? meanRedundantSlotsOff.toFixed(2) : "n/a"} redundant slots (OFF) | >1.5 avg | ${redundancyTriggered ? "**YES**" : "no"} | Tighten \`dedupMergeThreshold\` or add a diversity penalty |`);
+    L.push(`| Scope ON−OFF delta | ${Number.isFinite(scopeDeltaPts) ? scopeDeltaPts.toFixed(1) + "pts" : "n/a"} (empirical 95% threshold ±${p6off && p6on ? (1.96 * Math.sqrt(ci95(p6off.perPrompt).se ** 2 + ci95(p6on.perPrompt).se ** 2) * 100).toFixed(1) : "n/a"}pts; spec's illustrative ±8.5pts) | >8.5pts (spec illustrative) / empirical 95% CI | ${distinguishable === "YES" ? "**YES — distinguishable**" : "no — within noise"} | ${distinguishable === "YES" ? "Declaring mission domains in production is justified" : "Report \"within noise\" — do NOT ship scoping on this evidence"} |`);
+    L.push("");
+    // =====================================================================
+    // §17 — what to tune next
+    // =====================================================================
+    L.push("## 17. What to tune next\n");
+    const actionable = [];
+    const withinNoise = [];
+    if (noiseHighTriggered)
+        actionable.push(`\`recallMinSimilarity\`: overall noise-rate ${pct(noiseRate6off)} > 30% at prod K=6 (OFF).`);
+    else
+        withinNoise.push(`Overall noise-rate at prod K=6 (${pct(noiseRate6off)}) is under the 30% action threshold.`);
+    if (bandTriggered && worstBand)
+        actionable.push(`\`recallMinSimilarity\` → ~${worstBand.label.split("–")[1] ?? worstBand.label}: similarity band ${worstBand.label} is ${pct(worstBand.rate)} noise (n=${worstBand.n}).`);
+    if (ftsTriggered)
+        actionable.push(`\`passesRelevanceGate\` (recall-index.ts): FTS-matched noise-rate (${pct(ftsNoiseRate)}) is ${ftsRatio.toFixed(2)}× vector's (${pct(vecNoiseRate)}) — stop letting FTS bypass the similarity floor.`);
+    else if (Number.isFinite(ftsRatio))
+        withinNoise.push(`FTS-vs-vector noise ratio (${ftsRatio.toFixed(2)}×) is under the 1.5× action threshold.`);
+    if (snippetFailTriggered)
+        actionable.push(`\`maxLen\` (recall-index.ts \`memoryTitle\`): snippet_failure_rate ${pct(snippetFailOffRate)} (OFF) > 20% — the ${PROD_MENU_K}-line menu is losing real information at the render layer, not the retrieval layer.`);
+    else
+        withinNoise.push(`snippet_failure_rate (${pct(snippetFailOffRate)}, OFF) is under the 20% action threshold.`);
+    if (!lengthHelpsTriggered)
+        withinNoise.push(`Length sweep ${PROD_TITLE_CHARS}→200: Δ ${Number.isFinite(deltaBaseToLong) ? pts(deltaBaseToLong) : "n/a"} — within noise at N=${sweepSubsetIdx.length} (±~10pts CI); do not raise \`maxLen\` further on this evidence alone. If snippet_failure IS high, the fix is likely distillation quality (writing tighter titles), not raw length.`);
+    else
+        actionable.push(`\`maxLen\`: ${PROD_TITLE_CHARS}→200 improved line-relevance by ${pts(deltaBaseToLong)} (≥5pts) at N=${sweepSubsetIdx.length} — worth a targeted follow-up at full N before shipping.`);
+    if (canLowerTo120)
+        actionable.push(`\`maxLen\`: ${PROD_TITLE_CHARS}→120 shows Δ ${pts(deltaBaseToShort)} (no meaningful loss, ≥-5pts) at N=${sweepSubsetIdx.length} — a token-saving downgrade the owner asked about is evidence-backed.`);
+    else
+        withinNoise.push(`\`maxLen\` ${PROD_TITLE_CHARS}→120: Δ ${Number.isFinite(deltaBaseToShort) ? pts(deltaBaseToShort) : "n/a"} — do NOT lower to 120 on this evidence, it measurably costs signal.`);
+    if (bubbleWithin5)
+        actionable.push(`\`recallMaxItems\` 6→8: bubble (Q7-8, ${pct(offBubRel.rate)}) is within 5pts of the menu (${pct(offMenuRel.rate)}) — check token cost (§12) before raising.`);
+    else if (bubbleMuchLower)
+        withinNoise.push(`Bubble relevant-rate (${pct(offBubRel.rate)}) is >15pts below the menu (${pct(offMenuRel.rate)}) — \`recallMaxItems=6\` is not leaving good memories on the table.`);
+    if (topLoaded)
+        actionable.push(`\`computeScore\` (retrieval.ts): noise is top-loaded (Q1-3 ${pct(offTop.rate)} > Q4-6 ${pct(offBot.rate)}) — this is a RANKING bug, not a cap-size question.`);
+    else
+        withinNoise.push(`Top-3 vs bottom-3 noise split (${pct(offTop.rate)} vs ${pct(offBot.rate)}) shows no top-loading.`);
+    if (chronicTriggered && topChronicNoise)
+        actionable.push(`Dedup/demote/re-distill \`${topChronicNoise.memory_id.slice(0, 8)}\` (${topChronicNoise.memory_type ?? "unknown type"}): pollutes ${topChronicNoise.noisePrompts} prompts.`);
+    else
+        withinNoise.push(`No single memory pollutes >10 prompts (top offender: ${topChronicNoise ? `${topChronicNoise.noisePrompts} prompts` : "none"}).`);
+    if (redundancyTriggered)
+        actionable.push(`\`dedupMergeThreshold\` / diversity penalty: mean ${meanRedundantSlotsOff.toFixed(2)} redundant slots in the production 6 (OFF) > 1.5.`);
+    else
+        withinNoise.push(`Redundancy in the production 6 (mean ${Number.isFinite(meanRedundantSlotsOff) ? meanRedundantSlotsOff.toFixed(2) : "n/a"} redundant slots, OFF) is under the 1.5 action threshold.`);
+    if (distinguishable === "YES")
+        actionable.push(`Scope (\`missionDomains\`/\`project\` affinity): ON−OFF delta ${deltaStr} is distinguishable at 95% — declaring mission domains in production is evidence-backed.`);
+    else
+        withinNoise.push(`Scope ON−OFF delta (${deltaStr}) is within noise at N=${prompts.length} — do NOT ship mission-domain scoping on this evidence; it would need N≈1800 to detect a ~2pt true effect (spec §8).`);
+    L.push("**Actionable now:**\n");
+    if (actionable.length === 0)
+        L.push("_(none crossed a decision threshold this run)_\n");
+    else
+        for (const a of actionable)
+            L.push(`- ${a}`);
+    L.push("\n**Within noise (not actionable on this evidence):**\n");
+    if (withinNoise.length === 0)
+        L.push("_(none)_\n");
+    else
+        for (const w of withinNoise)
+            L.push(`- ${w}`);
+    L.push("");
+    if (judgeState.errorUnits > 0) {
+        L.push(`_${judgeState.errorUnits}/${judgeState.completedUnits} judge units returned \`judge_error\` (excluded from every ` +
+            `denominator above) — see \`${judgeState.sidecarPath}\` for the raw error messages._\n`);
+    }
+    return L.join("\n");
+}
+// ---------------------------------------------------------------------------
+// main
+// ---------------------------------------------------------------------------
+function usage() {
+    console.error("Usage: relevance-eval.js <snapshot.db> [prompts.json] [report.md] " +
+        "[--judge-delay-ms=2000] [--max-calls=900] [--resume] " +
+        "[--verdicts-json=path] [--verdicts-jsonl=path]\n" +
+        "  <snapshot.db>  — REQUIRED. Readonly bedrock snapshot (opened via openSnapshot).\n" +
+        "  [prompts.json] — omitted: build the v2 corpus fresh (20/source × 5 sources, spec §2),\n" +
+        "                   saved to data/prompts.json. A valid existing v2 set is reused verbatim.\n" +
+        "                   <path.json>: load a saved stable set verbatim (no resample).\n" +
+        "  [report.md]    — output report path (default data/relevance-eval-report.md).\n" +
+        "  --verdicts-json / --verdicts-jsonl — override the raw-verdicts JSON / checkpoint-sidecar\n" +
+        "                   JSONL paths (default data/relevance-verdicts.json[l]). ALWAYS override\n" +
+        "                   these for a re-run against a new snapshot — otherwise a re-run silently\n" +
+        "                   clobbers a prior baseline's raw verdicts.\n" +
+        "Env: ZAI_API_KEY (required — GLM-5.2 via z.ai is the judge)\n" +
+        "Preconditions: /tmp/hermes-{lenny,raider,nano}-state.db (readonly Hermes state DBs) when\n" +
+        "building the corpus fresh; ~/.claude/projects/.../infrastructure + .../aironic-marine dirs.");
+    process.exitCode = 1;
+    process.exit(1);
+}
+async function main() {
+    // Pin all retrieval knobs to shipped defaults — measure the SHIPPED ranker.
+    (0, retrieval_js_1.configureScoring)();
+    (0, retrieval_js_1.configureDecay)();
+    (0, retrieval_js_1.configureRecall)();
+    (0, retrieval_js_1.configureSessionIntent)();
+    const { positionals, judgeDelayMs, maxCalls, resume, verdictsJsonPath, verdictsJsonlPath } = parseArgs(process.argv.slice(2));
+    const [snapshotArg, promptsSourceArg, reportArg] = positionals;
+    if (!snapshotArg)
+        usage();
+    const apiKey = process.env.ZAI_API_KEY;
+    if (!apiKey) {
+        console.error("relevance-eval: ZAI_API_KEY env var is required (GLM-5.2 via z.ai is the judge)");
+        process.exitCode = 1;
+        return;
+    }
+    // ---- prompt corpus (spec §2) ----
+    let prompts;
+    let promptsPath;
+    if (promptsSourceArg) {
+        promptsPath = promptsSourceArg;
+        prompts = loadPromptsJson(promptsSourceArg);
+        console.log(`[relevance-eval] loaded ${prompts.length} prompts from ${promptsPath}`);
+    }
+    else {
+        promptsPath = PROMPTS_SAVE_PATH;
+        if ((0, node_fs_1.existsSync)(promptsPath)) {
+            const existing = loadPromptsJson(promptsPath);
+            if (isValidV2PromptSet(existing)) {
+                prompts = existing;
+                console.log(`[relevance-eval] reusing existing v2 prompt set: ${promptsPath} (${prompts.length} prompts, stable across runs)`);
+            }
+            else {
+                console.log(`[relevance-eval] existing ${promptsPath} does not satisfy the v2 corpus contract (5 sources) — resampling`);
+                prompts = buildV2Corpus();
+                savePromptsJson(promptsPath, prompts);
+            }
+        }
+        else {
+            prompts = buildV2Corpus();
+            savePromptsJson(promptsPath, prompts);
+        }
+        console.log(`[relevance-eval] corpus: ${prompts.length} prompts across ${new Set(prompts.map((p) => p.sourceTag)).size}/${SOURCE_TAGS.length} sources; saved to ${promptsPath}`);
+    }
+    const sweepSubsetIdx = computeSweepSubsetIndices(prompts);
+    const db = (0, eval_db_js_1.openSnapshot)(snapshotArg);
+    try {
+        console.log(`[relevance-eval] snapshot opened readonly: ${snapshotArg} (${prompts.length} prompts × 2 modes = ${prompts.length * 2} retrieve() calls)`);
+        console.log("[relevance-eval] loading bge-small-en-v1.5 + replaying retrieve()...");
+        const t0 = Date.now();
+        const results = await runRetrieveSweep(db, prompts);
+        console.log(`[relevance-eval] retrieve sweep done in ${Date.now() - t0}ms (${results.length} prompt×mode batches, LLM-free)`);
+        // ---- judge run state (rate limiting + resumability, spec §6b) ----
+        (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(verdictsJsonlPath), { recursive: true });
+        if (!resume)
+            (0, node_fs_1.writeFileSync)(verdictsJsonlPath, "", "utf-8"); // fresh run: don't let a stale sidecar leak in.
+        const resumedMap = resume ? loadResumeMap(verdictsJsonlPath) : new Map();
+        let preResumedErrors = 0;
+        for (const v of resumedMap.values()) {
+            if (v && typeof v === "object" && "judgeError" in v)
+                preResumedErrors++;
+        }
+        const judgeState = {
+            apiKey,
+            judgeDelayMs,
+            maxCalls,
+            totalCalls: 0,
+            totalRetries: 0,
+            completedUnits: resumedMap.size,
+            errorUnits: preResumedErrors,
+            startTime: Date.now(),
+            resumed: resumedMap,
+            sidecarPath: verdictsJsonlPath,
+            aborted: null,
+        };
+        if (resume) {
+            console.log(`[relevance-eval] --resume: ${resumedMap.size} units already checkpointed in ${verdictsJsonlPath}`);
+        }
+        const sweepRows = [];
+        try {
+            // ---- Phase 1: full_verdict (all prompts × both modes) ----
+            console.log(`[relevance-eval] phase 1/4: full-content judging (${results.length} units)...`);
+            for (const r of results) {
+                const key = `full|${prompts.indexOf(r.prompt)}|${r.mode}`;
+                const expectedLabels = new Set(r.topIds.map((_, i) => `Q${i + 1}`));
+                const promptText = buildFullContentJudgePrompt(r.prompt.prompt, r.topIds, r.contents);
+                r.fullVerdicts = await judgeVerdictUnit(judgeState, key, promptText, expectedLabels);
+            }
+            // ---- Phase 2: line_verdict at maxLen=PROD_TITLE_CHARS (production render) ----
+            console.log(`[relevance-eval] phase 2/4: line-only judging at production maxLen=${PROD_TITLE_CHARS} (${results.length} units)...`);
+            for (const r of results) {
+                const key = `line|${prompts.indexOf(r.prompt)}|${r.mode}`;
+                const expectedLabels = new Set(r.topIds.map((_, i) => `Q${i + 1}`));
+                const lines = {};
+                for (const id of r.topIds)
+                    lines[id] = (0, recall_index_js_1.formatIndexLine)(r.rows[id], PROD_TITLE_CHARS);
+                const promptText = buildLineJudgePrompt(r.prompt.prompt, r.topIds, lines);
+                r.lineVerdicts = await judgeVerdictUnit(judgeState, key, promptText, expectedLabels);
+            }
+            // ---- Phase 3: snippet-length sweep (40-subset, scope OFF, 4 variants) ----
+            const offResultByPromptIdx = new Map();
+            for (const r of results) {
+                if (r.mode === "off")
+                    offResultByPromptIdx.set(prompts.indexOf(r.prompt), r);
+            }
+            console.log(`[relevance-eval] phase 3/4: snippet-length sweep (${sweepSubsetIdx.length} prompts × ${SWEEP_VARIANTS.length} variants)...`);
+            for (const promptIdx of sweepSubsetIdx) {
+                const r = offResultByPromptIdx.get(promptIdx);
+                if (!r)
+                    continue;
+                for (const variant of SWEEP_VARIANTS) {
+                    const key = `sweep|${promptIdx}|off|${variant}`;
+                    const expectedLabels = new Set(r.topIds.map((_, i) => `Q${i + 1}`));
+                    const lines = {};
+                    for (const id of r.topIds)
+                        lines[id] = renderVariantLine(r.rows[id], variant);
+                    const promptText = buildLineJudgePrompt(r.prompt.prompt, r.topIds, lines);
+                    const verdicts = await judgeVerdictUnit(judgeState, key, promptText, expectedLabels);
+                    sweepRows.push({ promptIdx, variant, verdicts, lines });
+                }
+            }
+            // ---- Phase 4: redundancy over the production 6 (per prompt × mode) ----
+            console.log(`[relevance-eval] phase 4/4: redundancy judging (${results.length} units, some skipped when <2 shown)...`);
+            for (const r of results) {
+                const rowsArr = r.topIds.map((id) => r.rows[id]).filter(Boolean);
+                const { shownIds, lines } = renderProductionBlock(rowsArr, PROD_MIN_SIMILARITY, PROD_MENU_K, PROD_TITLE_CHARS);
+                if (shownIds.length < 2) {
+                    r.redundancy = { skipped: true, shown: shownIds.length };
+                    continue;
+                }
+                const key = `redundancy|${prompts.indexOf(r.prompt)}|${r.mode}`;
+                const shownLines = shownIds.map((id) => lines[id]);
+                const promptText = buildRedundancyPrompt(r.prompt.prompt, shownLines);
+                const parsed = await judgeRedundancyUnit(judgeState, key, promptText, shownIds.length);
+                r.redundancy = "judgeError" in parsed ? parsed : { shown: shownIds.length, ...parsed };
+            }
+        }
+        catch (err) {
+            if (err instanceof RunAbortedError) {
+                console.error(`[relevance-eval] RUN ABORTED: ${err.message}`);
+                judgeState.aborted = judgeState.aborted ?? { reason: err.message };
+            }
+            else {
+                throw err;
+            }
+        }
+        // ---- persist raw verdicts (source of truth, spec §6) ----
+        const verdictRows = buildVerdictRows(prompts, results, sweepRows);
+        (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(verdictsJsonPath), { recursive: true });
+        (0, node_fs_1.writeFileSync)(verdictsJsonPath, JSON.stringify(verdictRows, null, 2), "utf-8");
+        console.log(`[relevance-eval] raw verdicts saved: ${verdictsJsonPath} (${verdictRows.length} rows), checkpoint sidecar: ${verdictsJsonlPath}`);
+        const report = renderReport({
+            snapshotPath: snapshotArg,
+            promptsPath,
+            verdictsJsonPath,
+            prompts,
+            results,
+            sweepRows,
+            sweepSubsetIdx,
+            verdictRows,
+            judgeState,
+            generatedAt: new Date().toISOString(),
+            db,
+        });
+        const reportPath = reportArg ?? DEFAULT_REPORT_PATH;
+        (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(reportPath), { recursive: true });
+        (0, node_fs_1.writeFileSync)(reportPath, report, "utf-8");
+        console.log("\n=== relevance-eval summary ===");
+        const off6 = computeMetricsAtK(results.filter((r) => r.mode === "off"), PROD_MENU_K);
+        const on6 = computeMetricsAtK(results.filter((r) => r.mode === "on"), PROD_MENU_K);
+        console.log(`precision@${PROD_MENU_K} (menu)  OFF ${off6 ? pct(off6.precision) : "n/a"} → ON ${on6 ? pct(on6.precision) : "n/a"}`);
+        const gapOffSummary = computeSnippetGap(results.filter((r) => r.mode === "off"));
+        console.log(`snippet_failure_rate (OFF) ${gapOffSummary.fullRelevantTotal > 0 ? pct(gapOffSummary.snippetFailure / gapOffSummary.fullRelevantTotal) : "n/a"}`);
+        console.log(`judge units: ${judgeState.completedUnits} completed, ${judgeState.errorUnits} judge_error, ${judgeState.totalCalls} fresh calls, ${judgeState.totalRetries} retries`);
+        if (judgeState.aborted)
+            console.log(`ABORTED: ${judgeState.aborted.reason}`);
+        console.log(`\n[relevance-eval] full report: ${reportPath}`);
+        if (judgeState.aborted)
+            process.exitCode = 1;
+    }
+    finally {
+        try {
+            db.close();
+        }
+        catch {
+            /* already closed */
+        }
+    }
+}
+main().catch((err) => {
+    console.error("[relevance-eval] FAILED:", err instanceof Error ? err.stack : String(err));
+    process.exitCode = 1;
+});