lastlight-evals 0.10.0 → 0.12.0

package/dist/run.js

@@ -26,10 +26,13 @@ import chalk from "chalk";
 import { loadDotEnv, hasProviderKey, evalModels, compareModels, modelLabels, resolveModel, setModelsPath } from "./env.js";
 import { runInstance, applyEvalEnv, slug } from "./run-instance.js";
 import { modelsArm, configArm, releaseOverlayGuard } from "./arm.js";
+import { mapPool } from "./pool.js";
 import { summarize, writeArtifacts, writeScorecard, aggregateTrials, loadMartianSidecar, computeMartianRanking, } from "./report.js";
-import { bootstrapAssets } from "./bootstrap.js";
+import { bootstrapAssets, describeCore } from "./bootstrap.js";
 import { discoverTiers, loadInstances, workflowFor } from "./discovery.js";
-import { builtinDatasetsRoot, tierResultsDir, makeRunId, gitShortSha, resultsRoot, dashboardDistRoot } from "./paths.js";
+import { builtinDatasetsRoot, tierResultsDir, makeRunId, assignRunIds, gitShortSha, resultsRoot, dashboardDistRoot, packageRoot, harnessVersion, resolveFactsBin, factsToolchainStamp, } from "./paths.js";
+import { defaultJudgeModel } from "./judge.js";
+import { defaultBeta } from "./grade.js";
 import { startServer } from "./serve.js";
 import { runInit } from "./init.js";
 import { runAddCase } from "./add-case.js";
@@ -186,7 +189,7 @@ function strFlagAll(name) {
     return out;
 }
 /** CLI flags that take a following value (so it isn't read as a tier name). */
-const VALUE_FLAGS = new Set(["--runs", "--model", "--models", "--mode", "--overlay", "--datasets", "--models-file", "--instance", "--limit", "--f-beta", "--sandbox"]);
+const VALUE_FLAGS = new Set(["--runs", "--repeats", "--model", "--models", "--mode", "--overlay", "--datasets", "--models-file", "--instance", "--limit", "--f-beta", "--sandbox", "--concurrency", "--repeat-concurrency"]);
 async function runEval() {
     loadDotEnv();
     p.intro(chalk.bold(`Last Light ${chalk.yellow("·")} eval`));
@@ -254,6 +257,13 @@ async function runEval() {
     if (overlayDir && overlayDir === autoOverlay)
         p.log.info(`overlay → ${chalk.cyan("./instance")} ${chalk.dim("(auto-detected)")}`);
     const { builtInRoot } = bootstrapAssets({ overlayDir });
+    // Say WHICH core is about to run, at the top, every time. A published core and
+    // a working tree can report the same version, so an arm measuring unreleased
+    // engine work would otherwise look identical to one that never loaded it —
+    // and every gate in the deterministic-pr-levers plan is a delta against a
+    // stored baseline, where that reads as a real negative result.
+    const core = describeCore(builtInRoot);
+    p.log.info(`core → ${chalk.cyan(core.version)} ${core.published ? chalk.yellow("(published package)") : chalk.green("(working tree)")} ${chalk.dim(core.root)}`);
     // A user/overlay can ship its own model registry too: explicit --models-file
     // wins, else an overlay's `evals/models.json` if present, else the built-in.
     const overlayModels = overlayDir ? join(overlayDir, "evals", "models.json") : undefined;
@@ -276,7 +286,25 @@ async function runEval() {
         p.outro(chalk.red("aborted"));
         return 1;
     }
-    const noOpen = process.argv.includes("--no-open") || !!process.env.CI;
+    // `--repeats N` — run the WHOLE arm N times, as N sibling runs, so a result can
+    // be reported as a band rather than a point. Three identical runs of one arm
+    // measured micro-recall 0.320 / 0.080 / 0.200 (union 0.440, intersection 0.040
+    // — 1 of 25 gold findings found by all three), and `diff-runs` returned KEEP on
+    // one and REVERT on the other two FROM ONE CONFIGURATION. A single arm is not
+    // a measurement of that arm.
+    //
+    // Emphatically NOT `--runs N`, which repeats each CASE and folds the trials
+    // into one worst-case result with mean metrics (`aggregateTrials`): that
+    // destroys the per-trial hit vectors, so it can never produce a band. The two
+    // compose (`--repeats 3 --runs 2` = 3 bands of 2-trial cases), they do not
+    // substitute.
+    const repeats = intFlag("repeats", 1);
+    // A repeat loop must never leave a dashboard server behind: `run` holds one open
+    // FOREVER at the end (see the server-hold block), and on 2026-08-23 seven leaked
+    // servers had accumulated overnight from ordinary single runs. N of them would be
+    // worse, and the band is read afterwards anyway — so `--repeats` implies
+    // `--no-open` and prints the standalone `serve` command once instead.
+    const noOpen = process.argv.includes("--no-open") || !!process.env.CI || repeats > 1;
     const runs = intFlag("runs", 1);
     // Positional tier names — skip flags AND the values that follow value-flags.
     const argv = process.argv.slice(2);
@@ -367,7 +395,11 @@ async function runEval() {
             : compare
                 ? compareModels().map((m) => ({ id: m.id, family: m.envKey ?? m.provider ?? "default" }))
                 : evalModels().map((id) => ({ id, family: "default" }));
-        arms = entries.map((e) => modelsArm(e.id, e.family));
+        // The overlay goes in even though the model is forced: it also carries the
+        // arm's `review:` policy (`review.analysis.enabled` — the review evidence
+        // pipeline switch), which is a deployment fact a `--model` flag says nothing
+        // about. Same overlay `bootstrapAssets` already wired above.
+        arms = entries.map((e) => modelsArm(e.id, e.family, overlayDir));
     }
     if (!arms.length) {
         p.log.error("No comparison models available — set provider keys (OPENAI_API_KEY / ANTHROPIC_API_KEY /\n" +
@@ -410,6 +442,17 @@ async function runEval() {
     // run-instance's injection step. ON by default; `--no-inject-context` (or
     // `EVAL_INJECT_CONTEXT=0`) forces a clean control run for an A/B.
     const injectContext = !process.argv.includes("--no-inject-context") && process.env.EVAL_INJECT_CONTEXT !== "0";
+    // Keep each trial's workspace instead of deleting it (`runInstance`'s
+    // `finally`). The evidence pipeline's deliverable is files —
+    // `.lastlight/pr-review/{facts.json,obligations/,hypotheses/,probes/}` — and
+    // without this flag the only way to read one was to catch a live run
+    // mid-flight, which is not an instrument. OFF by default: a kept workspace is
+    // a full checkout, plus `node_modules` once `prepare` installs.
+    //
+    // `--repeats` forces it on: when the repeats disagree, the question is always
+    // *which* evidence each one produced, and a previous repeat was already lost
+    // for want of the flag. A band you cannot open is a band you cannot explain.
+    const keepWorkspace = process.argv.includes("--keep-workspace") || repeats > 1;
     // Execution sandbox backend (or EVAL_SANDBOX). Default `none` (in-process, no
     // QEMU dependency — the fast/CI path). `gondolin` isolates the agent's tools
     // in a QEMU micro-VM so it can't read host gold data, while keeping the fake
@@ -434,6 +477,68 @@ async function runEval() {
             return 1;
         }
     }
+    // `--concurrency N`: how many CASES of ONE arm run at once. Default 1, so an
+    // invocation without the flag takes the exact serial path it always did.
+    //
+    // Within-arm serialism was never a correctness constraint — it is the
+    // rate-limit choice recorded above `byFamily` below. Everything that makes
+    // in-process concurrency safe is already true per case: a fresh `mkdtemp`
+    // stateDir, its own fake-GitHub port, a threaded per-run `cwd` with no
+    // `process.chdir`, and the eval env hoisted once for the whole batch. What is
+    // NOT safe is two ARMS at once — see the overlay note where the pool is driven.
+    const concurrencyRaw = strFlag("concurrency");
+    let concurrency = concurrencyRaw !== undefined ? Number(concurrencyRaw) : 1;
+    if (!Number.isFinite(concurrency) || !Number.isInteger(concurrency) || concurrency < 1) {
+        p.log.error(`--concurrency must be a positive integer, got "${concurrencyRaw}".`);
+        return 1;
+    }
+    // gondolin boots a QEMU micro-VM per case. Clamp rather than error: the flag is
+    // about throughput and the right answer on this backend is "one", not "refuse".
+    if (sandbox === "gondolin" && concurrency > 1) {
+        p.log.warn(`--concurrency ${concurrency} ignored: --sandbox gondolin runs a QEMU micro-VM per case. Using 1.`);
+        concurrency = 1;
+    }
+    // `--repeat-concurrency N`: how many of the `--repeats` REPEATS run at once.
+    // Default 1 = today's strictly-sequential repeat loop, untouched. With N>1 the
+    // repeats of the arm overlap — safe because every repeat runs the SAME arm(s)
+    // over ONE overlay, so the process-global asset-root guard (ADR 0001) is taken
+    // once for the whole batch, exactly as `--concurrency` holds it per arm.
+    //
+    // What overlap costs is the LATENCY instrument: concurrent repeats contend for
+    // CPU/network, so per-phase `durationMs`/`agentMs` on such runs are
+    // contaminated. That trade is accepted and made legible — each repeat's
+    // `meta.repeat.concurrency` records it so latency reads can be discounted.
+    // Verdicts, cost, and recall/precision are unaffected.
+    //
+    // Composes with `--concurrency` (case-level): total in-flight cases =
+    // repeatConcurrency × concurrency. No cap — size the product against the
+    // provider's rate limit.
+    const repeatConcurrencyRaw = strFlag("repeat-concurrency");
+    let repeatConcurrency = repeatConcurrencyRaw !== undefined ? Number(repeatConcurrencyRaw) : 1;
+    if (!Number.isFinite(repeatConcurrency) || !Number.isInteger(repeatConcurrency) || repeatConcurrency < 1) {
+        p.log.error(`--repeat-concurrency must be a positive integer, got "${repeatConcurrencyRaw}".`);
+        return 1;
+    }
+    // Same clamp-not-refuse policy as --concurrency: one QEMU micro-VM at a time.
+    if (sandbox === "gondolin" && repeatConcurrency > 1) {
+        p.log.warn(`--repeat-concurrency ${repeatConcurrency} ignored: --sandbox gondolin runs a QEMU micro-VM per case. Using 1.`);
+        repeatConcurrency = 1;
+    }
+    // A multi-overlay `config` run switches the process-global asset root between
+    // arms, so its repeats cannot overlap: two repeats in different arms would be
+    // two overlays at once (the guard would throw — ADR 0001). One overlay per
+    // batch is the invariant the concurrent branch is built on.
+    if (runType === "config" && arms.length > 1 && repeatConcurrency > 1) {
+        p.log.warn(`--repeat-concurrency ${repeatConcurrency} ignored: a multi-overlay config run switches the ` +
+            `process-global asset root between arms (ADR 0001), so repeats must stay sequential. Using 1.`);
+        repeatConcurrency = 1;
+    }
+    if (repeats === 1 && repeatConcurrency > 1) {
+        p.log.warn(`--repeat-concurrency ${repeatConcurrency} has nothing to overlap without --repeats N>1 — ignored.`);
+        repeatConcurrency = 1;
+    }
+    /** True when the repeat loop actually overlaps (the concurrent branch below). */
+    const repeatsConcurrent = repeats > 1 && repeatConcurrency > 1;
     // Instances per tier, resolved ONCE — the case set is identical across arms
     // (arms vary only the model selection / assets, never the cases).
     const tierInstances = new Map();
@@ -582,8 +687,33 @@ async function runEval() {
     const tierKeyFor = (tier) => `${tier}${runType === "config" ? "-config" : compare ? "-compare" : ""}`;
     // One shared runId, checked free in the first tier's dir (collisions in the
     // same second across runs are what the suffix guards — rare, one dir suffices).
-    const runId = makeRunId(new Date(), gitSha, tierResultsDir(tierKeyFor(tiers[0])));
-    const resultsDirFor = (tier) => join(tierResultsDir(tierKeyFor(tier)), runId);
+    //
+    // `let`, not `const`, and reassigned once per `--repeats` iteration. Every
+    // closure below that names a run directory — `resultsDirFor`, `baseMetaFor`,
+    // the session paths built from them — reads `runId` at CALL time, so a single
+    // reassignment repoints all of them with no code movement. Repeats are SIBLING
+    // dirs (`<tierKey>/<runId>/`, a fresh id each time): `indexTier` and `clean`
+    // both walk exactly two levels, so a nested `<runId>/rep-2/` would be invisible
+    // to the dashboard AND uncleanable. `makeRunId`'s existing `-2`/`-3` suffix
+    // already handles two repeats finishing inside the same second.
+    //
+    // CONCURRENT repeats (`--repeat-concurrency N>1`) can't assign lazily — N
+    // launches inside one second would all resolve the same id, because the ids
+    // exist nowhere on disk yet for `makeRunId` to dedupe against. So the whole
+    // band's ids are PRE-ASSIGNED here (`assignRunIds` bumps the second per id,
+    // keeping every id `<timestamp>-<sha>`-shaped with a batch-unique stamp) and
+    // each repeat's state carries its own. The sequential path keeps lazy
+    // assignment, one fresh id per repeat, exactly as before.
+    const preassignedRunIds = repeatsConcurrent
+        ? assignRunIds(repeats, new Date(), gitSha, tierResultsDir(tierKeyFor(tiers[0])))
+        : undefined;
+    let runId = preassignedRunIds?.[0] ?? makeRunId(new Date(), gitSha, tierResultsDir(tierKeyFor(tiers[0])));
+    /** The band's id = the FIRST repeat's runId (see `RepeatRef`) — pre-assigned
+     * before any repeat launches, so the group label never depends on run order. */
+    const repeatGroup = runId;
+    /** 1-based position in the band; bumped by the (sequential) repeat loop. */
+    let repeatIndex = 1;
+    const resultsDirFor = (tier, rid = runId) => join(tierResultsDir(tierKeyFor(tier)), rid);
     // Each case's session logs live under `sessions/<id>__<model>/trial-<N>/`
     // (relative to the tier run dir) — the model is in the name since several
     // models share a run dir; per-trial keeps every `--runs N` trial. `full.jsonl`
@@ -595,14 +725,77 @@ async function runEval() {
     // identity, labels, and live state straight off disk). `live`/`progress`/
     // `pending`/`generatedAt` are layered on per write; `tiers` is the single tier.
     const armLabels = arms.map((a) => labels[a.label] ?? a.label);
-    const baseMetaFor = (tier) => ({
-        runId,
+    // The invocation, recorded beside the numbers it produced (see `RunProvenance`).
+    // Resolved ONCE — none of it varies per repeat, and the facts-toolchain probe
+    // spawns a process. `judgeModel`/`factsBin` are what this run WOULD use, which
+    // is the point: an arm reporting `coverage: "none"` is explained by a null
+    // `factsBin`, and a scorecard whose judge silently changed between two runs is
+    // otherwise indistinguishable from one where the models changed.
+    const factsBin = resolveFactsBin();
+    // A pr-review tier with no facts binary runs a DIFFERENT pipeline than the
+    // one the scorecard claims: the survey seed, the adjudicate conservation gate
+    // and the reconcile floor all shell out to `lastlight-facts` and exit 127
+    // without it — measured 2026-08-25, every adjudication ran to max_iterations
+    // and shipped unrepaired findings while the run reported all phases green.
+    // Warn loudly rather than refuse: an analysis-DISABLED pr-review arm (the
+    // shipped baseline) never touches the binary and is fine without it.
+    if (!factsBin && tiers.some((t) => tierKeyFor(t).includes("pr-review"))) {
+        p.log.warn("lastlight-facts is UNRESOLVABLE (factsBin: null). Any analysis-enabled pr-review arm will run with " +
+            "its conservation gate and reconcile floor dead (exit 127) — adjudication loops to max_iterations and " +
+            "findings ship unrepaired. Set LASTLIGHT_FACTS_BIN or build packages/code-facts before trusting this run.");
+    }
+    let judgeModel;
+    try {
+        judgeModel = defaultJudgeModel();
+    }
+    catch {
+        judgeModel = undefined; // no provider key resolves one; already fatal above for models
+    }
+    const provenance = {
+        overlay: overlayDir,
+        overlays: overlays.length ? overlays : undefined,
+        datasets: userDatasetsDir,
+        sandbox,
+        fBeta: fBeta ?? defaultBeta(),
+        judgeWithDiff: judge.withDiff,
+        injectContext,
+        keepWorkspace,
+        instances: instanceFilter.length ? instanceFilter : undefined,
+        limit,
+        repeats: repeats > 1 ? repeats : undefined,
+        judgeModel,
+        factsBin,
+        toolchain: factsToolchainStamp(factsBin),
+        harness: { version: harnessVersion(), root: packageRoot() },
+        argv: process.argv.slice(2),
+    };
+    // `rid`/`idx` default to the sequential loop's shared `runId`/`repeatIndex`;
+    // a CONCURRENT repeat passes its own (two repeats in flight cannot share one).
+    const baseMetaFor = (tier, rid = runId, idx = repeatIndex) => ({
+        // Flat on `meta`, exactly like `gitSha`/`concurrency`/`core`.
+        ...provenance,
+        runId: rid,
         runType,
         tiers: [tier],
         models: armLabels,
         runs,
+        concurrency,
+        // A band of one is not a band — an ordinary run's meta stays exactly as it
+        // was, and `undefined` reads as "not repeated" rather than "repeated once".
+        // `repeat.concurrency` is stamped only when the repeats actually overlapped
+        // (`--repeat-concurrency N>1`) — the flag that contaminates per-phase
+        // latency, recorded so `durationMs`/`agentMs` reads can be discounted.
+        repeat: repeats > 1
+            ? {
+                group: repeatGroup,
+                index: idx,
+                of: repeats,
+                ...(repeatsConcurrent ? { concurrency: repeatConcurrency } : {}),
+            }
+            : undefined,
         gitSha,
         labels,
+        core: describeCore(builtInRoot),
     });
     // pr-review: load each tier's Martian leaderboard sidecar ONCE. Lets every
     // scorecard carry "where would we rank" over the PRs it covered (recomputed per
@@ -631,22 +824,32 @@ async function runEval() {
         `${chalk.bold("tiers")}   ${tiers.join(", ")}\n` +
         `${chalk.bold("cases")}   ${work.length}${runs > 1
             ? chalk.dim(` × ${runs} trials = ${work.length * runs} runs · worst-case verdict, mean cost`)
+            : ""}${repeats > 1
+            ? `\n${chalk.bold("repeats")} ${repeats}${chalk.dim(` × the whole arm, ${repeatsConcurrent
+                ? `up to ${repeatConcurrency} at once (per-phase latency contends — stamped meta.repeat.concurrency)`
+                : "sequentially"} · ${repeats} sibling runs, read as a band (implies --keep-workspace, --no-open)`)}`
             : ""}`, "plan");
     // `total` counts individual trials so live progress advances per model call.
     const total = work.length * runs;
-    // Seed an empty live scorecard per tier so the dashboard has something to poll,
-    // then start the server and open the SPA deep-linked at this run. The server is
-    // skipped entirely when not opening (CI / --no-open) — we only write JSON.
-    for (const tier of tiers) {
-        writeScorecard(resultsDirFor(tier), withMeta(summarize([]), {
-            ...baseMetaFor(tier),
-            generatedAt: new Date().toISOString(),
-            live: true,
-            pid: process.pid,
-            heartbeat: new Date().toISOString(),
-            progress: `0/${total}`,
-        }));
-    }
+    // Seed an empty live scorecard per tier so the dashboard has something to poll.
+    // Called once per repeat, AFTER `runId` has been repointed, so each sibling run
+    // dir exists from the moment its repeat starts rather than only when it ends.
+    const seedScorecards = (rid = runId, idx = repeatIndex) => {
+        for (const tier of tiers) {
+            writeScorecard(resultsDirFor(tier, rid), withMeta(summarize([]), {
+                ...baseMetaFor(tier, rid, idx),
+                generatedAt: new Date().toISOString(),
+                live: true,
+                pid: process.pid,
+                heartbeat: new Date().toISOString(),
+                progress: `0/${total}`,
+            }));
+        }
+    };
+    // Repeat 1's seed lands BEFORE the server starts (the SPA must have something
+    // to poll the instant it opens deep-linked at this run) and before the possibly
+    // slow prefetch.
+    seedScorecards();
     let server;
     if (!noOpen) {
         try {
@@ -669,23 +872,38 @@ async function runEval() {
         p.outro(chalk.red("aborted"));
         return 1;
     }
+    // The per-repeat accumulators. Declared ONCE, outside the repeat loop, because
+    // `refresh`/`runItem` close over them — the loop RESETS them in place rather
+    // than rebinding, so every closure keeps pointing at the live objects.
     const all = [];
     let harnessErrors = 0;
     let completed = 0;
+    /** Harness errors across ALL repeats — what the exit code is read from. A band
+     * whose first repeat blew up and whose last was clean is not a clean run. */
+    let totalHarnessErrors = 0;
+    /** Trials + cases summed over every repeat, for the closing line. */
+    let totalCompleted = 0;
+    let totalCases = 0;
     // Track in-flight cases so the live report can show running / queued rows.
     const caseKey = (tier, model, id) => `${tier}|${model}|${id}`;
     const running = new Set();
     // Current trial number per running case, so the live "follow" link points at
     // the right `trial-<N>/full.jsonl` (matters only for `--runs N>1`).
     const trialOf = new Map();
+    /** The concurrent repeats currently in flight — what the heartbeat re-stamps. */
+    const activeStates = new Set();
     // writeScorecard/summarize/all.push run synchronously to completion inside one
     // event-loop turn, so even with concurrent families they never interleave; the
     // temp-file+rename keeps a polling dashboard from reading a half-written file.
-    const refresh = () => {
-        const done = new Set(all.map((r) => caseKey(r.tier ?? "", r.model, r.instance_id)));
+    const refresh = (st) => {
+        // A concurrent repeat's own accumulators, else the sequential shared ones.
+        const results = st?.all ?? all;
+        const inFlight = st?.running ?? running;
+        const trialNo = st?.trialOf ?? trialOf;
+        const done = new Set(results.map((r) => caseKey(r.tier ?? "", r.model, r.instance_id)));
         const now = new Date().toISOString();
         for (const tier of tiers) {
-            const tierResults = all.filter((r) => (r.tier ?? "") === tier);
+            const tierResults = results.filter((r) => (r.tier ?? "") === tier);
             const pending = work
                 .filter((w) => w.tierName === tier)
                 .map((w) => ({ w, k: caseKey(w.tierName, w.arm.label, w.inst.instance_id) }))
@@ -694,18 +912,18 @@ async function runEval() {
                 tier: w.tierName,
                 model: w.arm.label,
                 instance_id: w.inst.instance_id,
-                status: running.has(k) ? "running" : "pending",
+                status: inFlight.has(k) ? "running" : "pending",
                 // Only a running case has a (live-updating) transcript to follow —
                 // point at the current trial's consolidated `full.jsonl`.
-                sessionLog: running.has(k)
-                    ? `${trialRelFor(w.inst.instance_id, w.arm.label, trialOf.get(k) ?? 1)}/full.jsonl`
+                sessionLog: inFlight.has(k)
+                    ? `${trialRelFor(w.inst.instance_id, w.arm.label, trialNo.get(k) ?? 1)}/full.jsonl`
                     : undefined,
             }));
             // Per-tier progress (cases), not the global trial count — each tier's
             // scorecard stands alone, so "0/5" across both tiers was misleading.
             const tierCases = work.filter((w) => w.tierName === tier).length;
-            writeScorecard(resultsDirFor(tier), withMeta(summarize(tierResults), {
-                ...baseMetaFor(tier),
+            writeScorecard(resultsDirFor(tier, st?.runId), withMeta(summarize(tierResults), {
+                ...baseMetaFor(tier, st?.runId, st?.index),
                 generatedAt: now,
                 live: true,
                 // Liveness signals: a `live` run whose heartbeat goes stale (writer
@@ -719,13 +937,23 @@ async function runEval() {
             }));
         }
     };
+    // NOT reset between repeats: its only consumer is the end-of-run summary, and
+    // the whole reason `--repeats` forces `--keep-workspace` is that you need to
+    // open EVERY repeat's evidence when they disagree. A per-repeat reset would
+    // print only the last one's paths and silently orphan the rest.
+    const keptWorkspaces = [];
     // Run one case `runs` times and fold the trials into a single result
-    // (worst-case verdict, mean metrics). `onTrial` ticks per model call.
-    const runItem = async (w, onTrial) => {
+    // (worst-case verdict, mean metrics). `onTrial` ticks per model call. `st` is
+    // a concurrent repeat's own state (`--repeat-concurrency N>1`); absent, the
+    // sequential shared accumulators are used, unchanged.
+    const runItem = async (w, onTrial, st) => {
         const k = caseKey(w.tierName, w.arm.label, w.inst.instance_id);
         const trials = [];
+        // Collected per TRIAL, not off the aggregate: `aggregateTrials` carries
+        // trial 1's fields through, so with `--runs 3` the other two workspaces
+        // would still be on disk and named nowhere.
         for (let t = 1; t <= runs; t++) {
-            trialOf.set(k, t); // so the live "follow" link targets this trial
+            (st?.trialOf ?? trialOf).set(k, t); // so the live "follow" link targets this trial
             const trialRel = trialRelFor(w.inst.instance_id, w.arm.label, t);
             const r = await runInstance(w.inst, {
                 // The arm carries all model selection (forced model / merged config) +
@@ -740,15 +968,21 @@ async function runEval() {
                 defaultWorkflow: w.defaultWorkflow,
                 manageEnv: false,
                 // Per-trial dir: `full.jsonl` (consolidated, live) + `NN-<phase>.jsonl`.
-                sessionTrialDir: join(resultsDirFor(w.tierName), trialRel),
+                sessionTrialDir: join(resultsDirFor(w.tierName, st?.runId), trialRel),
                 sessionTrialRel: trialRel,
                 trial: t,
                 judge,
                 sandbox,
+                keepWorkspace,
             });
             r.tier = w.tierName;
+            if (r.workspaceDir)
+                keptWorkspaces.push({ id: w.inst.instance_id, trial: t, repeat: st?.index ?? repeatIndex, path: r.workspaceDir });
             trials.push(r);
-            completed++;
+            if (st)
+                st.completed++;
+            else
+                completed++;
             onTrial();
         }
         const agg = aggregateTrials(trials);
@@ -763,112 +997,359 @@ async function runEval() {
     // Heartbeat: re-stamp the live scorecard every 20s so a long-running single
     // phase keeps its `heartbeat` fresh; if the process dies the stamp goes stale
     // and the index reclassifies the run as interrupted. `unref` so the timer
-    // never keeps the process alive on its own.
-    const heartbeat = setInterval(() => refresh(), 20_000);
+    // never keeps the process alive on its own. With concurrent repeats each
+    // in-flight repeat's OWN scorecard is re-stamped (never the shared one — its
+    // accumulators are unused there, and a shared refresh would overwrite repeat
+    // 1's card with an empty live one).
+    const heartbeat = setInterval(() => {
+        if (repeatsConcurrent)
+            for (const st of activeStates)
+                refresh(st);
+        else
+            refresh();
+    }, 20_000);
     heartbeat.unref?.();
+    /** Every repeat's per-tier artifact dirs, for the closing summary. */
+    const repeatDirs = [];
     try {
-        if (parallel) {
-            // Per-family progress for the aggregate spinner line.
-            const fam = new Map();
-            for (const [f, items] of byFamily)
-                fam.set(f, { done: 0, total: items.length * runs });
-            const status = () => {
-                const segs = [...fam].map(([f, c]) => {
-                    const done = c.done === c.total ? chalk.green(`${c.done}/${c.total}`) : `${c.done}/${c.total}`;
-                    return `${familyLabel(f)} ${done}`;
-                });
-                return `${chalk.dim(`${completed}/${total}`)}  ${segs.join(chalk.dim(" · "))}`;
-            };
+        if (repeatsConcurrent) {
+            // ── `--repeat-concurrency N>1`: up to N whole-arm repeats in flight ──
+            //
+            // Everything the repeats share is BATCH-scoped, the same shape as the
+            // case-concurrent branch below, widened to span the repeats:
+            //
+            //  - ONE overlay for the whole batch. Every repeat runs the same arm
+            //    list, so the process-global asset root (ADR 0001) is activated once
+            //    here and released once in the finally — the guard window covers the
+            //    entire batch. (Multi-overlay config runs clamped the flag to 1 up
+            //    top: they have no single overlay for the window to hold.
+            //    `activate()` is a no-op for models arms.)
+            //  - Console is silenced ONCE (the per-run `quiet()` swap is not
+            //    concurrency-safe) and one spinner reports every repeat — the same
+            //    trade the parallel branches below make.
+            //  - RunIds were pre-assigned (`preassignedRunIds`), so simultaneous
+            //    launches cannot collide on the second; each repeat's `RepeatState`
+            //    carries its own id, index, and accumulators.
+            //
+            // Within a repeat, cases run through the same arms-outer work list with
+            // `--concurrency` bounding cases in flight — so the total is
+            // repeatConcurrency × concurrency. In models mode several arms' cases
+            // interleave here (no overlay is at stake, only the provider rate limit);
+            // the per-family grouping of the parallel branch is deliberately not
+            // reproduced — repeat-level parallelism is the throughput lever.
+            const states = preassignedRunIds.map((rid, i) => ({
+                runId: rid,
+                index: i + 1,
+                all: [],
+                harnessErrors: 0,
+                completed: 0,
+                running: new Set(),
+                trialOf: new Map(),
+            }));
+            p.log.step(`${chalk.bold(`repeats ×${repeats}, up to ${repeatConcurrency} at once`)}\n` +
+                chalk.dim(states.map((st) => `  ${st.index}/${repeats} ${st.runId}`).join("\n")));
+            for (const arm of arms)
+                arm.activate();
+            const grandTotal = total * repeats;
+            const grand = () => states.reduce((n, st) => n + st.completed, 0);
             const s = makeSpinner();
+            const status = () => `${chalk.dim(`${grand()}/${grandTotal}`)}  ${states
+                .map((st) => {
+                const seg = `r${st.index} ${st.completed}/${total}`;
+                return st.completed === total ? chalk.green(seg) : seg;
+            })
+                .join(chalk.dim(" · "))}`;
             s.start(status());
             const restoreConsole = silenceConsole();
             const verdicts = [];
             try {
-                await Promise.all([...byFamily].map(async ([f, items]) => {
-                    for (const w of items) {
-                        const k = caseKey(w.tierName, w.arm.label, w.inst.instance_id);
-                        running.add(k);
-                        refresh();
-                        const result = await runItem(w, () => {
-                            fam.get(f).done++;
+                await mapPool(states, repeatConcurrency, async (st) => {
+                    // Repeat 1's live scorecard was seeded before the server started /
+                    // the prefetch ran, like any run's; a later repeat seeds its sibling
+                    // dir the moment it LAUNCHES here, so the dashboard sees it live.
+                    if (st.index > 1)
+                        seedScorecards(st.runId, st.index);
+                    activeStates.add(st);
+                    try {
+                        await mapPool(work, concurrency, async (w) => {
+                            const k = caseKey(w.tierName, w.arm.label, w.inst.instance_id);
+                            st.running.add(k);
+                            refresh(st);
+                            const result = await runItem(w, () => {
+                                s.message(status());
+                                refresh(st);
+                            }, st);
+                            st.running.delete(k);
+                            st.all.push(result);
+                            if (result.error)
+                                st.harnessErrors++;
+                            const mark = result.error ? chalk.red("✗") : result.blocked ? chalk.yellow("■") : chalk.green("✓");
+                            verdicts.push(`${mark} ${chalk.dim(`r${st.index}/${repeats}`)}  ${verdictLine(w.tierName, w.inst, result)}`);
                             s.message(status());
-                            refresh();
+                            refresh(st);
                         });
-                        running.delete(k);
-                        all.push(result);
-                        if (result.error)
-                            harnessErrors++;
-                        const mark = result.error ? chalk.red("✗") : result.blocked ? chalk.yellow("■") : chalk.green("✓");
-                        verdicts.push(`${mark} ${chalk.dim(familyLabel(f))}  ${verdictLine(w.tierName, w.inst, result)}`);
-                        refresh();
                     }
-                }));
+                    finally {
+                        activeStates.delete(st);
+                    }
+                    // Finalize THIS repeat as it lands — its siblings may still be running.
+                    const generatedAt = new Date().toISOString();
+                    for (const tier of tiers) {
+                        const tierResults = st.all.filter((r) => (r.tier ?? "") === tier);
+                        writeArtifacts(resultsDirFor(tier, st.runId), withMeta(summarize(tierResults), {
+                            ...baseMetaFor(tier, st.runId, st.index),
+                            generatedAt,
+                            live: false,
+                            martian: martianFor(tier, tierResults),
+                        }));
+                        repeatDirs.push(resultsDirFor(tier, st.runId));
+                    }
+                    totalHarnessErrors += st.harnessErrors;
+                    totalCompleted += st.completed;
+                    totalCases += st.all.length;
+                });
             }
             finally {
                 restoreConsole();
+                // The batch-wide overlay window closes exactly once, whatever happened.
+                releaseOverlayGuard();
             }
-            s.stop(`${chalk.dim(`${completed}/${total}`)} ${chalk.green("done")}`);
+            s.stop(`${chalk.dim(`${grand()}/${grandTotal}`)} ${chalk.green("done")}`);
             p.log.message(verdicts.join("\n"));
         }
-        else {
-            // Serial: one spinner per case (updates per trial) + a verdict line. On
-            // each arm change `arm.activate()` repoints the (process-global) asset root
-            // to that arm's overlay (a no-op for models arms); work is arms-outer, so
-            // it fires at most once per arm. `releaseOverlayGuard()` lets the next arm
-            // switch overlays — without it the guard treats the switch as a concurrent
-            // overlay and throws (ADR 0001).
-            let currentArm;
-            for (let i = 0; i < work.length; i++) {
-                const w = work[i];
-                if (w.arm !== currentArm) {
-                    if (currentArm)
-                        releaseOverlayGuard();
-                    w.arm.activate();
-                    currentArm = w.arm;
+        else
+            // `--repeats N` (sequential, the default): the WHOLE arm, N times, one
+            // after another. Overlapping is opt-in (`--repeat-concurrency`, the branch
+            // above) because concurrent repeats contend for CPU/network and the same
+            // provider rate limit — which contaminates the per-phase latency this
+            // sequential path keeps clean.
+            for (repeatIndex = 1; repeatIndex <= repeats; repeatIndex++) {
+                if (repeatIndex > 1) {
+                    // A fresh SIBLING run id. Every closure that names a run dir reads
+                    // `runId` at call time, so this one assignment repoints all of them.
+                    // `makeRunId`'s `-2`/`-3` suffix covers two repeats landing in the same
+                    // second (a `--limit 1` smoke run can).
+                    runId = makeRunId(new Date(), gitSha, tierResultsDir(tierKeyFor(tiers[0])));
+                    // Reset the accumulators IN PLACE — `refresh`/`runItem` closed over these
+                    // objects, so rebinding them would leave the closures on the old ones.
+                    // `keptWorkspaces` is deliberately NOT reset (see its declaration).
+                    all.length = 0;
+                    harnessErrors = 0;
+                    completed = 0;
+                    running.clear();
+                    trialOf.clear();
+                    seedScorecards();
+                    p.log.step(`${chalk.bold(`repeat ${repeatIndex}/${repeats}`)} ${chalk.dim(runId)}`);
+                }
+                try {
+                    if (parallel) {
+                        // Per-family progress for the aggregate spinner line.
+                        const fam = new Map();
+                        for (const [f, items] of byFamily)
+                            fam.set(f, { done: 0, total: items.length * runs });
+                        const status = () => {
+                            const segs = [...fam].map(([f, c]) => {
+                                const done = c.done === c.total ? chalk.green(`${c.done}/${c.total}`) : `${c.done}/${c.total}`;
+                                return `${familyLabel(f)} ${done}`;
+                            });
+                            return `${chalk.dim(`${completed}/${total}`)}  ${segs.join(chalk.dim(" · "))}`;
+                        };
+                        const s = makeSpinner();
+                        s.start(status());
+                        const restoreConsole = silenceConsole();
+                        const verdicts = [];
+                        try {
+                            await Promise.all([...byFamily].map(async ([f, items]) => {
+                                // Within a family, `--concurrency` bounds how many of its cases are
+                                // in flight. `models` arms never switch overlays (`activate()` is a
+                                // no-op for them), which is why this branch needs no arm grouping —
+                                // see the `config`/overlay note in the serial branch below.
+                                await mapPool(items, concurrency, async (w) => {
+                                    const k = caseKey(w.tierName, w.arm.label, w.inst.instance_id);
+                                    running.add(k);
+                                    refresh();
+                                    const result = await runItem(w, () => {
+                                        fam.get(f).done++;
+                                        s.message(status());
+                                        refresh();
+                                    });
+                                    running.delete(k);
+                                    all.push(result);
+                                    if (result.error)
+                                        harnessErrors++;
+                                    const mark = result.error ? chalk.red("✗") : result.blocked ? chalk.yellow("■") : chalk.green("✓");
+                                    verdicts.push(`${mark} ${chalk.dim(familyLabel(f))}  ${verdictLine(w.tierName, w.inst, result)}`);
+                                    refresh();
+                                });
+                            }));
+                        }
+                        finally {
+                            restoreConsole();
+                        }
+                        s.stop(`${chalk.dim(`${completed}/${total}`)} ${chalk.green("done")}`);
+                        p.log.message(verdicts.join("\n"));
+                    }
+                    else if (concurrency > 1) {
+                        // Concurrent WITHIN one arm, strictly serial ACROSS arms.
+                        //
+                        // The asset root core resolves workflows/skills/prompts from is a module
+                        // GLOBAL (ADR 0001), so two overlays can never be live at once — which is
+                        // what `activateOverlay`/`releaseOverlayGuard` enforce by throwing. Running
+                        // arms concurrently would therefore either throw or, worse, measure one
+                        // arm's cases against another arm's prompts. So the pool is opened and
+                        // closed inside each arm's overlay window, never across it.
+                        //
+                        // Cases within one arm have nothing global between them: per-run `mkdtemp`
+                        // stateDir, per-run fake-GitHub port, a threaded `cwd` (never
+                        // `process.chdir`), and the static-token env installed once for the batch.
+                        const groups = [];
+                        for (const w of work) {
+                            const tail = groups[groups.length - 1];
+                            if (tail && tail.arm === w.arm)
+                                tail.items.push(w);
+                            else
+                                groups.push({ arm: w.arm, items: [w] });
+                        }
+                        const s = makeSpinner();
+                        const status = () => `${chalk.dim(`${completed}/${total}`)}  ${chalk.dim(`×${concurrency}`)}`;
+                        s.start(status());
+                        // The per-case `quiet()` swap saves/restores `console` and is NOT
+                        // re-entrant, so concurrency drops console for the batch instead — the
+                        // same trade the family-parallel branch makes.
+                        const restoreConsole = silenceConsole();
+                        const verdicts = [];
+                        try {
+                            for (const g of groups) {
+                                g.arm.activate();
+                                try {
+                                    await mapPool(g.items, concurrency, async (w) => {
+                                        // The invariant this whole branch is built on, asserted rather
+                                        // than assumed: one overlay is live and it is this arm's.
+                                        if (w.arm !== g.arm)
+                                            throw new Error(`concurrency crossed an arm boundary: ${w.arm.label} in ${g.arm.label}`);
+                                        const k = caseKey(w.tierName, w.arm.label, w.inst.instance_id);
+                                        running.add(k);
+                                        refresh();
+                                        const result = await runItem(w, () => {
+                                            s.message(status());
+                                            refresh();
+                                        });
+                                        running.delete(k);
+                                        all.push(result);
+                                        if (result.error)
+                                            harnessErrors++;
+                                        const mark = result.error ? chalk.red("✗") : result.blocked ? chalk.yellow("■") : chalk.green("✓");
+                                        verdicts.push(`${mark} ${chalk.dim(labels[w.arm.label] ?? w.arm.label)}  ${verdictLine(w.tierName, w.inst, result)}`);
+                                        s.message(status());
+                                        refresh();
+                                    });
+                                }
+                                finally {
+                                    // Release before the next arm activates, or the guard reads the
+                                    // switch as a second concurrent overlay and throws.
+                                    releaseOverlayGuard();
+                                }
+                            }
+                        }
+                        finally {
+                            restoreConsole();
+                        }
+                        s.stop(`${chalk.dim(`${completed}/${total}`)} ${chalk.green("done")}`);
+                        p.log.message(verdicts.join("\n"));
+                    }
+                    else {
+                        // Serial: one spinner per case (updates per trial) + a verdict line. On
+                        // each arm change `arm.activate()` repoints the (process-global) asset root
+                        // to that arm's overlay (a no-op for models arms); work is arms-outer, so
+                        // it fires at most once per arm. `releaseOverlayGuard()` lets the next arm
+                        // switch overlays — without it the guard treats the switch as a concurrent
+                        // overlay and throws (ADR 0001).
+                        let currentArm;
+                        for (let i = 0; i < work.length; i++) {
+                            const w = work[i];
+                            if (w.arm !== currentArm) {
+                                if (currentArm)
+                                    releaseOverlayGuard();
+                                w.arm.activate();
+                                currentArm = w.arm;
+                            }
+                            const s = makeSpinner();
+                            const head = `${chalk.dim(`[${i + 1}/${work.length}]`)} ${chalk.cyan(w.tierName)}/${w.inst.instance_id}  ${chalk.dim(labels[w.arm.label] ?? w.arm.label)}`;
+                            s.start(head);
+                            const k = caseKey(w.tierName, w.arm.label, w.inst.instance_id);
+                            running.add(k);
+                            refresh();
+                            let t = 0;
+                            const { value: result, logs } = await quiet(() => runItem(w, () => {
+                                t++;
+                                if (runs > 1)
+                                    s.message(`${head}  ${chalk.dim(`trial ${t}/${runs}`)}`);
+                                refresh();
+                            }));
+                            running.delete(k);
+                            all.push(result);
+                            if (result.error)
+                                harnessErrors++;
+                            const mark = result.error ? chalk.red("✗") : result.blocked ? chalk.yellow("■") : chalk.green("✓");
+                            s.stop(`${chalk.dim(`[${i + 1}/${work.length}]`)} ${mark} ${verdictLine(w.tierName, w.inst, result)}`);
+                            if (result.error) {
+                                p.log.error(chalk.dim(result.error));
+                                const tail = logs.split("\n").filter(Boolean).slice(-12).join("\n");
+                                if (tail)
+                                    p.log.message(chalk.dim(tail));
+                            }
+                            refresh();
+                        }
+                    }
+                }
+                finally {
+                    // Release the overlay guard at the END OF EVERY REPEAT, not just on an
+                    // arm change. The serial branch above declares `currentArm` inside itself
+                    // and only releases when the arm CHANGES (never after the last one), so
+                    // repeat 2 would call `w.arm.activate()` while repeat 1's overlay is
+                    // still marked active and `activateOverlay` would THROW (arm.ts). The
+                    // concurrent branch already releases per arm in its own `finally`, and a
+                    // second release is a harmless no-op — the guard is two booleans.
+                    releaseOverlayGuard();
                 }
-                const s = makeSpinner();
-                const head = `${chalk.dim(`[${i + 1}/${work.length}]`)} ${chalk.cyan(w.tierName)}/${w.inst.instance_id}  ${chalk.dim(labels[w.arm.label] ?? w.arm.label)}`;
-                s.start(head);
-                const k = caseKey(w.tierName, w.arm.label, w.inst.instance_id);
-                running.add(k);
-                refresh();
-                let t = 0;
-                const { value: result, logs } = await quiet(() => runItem(w, () => {
-                    t++;
-                    if (runs > 1)
-                        s.message(`${head}  ${chalk.dim(`trial ${t}/${runs}`)}`);
-                    refresh();
-                }));
-                running.delete(k);
-                all.push(result);
-                if (result.error)
-                    harnessErrors++;
-                const mark = result.error ? chalk.red("✗") : result.blocked ? chalk.yellow("■") : chalk.green("✓");
-                s.stop(`${chalk.dim(`[${i + 1}/${work.length}]`)} ${mark} ${verdictLine(w.tierName, w.inst, result)}`);
-                if (result.error) {
-                    p.log.error(chalk.dim(result.error));
-                    const tail = logs.split("\n").filter(Boolean).slice(-12).join("\n");
-                    if (tail)
-                        p.log.message(chalk.dim(tail));
+                // Final, static scorecard + machine artifacts FOR THIS REPEAT. The
+                // run-level metadata is persisted into scorecard.json so the dashboard can
+                // label, order, and (no longer) live-poll the run without re-deriving from
+                // the current config.
+                const generatedAt = new Date().toISOString();
+                for (const tier of tiers) {
+                    const tierResults = all.filter((r) => (r.tier ?? "") === tier);
+                    writeArtifacts(resultsDirFor(tier), withMeta(summarize(tierResults), { ...baseMetaFor(tier), generatedAt, live: false, martian: martianFor(tier, tierResults) }));
+                    repeatDirs.push(resultsDirFor(tier));
                 }
-                refresh();
+                totalHarnessErrors += harnessErrors;
+                totalCompleted += completed;
+                totalCases += all.length;
             }
-        }
     }
     finally {
         clearInterval(heartbeat);
         restoreEvalEnv();
     }
-    // Final, static scorecard + machine artifacts. The run-level metadata is
-    // persisted into scorecard.json so the dashboard can label, order, and (no
-    // longer) live-poll the run without re-deriving from the current config.
-    const generatedAt = new Date().toISOString();
-    for (const tier of tiers) {
-        const tierResults = all.filter((r) => (r.tier ?? "") === tier);
-        writeArtifacts(resultsDirFor(tier), withMeta(summarize(tierResults), { ...baseMetaFor(tier), generatedAt, live: false, martian: martianFor(tier, tierResults) }));
+    p.log.success(`Artifacts → ${chalk.cyan(repeatDirs.join("\n             "))}\n             /{scorecard.json,predictions.jsonl,sessions/}`);
+    // `--keep-workspace` is only useful if you can find what it kept. The paths
+    // are on every result too (`workspaceDir`); this is the line that stops a
+    // kept batch from being N anonymous temp dirs, and the reminder that nothing
+    // else will ever delete them.
+    if (keptWorkspaces.length) {
+        p.log.warn(`Kept ${keptWorkspaces.length} workspace${keptWorkspaces.length === 1 ? "" : "s"} (--keep-workspace) — nothing else will remove them:\n` +
+            keptWorkspaces
+                .map((w) => `  ${w.id}${repeats > 1 ? ` (repeat ${w.repeat})` : ""}${runs > 1 ? ` (trial ${w.trial})` : ""} · ${chalk.cyan(w.path)}`)
+                .join("\n"));
+    }
+    // `--repeats` suppressed the auto-served dashboard (see the flag's note), so
+    // say — once, at the end — how to bring one up over the whole band instead of
+    // leaving N servers running for the length of the run.
+    if (repeats > 1) {
+        p.log.info(`${repeats} repeats written as sibling runs (band ${chalk.cyan(repeatGroup)}). Browse the band:\n` +
+            `  ${chalk.cyan(`cd apps/evals && LASTLIGHT_EVALS_OUT=${resultsRoot()} npx tsx src/run.ts serve --port 4400`)}`);
     }
-    p.log.success(`Artifacts → ${chalk.cyan(tiers.map((t) => resultsDirFor(t)).join("\n             "))}\n             /{scorecard.json,predictions.jsonl,sessions/}`);
-    const ran = runs > 1 ? `${completed} runs (${all.length} cases × ${runs})` : `${all.length} runs`;
+    const ran = runs > 1 ? `${totalCompleted} runs (${totalCases} cases × ${runs})` : `${totalCases} runs`;
     // Keep the dashboard server alive so the just-finished run stays viewable —
     // until the user stops it (Ctrl-C, or `kill`/stop for a detached run). The
     // dashboard is opt-in already (`--no-open`, or CI, ⇒ no server was started), so
@@ -879,21 +1360,22 @@ async function runEval() {
     if (server) {
         const runUrl = `${server.url}/#/${encodeURIComponent(tierKeyFor(tiers[0]))}/${encodeURIComponent(runId)}`;
         p.log.success(`Dashboard → ${chalk.cyan(runUrl)} ${chalk.dim("(serving · Ctrl-C or kill to stop)")}`);
-        const tail = harnessErrors > 0 ? chalk.yellow(`done — ${ran}, ${harnessErrors} harness error${harnessErrors === 1 ? "" : "s"} (see above)`) : chalk.green(`done — ${ran}`);
+        const tail = totalHarnessErrors > 0 ? chalk.yellow(`done — ${ran}, ${totalHarnessErrors} harness error${totalHarnessErrors === 1 ? "" : "s"} (see above)`) : chalk.green(`done — ${ran}`);
         p.outro(tail);
         await waitForSigint();
         await server.close();
     }
     else {
-        if (harnessErrors > 0) {
-            p.outro(chalk.yellow(`done — ${ran}, ${harnessErrors} harness error${harnessErrors === 1 ? "" : "s"} (see above)`));
+        if (totalHarnessErrors > 0) {
+            p.outro(chalk.yellow(`done — ${ran}, ${totalHarnessErrors} harness error${totalHarnessErrors === 1 ? "" : "s"} (see above)`));
         }
         else {
             p.outro(chalk.green(`done — ${ran}`));
         }
     }
-    // Non-zero ONLY on harness failure — model quality is the measurement.
-    return harnessErrors > 0 ? 1 : 0;
+    // Non-zero ONLY on harness failure — model quality is the measurement. Summed
+    // over every repeat: a band whose middle repeat blew up did not run clean.
+    return totalHarnessErrors > 0 ? 1 : 0;
 }
 /** Resolve on the first SIGINT (Ctrl-C) so `run`/`serve` can keep a server up
  * for browsing and shut it down cleanly when the user is done. */
@@ -969,7 +1451,31 @@ Run options:
   --compare            Cross-vendor set (only models whose provider key is present)
   --instance <id[,id]> Only run these instance_id(s) (or set EVAL_INSTANCE). Exact match.
   --limit <n>          Run only the first n instances per tier (after --instance).
-  --runs <n>           Repeat each case n× (worst-case verdict, mean metrics)
+  --runs <n>           Repeat each CASE n× (worst-case verdict, mean metrics).
+                       Folds the trials into one result — no per-trial detail.
+  --repeats <n>        Repeat the whole ARM n×, sequentially, as n SIBLING runs,
+                       so a result reads as a band instead of a point. Three
+                       identical arms measured 0.320/0.080/0.200 micro-recall, so
+                       one arm is not a measurement of that arm. Each repeat is a
+                       normal run (its own runId + scorecard) tagged with
+                       meta.repeat={group,index,of}; scripts/band.ts and the
+                       dashboard roll them up into a band (mean/min/max + union
+                       and intersection recall). Implies --keep-workspace (you will
+                       want each repeat's evidence) and --no-open (a run holds a
+                       dashboard server open forever; n of them leak). Or
+                       EVAL_REPEATS.
+  --repeat-concurrency <n>  Run up to n of the --repeats repeats at once
+                       (default 1 = sequential, the exact path above). RunIds are
+                       pre-assigned so simultaneous launches never collide; each
+                       repeat stays a normal sibling run. Repeats share ONE
+                       overlay, so multi-overlay --mode config runs clamp to 1
+                       (as does --sandbox gondolin). Composes with --concurrency:
+                       total in-flight cases = repeat-concurrency × concurrency
+                       (no cap — size the product against your rate limit).
+                       CAVEAT: concurrent repeats contend for CPU/network, so
+                       per-phase durationMs/agentMs are contaminated; each
+                       repeat's meta.repeat.concurrency records n so latency
+                       reads can be discounted. Verdicts/cost/recall unaffected.
   --f-beta <n>         pr-review F-beta β (default 1 = F1; 0.5 = precision 2×). Or EVAL_F_BETA.
   --judge-with-diff    pr-review: feed the PR diff to the judge (higher fidelity,
                        off by default — Martian's offline judge is diff-blind)
@@ -981,6 +1487,15 @@ Run options:
                        in-process (fast, CI). gondolin isolates the agent's tools
                        in a QEMU micro-VM so it can't read host gold data (needs
                        QEMU natively — brew install qemu). Or EVAL_SANDBOX.
+  --keep-workspace     Don't delete each trial's workspace. Keeps the evidence
+                       pipeline's artifacts (.lastlight/pr-review/facts.json,
+                       obligations/, hypotheses/, probes/) readable after the run;
+                       the path lands on each result as workspaceDir. Costs disk.
+  --concurrency <n>    Run n cases of the SAME arm at once (default 1 = serial).
+                       Arms always stay serial (one overlay at a time, ADR 0001);
+                       --runs trials within a case stay serial too. Bounded by the
+                       provider's rate limit, not by correctness. Forced to 1 on
+                       --sandbox gondolin (a QEMU micro-VM per case).
   --serial             Force serial execution across provider families
   --datasets <dir>     Extra datasets root to discover tiers from
   --models-file <f>    Use an explicit models.json