lastlight-evals 0.1.2 → 0.2.1

package/dist/run.js

@@ -17,18 +17,19 @@
  * The deterministic, AI-free plumbing is covered separately by
  * `evals/mechanism.test.ts` in the normal `npm test` suite.
  */
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync } from "node:fs";
 import { join } from "node:path";
 import { spawn } from "node:child_process";
 import * as p from "@clack/prompts";
 import chalk from "chalk";
 import { loadDotEnv, hasProviderKey, evalModels, compareModels, modelLabels, resolveModel, setModelsPath } from "./env.js";
-import { runInstance, applyEvalEnv } from "./run-instance.js";
-import { summarize, writeArtifacts, aggregateTrials } from "./report.js";
-import { writeHtml } from "./html-report.js";
+import { runInstance, applyEvalEnv, slug } from "./run-instance.js";
+import { modelsArm, configArm, releaseOverlayGuard } from "./arm.js";
+import { summarize, writeArtifacts, writeScorecard, aggregateTrials, } from "./report.js";
 import { bootstrapAssets } from "./bootstrap.js";
 import { discoverTiers, loadInstances, workflowFor } from "./discovery.js";
-import { builtinDatasetsRoot, resultsRoot } from "./paths.js";
+import { builtinDatasetsRoot, tierResultsDir, makeRunId, gitShortSha, resultsRoot, dashboardDistRoot } from "./paths.js";
+import { startServer } from "./serve.js";
 import { runInit } from "./init.js";
 /**
  * A clack spinner in a TTY; in non-TTY (CI / piped / agent) a quiet stub that
@@ -48,9 +49,8 @@ function makeSpinner() {
         },
     };
 }
-/** Open a file in the OS default browser (best-effort, never throws). */
-function openInBrowser(file) {
-    const url = `file://${file}`;
+/** Open a URL in the OS default browser (best-effort, never throws). */
+function openInBrowser(url) {
     const [cmd, args] = process.platform === "darwin"
         ? ["open", [url]]
         : process.platform === "win32"
@@ -99,6 +99,12 @@ function fmtMs(ms) {
 function familyLabel(envKey) {
     return envKey.replace(/_API_KEY$/i, "").toLowerCase() || "default";
 }
+/** Attach run metadata to a scorecard (mutate-and-return; `summarize` hands back
+ * a fresh object each call, so this is safe). */
+function withMeta(card, meta) {
+    card.meta = meta;
+    return card;
+}
 /**
  * Silence `console.*` for the whole batch (parallel mode). The per-run
  * `quiet()` swap saves/restores console and would corrupt under concurrent
@@ -116,6 +122,8 @@ function verdictLine(tierName, inst, r) {
     const head = `${chalk.cyan(tierName)}/${inst.instance_id}`;
     if (r.error)
         return `${head}  ${chalk.red("harness error")}`;
+    if (r.blocked)
+        return `${head}  ${chalk.yellow("blocked")} ${chalk.dim("(workflow gate)")}`;
     const count = (pass) => (pass !== undefined && r.trials ? chalk.dim(` ${pass}/${r.trials}`) : "");
     const parts = [];
     if (r.resolved !== undefined)
@@ -154,22 +162,93 @@ function strFlag(name) {
     }
     return process.env[`EVAL_${name.toUpperCase()}`];
 }
+/** Collect ALL occurrences of a repeatable string flag (`--name a --name b` or
+ * `--name=a`). Used by `--overlay`, which may repeat in `config` mode to compare
+ * one arm per overlay. */
+function strFlagAll(name) {
+    const out = [];
+    const argv = process.argv;
+    for (let i = 0; i < argv.length; i++) {
+        if (argv[i] === `--${name}` && argv[i + 1] !== undefined) {
+            out.push(argv[i + 1]);
+            i++;
+            continue;
+        }
+        const m = argv[i].match(new RegExp(`^--${name}=(.+)$`));
+        if (m)
+            out.push(m[1]);
+    }
+    return out;
+}
 /** CLI flags that take a following value (so it isn't read as a tier name). */
-const VALUE_FLAGS = new Set(["--runs", "--model", "--models", "--overlay", "--datasets", "--models-file"]);
+const VALUE_FLAGS = new Set(["--runs", "--model", "--models", "--mode", "--overlay", "--datasets", "--models-file"]);
 async function runEval() {
     loadDotEnv();
     p.intro(chalk.bold(`Last Light ${chalk.yellow("·")} eval`));
+    // Run type — the comparison axis:
+    //   `models` — compare N models, each FORCED across every workflow step.
+    //   `config` — run an overlay's REAL per-step model config (what ships); the
+    //              arm is the config/overlay, compared across runs (or N overlays
+    //              side-by-side). `--mode` wins; an explicit `--model`/`--compare`
+    //              implies `models`; otherwise ask in a TTY (default `models`).
+    const modeFlag = strFlag("mode");
+    const modelArg = strFlag("model") ?? strFlag("models");
+    const compare = process.argv.includes("--compare");
+    let runType;
+    if (modeFlag === "config")
+        runType = "config";
+    else if (modeFlag === "models" || modelArg || compare)
+        runType = "models";
+    else if (process.stdin.isTTY) {
+        const picked = await p.select({
+            message: "What do you want to eval?",
+            options: [
+                { value: "models", label: "compare models", hint: "force each model across every workflow step" },
+                { value: "config", label: "eval config", hint: "an overlay's real per-step model config (what ships)" },
+            ],
+            initialValue: "models",
+        });
+        if (p.isCancel(picked)) {
+            p.cancel("aborted");
+            return 1;
+        }
+        runType = picked;
+    }
+    else
+        runType = "models";
     // Asset roots FIRST — before any getWorkflow/runWorkflow. `--overlay` (or
     // LASTLIGHT_OVERLAY_DIR) layers a deployment's own workflows/skills over the
     // built-ins, and also contributes its `evals/datasets/` (see discovery).
     // With neither set, auto-detect a local `./instance/` overlay checkout — the
     // Separate layout `init --clone` produces — so a bare run "just works".
+    // `--overlay` may REPEAT in config mode (one arm per overlay).
     const autoInstance = join(process.cwd(), "instance");
     const autoOverlay = existsSync(join(autoInstance, "config.yaml")) ? autoInstance : undefined;
-    const overlayDir = strFlag("overlay") ?? process.env.LASTLIGHT_OVERLAY_DIR ?? autoOverlay;
-    if (overlayDir === autoOverlay && autoOverlay)
+    const overlayFlags = strFlagAll("overlay");
+    let overlays = overlayFlags.length
+        ? overlayFlags
+        : [process.env.LASTLIGHT_OVERLAY_DIR ?? autoOverlay].filter((d) => !!d);
+    // In a config-mode TTY with no overlay given, ask for one (blank ⇒ core defaults).
+    if (runType === "config" && !overlayFlags.length && process.stdin.isTTY) {
+        const ans = await p.text({
+            message: "Overlay dir whose config.yaml drives per-step models (blank = core defaults):",
+            placeholder: overlays[0] ?? autoInstance,
+            initialValue: overlays[0] ?? "",
+        });
+        if (p.isCancel(ans)) {
+            p.cancel("aborted");
+            return 1;
+        }
+        const dir = ans.trim();
+        overlays = dir ? [dir] : [];
+    }
+    // The primary overlay wires discovery + the initial asset bootstrap. Config
+    // arms re-bootstrap their own overlay before running (the asset root is a
+    // process global — see the serial loop below).
+    const overlayDir = overlays[0];
+    if (overlayDir && overlayDir === autoOverlay)
         p.log.info(`overlay → ${chalk.cyan("./instance")} ${chalk.dim("(auto-detected)")}`);
-    bootstrapAssets({ overlayDir });
+    const { builtInRoot } = bootstrapAssets({ overlayDir });
     // A user/overlay can ship its own model registry too: explicit --models-file
     // wins, else an overlay's `evals/models.json` if present, else the built-in.
     const overlayModels = overlayDir ? join(overlayDir, "evals", "models.json") : undefined;
@@ -192,7 +271,6 @@ async function runEval() {
         p.outro(chalk.red("aborted"));
         return 1;
     }
-    const compare = process.argv.includes("--compare");
     const noOpen = process.argv.includes("--no-open") || !!process.env.CI;
     const runs = intFlag("runs", 1);
     // Positional tier names — skip flags AND the values that follow value-flags.
@@ -248,35 +326,54 @@ async function runEval() {
         if (!discovered.has(t))
             p.log.warn(`Unknown tier "${t}". Known: ${known.join(", ")}`);
     }
-    // Model selection precedence:
-    //   1. --model / --models (or EVAL_MODEL[S]) — an explicit list, fuzzy-matched
-    //      against models.json; lets you test one model quickly.
-    //   2. --compare — the full cross-vendor set (key-gated).
-    //   3. default single model from models.json.
-    // Each entry carries its provider family (env-key) for parallel grouping.
-    const modelArg = strFlag("model") ?? strFlag("models");
-    const mode = modelArg ? "select" : compare ? "compare" : "single";
-    const entries = modelArg
-        ? modelArg
-            .split(",")
-            .map((s) => s.trim())
-            .filter(Boolean)
-            .map((tok) => {
-            const r = resolveModel(tok);
-            return { id: r.id, family: r.family };
-        })
-        : compare
-            ? compareModels().map((m) => ({ id: m.id, family: m.envKey ?? m.provider ?? "default" }))
-            : evalModels().map((id) => ({ id, family: "default" }));
-    if (!entries.length) {
+    // An "arm" is one column of the comparison, behind the `Arm` seam (src/arm.ts):
+    // `models` runs build one `modelsArm` per model (forced across every step);
+    // `config` runs build one `configArm` per overlay (its merged per-step config
+    // drives selection). Both flow through the same work-list → scorecard →
+    // dashboard, keyed on the arm's `label`. run.ts owns *which* arms exist (the
+    // flag + registry resolution below); the adapters own *how* to build one.
+    //
+    // The model-selection sub-mode (only meaningful for `models` runs) is shown in
+    // the plan note; `config` runs report their arm count instead.
+    const mode = runType === "config" ? "config" : modelArg ? "select" : compare ? "compare" : "single";
+    let arms;
+    if (runType === "config") {
+        // One arm per overlay (or a single core-defaults arm when none). `--model`
+        // resolves to an id that overrides each merged config's `default` for quick
+        // what-if runs (the override is applied inside configArm).
+        const configOverlays = overlays.length ? overlays : [overlayDir];
+        const defaultOverride = modelArg ? resolveModel(modelArg).id : undefined;
+        arms = configOverlays.map((dir) => configArm(builtInRoot, dir, defaultOverride));
+    }
+    else {
+        // Model selection precedence:
+        //   1. --model / --models (or EVAL_MODEL[S]) — an explicit list, fuzzy-matched.
+        //   2. --compare — the full cross-vendor set (key-gated).
+        //   3. default single model from models.json.
+        const entries = modelArg
+            ? modelArg
+                .split(",")
+                .map((s) => s.trim())
+                .filter(Boolean)
+                .map((tok) => {
+                const r = resolveModel(tok);
+                return { id: r.id, family: r.family };
+            })
+            : compare
+                ? compareModels().map((m) => ({ id: m.id, family: m.envKey ?? m.provider ?? "default" }))
+                : evalModels().map((id) => ({ id, family: "default" }));
+        arms = entries.map((e) => modelsArm(e.id, e.family));
+    }
+    if (!arms.length) {
         p.log.error("No comparison models available — set provider keys (OPENAI_API_KEY / ANTHROPIC_API_KEY /\n" +
             "FIREWORKS_API_KEY …) for the entries in evals/models.json.");
         p.outro(chalk.red("aborted"));
         return 1;
     }
     const labels = modelLabels();
-    // Resolve the work-list up front so we can show deterministic progress.
-    const work = [];
+    // Instances per tier, resolved ONCE — the case set is identical across arms
+    // (arms vary only the model selection / assets, never the cases).
+    const tierInstances = new Map();
     for (const tierName of tiers) {
         const tier = discovered.get(tierName);
         const instances = loadInstances(tier);
@@ -284,7 +381,15 @@ async function runEval() {
             p.log.warn(`tier "${tierName}": no instances at ${tier.instancesPath} — skipping`);
             continue;
         }
-        for (const e of entries) {
+        tierInstances.set(tierName, instances);
+    }
+    // Resolve the work-list up front so we can show deterministic progress. Arms
+    // are the OUTER loop so a `config` run's per-arm overlay switches at most once
+    // (the serial loop re-bootstraps on arm change; see below).
+    const work = [];
+    for (const arm of arms) {
+        for (const [tierName, instances] of tierInstances) {
+            const tier = discovered.get(tierName);
             for (const inst of instances) {
                 // Per-instance workflow wins, else the tier's defaultWorkflow (throws if
                 // neither is set — surfaced as a harness error for that case).
@@ -292,8 +397,7 @@ async function runEval() {
                     tierName,
                     defaultWorkflow: workflowFor(tier, inst),
                     datasetDir: tier.root,
-                    model: e.id,
-                    family: e.family,
+                    arm,
                     inst,
                 });
             }
@@ -309,34 +413,84 @@ async function runEval() {
     // serial with --serial or when there's only one family.
     const byFamily = new Map();
     for (const w of work) {
-        const arr = byFamily.get(w.family);
+        const arr = byFamily.get(w.arm.family);
         if (arr)
             arr.push(w);
         else
-            byFamily.set(w.family, [w]);
+            byFamily.set(w.arm.family, [w]);
     }
-    const parallel = !process.argv.includes("--serial") && byFamily.size > 1;
-    const resultsDir = join(resultsRoot(), `${tiers.join("+")}${compare ? "-compare" : ""}`);
-    const htmlBase = {
-        models: entries.map((e) => labels[e.id] ?? e.id),
-        tiers,
-        labels,
+    // `config` runs stay SERIAL: arms may carry distinct overlays and the asset
+    // root is a process global, so the serial loop re-bootstraps per arm (below).
+    const parallel = runType === "models" && !process.argv.includes("--serial") && byFamily.size > 1;
+    // Each tier writes its OWN folder + scorecard (all sharing this run's id), so
+    // tiers stay separate in the dashboard instead of collapsing into one combined
+    // `<a+b>` entry. A single invocation appears as the same run under each tier it
+    // touched. The `-compare` suffix keeps cross-vendor runs on their own trend
+    // line, distinct from single-model runs of the same tier. The dashboard server
+    // indexes the whole tree; each tier dir is a `tierKey` segment the SPA routes on.
+    // `-compare` keeps cross-vendor runs on their own trend line; `-config` keeps
+    // config-eval runs on theirs — so the three run shapes never collapse together.
+    const gitSha = gitShortSha();
+    const tierKeyFor = (tier) => `${tier}${runType === "config" ? "-config" : compare ? "-compare" : ""}`;
+    // One shared runId, checked free in the first tier's dir (collisions in the
+    // same second across runs are what the suffix guards — rare, one dir suffices).
+    const runId = makeRunId(new Date(), gitSha, tierResultsDir(tierKeyFor(tiers[0])));
+    const resultsDirFor = (tier) => join(tierResultsDir(tierKeyFor(tier)), runId);
+    // Each case's session logs live under `sessions/<id>__<model>/trial-<N>/`
+    // (relative to the tier run dir) — the model is in the name since several
+    // models share a run dir; per-trial keeps every `--runs N` trial. `full.jsonl`
+    // is the consolidated live-followable transcript; `NN-<phase>.jsonl` are the
+    // per-phase splits written when the trial finishes.
+    const caseRelFor = (instanceId, model) => `sessions/${slug(instanceId)}__${slug(model)}`;
+    const trialRelFor = (instanceId, model, trial) => `${caseRelFor(instanceId, model)}/trial-${trial}`;
+    // Per-tier run metadata stamped into every scorecard write (the dashboard reads
+    // identity, labels, and live state straight off disk). `live`/`progress`/
+    // `pending`/`generatedAt` are layered on per write; `tiers` is the single tier.
+    const armLabels = arms.map((a) => labels[a.label] ?? a.label);
+    const baseMetaFor = (tier) => ({
+        runId,
+        runType,
+        tiers: [tier],
+        models: armLabels,
         runs,
-    };
+        gitSha,
+        labels,
+    });
+    // In `config` runs the axis is the config(s); show the merged per-step model
+    // map for a single-arm run so the plan is legible.
+    const axisLine = runType === "config"
+        ? `${chalk.bold("configs")} ${armLabels.join(", ")}`
+        : `${chalk.bold("models")}  ${armLabels.join(", ")}`;
+    // For a single-arm run, show the per-step model map (config arms) so the plan
+    // is legible. `describe()` returns the summary for config arms, undefined for
+    // models arms — the reach into the arm's config map stays behind the seam.
+    const armSummary = arms.length === 1 ? arms[0].describe() : undefined;
+    const phaseMapLine = armSummary ? `\n${chalk.bold("models")}  ${chalk.dim(armSummary)}` : "";
     p.note(`${chalk.bold("mode")}    ${mode}${parallel ? chalk.dim(` (parallel · ${byFamily.size} families)`) : ""}\n` +
-        `${chalk.bold("models")}  ${entries.map((e) => labels[e.id] ?? e.id).join(", ")}\n` +
+        `${axisLine}${phaseMapLine}\n` +
         `${chalk.bold("tiers")}   ${tiers.join(", ")}\n` +
         `${chalk.bold("cases")}   ${work.length}${runs > 1
             ? chalk.dim(` × ${runs} trials = ${work.length * runs} runs · worst-case verdict, mean cost`)
             : ""}`, "plan");
     // `total` counts individual trials so live progress advances per model call.
     const total = work.length * runs;
-    // Open the report immediately (live placeholder) so it fills in as we go.
-    writeHtml(resultsDir, summarize([]), { ...htmlBase, generatedAt: new Date().toISOString(), live: true, progress: `0/${total}` });
-    const htmlFile = join(resultsDir, "index.html");
+    // Seed an empty live scorecard per tier so the dashboard has something to poll,
+    // then start the server and open the SPA deep-linked at this run. The server is
+    // skipped entirely when not opening (CI / --no-open) — we only write JSON.
+    for (const tier of tiers) {
+        writeScorecard(resultsDirFor(tier), withMeta(summarize([]), { ...baseMetaFor(tier), generatedAt: new Date().toISOString(), live: true, progress: `0/${total}` }));
+    }
+    let server;
     if (!noOpen) {
-        openInBrowser(htmlFile);
-        p.log.info(`Live report → ${chalk.cyan(htmlFile)} ${chalk.dim("(auto-refreshing)")}`);
+        try {
+            server = await startServer({ resultsRoot: resultsRoot(), dashboardRoot: dashboardDistRoot() });
+            const runUrl = `${server.url}/#/${encodeURIComponent(tierKeyFor(tiers[0]))}/${encodeURIComponent(runId)}`;
+            openInBrowser(runUrl);
+            p.log.info(`Live dashboard → ${chalk.cyan(runUrl)}`);
+        }
+        catch (err) {
+            p.log.warn(`Couldn't start the dashboard server (${err.message}) — writing JSON only.`);
+        }
     }
     const all = [];
     let harnessErrors = 0;
@@ -344,44 +498,74 @@ async function runEval() {
     // Track in-flight cases so the live report can show running / queued rows.
     const caseKey = (tier, model, id) => `${tier}|${model}|${id}`;
     const running = new Set();
-    // writeHtml/summarize/all.push run synchronously to completion inside one
-    // event-loop turn, so even with concurrent families they never interleave.
+    // Current trial number per running case, so the live "follow" link points at
+    // the right `trial-<N>/full.jsonl` (matters only for `--runs N>1`).
+    const trialOf = new Map();
+    // writeScorecard/summarize/all.push run synchronously to completion inside one
+    // event-loop turn, so even with concurrent families they never interleave; the
+    // temp-file+rename keeps a polling dashboard from reading a half-written file.
     const refresh = () => {
         const done = new Set(all.map((r) => caseKey(r.tier ?? "", r.model, r.instance_id)));
-        const pending = work
-            .map((w) => ({ w, k: caseKey(w.tierName, w.model, w.inst.instance_id) }))
-            .filter(({ k }) => !done.has(k))
-            .map(({ w, k }) => ({
-            tier: w.tierName,
-            model: w.model,
-            instance_id: w.inst.instance_id,
-            status: (running.has(k) ? "running" : "pending"),
-        }));
-        writeHtml(resultsDir, summarize(all), {
-            ...htmlBase,
-            generatedAt: new Date().toISOString(),
-            live: true,
-            progress: `${completed}/${total}`,
-            pending,
-        });
+        const now = new Date().toISOString();
+        for (const tier of tiers) {
+            const tierResults = all.filter((r) => (r.tier ?? "") === tier);
+            const pending = work
+                .filter((w) => w.tierName === tier)
+                .map((w) => ({ w, k: caseKey(w.tierName, w.arm.label, w.inst.instance_id) }))
+                .filter(({ k }) => !done.has(k))
+                .map(({ w, k }) => ({
+                tier: w.tierName,
+                model: w.arm.label,
+                instance_id: w.inst.instance_id,
+                status: running.has(k) ? "running" : "pending",
+                // Only a running case has a (live-updating) transcript to follow —
+                // point at the current trial's consolidated `full.jsonl`.
+                sessionLog: running.has(k)
+                    ? `${trialRelFor(w.inst.instance_id, w.arm.label, trialOf.get(k) ?? 1)}/full.jsonl`
+                    : undefined,
+            }));
+            // Per-tier progress (cases), not the global trial count — each tier's
+            // scorecard stands alone, so "0/5" across both tiers was misleading.
+            const tierCases = work.filter((w) => w.tierName === tier).length;
+            writeScorecard(resultsDirFor(tier), withMeta(summarize(tierResults), {
+                ...baseMetaFor(tier),
+                generatedAt: now,
+                live: true,
+                progress: `${tierResults.length}/${tierCases}`,
+                pending,
+            }));
+        }
     };
     // Run one case `runs` times and fold the trials into a single result
     // (worst-case verdict, mean metrics). `onTrial` ticks per model call.
     const runItem = async (w, onTrial) => {
+        const k = caseKey(w.tierName, w.arm.label, w.inst.instance_id);
         const trials = [];
-        for (let t = 0; t < runs; t++) {
+        for (let t = 1; t <= runs; t++) {
+            trialOf.set(k, t); // so the live "follow" link targets this trial
+            const trialRel = trialRelFor(w.inst.instance_id, w.arm.label, t);
             const r = await runInstance(w.inst, {
-                model: w.model,
+                // The arm carries all model selection (forced model / merged config) +
+                // the axis label; runInstance calls its prepare()/recordPhaseModel().
+                arm: w.arm,
                 datasetDir: w.datasetDir,
                 defaultWorkflow: w.defaultWorkflow,
                 manageEnv: false,
+                // Per-trial dir: `full.jsonl` (consolidated, live) + `NN-<phase>.jsonl`.
+                sessionTrialDir: join(resultsDirFor(w.tierName), trialRel),
+                sessionTrialRel: trialRel,
+                trial: t,
             });
             r.tier = w.tierName;
             trials.push(r);
             completed++;
             onTrial();
         }
-        return aggregateTrials(trials);
+        const agg = aggregateTrials(trials);
+        // Keep every trial's per-phase sessions on the aggregate (aggregateTrials
+        // only carries trial 0's fields through).
+        agg.sessions = trials.map((r) => r.sessionTrial).filter((s) => !!s);
+        return agg;
     };
     // Install the eval's static-token env ONCE for the whole batch so concurrent
     // runs share one stable baseline (manageEnv:false on every runInstance).
@@ -406,7 +590,7 @@ async function runEval() {
             try {
                 await Promise.all([...byFamily].map(async ([f, items]) => {
                     for (const w of items) {
-                        const k = caseKey(w.tierName, w.model, w.inst.instance_id);
+                        const k = caseKey(w.tierName, w.arm.label, w.inst.instance_id);
                         running.add(k);
                         refresh();
                         const result = await runItem(w, () => {
@@ -418,7 +602,7 @@ async function runEval() {
                         all.push(result);
                         if (result.error)
                             harnessErrors++;
-                        const mark = result.error ? chalk.red("✗") : chalk.green("✓");
+                        const mark = result.error ? chalk.red("✗") : result.blocked ? chalk.yellow("■") : chalk.green("✓");
                         verdicts.push(`${mark} ${chalk.dim(familyLabel(f))}  ${verdictLine(w.tierName, w.inst, result)}`);
                         refresh();
                     }
@@ -431,13 +615,25 @@ async function runEval() {
             p.log.message(verdicts.join("\n"));
         }
         else {
-            // Serial: one spinner per case (updates per trial) + a verdict line.
+            // Serial: one spinner per case (updates per trial) + a verdict line. On
+            // each arm change `arm.activate()` repoints the (process-global) asset root
+            // to that arm's overlay (a no-op for models arms); work is arms-outer, so
+            // it fires at most once per arm. `releaseOverlayGuard()` lets the next arm
+            // switch overlays — without it the guard treats the switch as a concurrent
+            // overlay and throws (ADR 0001).
+            let currentArm;
             for (let i = 0; i < work.length; i++) {
                 const w = work[i];
+                if (w.arm !== currentArm) {
+                    if (currentArm)
+                        releaseOverlayGuard();
+                    w.arm.activate();
+                    currentArm = w.arm;
+                }
                 const s = makeSpinner();
-                const head = `${chalk.dim(`[${i + 1}/${work.length}]`)} ${chalk.cyan(w.tierName)}/${w.inst.instance_id}  ${chalk.dim(labels[w.model] ?? w.model)}`;
+                const head = `${chalk.dim(`[${i + 1}/${work.length}]`)} ${chalk.cyan(w.tierName)}/${w.inst.instance_id}  ${chalk.dim(labels[w.arm.label] ?? w.arm.label)}`;
                 s.start(head);
-                const k = caseKey(w.tierName, w.model, w.inst.instance_id);
+                const k = caseKey(w.tierName, w.arm.label, w.inst.instance_id);
                 running.add(k);
                 refresh();
                 let t = 0;
@@ -451,7 +647,7 @@ async function runEval() {
                 all.push(result);
                 if (result.error)
                     harnessErrors++;
-                const mark = result.error ? chalk.red("✗") : chalk.green("✓");
+                const mark = result.error ? chalk.red("✗") : result.blocked ? chalk.yellow("■") : chalk.green("✓");
                 s.stop(`${chalk.dim(`[${i + 1}/${work.length}]`)} ${mark} ${verdictLine(w.tierName, w.inst, result)}`);
                 if (result.error) {
                     p.log.error(chalk.dim(result.error));
@@ -466,47 +662,73 @@ async function runEval() {
     finally {
         restoreEvalEnv();
     }
-    // Final, static report + machine artifacts.
-    const card = summarize(all);
-    writeArtifacts(resultsDir, card);
-    const html = writeHtml(resultsDir, card, { ...htmlBase, generatedAt: new Date().toISOString() });
-    p.log.success(`Scorecard → ${chalk.cyan(html)}`);
-    p.log.success(`Artifacts → ${chalk.cyan(resultsDir)}/{scorecard.json,predictions.jsonl}`);
+    // Final, static scorecard + machine artifacts. The run-level metadata is
+    // persisted into scorecard.json so the dashboard can label, order, and (no
+    // longer) live-poll the run without re-deriving from the current config.
+    const generatedAt = new Date().toISOString();
+    for (const tier of tiers) {
+        const tierResults = all.filter((r) => (r.tier ?? "") === tier);
+        writeArtifacts(resultsDirFor(tier), withMeta(summarize(tierResults), { ...baseMetaFor(tier), generatedAt, live: false }));
+    }
+    p.log.success(`Artifacts → ${chalk.cyan(tiers.map((t) => resultsDirFor(t)).join("\n             "))}\n             /{scorecard.json,predictions.jsonl,sessions/}`);
     const ran = runs > 1 ? `${completed} runs (${all.length} cases × ${runs})` : `${all.length} runs`;
-    if (harnessErrors > 0) {
-        p.outro(chalk.yellow(`done — ${ran}, ${harnessErrors} harness error${harnessErrors === 1 ? "" : "s"} (see above)`));
+    // Keep the dashboard server alive so the just-finished run stays viewable.
+    // Only block in an interactive terminal — piped/non-TTY callers exit cleanly.
+    if (server && process.stdout.isTTY) {
+        const runUrl = `${server.url}/#/${encodeURIComponent(tierKeyFor(tiers[0]))}/${encodeURIComponent(runId)}`;
+        p.log.success(`Dashboard → ${chalk.cyan(runUrl)} ${chalk.dim("(serving · Ctrl-C to stop)")}`);
+        const tail = harnessErrors > 0 ? chalk.yellow(`done — ${ran}, ${harnessErrors} harness error${harnessErrors === 1 ? "" : "s"} (see above)`) : chalk.green(`done — ${ran}`);
+        p.outro(tail);
+        await waitForSigint();
+        await server.close();
     }
     else {
-        p.outro(chalk.green(`done — ${ran}, report at ${html}`));
+        if (server)
+            await server.close();
+        if (harnessErrors > 0) {
+            p.outro(chalk.yellow(`done — ${ran}, ${harnessErrors} harness error${harnessErrors === 1 ? "" : "s"} (see above)`));
+        }
+        else {
+            p.outro(chalk.green(`done — ${ran}`));
+        }
     }
     // Non-zero ONLY on harness failure — model quality is the measurement.
     return harnessErrors > 0 ? 1 : 0;
 }
+/** Resolve on the first SIGINT (Ctrl-C) so `run`/`serve` can keep a server up
+ * for browsing and shut it down cleanly when the user is done. */
+function waitForSigint() {
+    return new Promise((resolve) => {
+        const onSig = () => {
+            process.off("SIGINT", onSig);
+            resolve();
+        };
+        process.on("SIGINT", onSig);
+    });
+}
 /**
- * `report <dir>` — re-render `<dir>/index.html` from an existing
- * `<dir>/scorecard.json`, no models run. Lets you regenerate the HTML after a
- * report-template change (or to re-skin an old run) without paying for a fresh
- * eval. Models/tiers/labels are reconstructed from the scorecard itself.
+ * `serve` — start the dashboard server over `eval-results/` and open it in the
+ * browser to browse every past run (no models run). The same server `run` uses
+ * for the live report; blocks until Ctrl-C. There is no HTML to regenerate any
+ * more — the SPA reads the JSON directly.
  */
-function runReport(dir) {
-    if (!dir) {
-        console.error("usage: lastlight-evals report <results-dir>  (a dir holding scorecard.json)");
-        return 1;
+async function runServe() {
+    loadDotEnv();
+    const noOpen = process.argv.includes("--no-open");
+    const port = intFlag("port", 0) || undefined;
+    let server;
+    try {
+        server = await startServer({ resultsRoot: resultsRoot(), dashboardRoot: dashboardDistRoot(), port });
     }
-    const file = join(dir, "scorecard.json");
-    if (!existsSync(file)) {
-        console.error(`no scorecard.json in ${dir}`);
+    catch (err) {
+        console.error(`Couldn't start the dashboard server: ${err.message}`);
         return 1;
     }
-    const card = JSON.parse(readFileSync(file, "utf8"));
-    const labels = modelLabels();
-    // Tiers/models come straight off the saved results so the re-render matches
-    // exactly what that run measured (no dependence on current models.json).
-    const tiers = [...new Set(card.results.map((r) => r.tier ?? "triage"))];
-    const models = [...new Set(card.results.map((r) => r.model))].map((m) => labels[m] ?? m);
-    const runs = Math.max(1, ...card.results.map((r) => r.trials ?? 1));
-    const html = writeHtml(dir, card, { models, tiers, labels, runs, generatedAt: new Date().toISOString() });
-    console.log(`Scorecard → ${html}`);
+    console.log(`Dashboard → ${server.url}  (serving ${resultsRoot()} · Ctrl-C to stop)`);
+    if (!noOpen)
+        openInBrowser(server.url);
+    await waitForSigint();
+    await server.close();
     return 0;
 }
 const USAGE = `lastlight-evals — eval harness for Last Light workflows
@@ -514,21 +736,31 @@ const USAGE = `lastlight-evals — eval harness for Last Light workflows
 Usage:
   lastlight-evals [run] [tiers...] [options]   Run evals (default command)
   lastlight-evals init [dir] [options]         Scaffold an overlay+evals workspace
-  lastlight-evals report <results-dir>         Re-render index.html from scorecard.json
+  lastlight-evals serve [options]              Browse past runs in the dashboard (no models run)
 
 Run options:
-  --overlay <dir>      Layer a deployment's workflows/skills + evals/ over built-ins
-  --model <m[,m2]>     Model(s) to run (fuzzy-matched against models.json)
+  --mode <models|config>  Comparison axis. models (default): force each --model
+                          across every step. config: run an overlay's real
+                          per-step model config (its config.yaml). No flags in a
+                          TTY ⇒ asks.
+  --overlay <dir>      Layer a deployment's workflows/skills + evals/ over built-ins.
+                       Repeatable in --mode config (one arm per overlay).
+  --model <m[,m2]>     models: model(s) to run (fuzzy-matched against models.json).
+                       config: override each config's default model.
   --compare            Cross-vendor set (only models whose provider key is present)
   --runs <n>           Repeat each case n× (worst-case verdict, mean metrics)
   --serial             Force serial execution across provider families
   --datasets <dir>     Extra datasets root to discover tiers from
   --models-file <f>    Use an explicit models.json
-  --no-open            Don't open the HTML report (also implied by CI=1)
+  --no-open            Don't open / auto-serve the dashboard (also implied by CI=1)
+
+Serve options:
+  --port <n>           Preferred port for the dashboard server (default 4319)
+  --no-open            Start the server but don't open a browser
 
 Run \`lastlight-evals init --help\` for init-specific flags.
 GitHub is mocked end-to-end — no real GitHub token is needed, only a provider key.`;
-/** Top-level subcommand dispatcher: `run` (default) | `init` | `report`. */
+/** Top-level subcommand dispatcher: `run` (default) | `init` | `serve`. */
 async function main() {
     const sub = process.argv[2];
     // Top-level help — only when it's not standing in for a `run` tier name.
@@ -540,9 +772,9 @@ async function main() {
         // `init [dir] [flags]` — scaffold a fresh overlay+evals repo.
         return runInit(process.argv.slice(3));
     }
-    if (sub === "report") {
-        // `report <dir>` — re-render index.html from a saved scorecard.json.
-        return runReport(process.argv[3]);
+    if (sub === "serve") {
+        // `serve` — browse past runs; the live dashboard server, standalone.
+        return runServe();
     }
     // `run` is the default; allow an explicit leading `run` token too.
     if (sub === "run")