@hona/openeval 0.5.0 → 0.5.2

package/README.md

@@ -87,8 +87,8 @@ or does not provide a parameterized query.
 | Required recording is unavailable | **null** | **null** |
 
 Add an optional `Categories: misalignment` line under a heading to compare models
-by category in the viewer's radar chart and heatmap. Categories never trigger
-rejudging.
+by category in the viewer's radar chart, heatmap, and benchmark scorecard. Export
+the scorecard as a PNG from the results page. Categories never trigger rejudging.
 
 → [Write good rubrics](https://openev.al/docs/rubrics/) · [Download the SQL starter](https://openev.al/starter.zip)
 
@@ -143,7 +143,9 @@ bunx --bun @hona/openeval view
 ```
 
 The viewer opens at **http://127.0.0.1:4173**. `run` resumes the same aggregate;
-scope flags select work while retaining existing scores.
+scope flags select work while retaining existing scores. If a runner stops
+mid-run, the next `run` waits until its heartbeat is two minutes old, keeps the
+interrupted records, and collects that work again.
 
 To remove a model from an existing aggregate, remove its entry from
 `benchmark.ts`, then retire its active selections:

package/VERIFICATION.md

@@ -87,6 +87,9 @@ scores add `reason`, `evidence`, and JSON `measurements`. These details appear i
 the normal viewer beside verification receipts; arbitrary returned JSON remains
 inspectable. Declared code criteria must be returned exactly. Missing evidence
 references are judging errors, not candidate zeros.
+Arrays of named check observations render as readable tables, including author
+supplied pass/fail details. They do not introduce additional scores or weights.
+Large tables are explicitly bounded in the display; full returned JSON is retained.
 
 `recordEvidence({ workspace: { initial, final }, ... })` can retain constructed
 artifact controls without executing them. `judgeEvidence` then uses the same
@@ -97,4 +100,6 @@ Unused `criteria` labels/categories are removed from executable bundling. Debug
 source maps remain retained but do not affect code identity. If grading reads a
 metadata value, it is executable behavior and remains fingerprinted. Changes to
 criterion IDs, actual grading code, imported inputs, dependencies, or verification
-environment are not reporting-only edits.
+environment are not reporting-only edits. The verification environment is part
+of judge identity only for evals with `judge.ts`. Rebuilding the image does not
+rejudge Markdown-only evals.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.5.0",
+  "version": "0.5.2",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/input-fingerprints.ts

@@ -43,7 +43,8 @@ export function candidateProviders(
     : undefined;
 }
 
-/** Declared, candidate-visible inputs. Harness implementation hashes are provenance. */
+/** Declared, candidate-visible inputs. Harness implementation hashes are provenance.
+ * Early stopping is a judge-side policy; the planner re-collects sessions it cut short. */
 export function candidateFingerprint(
   definition: BenchmarkDefinition,
   evalId: string,
@@ -61,7 +62,6 @@ export function candidateFingerprint(
       timeoutMs: definition.candidate.timeoutMs,
       websearch: definition.candidate.websearch,
       provider: scope && { ...scope.settings, model: scope.override },
-      earlyStop: item.settings.earlyStop ?? false,
     },
     container: {
       engine: definition.container.engine,
@@ -70,22 +70,24 @@ export function candidateFingerprint(
     },
   });
 }
+/** Only judge.ts can run checks, so the verification runtime never changes an LLM-only judgment. */
 export const judgeFingerprint = (
   definition: BenchmarkDefinition,
   evalId: string,
-) =>
-  fingerprint({
-    rubric: definition.evals.find((item) => item.id === evalId)!.judge,
-    code: definition.evals.find((item) => item.id === evalId)!.code?.hash,
-    codeIds: definition.evals.find((item) => item.id === evalId)!.codeCriteria?.map(item => item.id).sort(),
-    agent: definition.evals.find((item) => item.id === evalId)!.judge
-      ? JUDGE_AGENT
-      : undefined,
-    judge: definition.evals.find((item) => item.id === evalId)!.judge
-      ? definition.judge
-      : { timeoutMs: definition.judge.timeoutMs, verification: definition.judge.verification },
+) => {
+  const item = definition.evals.find((item) => item.id === evalId)!;
+  const verification = item.code ? definition.judge.verification : undefined;
+  return fingerprint({
+    rubric: item.judge,
+    code: item.code?.hash,
+    codeIds: item.codeCriteria?.map((criterion) => criterion.id).sort(),
+    agent: item.judge ? JUDGE_AGENT : undefined,
+    judge: item.judge
+      ? { ...definition.judge, verification }
+      : { timeoutMs: definition.judge.timeoutMs, verification },
     protocol: JUDGE_PROTOCOL,
   });
+};
 export const savedJudgeFingerprint = (
   input: Pick<
     JudgeRunInput,
@@ -99,11 +101,12 @@ export const savedJudgeFingerprint = (
     | "codeCriteria"
     | "verification"
   >,
-) =>
-  fingerprint({
+) => {
+  const verification = input.code ? input.verification : undefined;
+  return fingerprint({
     rubric: input.rubric,
     code: input.code?.hash,
-    codeIds: input.codeCriteria?.map(item => item.id).sort(),
+    codeIds: input.codeCriteria?.map((criterion) => criterion.id).sort(),
     agent: input.agent,
     protocol: input.protocol,
     judge: input.rubric
@@ -111,7 +114,8 @@ export const savedJudgeFingerprint = (
           model: input.model,
           timeoutMs: input.timeoutMs,
           websearch: input.websearch,
-          verification: input.verification,
+          verification,
         }
-      : { timeoutMs: input.timeoutMs, verification: input.verification },
+      : { timeoutMs: input.timeoutMs, verification },
   });
+};

package/src/app/load-benchmark.ts

@@ -266,6 +266,7 @@ export async function loadBenchmark(
       (key) =>
         ![
           "name",
+          "suite",
           "models",
           "judge",
           "repetitions",
@@ -277,6 +278,13 @@ export async function loadBenchmark(
     )
   )
     throw new Error("benchmark.ts contains unsupported settings");
+  if (
+    definition.suite !== undefined &&
+    (typeof definition.suite !== "string" ||
+      !definition.suite.trim() ||
+      definition.suite.length > 80)
+  )
+    throw new Error("benchmark.ts suite must be a label of 1 to 80 characters");
   if (
     definition.categories !== undefined &&
     (!Array.isArray(definition.categories) || !definition.categories.length)
@@ -352,6 +360,7 @@ export async function loadBenchmark(
       throw new Error("Websearch must be exa or false");
   return {
     name: definition.name ?? basename(directory),
+    ...(definition.suite ? { suite: definition.suite.trim() } : {}),
     directory,
     models,
     evals,

package/src/app/plan-benchmark.ts

@@ -7,6 +7,7 @@ import type {
 } from "../types";
 import type { Results } from "../infra/sqlite";
 import { canJudgeEval } from "./eval-state";
+import { monitorPolicy } from "./monitor-policy";
 import {
   candidateFingerprint,
   judgeFingerprint,
@@ -21,14 +22,18 @@ export {
 export const slotId = (evalId: string, model: ModelRef, repetition: number) =>
   `${evalId}:${model}:${repetition}`;
 
-/** Plan only missing or changed work. A failed execution requires an explicit retry. */
+/** Plan only missing or changed work. A failed execution requires an explicit retry,
+ * except work a stopped runner interrupted. `stoppedRunner` plans open executions that way. */
 export function planBenchmark(
   definition: BenchmarkDefinition,
   runtime: BenchmarkRun["runtime"],
   results: Results,
+  options: { stoppedRunner?: boolean } = {},
 ): PlanItem[] {
   const previous = new Map(results.slots().map((slot) => [slot.id, slot]));
   const judgments = results.judgeRuns();
+  const interrupted = (run: { state: string; interrupted?: boolean }) =>
+    !!run.interrupted || (!!options.stoppedRunner && run.state === "running");
   return Array.from(
     { length: definition.repetitions },
     (_, index) => index + 1,
@@ -69,12 +74,39 @@ export function planBenchmark(
               : "New eval/model/repetition",
           };
         const execution = results.evalRun(slot.evalRunId)!;
+        if (interrupted(execution))
+          return {
+            slot: {
+              ...slot,
+              evalRunId: null,
+              judgeRunId: null,
+              previousEvalRunId: execution.id,
+            },
+            action: "candidate",
+            reason: "The runner stopped during this session; collecting it again",
+          };
         if (!canJudgeEval(execution))
           return {
             slot,
             action: "failed",
             reason: `Candidate ${execution.state}; explicit retry required`,
           };
+        // A session cut short is evidence only under a stopping rule that still applies.
+        if (
+          execution.state === "stopped" &&
+          execution.stop &&
+          !monitorPolicy(evalDefinition.settings.earlyStop, model)
+        )
+          return {
+            slot: {
+              ...slot,
+              evalRunId: null,
+              judgeRunId: null,
+              previousEvalRunId: execution.id,
+            },
+            action: "candidate",
+            reason: "Early stopping is off; the recorded session stopped early",
+          };
         if (slot.judgeRunId)
           if (
             results.judgeRun(slot.judgeRunId)?.judgment?.value === null &&
@@ -102,12 +134,13 @@ export function planBenchmark(
             action: "reuse",
             reason: "Candidate inputs and judgment unchanged",
           };
-        const failed = judgments.find(
+        const attempts = judgments.filter(
           (run) =>
             run.input.evalRunId === slot.evalRunId &&
             run.input.judgeHash === slot.judgeHash &&
             run.state !== "completed",
         );
+        const failed = attempts.find((run) => !interrupted(run));
         if (failed)
           return {
             slot,
@@ -117,9 +150,11 @@ export function planBenchmark(
         return {
           slot,
           action: "judge",
-          reason: !sameJudge
-            ? "Judge inputs changed"
-            : "Recorded work is ready for judging",
+          reason: attempts.length
+            ? "The runner stopped during judging; judging again"
+            : !sameJudge
+              ? "Judge inputs changed"
+              : "Recorded work is ready for judging",
         };
       }),
     ),

package/src/app/read-results.ts

@@ -58,6 +58,7 @@ const entryOf = (
   name: evalId
     ? (run.definition.evals.find((item) => item.id === evalId)?.name ?? evalId)
     : run.name,
+  ...(!evalId && run.definition.suite ? { suite: run.definition.suite } : {}),
   status: run.state,
   startedAt: run.createdAt,
   updatedAt: Date.parse(run.updatedAt),
@@ -78,6 +79,7 @@ export class ResultReader {
     benchmark: BenchmarkRun,
     now: number,
     candidate?: EvalRun,
+    judges?: readonly JudgeRun[],
   ) {
     const key = `${results.path}:${run.id}`;
     const cached = this.runtimes.get(key);
@@ -91,8 +93,7 @@ export class ResultReader {
         scheduled(slot, benchmark, now);
       if (current && "evalRunId" in run.input)
         current =
-          results
-            .judgeRuns()
+          (judges ?? results.judgeRuns())
             .filter(
               (judge) =>
                 judge.input.evalRunId === candidate!.id &&
@@ -279,7 +280,16 @@ export class ResultReader {
         evalId,
       ),
     );
-    const candidates = results.evalRuns(),
+    // Runtime and cost describe the work behind the active selections, not retired history.
+    const active = new Set(
+      results
+        .slots()
+        .filter((slot) => slot.active)
+        .map((slot) => slot.id),
+    );
+    const candidates = results
+        .evalRuns()
+        .filter((candidate) => active.has(candidate.slotId)),
       allJudges = results.judgeRuns();
     const now = Date.now(),
       candidateIndex = new Map(candidates.map((run) => [run.id, run]));
@@ -291,7 +301,9 @@ export class ResultReader {
           : (execution as EvalRun);
       if (!candidate) continue;
       const intervals = evalRuntime.get(candidate.input.evalId) ?? [];
-      intervals.push(...this.runtime(execution, results, run, now, candidate));
+      intervals.push(
+        ...this.runtime(execution, results, run, now, candidate, allJudges),
+      );
       evalRuntime.set(candidate.input.evalId, intervals);
     }
     for (const [id, intervals] of evalRuntime)
@@ -323,6 +335,9 @@ export class ResultReader {
       entry: entryOf(run, id, evalId),
       overview: {
         name: evalId ? evals[0].name : run.name,
+        ...(!evalId && run.definition.suite
+          ? { suite: run.definition.suite }
+          : {}),
         kind: evalId ? "eval" : "benchmark",
         status: run.state,
         startedAt: run.createdAt,
@@ -353,59 +368,69 @@ export class ResultReader {
     if (split < 0) throw new Error("Select an eval");
     const evalId = decodeURIComponent(id.slice(split + 1));
     using results = await this.database(id);
-    const benchmark = results.benchmark!,
-      executions = results.evalRuns(),
-      judges = results.judgeRuns();
-    const runs = results
-      .slots()
-      .filter((slot) => slot.active && slot.evalId === evalId)
-      .map((slot) => {
-        const candidate = executions.find((run) => run.id === slot.evalRunId);
-        const current = liveRun(
-          slot,
-          candidate,
-          judges,
-          benchmark,
-          results,
-          (run) => this.runtime(run, results, benchmark, Date.now(), candidate),
-        );
-        const related = executions.filter((run) => run.slotId === slot.id),
-          ids = new Set(related.map((run) => run.id));
-        return {
-          ...current,
-          cost: runCosts(
-            [
-              ...related,
-              ...judges.filter((run) => ids.has(run.input.evalRunId)),
-            ],
-            results,
-          ),
-          elapsedMs: runtimeMs(
-            [...current.eval.runtime, ...current.judge.runtime],
-            Date.now(),
-          ),
-        };
-      });
+    const benchmark = results.benchmark!;
     return {
       entry: entryOf(benchmark, id, evalId),
       eval: evalId,
       benchmarkId: benchmark.id,
-      runs,
+      runs: this.liveRuns(results, [evalId]).get(evalId)!,
     };
   }
+  /** Active slot rows for the requested evals, from one read of the store. */
+  private liveRuns(results: Results, evalIds: readonly string[]) {
+    const benchmark = results.benchmark!,
+      executions = results.evalRuns(),
+      judges = results.judgeRuns(),
+      now = Date.now();
+    const byId = new Map(executions.map((run) => [run.id, run]));
+    const bySlot = Map.groupBy(executions, (run) => run.slotId);
+    const judgesFor = Map.groupBy(judges, (run) => run.input.evalRunId);
+    const out = new Map<string, EvalRunSummary[]>(
+      evalIds.map((evalId) => [evalId, []]),
+    );
+    for (const slot of results.slots()) {
+      const rows = out.get(slot.evalId);
+      if (!slot.active || !rows) continue;
+      const candidate = slot.evalRunId ? byId.get(slot.evalRunId) : undefined;
+      const current = liveRun(
+        slot,
+        candidate,
+        candidate ? (judgesFor.get(candidate.id) ?? []) : [],
+        benchmark,
+        results,
+        (run) => this.runtime(run, results, benchmark, now, candidate, judges),
+      );
+      const related = bySlot.get(slot.id) ?? [];
+      rows.push({
+        ...current,
+        cost: runCosts(
+          [...related, ...related.flatMap((run) => judgesFor.get(run.id) ?? [])],
+          results,
+        ),
+        elapsedMs: runtimeMs(
+          [...current.eval.runtime, ...current.judge.runtime],
+          now,
+        ),
+      });
+    }
+    return out;
+  }
   async activity(): Promise<ActivityRun[]> {
     const index = await this.index(),
       out: ActivityRun[] = [];
     for (const entry of index.entries) {
+      const summary = await this.summary(entry.id);
       using results = await this.database(entry.id);
       const benchmark = results.benchmark!;
-      const summary = await this.summary(entry.id);
+      const live = this.liveRuns(
+        results,
+        benchmark.definition.evals.map((item) => item.id),
+      );
       for (const [
         evalOrder,
         evalDefinition,
       ] of benchmark.definition.evals.entries()) {
-        const id = evalResultId(benchmark.id, evalDefinition.id),
-          runs = await this.evalRuns(id);
+        const id = evalResultId(benchmark.id, evalDefinition.id);
         out.push({
           id,
           benchmarkId: benchmark.id,
@@ -415,15 +440,27 @@ export class ResultReader {
             id,
             eval: evalDefinition.id,
             benchmark: benchmark.id,
+            name: benchmark.name,
+            ...(benchmark.definition.suite
+              ? { suite: benchmark.definition.suite }
+              : {}),
             startedAt: benchmark.createdAt,
             heartbeatAt: benchmark.updatedAt,
             status: benchmark.state,
             concurrency: benchmark.definition.concurrency,
             evalOrder,
-            runs: runs.runs,
+            runs: live.get(evalDefinition.id)!,
             runtime: summary.related.find(
               (item) => item.eval === evalDefinition.id,
             )!.runtime,
+            ...(benchmark.execution
+              ? {
+                  invocation: {
+                    startedAt: benchmark.execution.startedAt,
+                    spentUSD: benchmark.execution.spentUSD,
+                  },
+                }
+              : {}),
           },
         });
       }

package/src/app/recover-run.ts

@@ -0,0 +1,61 @@
+import type { BenchmarkRun } from "../types";
+import type { Results } from "../infra/sqlite";
+
+/** A live runner saves its heartbeat every five seconds. */
+export const STOPPED_RUNNER_MS = 120_000;
+
+export const runnerStopped = (
+  benchmark: Pick<BenchmarkRun, "state" | "updatedAt">,
+  now = Date.now(),
+) =>
+  benchmark.state === "running" &&
+  now - Date.parse(benchmark.updatedAt) > STOPPED_RUNNER_MS;
+
+/** Close the executions a stopped runner left open. Their records stay; the planner repeats the work. */
+export function recoverStoppedRunner(results: Results, now = new Date()) {
+  const time = now.toISOString();
+  const elapsedMs = (startedAt: string) =>
+    Math.max(0, now.getTime() - Date.parse(startedAt));
+  const recovered = { evalRuns: 0, judgeRuns: 0 };
+  results.transaction(() => {
+    for (const run of results.judgeRuns()) {
+      if (run.state !== "running") continue;
+      for (const check of results.judgeChecks(run.id))
+        if (check.state === "running")
+          results.finishJudgeCheck({
+            ...check,
+            state: "cancelled",
+            completedAt: time,
+          });
+      results.finishJudge({
+        ...run,
+        state: "failed",
+        interrupted: true,
+        error: "The runner stopped before judging finished",
+        completedAt: time,
+        elapsedMs: elapsedMs(run.startedAt),
+      });
+      recovered.judgeRuns++;
+    }
+    for (const run of results.evalRuns()) {
+      if (run.state !== "running") continue;
+      results.finishEval({
+        ...run,
+        state: "failed",
+        interrupted: true,
+        error: "The runner stopped before this session finished",
+        completedAt: time,
+        elapsedMs: elapsedMs(run.startedAt),
+        ...(run.stop ? { stop: { ...run.stop, applied: false } } : {}),
+      });
+      recovered.evalRuns++;
+    }
+    results.saveBenchmark({
+      ...results.benchmark!,
+      state: "incomplete",
+      scheduledSlotIds: [],
+      updatedAt: time,
+    });
+  });
+  return recovered;
+}

package/src/app/rejudge.ts

@@ -8,6 +8,7 @@ import { executeQueue, finishBenchmark } from "./run-benchmark";
 import { judgingFingerprint } from "../infra/judging";
 import { canJudgeEval } from "./eval-state";
 import { verificationRuntime } from "../infra/verification/image";
+import { recoverStoppedRunner, runnerStopped } from "./recover-run";
 
 /** Rejudging changes only the selected judgment. The candidate execution is never repeated. */
 export async function judgeRun(
@@ -26,13 +27,16 @@ export async function judgeRuns(
 ): Promise<JudgeRun[]> {
   if (!evalRunIds.length) return [];
   using results = new Results(resolve(directory, "runner.db"));
-  const previous = results.benchmark!;
-  if (previous.mergedInto)
+  if (results.benchmark!.mergedInto)
     throw new Error(
-      `This run was merged into ${previous.mergedInto}; use the aggregate run`,
+      `This run was merged into ${results.benchmark!.mergedInto}; use the aggregate run`,
     );
-  if (previous.state === "running")
-    throw new Error("Wait for the active benchmark run");
+  if (results.benchmark!.state === "running") {
+    if (!runnerStopped(results.benchmark!))
+      throw new Error("Wait for the active benchmark run");
+    recoverStoppedRunner(results);
+  }
+  const previous = results.benchmark!;
   const candidates = [...new Set(evalRunIds)].map((evalRunId) => {
     const candidate = results.evalRun(evalRunId);
     if (!canJudgeEval(candidate))

package/src/app/retry-run.ts

@@ -6,6 +6,7 @@ import { candidateFingerprint } from "./plan-benchmark";
 import { runEvalPipeline } from "./run-eval-pipeline";
 import { ExecutionBudget } from "./execution-budget";
 import { finishBenchmark } from "./run-benchmark";
+import { recoverStoppedRunner, runnerStopped } from "./recover-run";
 
 export async function retryEvalRun(
   directory: string,
@@ -13,13 +14,16 @@ export async function retryEvalRun(
   onEvent?: ExecutionObserver,
 ) {
   using results = new Results(resolve(directory, "runner.db"));
-  const benchmark = results.benchmark!;
-  if (benchmark.mergedInto)
+  if (results.benchmark!.mergedInto)
     throw new Error(
-      `This run was merged into ${benchmark.mergedInto}; use the aggregate run`,
+      `This run was merged into ${results.benchmark!.mergedInto}; use the aggregate run`,
     );
-  if (benchmark.state === "running")
-    throw new Error("Wait for the active benchmark run");
+  if (results.benchmark!.state === "running") {
+    if (!runnerStopped(results.benchmark!))
+      throw new Error("Wait for the active benchmark run");
+    recoverStoppedRunner(results);
+  }
+  const benchmark = results.benchmark!;
   const previous = results.evalRun(evalRunId);
   if (!previous || previous.state === "completed")
     throw new Error(
@@ -28,7 +32,10 @@ export async function retryEvalRun(
   const slot = results.slot(previous.slotId)!;
   if (slot.evalRunId !== previous.id)
     throw new Error("Select the current eval run for this repetition");
-  const runtime = await runtimeFingerprint(benchmark.definition.container);
+  const runtime = await runtimeFingerprint(
+    benchmark.definition.container,
+    benchmark.definition.judge.verification,
+  );
   if (
     candidateFingerprint(
       benchmark.definition,

package/src/app/run-benchmark.ts

@@ -23,6 +23,7 @@ import { canJudgeEval } from "./eval-state";
 import { isScored } from "../judgment";
 import { benchmarkScores } from "./scores";
 import { CostBudget, estimateWork } from "./cost-plan";
+import { recoverStoppedRunner, runnerStopped } from "./recover-run";
 import {
   categoryKey,
   categoryList,
@@ -101,7 +102,8 @@ export function finishBenchmark(context: ExecutionContext) {
         (run) =>
           run.input.evalRunId === slot.evalRunId &&
           run.input.judgeHash === slot.judgeHash &&
-          ["failed", "timed_out"].includes(run.state),
+          ["failed", "timed_out"].includes(run.state) &&
+          !run.interrupted,
       ) &&
         !slot.judgeRunId),
   );
@@ -183,6 +185,7 @@ export async function runBenchmark(
     (!options.fresh
       ? await currentBenchmarkRun(definition.directory)
       : undefined);
+  let stopped = false;
   if (directory) {
     using previous = new Results(resolve(directory, "runner.db"), true);
     const saved = previous.benchmark!;
@@ -190,8 +193,9 @@ export async function runBenchmark(
       throw new Error(
         `This run was merged into ${saved.mergedInto}; use the aggregate run`,
       );
-    if (saved.state === "running")
+    if (saved.state === "running" && !runnerStopped(saved))
       throw new Error("This benchmark run is already running");
+    stopped = saved.state === "running";
     definition.models = [
       ...new Set([...saved.definition.models, ...definition.models]),
     ];
@@ -220,10 +224,16 @@ export async function runBenchmark(
       ? new Results(database, true)
       : new Results(":memory:")
     : new Results(database);
+  // A stopped runner leaves sessions open. A dry run plans them as interrupted work.
+  const recovered = stopped && !options.dryRun
+    ? recoverStoppedRunner(results)
+    : undefined;
   const previousSelections = new Map(
     results.slots().map((slot) => [slot.id, slot]),
   );
-  let plan = planBenchmark(definition, runtime, results);
+  let plan = planBenchmark(definition, runtime, results, {
+    stoppedRunner: stopped,
+  });
   if (selected || onlyEvals || options.onlyRepetitions) {
     const retained = new Map(results.slots().map((slot) => [slot.id, slot]));
     plan = plan.map((item) =>
@@ -260,7 +270,7 @@ export async function runBenchmark(
     fingerprint(previous.runtime) === fingerprint(runtime) &&
     fingerprint(previous.modelNames ?? {}) === fingerprint(modelNames)
   )
-    return { directory, plan, estimate, benchmark: previous };
+    return { directory, plan, estimate, benchmark: previous, recovered };
   const benchmark: BenchmarkRun = {
     id: previous?.id ?? `benchmark_${randomUUID()}`,
     name: definition.name,
@@ -397,7 +407,13 @@ export async function runBenchmark(
         deferred: plan.filter((item) => item.action === "deferred").length,
       },
     });
-    return { directory, plan, estimate, benchmark: finishBenchmark(context) };
+    return {
+      directory,
+      plan,
+      estimate,
+      benchmark: finishBenchmark(context),
+      recovered,
+    };
   } catch (error) {
     results.saveBenchmark({
       ...results.benchmark!,