@hona/openeval 0.5.1 → 0.5.2

package/README.md

@@ -143,7 +143,9 @@ bunx --bun @hona/openeval view
 ```
 
 The viewer opens at **http://127.0.0.1:4173**. `run` resumes the same aggregate;
-scope flags select work while retaining existing scores.
+scope flags select work while retaining existing scores. If a runner stops
+mid-run, the next `run` waits until its heartbeat is two minutes old, keeps the
+interrupted records, and collects that work again.
 
 To remove a model from an existing aggregate, remove its entry from
 `benchmark.ts`, then retire its active selections:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.5.1",
+  "version": "0.5.2",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/load-benchmark.ts

@@ -266,6 +266,7 @@ export async function loadBenchmark(
       (key) =>
         ![
           "name",
+          "suite",
           "models",
           "judge",
           "repetitions",
@@ -277,6 +278,13 @@ export async function loadBenchmark(
     )
   )
     throw new Error("benchmark.ts contains unsupported settings");
+  if (
+    definition.suite !== undefined &&
+    (typeof definition.suite !== "string" ||
+      !definition.suite.trim() ||
+      definition.suite.length > 80)
+  )
+    throw new Error("benchmark.ts suite must be a label of 1 to 80 characters");
   if (
     definition.categories !== undefined &&
     (!Array.isArray(definition.categories) || !definition.categories.length)
@@ -352,6 +360,7 @@ export async function loadBenchmark(
       throw new Error("Websearch must be exa or false");
   return {
     name: definition.name ?? basename(directory),
+    ...(definition.suite ? { suite: definition.suite.trim() } : {}),
     directory,
     models,
     evals,

package/src/app/plan-benchmark.ts

@@ -22,14 +22,18 @@ export {
 export const slotId = (evalId: string, model: ModelRef, repetition: number) =>
   `${evalId}:${model}:${repetition}`;
 
-/** Plan only missing or changed work. A failed execution requires an explicit retry. */
+/** Plan only missing or changed work. A failed execution requires an explicit retry,
+ * except work a stopped runner interrupted. `stoppedRunner` plans open executions that way. */
 export function planBenchmark(
   definition: BenchmarkDefinition,
   runtime: BenchmarkRun["runtime"],
   results: Results,
+  options: { stoppedRunner?: boolean } = {},
 ): PlanItem[] {
   const previous = new Map(results.slots().map((slot) => [slot.id, slot]));
   const judgments = results.judgeRuns();
+  const interrupted = (run: { state: string; interrupted?: boolean }) =>
+    !!run.interrupted || (!!options.stoppedRunner && run.state === "running");
   return Array.from(
     { length: definition.repetitions },
     (_, index) => index + 1,
@@ -70,6 +74,17 @@ export function planBenchmark(
               : "New eval/model/repetition",
           };
         const execution = results.evalRun(slot.evalRunId)!;
+        if (interrupted(execution))
+          return {
+            slot: {
+              ...slot,
+              evalRunId: null,
+              judgeRunId: null,
+              previousEvalRunId: execution.id,
+            },
+            action: "candidate",
+            reason: "The runner stopped during this session; collecting it again",
+          };
         if (!canJudgeEval(execution))
           return {
             slot,
@@ -119,12 +134,13 @@ export function planBenchmark(
             action: "reuse",
             reason: "Candidate inputs and judgment unchanged",
           };
-        const failed = judgments.find(
+        const attempts = judgments.filter(
           (run) =>
             run.input.evalRunId === slot.evalRunId &&
             run.input.judgeHash === slot.judgeHash &&
             run.state !== "completed",
         );
+        const failed = attempts.find((run) => !interrupted(run));
         if (failed)
           return {
             slot,
@@ -134,9 +150,11 @@ export function planBenchmark(
         return {
           slot,
           action: "judge",
-          reason: !sameJudge
-            ? "Judge inputs changed"
-            : "Recorded work is ready for judging",
+          reason: attempts.length
+            ? "The runner stopped during judging; judging again"
+            : !sameJudge
+              ? "Judge inputs changed"
+              : "Recorded work is ready for judging",
         };
       }),
     ),

package/src/app/read-results.ts

@@ -58,6 +58,7 @@ const entryOf = (
   name: evalId
     ? (run.definition.evals.find((item) => item.id === evalId)?.name ?? evalId)
     : run.name,
+  ...(!evalId && run.definition.suite ? { suite: run.definition.suite } : {}),
   status: run.state,
   startedAt: run.createdAt,
   updatedAt: Date.parse(run.updatedAt),
@@ -78,6 +79,7 @@ export class ResultReader {
     benchmark: BenchmarkRun,
     now: number,
     candidate?: EvalRun,
+    judges?: readonly JudgeRun[],
   ) {
     const key = `${results.path}:${run.id}`;
     const cached = this.runtimes.get(key);
@@ -91,8 +93,7 @@ export class ResultReader {
         scheduled(slot, benchmark, now);
       if (current && "evalRunId" in run.input)
         current =
-          results
-            .judgeRuns()
+          (judges ?? results.judgeRuns())
             .filter(
               (judge) =>
                 judge.input.evalRunId === candidate!.id &&
@@ -279,7 +280,16 @@ export class ResultReader {
         evalId,
       ),
     );
-    const candidates = results.evalRuns(),
+    // Runtime and cost describe the work behind the active selections, not retired history.
+    const active = new Set(
+      results
+        .slots()
+        .filter((slot) => slot.active)
+        .map((slot) => slot.id),
+    );
+    const candidates = results
+        .evalRuns()
+        .filter((candidate) => active.has(candidate.slotId)),
       allJudges = results.judgeRuns();
     const now = Date.now(),
       candidateIndex = new Map(candidates.map((run) => [run.id, run]));
@@ -291,7 +301,9 @@ export class ResultReader {
           : (execution as EvalRun);
       if (!candidate) continue;
       const intervals = evalRuntime.get(candidate.input.evalId) ?? [];
-      intervals.push(...this.runtime(execution, results, run, now, candidate));
+      intervals.push(
+        ...this.runtime(execution, results, run, now, candidate, allJudges),
+      );
       evalRuntime.set(candidate.input.evalId, intervals);
     }
     for (const [id, intervals] of evalRuntime)
@@ -323,6 +335,9 @@ export class ResultReader {
       entry: entryOf(run, id, evalId),
       overview: {
         name: evalId ? evals[0].name : run.name,
+        ...(!evalId && run.definition.suite
+          ? { suite: run.definition.suite }
+          : {}),
         kind: evalId ? "eval" : "benchmark",
         status: run.state,
         startedAt: run.createdAt,
@@ -353,59 +368,69 @@ export class ResultReader {
     if (split < 0) throw new Error("Select an eval");
     const evalId = decodeURIComponent(id.slice(split + 1));
     using results = await this.database(id);
-    const benchmark = results.benchmark!,
-      executions = results.evalRuns(),
-      judges = results.judgeRuns();
-    const runs = results
-      .slots()
-      .filter((slot) => slot.active && slot.evalId === evalId)
-      .map((slot) => {
-        const candidate = executions.find((run) => run.id === slot.evalRunId);
-        const current = liveRun(
-          slot,
-          candidate,
-          judges,
-          benchmark,
-          results,
-          (run) => this.runtime(run, results, benchmark, Date.now(), candidate),
-        );
-        const related = executions.filter((run) => run.slotId === slot.id),
-          ids = new Set(related.map((run) => run.id));
-        return {
-          ...current,
-          cost: runCosts(
-            [
-              ...related,
-              ...judges.filter((run) => ids.has(run.input.evalRunId)),
-            ],
-            results,
-          ),
-          elapsedMs: runtimeMs(
-            [...current.eval.runtime, ...current.judge.runtime],
-            Date.now(),
-          ),
-        };
-      });
+    const benchmark = results.benchmark!;
     return {
       entry: entryOf(benchmark, id, evalId),
       eval: evalId,
       benchmarkId: benchmark.id,
-      runs,
+      runs: this.liveRuns(results, [evalId]).get(evalId)!,
     };
   }
+  /** Active slot rows for the requested evals, from one read of the store. */
+  private liveRuns(results: Results, evalIds: readonly string[]) {
+    const benchmark = results.benchmark!,
+      executions = results.evalRuns(),
+      judges = results.judgeRuns(),
+      now = Date.now();
+    const byId = new Map(executions.map((run) => [run.id, run]));
+    const bySlot = Map.groupBy(executions, (run) => run.slotId);
+    const judgesFor = Map.groupBy(judges, (run) => run.input.evalRunId);
+    const out = new Map<string, EvalRunSummary[]>(
+      evalIds.map((evalId) => [evalId, []]),
+    );
+    for (const slot of results.slots()) {
+      const rows = out.get(slot.evalId);
+      if (!slot.active || !rows) continue;
+      const candidate = slot.evalRunId ? byId.get(slot.evalRunId) : undefined;
+      const current = liveRun(
+        slot,
+        candidate,
+        candidate ? (judgesFor.get(candidate.id) ?? []) : [],
+        benchmark,
+        results,
+        (run) => this.runtime(run, results, benchmark, now, candidate, judges),
+      );
+      const related = bySlot.get(slot.id) ?? [];
+      rows.push({
+        ...current,
+        cost: runCosts(
+          [...related, ...related.flatMap((run) => judgesFor.get(run.id) ?? [])],
+          results,
+        ),
+        elapsedMs: runtimeMs(
+          [...current.eval.runtime, ...current.judge.runtime],
+          now,
+        ),
+      });
+    }
+    return out;
+  }
   async activity(): Promise<ActivityRun[]> {
     const index = await this.index(),
       out: ActivityRun[] = [];
     for (const entry of index.entries) {
+      const summary = await this.summary(entry.id);
       using results = await this.database(entry.id);
       const benchmark = results.benchmark!;
-      const summary = await this.summary(entry.id);
+      const live = this.liveRuns(
+        results,
+        benchmark.definition.evals.map((item) => item.id),
+      );
       for (const [
         evalOrder,
         evalDefinition,
       ] of benchmark.definition.evals.entries()) {
-        const id = evalResultId(benchmark.id, evalDefinition.id),
-          runs = await this.evalRuns(id);
+        const id = evalResultId(benchmark.id, evalDefinition.id);
         out.push({
           id,
           benchmarkId: benchmark.id,
@@ -415,15 +440,27 @@ export class ResultReader {
             id,
             eval: evalDefinition.id,
             benchmark: benchmark.id,
+            name: benchmark.name,
+            ...(benchmark.definition.suite
+              ? { suite: benchmark.definition.suite }
+              : {}),
             startedAt: benchmark.createdAt,
             heartbeatAt: benchmark.updatedAt,
             status: benchmark.state,
             concurrency: benchmark.definition.concurrency,
             evalOrder,
-            runs: runs.runs,
+            runs: live.get(evalDefinition.id)!,
             runtime: summary.related.find(
               (item) => item.eval === evalDefinition.id,
             )!.runtime,
+            ...(benchmark.execution
+              ? {
+                  invocation: {
+                    startedAt: benchmark.execution.startedAt,
+                    spentUSD: benchmark.execution.spentUSD,
+                  },
+                }
+              : {}),
           },
         });
       }

package/src/app/recover-run.ts

@@ -0,0 +1,61 @@
+import type { BenchmarkRun } from "../types";
+import type { Results } from "../infra/sqlite";
+
+/** A live runner saves its heartbeat every five seconds. */
+export const STOPPED_RUNNER_MS = 120_000;
+
+export const runnerStopped = (
+  benchmark: Pick<BenchmarkRun, "state" | "updatedAt">,
+  now = Date.now(),
+) =>
+  benchmark.state === "running" &&
+  now - Date.parse(benchmark.updatedAt) > STOPPED_RUNNER_MS;
+
+/** Close the executions a stopped runner left open. Their records stay; the planner repeats the work. */
+export function recoverStoppedRunner(results: Results, now = new Date()) {
+  const time = now.toISOString();
+  const elapsedMs = (startedAt: string) =>
+    Math.max(0, now.getTime() - Date.parse(startedAt));
+  const recovered = { evalRuns: 0, judgeRuns: 0 };
+  results.transaction(() => {
+    for (const run of results.judgeRuns()) {
+      if (run.state !== "running") continue;
+      for (const check of results.judgeChecks(run.id))
+        if (check.state === "running")
+          results.finishJudgeCheck({
+            ...check,
+            state: "cancelled",
+            completedAt: time,
+          });
+      results.finishJudge({
+        ...run,
+        state: "failed",
+        interrupted: true,
+        error: "The runner stopped before judging finished",
+        completedAt: time,
+        elapsedMs: elapsedMs(run.startedAt),
+      });
+      recovered.judgeRuns++;
+    }
+    for (const run of results.evalRuns()) {
+      if (run.state !== "running") continue;
+      results.finishEval({
+        ...run,
+        state: "failed",
+        interrupted: true,
+        error: "The runner stopped before this session finished",
+        completedAt: time,
+        elapsedMs: elapsedMs(run.startedAt),
+        ...(run.stop ? { stop: { ...run.stop, applied: false } } : {}),
+      });
+      recovered.evalRuns++;
+    }
+    results.saveBenchmark({
+      ...results.benchmark!,
+      state: "incomplete",
+      scheduledSlotIds: [],
+      updatedAt: time,
+    });
+  });
+  return recovered;
+}

package/src/app/rejudge.ts

@@ -8,6 +8,7 @@ import { executeQueue, finishBenchmark } from "./run-benchmark";
 import { judgingFingerprint } from "../infra/judging";
 import { canJudgeEval } from "./eval-state";
 import { verificationRuntime } from "../infra/verification/image";
+import { recoverStoppedRunner, runnerStopped } from "./recover-run";
 
 /** Rejudging changes only the selected judgment. The candidate execution is never repeated. */
 export async function judgeRun(
@@ -26,13 +27,16 @@ export async function judgeRuns(
 ): Promise<JudgeRun[]> {
   if (!evalRunIds.length) return [];
   using results = new Results(resolve(directory, "runner.db"));
-  const previous = results.benchmark!;
-  if (previous.mergedInto)
+  if (results.benchmark!.mergedInto)
     throw new Error(
-      `This run was merged into ${previous.mergedInto}; use the aggregate run`,
+      `This run was merged into ${results.benchmark!.mergedInto}; use the aggregate run`,
     );
-  if (previous.state === "running")
-    throw new Error("Wait for the active benchmark run");
+  if (results.benchmark!.state === "running") {
+    if (!runnerStopped(results.benchmark!))
+      throw new Error("Wait for the active benchmark run");
+    recoverStoppedRunner(results);
+  }
+  const previous = results.benchmark!;
   const candidates = [...new Set(evalRunIds)].map((evalRunId) => {
     const candidate = results.evalRun(evalRunId);
     if (!canJudgeEval(candidate))

package/src/app/retry-run.ts

@@ -6,6 +6,7 @@ import { candidateFingerprint } from "./plan-benchmark";
 import { runEvalPipeline } from "./run-eval-pipeline";
 import { ExecutionBudget } from "./execution-budget";
 import { finishBenchmark } from "./run-benchmark";
+import { recoverStoppedRunner, runnerStopped } from "./recover-run";
 
 export async function retryEvalRun(
   directory: string,
@@ -13,13 +14,16 @@ export async function retryEvalRun(
   onEvent?: ExecutionObserver,
 ) {
   using results = new Results(resolve(directory, "runner.db"));
-  const benchmark = results.benchmark!;
-  if (benchmark.mergedInto)
+  if (results.benchmark!.mergedInto)
     throw new Error(
-      `This run was merged into ${benchmark.mergedInto}; use the aggregate run`,
+      `This run was merged into ${results.benchmark!.mergedInto}; use the aggregate run`,
     );
-  if (benchmark.state === "running")
-    throw new Error("Wait for the active benchmark run");
+  if (results.benchmark!.state === "running") {
+    if (!runnerStopped(results.benchmark!))
+      throw new Error("Wait for the active benchmark run");
+    recoverStoppedRunner(results);
+  }
+  const benchmark = results.benchmark!;
   const previous = results.evalRun(evalRunId);
   if (!previous || previous.state === "completed")
     throw new Error(

package/src/app/run-benchmark.ts

@@ -23,6 +23,7 @@ import { canJudgeEval } from "./eval-state";
 import { isScored } from "../judgment";
 import { benchmarkScores } from "./scores";
 import { CostBudget, estimateWork } from "./cost-plan";
+import { recoverStoppedRunner, runnerStopped } from "./recover-run";
 import {
   categoryKey,
   categoryList,
@@ -101,7 +102,8 @@ export function finishBenchmark(context: ExecutionContext) {
         (run) =>
           run.input.evalRunId === slot.evalRunId &&
           run.input.judgeHash === slot.judgeHash &&
-          ["failed", "timed_out"].includes(run.state),
+          ["failed", "timed_out"].includes(run.state) &&
+          !run.interrupted,
       ) &&
         !slot.judgeRunId),
   );
@@ -183,6 +185,7 @@ export async function runBenchmark(
     (!options.fresh
       ? await currentBenchmarkRun(definition.directory)
       : undefined);
+  let stopped = false;
   if (directory) {
     using previous = new Results(resolve(directory, "runner.db"), true);
     const saved = previous.benchmark!;
@@ -190,8 +193,9 @@ export async function runBenchmark(
       throw new Error(
         `This run was merged into ${saved.mergedInto}; use the aggregate run`,
       );
-    if (saved.state === "running")
+    if (saved.state === "running" && !runnerStopped(saved))
       throw new Error("This benchmark run is already running");
+    stopped = saved.state === "running";
     definition.models = [
       ...new Set([...saved.definition.models, ...definition.models]),
     ];
@@ -220,10 +224,16 @@ export async function runBenchmark(
       ? new Results(database, true)
       : new Results(":memory:")
     : new Results(database);
+  // A stopped runner leaves sessions open. A dry run plans them as interrupted work.
+  const recovered = stopped && !options.dryRun
+    ? recoverStoppedRunner(results)
+    : undefined;
   const previousSelections = new Map(
     results.slots().map((slot) => [slot.id, slot]),
   );
-  let plan = planBenchmark(definition, runtime, results);
+  let plan = planBenchmark(definition, runtime, results, {
+    stoppedRunner: stopped,
+  });
   if (selected || onlyEvals || options.onlyRepetitions) {
     const retained = new Map(results.slots().map((slot) => [slot.id, slot]));
     plan = plan.map((item) =>
@@ -260,7 +270,7 @@ export async function runBenchmark(
     fingerprint(previous.runtime) === fingerprint(runtime) &&
     fingerprint(previous.modelNames ?? {}) === fingerprint(modelNames)
   )
-    return { directory, plan, estimate, benchmark: previous };
+    return { directory, plan, estimate, benchmark: previous, recovered };
   const benchmark: BenchmarkRun = {
     id: previous?.id ?? `benchmark_${randomUUID()}`,
     name: definition.name,
@@ -397,7 +407,13 @@ export async function runBenchmark(
         deferred: plan.filter((item) => item.action === "deferred").length,
       },
     });
-    return { directory, plan, estimate, benchmark: finishBenchmark(context) };
+    return {
+      directory,
+      plan,
+      estimate,
+      benchmark: finishBenchmark(context),
+      recovered,
+    };
   } catch (error) {
     results.saveBenchmark({
       ...results.benchmark!,

package/src/app/scores.ts

@@ -14,6 +14,15 @@ export function benchmarkScores(
   const evals = definition.evals.filter(
     (item) => !evalId || item.id === evalId,
   );
+  // A code judgment counts only when it ran the eval's current judge.ts to completion.
+  const current = (item: (typeof evals)[number], slot: Slot) => {
+    const judge = slot.judgeRunId ? index.get(slot.judgeRunId) : undefined;
+    return !item.code ||
+      (judge?.code?.state === "completed" &&
+        judge.input.code?.hash === item.code.hash)
+      ? judge
+      : undefined;
+  };
   const keys = definition.categories?.map(categoryKey);
   const definitions = new Map(
     evals.map((item) => {
@@ -58,9 +67,7 @@ export function benchmarkScores(
           const values = selected
             .map(
               (slot) =>
-                (slot.judgeRunId
-                  ? index.get(slot.judgeRunId)?.judgment?.scores
-                  : undefined)?.[criterion.id]?.value,
+                current(item, slot)?.judgment?.scores?.[criterion.id]?.value,
             )
             .filter(isScored);
           const scoredSum = values.reduce<number>(
@@ -86,26 +93,19 @@ export function benchmarkScores(
         });
       }),
       evals.map((item) => item.id),
+      // Without a declared `criteria` export, judge.ts reveals its criteria only by running.
       evals
         .filter(
           (item) =>
             item.code &&
-            slots
-              .filter(
-                (slot) =>
-                  slot.active &&
-                  slot.evalId === item.id &&
-                  slot.model === model,
-              )
-              .some((slot) => {
-                const judge = slot.judgeRunId
-                  ? index.get(slot.judgeRunId)
-                  : undefined;
-                return (
-                  judge?.code?.state !== "completed" ||
-                  judge.input.code?.hash !== item.code!.hash
-                );
-              }),
+            !item.codeCriteria?.length &&
+            slots.some(
+              (slot) =>
+                slot.active &&
+                slot.evalId === item.id &&
+                slot.model === model &&
+                !current(item, slot),
+            ),
         )
         .map((item) => item.id),
     ),

package/src/app/serve-results.ts

@@ -236,11 +236,16 @@ export async function serveResults(options: {
           if (path !== assets && !path.startsWith(assets + sep))
             return new Response("Not found", { status: 404 });
           const file = Bun.file(path);
-          return new Response(
-            url.pathname !== "/" && (await file.exists())
-              ? file
-              : Bun.file(resolve(assets, "index.html")),
-          );
+          const found = url.pathname !== "/" && (await file.exists());
+          return new Response(found ? file : Bun.file(resolve(assets, "index.html")), {
+            // Built assets have content-hashed names; the page itself must revalidate.
+            headers: {
+              "Cache-Control":
+                found && url.pathname.startsWith("/assets/")
+                  ? "public, max-age=31536000, immutable"
+                  : "no-cache",
+            },
+          });
         }
         return new Response("Not found", { status: 404 });
       } catch (error) {

package/src/cli.ts

@@ -133,6 +133,9 @@ Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
             reason: item.reason,
           })),
           status: result.benchmark?.state,
+          ...("recovered" in result && result.recovered
+            ? { recovered: result.recovered }
+            : {}),
         },
         null,
         2,

package/src/types.ts

@@ -36,6 +36,8 @@ export type Judge = {
 };
 export type Benchmark = {
   name?: string;
+  /** Display label for one benchmark.ts among several that share `name`, such as "Frontier". */
+  suite?: string;
   models: readonly ModelRef[];
   judge?: Judge;
   repetitions?: number;
@@ -101,6 +103,7 @@ export type EvalDefinition = {
 };
 export type BenchmarkDefinition = {
   name: string;
+  suite?: string;
   directory: string;
   models: ModelRef[];
   judge: { model?: ModelRef; timeoutMs: number; websearch: "exa" | false; verification?: VerificationRuntime | Required<VerificationEnvironment> };
@@ -220,6 +223,8 @@ export type EvalRun = {
   error?: string;
   replaces?: string;
   stop?: EvalStop;
+  /** The runner stopped before this session finished; the planner collects it again. */
+  interrupted?: boolean;
 };
 export type CriterionDefinition = { id: string; name: string };
 export type EvidenceCitation = (
@@ -282,6 +287,8 @@ export type JudgeRun = {
   monitorError?: string;
   monitorErrorAt?: string;
   monitorLimit?: string;
+  /** The runner stopped before judging finished; the planner judges the evidence again. */
+  interrupted?: boolean;
 };
 export type Slot = {
   id: string;