@hona/openeval 0.5.2 → 0.5.4

package/JUDGING.md

@@ -47,11 +47,13 @@ cannot silently disappear from the benchmark denominator. Use consistent score
 IDs across models and repetitions; a missing required score stays unresolved.
 There are no built-in task-specific scorers, registration steps, or builder APIs.
 
-The host bundles local imports before execution, captures source maps and
-dependency manifests/lockfiles, and records their fingerprint. Code, imported
-references, or dependency changes schedule rejudging rather than candidate
-execution. Use static imports for reference data; make runtime network and file
-inputs reproducible when an author-owned judge uses them.
+The host bundles local imports before execution, captures source maps, and
+fingerprints the bundled code with the name and version of each imported
+package. Code, imported references, or package version changes schedule
+rejudging rather than candidate execution. The checkout location, working
+directory, install layout, lockfiles, and SDK version do not. Use static imports
+for reference data; make runtime network and file inputs reproducible when an
+author-owned judge uses them.
 
 The viewer shows normalized criterion scores, original returned JSON, frozen
 source, process logs, and recorded candidate metrics. A synchronous loop can be

package/VERIFICATION.md

@@ -8,7 +8,7 @@ run it in a disposable OCI container rather than on the runner host.
 import { VERIFICATION_IMAGE, type Benchmark } from "@hona/openeval";
 export default {
   models: ["example/model"],
-  judge: { verification: { image: VERIFICATION_IMAGE, cpus: 2, memoryMiB: 4096 } },
+  judge: { verification: { image: VERIFICATION_IMAGE, cpus: 2, memoryMiB: 4096, workspaceMiB: 2048 } },
 } satisfies Benchmark;
 ```
 
@@ -61,6 +61,13 @@ prefix does not necessarily establish final-task failure.
 - Containers self-expire. The parent also removes containers bearing only its
   unique execution label after worker interruption. No shared/broad cleanup.
 
+The workspace tmpfs defaults to 512 MiB. Set `judge.verification.workspaceMiB`
+for a larger restored project or prepared dependency tree. The value must be a
+positive integer no larger than `memoryMiB` (which defaults to 4096). It is
+retained in verification receipts and code-judge identity. Omitting it preserves
+existing identities and limits. Network, credentials, privileges, transfer caps,
+and the read-only image filesystem are unchanged.
+
 The standard image includes Bun 1.4.2, Python, Git, Node, and Playwright 1.63.0
 with Chromium. Import Playwright from
 `/opt/verify/node_modules/playwright/index.mjs`. Dependencies needed by a task

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.5.2",
+  "version": "0.5.4",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/input-fingerprints.ts

@@ -43,6 +43,14 @@ export function candidateProviders(
     : undefined;
 }
 
+/** The candidate session limit for one eval: its eval.ts override, else the benchmark value. */
+export const candidateTimeout = (
+  definition: BenchmarkDefinition,
+  evalId: string,
+) =>
+  definition.evals.find((item) => item.id === evalId)?.settings.candidate
+    ?.timeoutMs ?? definition.candidate.timeoutMs;
+
 /** Declared, candidate-visible inputs. Harness implementation hashes are provenance.
  * Early stopping is a judge-side policy; the planner re-collects sessions it cut short. */
 export function candidateFingerprint(
@@ -59,7 +67,7 @@ export function candidateFingerprint(
     model,
     image: runtime.imageId,
     candidate: {
-      timeoutMs: definition.candidate.timeoutMs,
+      timeoutMs: candidateTimeout(definition, evalId),
       websearch: definition.candidate.websearch,
       provider: scope && { ...scope.settings, model: scope.override },
     },

package/src/app/load-benchmark.ts

@@ -8,7 +8,7 @@ import type {
   EvalDefinition,
   ModelRef,
 } from "../types";
-import { CANDIDATE_TIMEOUT_MS } from "../types";
+import { CANDIDATE_TIMEOUT_MS, MAX_CANDIDATE_TIMEOUT_MS } from "../types";
 import { rubricCriteria } from "../judgment";
 import {
   categoryKey,
@@ -32,6 +32,12 @@ const positive = (value: unknown, fallback: number, label: string) => {
     throw new Error(`${label} must be a positive integer`);
   return result;
 };
+const timeLimit = (value: unknown, label: string) => {
+  const timeoutMs = positive(value, CANDIDATE_TIMEOUT_MS, label);
+  if (timeoutMs > MAX_CANDIDATE_TIMEOUT_MS)
+    throw new Error(`${label} cannot exceed 12 hours`);
+  return timeoutMs;
+};
 export const modelRef = (value: unknown): ModelRef => {
   if (
     typeof value !== "string" ||
@@ -66,10 +72,23 @@ async function loadEval(directory: string): Promise<EvalDefinition> {
     throw new Error(`${id}/eval.ts must export a declaration`);
   if (
     Object.keys(settings).some(
-      (key) => !["workspace", "prepare", "earlyStop"].includes(key),
+      (key) =>
+        !["workspace", "prepare", "earlyStop", "candidate"].includes(key),
     )
   )
     throw new Error(`${id}/eval.ts has unsupported settings`);
+  if (settings.candidate !== undefined) {
+    const candidate: unknown = settings.candidate;
+    if (
+      !candidate ||
+      typeof candidate !== "object" ||
+      Array.isArray(candidate) ||
+      Object.keys(candidate).some((key) => key !== "timeoutMs")
+    )
+      throw new Error(`${id}: candidate contains unsupported settings`);
+    if (settings.candidate.timeoutMs !== undefined)
+      timeLimit(settings.candidate.timeoutMs, `${id}: candidate timeout`);
+  }
   monitorPolicy(settings.earlyStop);
   if (hasCode && settings.earlyStop)
     throw new Error(
@@ -308,13 +327,10 @@ export async function loadBenchmark(
     );
   if (new Set(models).size !== models.length)
     throw new Error("Benchmark contains duplicate models");
-  const timeoutMs = positive(
+  const timeoutMs = timeLimit(
     definition.candidate?.timeoutMs,
-    CANDIDATE_TIMEOUT_MS,
     "Candidate timeout",
   );
-  if (timeoutMs > CANDIDATE_TIMEOUT_MS)
-    throw new Error("Candidate timeout cannot exceed 45 minutes");
   const evalRoot = resolve(directory, "evals");
   const directories = (await readdir(evalRoot, { withFileTypes: true }))
     .filter((entry) => entry.isDirectory() && !entry.name.startsWith("."))
@@ -348,10 +364,14 @@ export async function loadBenchmark(
   const verification = definition.judge?.verification;
   if (verification && (
     typeof verification !== "object" || Array.isArray(verification) ||
-    Object.keys(verification).some(key => !["image", "engine", "cpus", "memoryMiB"].includes(key)) ||
+    Object.keys(verification).some(key => !["image", "engine", "cpus", "memoryMiB", "workspaceMiB"].includes(key)) ||
     typeof verification.image !== "string" || !verification.image.trim() || /\s/.test(verification.image) ||
     !["docker", "podman"].includes(verification.engine ?? engine)
   )) throw new Error("judge.verification requires an image and valid container limits");
+  const verificationMemory = positive(verification?.memoryMiB, 4096, "Verification memory");
+  if (verification?.workspaceMiB !== undefined &&
+    (!Number.isSafeInteger(verification.workspaceMiB) || verification.workspaceMiB < 1 || verification.workspaceMiB > verificationMemory))
+    throw new Error("Verification workspaceMiB must be a positive integer within memoryMiB");
   for (const search of [
     definition.candidate?.websearch,
     definition.judge?.websearch,
@@ -387,7 +407,8 @@ export async function loadBenchmark(
         image: verification.image,
         engine: verification.engine ?? engine,
         cpus: positive(verification.cpus, 2, "Verification CPU count"),
-        memoryMiB: positive(verification.memoryMiB, 4096, "Verification memory"),
+        memoryMiB: verificationMemory,
+        ...(verification.workspaceMiB === undefined ? {} : { workspaceMiB: verification.workspaceMiB }),
       } } : {}),
     },
     container: {

package/src/app/read-results.ts

@@ -72,7 +72,22 @@ export const evalResultId = (benchmarkId: string, evalId: string) =>
 export class ResultReader {
   private paths = new Map<string, string>();
   private runtimes = new Map<string, RuntimeInterval[]>();
+  private finishedJudges = new Map<string, Map<string, JudgeRun>>();
   constructor(readonly root: string) {}
+  /** Every judge run; parses only running or unseen records, since finalized runs never change. */
+  private judgeRunsOf(results: Results) {
+    const cache =
+      this.finishedJudges.get(results.path) ?? new Map<string, JudgeRun>();
+    this.finishedJudges.set(results.path, cache);
+    return results.judgeRunStates().flatMap(({ id }) => {
+      const cached = cache.get(id);
+      if (cached) return [cached];
+      const run = results.judgeRun(id);
+      if (!run) return [];
+      if (run.state !== "running") cache.set(id, run);
+      return [run];
+    });
+  }
   private runtime(
     run: EvalRun | JudgeRun,
     results: Results,
@@ -93,7 +108,7 @@ export class ResultReader {
         scheduled(slot, benchmark, now);
       if (current && "evalRunId" in run.input)
         current =
-          (judges ?? results.judgeRuns())
+          (judges ?? this.judgeRunsOf(results))
             .filter(
               (judge) =>
                 judge.input.evalRunId === candidate!.id &&
@@ -272,25 +287,17 @@ export class ResultReader {
       (item) => !evalId || item.id === evalId,
     );
     if (!evals.length) throw new Error("Unknown eval");
-    const scores = results.readSnapshot(() =>
-      benchmarkScores(
-        run.definition,
-        results.slots(),
-        results.judgeRuns(),
-        evalId,
-      ),
+    const [slots, allJudges] = results.readSnapshot(
+      () => [results.slots(), this.judgeRunsOf(results)] as const,
     );
+    const scores = benchmarkScores(run.definition, slots, allJudges, evalId);
     // Runtime and cost describe the work behind the active selections, not retired history.
     const active = new Set(
-      results
-        .slots()
-        .filter((slot) => slot.active)
-        .map((slot) => slot.id),
+      slots.filter((slot) => slot.active).map((slot) => slot.id),
     );
     const candidates = results
-        .evalRuns()
-        .filter((candidate) => active.has(candidate.slotId)),
-      allJudges = results.judgeRuns();
+      .evalRuns()
+      .filter((candidate) => active.has(candidate.slotId));
     const now = Date.now(),
       candidateIndex = new Map(candidates.map((run) => [run.id, run]));
     const evalRuntime = new Map<string, RuntimeInterval[]>();
@@ -380,7 +387,7 @@ export class ResultReader {
   private liveRuns(results: Results, evalIds: readonly string[]) {
     const benchmark = results.benchmark!,
       executions = results.evalRuns(),
-      judges = results.judgeRuns(),
+      judges = this.judgeRunsOf(results),
       now = Date.now();
     const byId = new Map(executions.map((run) => [run.id, run]));
     const bySlot = Map.groupBy(executions, (run) => run.slotId);

package/src/app/run-eval.ts

@@ -5,7 +5,7 @@ import { executeCandidate } from "../infra/containers/candidate";
 import type { EvidenceFeed } from "../evidence";
 import { CandidateEvidence } from "../infra/evidence";
 import { measureRecording } from "../infra/recording/metrics";
-import { candidateProviders } from "./input-fingerprints";
+import { candidateProviders, candidateTimeout } from "./input-fingerprints";
 
 /** The normal candidate use case: record its inputs, execute, finalize its evidence. */
 export async function runEval(
@@ -20,6 +20,7 @@ export async function runEval(
   const definition = context.definition.evals.find(
     (item) => item.id === slot.evalId,
   )!;
+  const timeoutMs = candidateTimeout(context.definition, slot.evalId);
   const run = context.results.startEval(slot, {
     evalId: slot.evalId,
     model: slot.model,
@@ -28,7 +29,7 @@ export async function runEval(
     candidateHash: slot.candidateHash,
     sourceHash: definition.sourceHash,
     imageId: context.runtime.imageId,
-    timeoutMs: context.definition.candidate.timeoutMs,
+    timeoutMs,
     earlyStop: !!controls.onEvidence,
     runtime: context.runtime,
   });
@@ -39,7 +40,7 @@ export async function runEval(
       prompt: definition.prompt,
       workspace,
       prepare: definition.settings.prepare ?? [],
-      timeoutMs: context.definition.candidate.timeoutMs,
+      timeoutMs,
       websearch: context.definition.candidate.websearch,
       container: context.definition.container,
       imageId: context.runtime.imageId,

package/src/app/serve-results.ts

@@ -16,6 +16,16 @@ export async function serveResults(options: {
   await mkdir(root, { recursive: true });
   const reader = new ResultReader(root),
     encoder = new TextEncoder();
+  // Open tabs refresh on every change. Share one recent read so polling cannot saturate the server.
+  const recent = new Map<string, { at: number; value: Promise<unknown> }>();
+  const shared = <T>(key: string, read: () => Promise<T>): Promise<T> => {
+    const hit = recent.get(key);
+    if (hit && Date.now() - hit.at < 2000) return hit.value as Promise<T>;
+    const value = read();
+    recent.set(key, { at: Date.now(), value });
+    value.catch(() => recent.delete(key));
+    return value;
+  };
   const clients = new Set<ReadableStreamDefaultController<Uint8Array>>(),
     streams = new Set<() => void>();
   let changed: ReturnType<typeof setTimeout> | undefined;
@@ -44,7 +54,7 @@ export async function serveResults(options: {
         if (url.pathname === "/api/results")
           return Response.json(await reader.index());
         if (url.pathname === "/api/activity")
-          return Response.json(await reader.activity());
+          return Response.json(await shared("activity", () => reader.activity()));
         if (url.pathname === "/api/judge-checks")
           return Response.json(
             await reader.judgeAudit(
@@ -93,12 +103,14 @@ export async function serveResults(options: {
             ),
           );
         }
-        if (url.pathname === "/api/result")
+        if (url.pathname === "/api/result") {
+          const id = url.searchParams.get("id") ?? "";
           return Response.json(
             url.searchParams.get("format") === "runs"
-              ? await reader.evalRuns(url.searchParams.get("id") ?? "")
-              : await reader.summary(url.searchParams.get("id") ?? ""),
+              ? await shared(`runs:${id}`, () => reader.evalRuns(id))
+              : await shared(`summary:${id}`, () => reader.summary(id)),
           );
+        }
         if (url.pathname === "/api/events") {
           let owner: ReadableStreamDefaultController<Uint8Array>;
           return new Response(

package/src/index.ts

@@ -1,5 +1,6 @@
 export type {
   Benchmark,
+  CandidateLimits,
   Eval,
   ModelRef,
   ProviderDefinitions,
@@ -30,7 +31,7 @@ export type {
   EvidenceQuery,
   JudgeSession,
 } from "./evidence";
-export { CANDIDATE_TIMEOUT_MS } from "./types";
+export { CANDIDATE_TIMEOUT_MS, MAX_CANDIDATE_TIMEOUT_MS } from "./types";
 export { rubricCriteria, criterionMean, isScored } from "./judgment";
 export {
   categoryKey,

package/src/infra/judging/code-source.ts

@@ -1,18 +1,34 @@
-import { dirname, isAbsolute, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
+import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
 import type { CodeJudgeDefinition } from "../../judge-context";
 import { fingerprint } from "../files";
 
+/** Raise when a release changes what judge.ts receives or how it runs; every code judge then runs again. */
+const CODE_JUDGE_PROTOCOL = 1;
+const printer = new Bun.Transpiler({
+  loader: "js",
+  target: "bun",
+  deadCodeElimination: false,
+});
+
+/** The package that provides an external import, as `name@version/path`. */
+async function packageSpecifier(path: string, from: string) {
+  for (let directory = dirname(path); ; directory = dirname(directory)) {
+    const manifest = Bun.file(resolve(directory, "package.json"));
+    const { name, version } = (await manifest.exists())
+      ? await manifest.json().catch(() => ({}))
+      : {};
+    if (typeof name === "string" && name)
+      return `${name}@${version ?? ""}/${relative(directory, path).split(sep).join("/")}`;
+    if (dirname(directory) === directory)
+      return relative(from, path).split(sep).join("/");
+  }
+}
+
 /** Bundle local imports without executing author code during planning. */
 export async function compileCodeJudge(
   file: string,
 ): Promise<CodeJudgeDefinition> {
   file = resolve(file);
-  const dependencies: Record<string, string> = {};
-  const sdkManifest = fileURLToPath(
-    new URL("../../../package.json", import.meta.url),
-  );
-  dependencies[sdkManifest] = await Bun.file(sdkManifest).text();
   const external = new Set<string>();
   const built = await Bun.build({
     // Import only the judge function: unused reporting metadata is tree-shaken.
@@ -65,40 +81,32 @@ export async function compileCodeJudge(
     (await built.outputs
       .find((output) => output.kind === "sourcemap")
       ?.text()) ?? "";
-  const collect = async (directory: string, stopAtPackage: boolean) => {
-    for (;;) {
-      let found = false;
-      for (const name of [
-        "package.json",
-        "bun.lock",
-        "bun.lockb",
-        "package-lock.json",
-        "pnpm-lock.yaml",
-        "yarn.lock",
-      ]) {
-        const path = resolve(directory, name);
-        if (await Bun.file(path).exists()) {
-          dependencies[path] =
-            name === "bun.lockb"
-              ? Buffer.from(await Bun.file(path).bytes()).toString("base64")
-              : await Bun.file(path).text();
-          found ||= name === "package.json";
-        }
-      }
-      if (found && stopAtPackage) return;
-      const parent = dirname(directory);
-      if (parent === directory) return;
-      directory = parent;
-    }
-  };
-  await collect(dirname(file), false);
-  for (const path of external) await collect(dirname(path), true);
+  // The executable keeps absolute import paths; identity uses the package name and version.
+  const dependencies: Record<string, string> = {};
+  let portable = source;
+  for (const path of external) {
+    const specifier = await packageSpecifier(path, dirname(file));
+    dependencies[specifier] = path;
+    portable = portable.replaceAll(
+      JSON.stringify(path),
+      JSON.stringify(specifier),
+    );
+  }
+  // Bun writes __dirname and __filename as absolute paths. Like other runtime file inputs, they are not identity.
+  portable = portable.replace(
+    /^\s*var __(?:dirname|filename) = .*$/gm,
+    (line) => line.replace(/"(?:[^"\\]|\\.)*"/g, '""'),
+  );
   return {
     file,
     source,
     sourceMap,
     dependencies,
-    // Debug mappings and original-source metadata are retained, but not executable inputs.
-    hash: fingerprint({ source, dependencies }),
+    // Reprinting drops comments: module comments name files relative to the working
+    // directory, and the debug ID depends on them. Source maps are retained, not hashed.
+    hash: fingerprint({
+      protocol: CODE_JUDGE_PROTOCOL,
+      source: printer.transformSync(portable),
+    }),
   };
 }

package/src/infra/sqlite/index.ts

@@ -166,6 +166,13 @@ export class Results {
       .all()
       .map((row) => row.value);
   }
+  /** IDs and states in judgeRuns() order. Triggers keep finalized runs immutable, so readers can cache them. */
+  judgeRunStates() {
+    return this.sqlite.query("SELECT id, state FROM judge_runs").all() as Array<{
+      id: string;
+      state: JudgeRun["state"];
+    }>;
+  }
   evalRun(id: string) {
     return this.db
       .select()

package/src/infra/verification/image.ts

@@ -21,6 +21,10 @@ export async function verificationRuntime(environment?: VerificationEnvironment)
   if (!["docker", "podman"].includes(engine) || !environment.image || /\s/.test(environment.image) ||
     [environment.cpus ?? 2, environment.memoryMiB ?? 4096].some(value => !Number.isSafeInteger(value) || value < 1))
     throw new Error("Verification requires a valid image, engine, and positive integer resource limits");
+  if (environment.workspaceMiB !== undefined &&
+    (!Number.isSafeInteger(environment.workspaceMiB) || environment.workspaceMiB < 1 ||
+      environment.workspaceMiB > (environment.memoryMiB ?? 4096)))
+    throw new Error("Verification workspaceMiB must be a positive integer within memoryMiB");
   const output = await engineCommand(engine, [
     "image", "inspect", environment.image, "--format",
     '{{.Id}}|{{index .Config.Labels "openeval.verification"}}',
@@ -29,5 +33,9 @@ export async function verificationRuntime(environment?: VerificationEnvironment)
   if (!/^sha256:[a-f0-9]{64}$/.test(imageId)) throw new Error("Verification image has no immutable image ID");
   if (environment.image === VERIFICATION_IMAGE && label !== await treeHash(directory))
     throw new Error("Verification runtime changed. Run image --verification first.");
-  return { engine, image: environment.image, imageId, cpus: environment.cpus ?? 2, memoryMiB: environment.memoryMiB ?? 4096 };
+  return {
+    engine, image: environment.image, imageId,
+    cpus: environment.cpus ?? 2, memoryMiB: environment.memoryMiB ?? 4096,
+    ...(environment.workspaceMiB === undefined ? {} : { workspaceMiB: environment.workspaceMiB }),
+  };
 }

package/src/infra/verification/session.ts

@@ -26,7 +26,7 @@ export function verificationArgs(runtime: VerificationRuntime, owner: string, na
     "--network=none", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--read-only",
     "--pids-limit", "128", "--cpus", String(runtime.cpus), "--memory", `${runtime.memoryMiB}m`,
     "--memory-swap", `${runtime.memoryMiB}m`, "--user", "10001:10001",
-    "--tmpfs", "/workspace:rw,exec,nosuid,nodev,mode=1777,size=512m",
+    "--tmpfs", `/workspace:rw,exec,nosuid,nodev,mode=1777,size=${runtime.workspaceMiB ?? 512}m`,
     "--tmpfs", "/verification:rw,exec,nosuid,nodev,mode=0755,size=64m",
     "--tmpfs", "/tmp:rw,exec,nosuid,nodev,mode=1777,size=256m",
     "--env", "HOME=/tmp/home", "--env", "CI=1",

package/src/judge-context.ts

@@ -39,6 +39,8 @@ export type VerificationEnvironment = {
   image: string;
   cpus?: number;
   memoryMiB?: number;
+  /** Workspace tmpfs capacity in MiB; default 512. A custom value must fit within memoryMiB. */
+  workspaceMiB?: number;
 };
 export type VerificationRuntime = {
   engine: Engine;
@@ -46,6 +48,7 @@ export type VerificationRuntime = {
   imageId: string;
   cpus: number;
   memoryMiB: number;
+  workspaceMiB?: number;
 };
 export type VerificationRequest = {
   revision?: Revision;
@@ -185,9 +188,11 @@ export interface JudgeContext {
 /** Frozen judging source and dependency identity, kept in host-side run records. */
 export type CodeJudgeDefinition = {
   file: string;
+  /** Executable identity: bundled code and imported package versions, without host paths or comments. */
   hash: string;
   source: string;
   sourceMap: string;
+  /** Packages imported at run time, as `name@version/path`, with the file each resolved to. */
   dependencies: Record<string, string>;
 };
 export type CodeJudgeExecution = {

package/src/types.ts

@@ -15,6 +15,7 @@ import type {
 export type OpenCodeStreamEvent = OpenCodeEvent | SessionLogItem;
 
 export const CANDIDATE_TIMEOUT_MS = 45 * 60 * 1000;
+export const MAX_CANDIDATE_TIMEOUT_MS = 12 * 60 * 60 * 1000;
 export type ModelRef = `${string}/${string}`;
 export type Engine = "docker" | "podman";
 export type ProviderDefinitions = NonNullable<
@@ -34,6 +35,11 @@ export type Judge = {
   websearch?: "exa" | false;
   verification?: VerificationEnvironment;
 };
+/** Execution controls for one candidate attempt. Eval values override benchmark values. */
+export type CandidateLimits = {
+  /** Agent session limit, after preparation; default 45 minutes, at most 12 hours. */
+  timeoutMs?: number;
+};
 export type Benchmark = {
   name?: string;
   /** Display label for one benchmark.ts among several that share `name`, such as "Frontier". */
@@ -42,8 +48,7 @@ export type Benchmark = {
   judge?: Judge;
   repetitions?: number;
   concurrency?: number;
-  candidate?: {
-    timeoutMs?: number;
+  candidate?: CandidateLimits & {
     websearch?: "exa" | false;
     providers?: ProviderDefinitions;
   };
@@ -82,6 +87,8 @@ export type Eval = {
     overlay?: string;
   };
   prepare?: readonly PreparationStep[];
+  /** Overrides benchmark.candidate for this eval. */
+  candidate?: CandidateLimits;
 };
 export type EvalDefinition = {
   id: string;
@@ -106,7 +113,7 @@ export type BenchmarkDefinition = {
   suite?: string;
   directory: string;
   models: ModelRef[];
-  judge: { model?: ModelRef; timeoutMs: number; websearch: "exa" | false; verification?: VerificationRuntime | Required<VerificationEnvironment> };
+  judge: { model?: ModelRef; timeoutMs: number; websearch: "exa" | false; verification?: VerificationRuntime | Omit<VerificationRuntime, "imageId"> };
   repetitions: number;
   concurrency: number;
   candidate: {

package/src/view.ts

@@ -171,13 +171,33 @@ function componentBounds(
     coverage: coverage * scale,
   };
 }
-export const sumCosts = (costs: readonly (Cost | undefined)[]): Cost => {
+/** A cost summed over executions. `unaccounted` counts executions without a final cost. */
+export type CostTotal = Cost & { unaccounted: number };
+/** Sum execution costs or totals; a single execution without a final cost counts once. */
+export const sumCosts = (
+  costs: readonly (Cost | CostTotal | undefined)[],
+): CostTotal => {
   const reportedUSD = costs.reduce(
     (sum, cost) => sum + (cost?.reportedUSD ?? 0),
     0,
   );
   const complete = costs.every((cost) => cost?.complete === true);
-  return { reportedUSD, complete, usd: complete ? reportedUSD : null };
+  const unaccounted = costs.reduce(
+    (sum, cost) =>
+      sum +
+      (cost && "unaccounted" in cost
+        ? cost.unaccounted
+        : cost?.complete
+          ? 0
+          : 1),
+    0,
+  );
+  return {
+    reportedUSD,
+    complete,
+    usd: complete ? reportedUSD : null,
+    unaccounted,
+  };
 };
 export type ResultEntry = {
   id: string;
@@ -209,8 +229,8 @@ export type Overview = {
   evalNames: Record<string, string>;
   modelNames: ModelNames;
   categories: CategorySummary[];
-  cost: Cost;
-  evalCosts: Record<string, Cost>;
+  cost: CostTotal;
+  evalCosts: Record<string, CostTotal>;
   runtime: RuntimeInterval[];
 };
 export type ResultSummary = {
@@ -305,7 +325,10 @@ export type JudgeAudit = {
     Pick<JudgeRun, "id" | "state" | "startedAt"> & { mode: "monitor" | "final" }
   >;
 };
-export type EvalRunSummary = LiveEvalRun & { cost: Cost; elapsedMs: number };
+export type EvalRunSummary = LiveEvalRun & {
+  cost: CostTotal;
+  elapsedMs: number;
+};
 export type EvalRunIndex = {
   entry: ResultEntry;
   eval: string;
@@ -316,7 +339,7 @@ export type ActivityRun = {
   id: string;
   benchmarkId: string;
   resultId: string;
-  cost: Cost;
+  cost: CostTotal;
   state: {
     id: string;
     eval: string;

package/viewer/assets/{abnfDiagram-VCTEODGH-DheibVLk.js → abnfDiagram-VCTEODGH-ug3iRKuR.js}

@@ -1 +1 @@
-import{m as e,t}from"./mermaid-parser.core-CiGT3tRM.js";import{n}from"./chunk-Y2CYZVJY-DsF7k-Jl.js";import{m as r}from"./src-BnSwVg9U.js";import"./chunk-DU6HZSFF-Cty-qidK.js";import{n as i,r as a,t as o}from"./chunk-SVP7TREG-BEMJeRHW.js";import{t as s}from"./chunk-JWPE2WC7-DVXcaiue.js";import"./mermaid.core-BfFqBx2V.js";var c=e().RailroadAbnf.parser.LangiumParser,l=n(e=>{let t=e.alternatives.map(u);return t.length===1?t[0]:{type:`choice`,alternatives:t}},`transformAlternation`),u=n(e=>{let t=e.elements.map(f);return t.length===1?t[0]:{type:`sequence`,elements:t}},`transformConcatenation`),d=n(e=>{if(e.includes(`*`)){let[t,n]=e.split(`*`);return{min:t?parseInt(t,10):0,max:n?parseInt(n,10):1/0}}let t=parseInt(e,10);return{min:t,max:t}},`parseRepeat`),f=n(e=>{let t=p(e.primary);if(!e.repeat)return t;let{min:n,max:r}=d(e.repeat);return n===0&&r===1?{type:`optional`,element:t}:{type:`repetition`,element:t,min:n,max:r}},`transformElement`),p=n(e=>{switch(e.$type){case`AbnfStringLiteral`:return{type:`terminal`,value:e.value};case`AbnfNumVal`:return{type:`terminal`,value:e.value};case`AbnfRuleName`:return{type:`nonterminal`,name:e.name};case`AbnfGroup`:return l(e.element);case`AbnfOptionalGroup`:return{type:`optional`,element:l(e.element)};default:throw Error(`Unsupported ABNF primary node: ${e.$type}`)}},`transformPrimary`),m=n(e=>({name:e.name,definition:l(e.definition)}),`transformRule`),h=n(e=>{s(e,o),e.title&&o.setTitle(e.title),e.rules.map(e=>o.addRule(m(e)))},`populateDb`),g={parser:{parse:n(e=>{o.clear(),r.debug(`[ABNF Parser] Starting Langium parse`);let n=c.parse(e);if(n.lexerErrors.length>0||n.parserErrors.length>0)throw new t(n);let i=n.value;r.debug(`[ABNF Parser] Parsed rules:`,i.rules.length),h(i),r.debug(`[ABNF Parser] Parse complete`)},`parse`),parser:{yy:o}},db:o,renderer:a,styles:i};export{g as diagram};
+import{m as e,t}from"./mermaid-parser.core-VTfqiyyK.js";import{n}from"./chunk-Y2CYZVJY-DsF7k-Jl.js";import{m as r}from"./src-C171e0nu.js";import"./chunk-DU6HZSFF-BI4SVpji.js";import{n as i,r as a,t as o}from"./chunk-SVP7TREG-CGuEYo-A.js";import{t as s}from"./chunk-JWPE2WC7-DVXcaiue.js";import"./mermaid.core-eoOgKmWC.js";var c=e().RailroadAbnf.parser.LangiumParser,l=n(e=>{let t=e.alternatives.map(u);return t.length===1?t[0]:{type:`choice`,alternatives:t}},`transformAlternation`),u=n(e=>{let t=e.elements.map(f);return t.length===1?t[0]:{type:`sequence`,elements:t}},`transformConcatenation`),d=n(e=>{if(e.includes(`*`)){let[t,n]=e.split(`*`);return{min:t?parseInt(t,10):0,max:n?parseInt(n,10):1/0}}let t=parseInt(e,10);return{min:t,max:t}},`parseRepeat`),f=n(e=>{let t=p(e.primary);if(!e.repeat)return t;let{min:n,max:r}=d(e.repeat);return n===0&&r===1?{type:`optional`,element:t}:{type:`repetition`,element:t,min:n,max:r}},`transformElement`),p=n(e=>{switch(e.$type){case`AbnfStringLiteral`:return{type:`terminal`,value:e.value};case`AbnfNumVal`:return{type:`terminal`,value:e.value};case`AbnfRuleName`:return{type:`nonterminal`,name:e.name};case`AbnfGroup`:return l(e.element);case`AbnfOptionalGroup`:return{type:`optional`,element:l(e.element)};default:throw Error(`Unsupported ABNF primary node: ${e.$type}`)}},`transformPrimary`),m=n(e=>({name:e.name,definition:l(e.definition)}),`transformRule`),h=n(e=>{s(e,o),e.title&&o.setTitle(e.title),e.rules.map(e=>o.addRule(m(e)))},`populateDb`),g={parser:{parse:n(e=>{o.clear(),r.debug(`[ABNF Parser] Starting Langium parse`);let n=c.parse(e);if(n.lexerErrors.length>0||n.parserErrors.length>0)throw new t(n);let i=n.value;r.debug(`[ABNF Parser] Parsed rules:`,i.rules.length),h(i),r.debug(`[ABNF Parser] Parse complete`)},`parse`),parser:{yy:o}},db:o,renderer:a,styles:i};export{g as diagram};