@hona/openeval 0.5.3 → 0.5.5

package/JUDGING.md

@@ -47,11 +47,13 @@ cannot silently disappear from the benchmark denominator. Use consistent score
 IDs across models and repetitions; a missing required score stays unresolved.
 There are no built-in task-specific scorers, registration steps, or builder APIs.
 
-The host bundles local imports before execution, captures source maps and
-dependency manifests/lockfiles, and records their fingerprint. Code, imported
-references, or dependency changes schedule rejudging rather than candidate
-execution. Use static imports for reference data; make runtime network and file
-inputs reproducible when an author-owned judge uses them.
+The host bundles local imports before execution, captures source maps, and
+fingerprints the bundled code with the name and version of each imported
+package. Code, imported references, or package version changes schedule
+rejudging rather than candidate execution. The checkout location, working
+directory, install layout, lockfiles, and SDK version do not. Use static imports
+for reference data; make runtime network and file inputs reproducible when an
+author-owned judge uses them.
 
 The viewer shows normalized criterion scores, original returned JSON, frozen
 source, process logs, and recorded candidate metrics. A synchronous loop can be

package/VERIFICATION.md

@@ -8,7 +8,7 @@ run it in a disposable OCI container rather than on the runner host.
 import { VERIFICATION_IMAGE, type Benchmark } from "@hona/openeval";
 export default {
   models: ["example/model"],
-  judge: { verification: { image: VERIFICATION_IMAGE, cpus: 2, memoryMiB: 4096 } },
+  judge: { verification: { image: VERIFICATION_IMAGE, cpus: 2, memoryMiB: 4096, workspaceMiB: 2048, inputMiB: 256 } },
 } satisfies Benchmark;
 ```
 
@@ -56,11 +56,27 @@ prefix does not necessarily establish final-task failure.
 - Commands, exit statuses, image/resource identity, logs, and requested regular
   output files are retained with the JudgeRun. Viewer previews never execute HTML
   or SVG from the delivered artifact.
-- Workspace transfer is limited to 128 MiB; each output archive is limited to
+- Workspace transfer defaults to 128 MiB; each output archive is limited to
   32 MiB; logs are limited to 1 MiB per stream and are marked when truncated.
 - Containers self-expire. The parent also removes containers bearing only its
   unique execution label after worker interruption. No shared/broad cleanup.
 
+The workspace tmpfs defaults to 512 MiB. Set `judge.verification.workspaceMiB`
+for a larger restored project or prepared dependency tree. The value must be a
+positive integer no larger than `memoryMiB` (which defaults to 4096). It is
+retained in verification receipts and code-judge identity. Omitting it preserves
+existing identities and limits. Network, credentials, privileges, transfer caps,
+and the read-only image filesystem are unchanged.
+
+For larger source snapshots, set `judge.verification.inputMiB` to a positive
+integer archive limit. This is separate from `workspaceMiB`: increasing storage
+alone does not increase the transfer limit. The configured byte count must be a
+safe integer. Non-default limits are retained in receipts and code-judge identity;
+omitting the field or setting it to 128 preserves existing identities. Output and
+log caps, filesystem permissions, network access, and container isolation stay
+unchanged. Provision enough workspace storage for the actual restored files and
+any verification-generated data.
+
 The standard image includes Bun 1.4.2, Python, Git, Node, and Playwright 1.63.0
 with Chromium. Import Playwright from
 `/opt/verify/node_modules/playwright/index.mjs`. Dependencies needed by a task

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.5.3",
+  "version": "0.5.5",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/input-fingerprints.ts

@@ -43,6 +43,14 @@ export function candidateProviders(
     : undefined;
 }
 
+/** The candidate session limit for one eval: its eval.ts override, else the benchmark value. */
+export const candidateTimeout = (
+  definition: BenchmarkDefinition,
+  evalId: string,
+) =>
+  definition.evals.find((item) => item.id === evalId)?.settings.candidate
+    ?.timeoutMs ?? definition.candidate.timeoutMs;
+
 /** Declared, candidate-visible inputs. Harness implementation hashes are provenance.
  * Early stopping is a judge-side policy; the planner re-collects sessions it cut short. */
 export function candidateFingerprint(
@@ -59,7 +67,7 @@ export function candidateFingerprint(
     model,
     image: runtime.imageId,
     candidate: {
-      timeoutMs: definition.candidate.timeoutMs,
+      timeoutMs: candidateTimeout(definition, evalId),
       websearch: definition.candidate.websearch,
       provider: scope && { ...scope.settings, model: scope.override },
     },

package/src/app/load-benchmark.ts

@@ -8,7 +8,7 @@ import type {
   EvalDefinition,
   ModelRef,
 } from "../types";
-import { CANDIDATE_TIMEOUT_MS } from "../types";
+import { CANDIDATE_TIMEOUT_MS, MAX_CANDIDATE_TIMEOUT_MS } from "../types";
 import { rubricCriteria } from "../judgment";
 import {
   categoryKey,
@@ -18,6 +18,7 @@ import {
 import { compileCodeJudge } from "../infra/judging/code-source";
 import { readCodeCriteria } from "../infra/judging/code-criteria";
 import { RUNTIME_IMAGE } from "../infra/opencode/version";
+import { DEFAULT_INPUT_MIB, verificationInputMiB } from "../infra/verification/limits";
 import { monitorPolicy } from "./monitor-policy";
 import {
   fingerprint,
@@ -32,6 +33,12 @@ const positive = (value: unknown, fallback: number, label: string) => {
     throw new Error(`${label} must be a positive integer`);
   return result;
 };
+const timeLimit = (value: unknown, label: string) => {
+  const timeoutMs = positive(value, CANDIDATE_TIMEOUT_MS, label);
+  if (timeoutMs > MAX_CANDIDATE_TIMEOUT_MS)
+    throw new Error(`${label} cannot exceed 12 hours`);
+  return timeoutMs;
+};
 export const modelRef = (value: unknown): ModelRef => {
   if (
     typeof value !== "string" ||
@@ -66,10 +73,23 @@ async function loadEval(directory: string): Promise<EvalDefinition> {
     throw new Error(`${id}/eval.ts must export a declaration`);
   if (
     Object.keys(settings).some(
-      (key) => !["workspace", "prepare", "earlyStop"].includes(key),
+      (key) =>
+        !["workspace", "prepare", "earlyStop", "candidate"].includes(key),
     )
   )
     throw new Error(`${id}/eval.ts has unsupported settings`);
+  if (settings.candidate !== undefined) {
+    const candidate: unknown = settings.candidate;
+    if (
+      !candidate ||
+      typeof candidate !== "object" ||
+      Array.isArray(candidate) ||
+      Object.keys(candidate).some((key) => key !== "timeoutMs")
+    )
+      throw new Error(`${id}: candidate contains unsupported settings`);
+    if (settings.candidate.timeoutMs !== undefined)
+      timeLimit(settings.candidate.timeoutMs, `${id}: candidate timeout`);
+  }
   monitorPolicy(settings.earlyStop);
   if (hasCode && settings.earlyStop)
     throw new Error(
@@ -308,13 +328,10 @@ export async function loadBenchmark(
     );
   if (new Set(models).size !== models.length)
     throw new Error("Benchmark contains duplicate models");
-  const timeoutMs = positive(
+  const timeoutMs = timeLimit(
     definition.candidate?.timeoutMs,
-    CANDIDATE_TIMEOUT_MS,
     "Candidate timeout",
   );
-  if (timeoutMs > CANDIDATE_TIMEOUT_MS)
-    throw new Error("Candidate timeout cannot exceed 45 minutes");
   const evalRoot = resolve(directory, "evals");
   const directories = (await readdir(evalRoot, { withFileTypes: true }))
     .filter((entry) => entry.isDirectory() && !entry.name.startsWith("."))
@@ -348,10 +365,15 @@ export async function loadBenchmark(
   const verification = definition.judge?.verification;
   if (verification && (
     typeof verification !== "object" || Array.isArray(verification) ||
-    Object.keys(verification).some(key => !["image", "engine", "cpus", "memoryMiB"].includes(key)) ||
+    Object.keys(verification).some(key => !["image", "engine", "cpus", "memoryMiB", "workspaceMiB", "inputMiB"].includes(key)) ||
     typeof verification.image !== "string" || !verification.image.trim() || /\s/.test(verification.image) ||
     !["docker", "podman"].includes(verification.engine ?? engine)
   )) throw new Error("judge.verification requires an image and valid container limits");
+  const verificationMemory = positive(verification?.memoryMiB, 4096, "Verification memory");
+  if (verification?.workspaceMiB !== undefined &&
+    (!Number.isSafeInteger(verification.workspaceMiB) || verification.workspaceMiB < 1 || verification.workspaceMiB > verificationMemory))
+    throw new Error("Verification workspaceMiB must be a positive integer within memoryMiB");
+  const inputMiB = verificationInputMiB(verification?.inputMiB);
   for (const search of [
     definition.candidate?.websearch,
     definition.judge?.websearch,
@@ -387,7 +409,9 @@ export async function loadBenchmark(
         image: verification.image,
         engine: verification.engine ?? engine,
         cpus: positive(verification.cpus, 2, "Verification CPU count"),
-        memoryMiB: positive(verification.memoryMiB, 4096, "Verification memory"),
+        memoryMiB: verificationMemory,
+        ...(verification.workspaceMiB === undefined ? {} : { workspaceMiB: verification.workspaceMiB }),
+        ...(inputMiB === DEFAULT_INPUT_MIB ? {} : { inputMiB }),
       } } : {}),
     },
     container: {

package/src/app/run-eval.ts

@@ -5,7 +5,7 @@ import { executeCandidate } from "../infra/containers/candidate";
 import type { EvidenceFeed } from "../evidence";
 import { CandidateEvidence } from "../infra/evidence";
 import { measureRecording } from "../infra/recording/metrics";
-import { candidateProviders } from "./input-fingerprints";
+import { candidateProviders, candidateTimeout } from "./input-fingerprints";
 
 /** The normal candidate use case: record its inputs, execute, finalize its evidence. */
 export async function runEval(
@@ -20,6 +20,7 @@ export async function runEval(
   const definition = context.definition.evals.find(
     (item) => item.id === slot.evalId,
   )!;
+  const timeoutMs = candidateTimeout(context.definition, slot.evalId);
   const run = context.results.startEval(slot, {
     evalId: slot.evalId,
     model: slot.model,
@@ -28,7 +29,7 @@ export async function runEval(
     candidateHash: slot.candidateHash,
     sourceHash: definition.sourceHash,
     imageId: context.runtime.imageId,
-    timeoutMs: context.definition.candidate.timeoutMs,
+    timeoutMs,
     earlyStop: !!controls.onEvidence,
     runtime: context.runtime,
   });
@@ -39,7 +40,7 @@ export async function runEval(
       prompt: definition.prompt,
       workspace,
       prepare: definition.settings.prepare ?? [],
-      timeoutMs: context.definition.candidate.timeoutMs,
+      timeoutMs,
       websearch: context.definition.candidate.websearch,
       container: context.definition.container,
       imageId: context.runtime.imageId,

package/src/index.ts

@@ -1,5 +1,6 @@
 export type {
   Benchmark,
+  CandidateLimits,
   Eval,
   ModelRef,
   ProviderDefinitions,
@@ -30,7 +31,7 @@ export type {
   EvidenceQuery,
   JudgeSession,
 } from "./evidence";
-export { CANDIDATE_TIMEOUT_MS } from "./types";
+export { CANDIDATE_TIMEOUT_MS, MAX_CANDIDATE_TIMEOUT_MS } from "./types";
 export { rubricCriteria, criterionMean, isScored } from "./judgment";
 export {
   categoryKey,

package/src/infra/judging/code-source.ts

@@ -1,18 +1,34 @@
-import { dirname, isAbsolute, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
+import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
 import type { CodeJudgeDefinition } from "../../judge-context";
 import { fingerprint } from "../files";
 
+/** Raise when a release changes what judge.ts receives or how it runs; every code judge then runs again. */
+const CODE_JUDGE_PROTOCOL = 1;
+const printer = new Bun.Transpiler({
+  loader: "js",
+  target: "bun",
+  deadCodeElimination: false,
+});
+
+/** The package that provides an external import, as `name@version/path`. */
+async function packageSpecifier(path: string, from: string) {
+  for (let directory = dirname(path); ; directory = dirname(directory)) {
+    const manifest = Bun.file(resolve(directory, "package.json"));
+    const { name, version } = (await manifest.exists())
+      ? await manifest.json().catch(() => ({}))
+      : {};
+    if (typeof name === "string" && name)
+      return `${name}@${version ?? ""}/${relative(directory, path).split(sep).join("/")}`;
+    if (dirname(directory) === directory)
+      return relative(from, path).split(sep).join("/");
+  }
+}
+
 /** Bundle local imports without executing author code during planning. */
 export async function compileCodeJudge(
   file: string,
 ): Promise<CodeJudgeDefinition> {
   file = resolve(file);
-  const dependencies: Record<string, string> = {};
-  const sdkManifest = fileURLToPath(
-    new URL("../../../package.json", import.meta.url),
-  );
-  dependencies[sdkManifest] = await Bun.file(sdkManifest).text();
   const external = new Set<string>();
   const built = await Bun.build({
     // Import only the judge function: unused reporting metadata is tree-shaken.
@@ -65,40 +81,32 @@ export async function compileCodeJudge(
     (await built.outputs
       .find((output) => output.kind === "sourcemap")
       ?.text()) ?? "";
-  const collect = async (directory: string, stopAtPackage: boolean) => {
-    for (;;) {
-      let found = false;
-      for (const name of [
-        "package.json",
-        "bun.lock",
-        "bun.lockb",
-        "package-lock.json",
-        "pnpm-lock.yaml",
-        "yarn.lock",
-      ]) {
-        const path = resolve(directory, name);
-        if (await Bun.file(path).exists()) {
-          dependencies[path] =
-            name === "bun.lockb"
-              ? Buffer.from(await Bun.file(path).bytes()).toString("base64")
-              : await Bun.file(path).text();
-          found ||= name === "package.json";
-        }
-      }
-      if (found && stopAtPackage) return;
-      const parent = dirname(directory);
-      if (parent === directory) return;
-      directory = parent;
-    }
-  };
-  await collect(dirname(file), false);
-  for (const path of external) await collect(dirname(path), true);
+  // The executable keeps absolute import paths; identity uses the package name and version.
+  const dependencies: Record<string, string> = {};
+  let portable = source;
+  for (const path of external) {
+    const specifier = await packageSpecifier(path, dirname(file));
+    dependencies[specifier] = path;
+    portable = portable.replaceAll(
+      JSON.stringify(path),
+      JSON.stringify(specifier),
+    );
+  }
+  // Bun writes __dirname and __filename as absolute paths. Like other runtime file inputs, they are not identity.
+  portable = portable.replace(
+    /^\s*var __(?:dirname|filename) = .*$/gm,
+    (line) => line.replace(/"(?:[^"\\]|\\.)*"/g, '""'),
+  );
   return {
     file,
     source,
     sourceMap,
     dependencies,
-    // Debug mappings and original-source metadata are retained, but not executable inputs.
-    hash: fingerprint({ source, dependencies }),
+    // Reprinting drops comments: module comments name files relative to the working
+    // directory, and the debug ID depends on them. Source maps are retained, not hashed.
+    hash: fingerprint({
+      protocol: CODE_JUDGE_PROTOCOL,
+      source: printer.transformSync(portable),
+    }),
   };
 }

package/src/infra/verification/image.ts

@@ -2,6 +2,7 @@ import { fileURLToPath } from "node:url";
 import type { VerificationEnvironment, VerificationRuntime } from "../../judge-context";
 import { engineCommand } from "../containers/docker";
 import { treeHash } from "../files";
+import { DEFAULT_INPUT_MIB, verificationInputMiB } from "./limits";
 
 export const VERIFICATION_IMAGE = "openeval-verification:0.5.0";
 const directory = fileURLToPath(new URL("./runtime/", import.meta.url));
@@ -21,6 +22,11 @@ export async function verificationRuntime(environment?: VerificationEnvironment)
   if (!["docker", "podman"].includes(engine) || !environment.image || /\s/.test(environment.image) ||
     [environment.cpus ?? 2, environment.memoryMiB ?? 4096].some(value => !Number.isSafeInteger(value) || value < 1))
     throw new Error("Verification requires a valid image, engine, and positive integer resource limits");
+  if (environment.workspaceMiB !== undefined &&
+    (!Number.isSafeInteger(environment.workspaceMiB) || environment.workspaceMiB < 1 ||
+      environment.workspaceMiB > (environment.memoryMiB ?? 4096)))
+    throw new Error("Verification workspaceMiB must be a positive integer within memoryMiB");
+  const inputMiB = verificationInputMiB(environment.inputMiB);
   const output = await engineCommand(engine, [
     "image", "inspect", environment.image, "--format",
     '{{.Id}}|{{index .Config.Labels "openeval.verification"}}',
@@ -29,5 +35,10 @@ export async function verificationRuntime(environment?: VerificationEnvironment)
   if (!/^sha256:[a-f0-9]{64}$/.test(imageId)) throw new Error("Verification image has no immutable image ID");
   if (environment.image === VERIFICATION_IMAGE && label !== await treeHash(directory))
     throw new Error("Verification runtime changed. Run image --verification first.");
-  return { engine, image: environment.image, imageId, cpus: environment.cpus ?? 2, memoryMiB: environment.memoryMiB ?? 4096 };
+  return {
+    engine, image: environment.image, imageId,
+    cpus: environment.cpus ?? 2, memoryMiB: environment.memoryMiB ?? 4096,
+    ...(environment.workspaceMiB === undefined ? {} : { workspaceMiB: environment.workspaceMiB }),
+    ...(inputMiB === DEFAULT_INPUT_MIB ? {} : { inputMiB }),
+  };
 }

package/src/infra/verification/limits.ts

@@ -0,0 +1,14 @@
+export const DEFAULT_INPUT_MIB = 128;
+const MIB = 1024 * 1024;
+
+/** Restored workspace archives stay bounded, including opt-in larger inputs. */
+export function verificationInputMiB(value?: number): number {
+  const limit = value === undefined ? DEFAULT_INPUT_MIB : value;
+  if (!Number.isSafeInteger(limit) || limit < 1 || !Number.isSafeInteger(limit * MIB))
+    throw new Error("Verification inputMiB must be a positive integer with a safe byte count");
+  return limit;
+}
+
+export function verificationInputBytes(value?: number): number {
+  return verificationInputMiB(value) * MIB;
+}

package/src/infra/verification/session.ts

@@ -6,10 +6,10 @@ import type { Engine } from "../../types";
 import { engineCommand } from "../containers/docker";
 import { extractWorkspaceArchive } from "../containers/transfer";
 import { contained, hash, relativePath, writeJson } from "../files";
+import { verificationInputBytes, verificationInputMiB } from "./limits";
 
 const LOG_BYTES = 1024 * 1024;
 const OUTPUT_BYTES = 32 * 1024 * 1024;
-const INPUT_BYTES = 128 * 1024 * 1024;
 
 export function verificationPath(value: string): string {
   relativePath(value, "Verification path");
@@ -26,7 +26,7 @@ export function verificationArgs(runtime: VerificationRuntime, owner: string, na
     "--network=none", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--read-only",
     "--pids-limit", "128", "--cpus", String(runtime.cpus), "--memory", `${runtime.memoryMiB}m`,
     "--memory-swap", `${runtime.memoryMiB}m`, "--user", "10001:10001",
-    "--tmpfs", "/workspace:rw,exec,nosuid,nodev,mode=1777,size=512m",
+    "--tmpfs", `/workspace:rw,exec,nosuid,nodev,mode=1777,size=${runtime.workspaceMiB ?? 512}m`,
     "--tmpfs", "/verification:rw,exec,nosuid,nodev,mode=0755,size=64m",
     "--tmpfs", "/tmp:rw,exec,nosuid,nodev,mode=1777,size=256m",
     "--env", "HOME=/tmp/home", "--env", "CI=1",
@@ -116,8 +116,8 @@ export function verificationSession(options: {
         const archive = resolve(staging, "workspace.tar");
         const pack = Bun.spawn(["tar", "-cf", archive, "-C", workspace, "."], { stdout: "ignore", stderr: "pipe" });
         const [packed, packError] = await Promise.all([pack.exited, new Response(pack.stderr).text()]);
-        if (packed || (await lstat(archive)).size > INPUT_BYTES)
-          throw new Error(packed ? `Verification transfer failed: ${packError}` : "Restored verification workspace exceeds 128 MiB");
+        if (packed || (await lstat(archive)).size > verificationInputBytes(runtime.inputMiB))
+          throw new Error(packed ? `Verification transfer failed: ${packError}` : `Restored verification workspace exceeds ${verificationInputMiB(runtime.inputMiB)} MiB`);
         const name = `openeval-verify-${randomUUID()}`;
         const startedAt = new Date().toISOString();
         const deadlineAt = Date.now() + timeoutMs;

package/src/judge-context.ts

@@ -39,6 +39,10 @@ export type VerificationEnvironment = {
   image: string;
   cpus?: number;
   memoryMiB?: number;
+  /** Workspace tmpfs capacity in MiB; default 512. A custom value must fit within memoryMiB. */
+  workspaceMiB?: number;
+  /** Maximum restored workspace archive size in MiB; default 128. */
+  inputMiB?: number;
 };
 export type VerificationRuntime = {
   engine: Engine;
@@ -46,6 +50,8 @@ export type VerificationRuntime = {
   imageId: string;
   cpus: number;
   memoryMiB: number;
+  workspaceMiB?: number;
+  inputMiB?: number;
 };
 export type VerificationRequest = {
   revision?: Revision;
@@ -185,9 +191,11 @@ export interface JudgeContext {
 /** Frozen judging source and dependency identity, kept in host-side run records. */
 export type CodeJudgeDefinition = {
   file: string;
+  /** Executable identity: bundled code and imported package versions, without host paths or comments. */
   hash: string;
   source: string;
   sourceMap: string;
+  /** Packages imported at run time, as `name@version/path`, with the file each resolved to. */
   dependencies: Record<string, string>;
 };
 export type CodeJudgeExecution = {

package/src/types.ts

@@ -15,6 +15,7 @@ import type {
 export type OpenCodeStreamEvent = OpenCodeEvent | SessionLogItem;
 
 export const CANDIDATE_TIMEOUT_MS = 45 * 60 * 1000;
+export const MAX_CANDIDATE_TIMEOUT_MS = 12 * 60 * 60 * 1000;
 export type ModelRef = `${string}/${string}`;
 export type Engine = "docker" | "podman";
 export type ProviderDefinitions = NonNullable<
@@ -34,6 +35,11 @@ export type Judge = {
   websearch?: "exa" | false;
   verification?: VerificationEnvironment;
 };
+/** Execution controls for one candidate attempt. Eval values override benchmark values. */
+export type CandidateLimits = {
+  /** Agent session limit, after preparation; default 45 minutes, at most 12 hours. */
+  timeoutMs?: number;
+};
 export type Benchmark = {
   name?: string;
   /** Display label for one benchmark.ts among several that share `name`, such as "Frontier". */
@@ -42,8 +48,7 @@ export type Benchmark = {
   judge?: Judge;
   repetitions?: number;
   concurrency?: number;
-  candidate?: {
-    timeoutMs?: number;
+  candidate?: CandidateLimits & {
     websearch?: "exa" | false;
     providers?: ProviderDefinitions;
   };
@@ -82,6 +87,8 @@ export type Eval = {
     overlay?: string;
   };
   prepare?: readonly PreparationStep[];
+  /** Overrides benchmark.candidate for this eval. */
+  candidate?: CandidateLimits;
 };
 export type EvalDefinition = {
   id: string;
@@ -106,7 +113,7 @@ export type BenchmarkDefinition = {
   suite?: string;
   directory: string;
   models: ModelRef[];
-  judge: { model?: ModelRef; timeoutMs: number; websearch: "exa" | false; verification?: VerificationRuntime | Required<VerificationEnvironment> };
+  judge: { model?: ModelRef; timeoutMs: number; websearch: "exa" | false; verification?: VerificationRuntime | Omit<VerificationRuntime, "imageId"> };
   repetitions: number;
   concurrency: number;
   candidate: {

package/src/view.ts

@@ -171,13 +171,33 @@ function componentBounds(
     coverage: coverage * scale,
   };
 }
-export const sumCosts = (costs: readonly (Cost | undefined)[]): Cost => {
+/** A cost summed over executions. `unaccounted` counts executions without a final cost. */
+export type CostTotal = Cost & { unaccounted: number };
+/** Sum execution costs or totals; a single execution without a final cost counts once. */
+export const sumCosts = (
+  costs: readonly (Cost | CostTotal | undefined)[],
+): CostTotal => {
   const reportedUSD = costs.reduce(
     (sum, cost) => sum + (cost?.reportedUSD ?? 0),
     0,
   );
   const complete = costs.every((cost) => cost?.complete === true);
-  return { reportedUSD, complete, usd: complete ? reportedUSD : null };
+  const unaccounted = costs.reduce(
+    (sum, cost) =>
+      sum +
+      (cost && "unaccounted" in cost
+        ? cost.unaccounted
+        : cost?.complete
+          ? 0
+          : 1),
+    0,
+  );
+  return {
+    reportedUSD,
+    complete,
+    usd: complete ? reportedUSD : null,
+    unaccounted,
+  };
 };
 export type ResultEntry = {
   id: string;
@@ -209,8 +229,8 @@ export type Overview = {
   evalNames: Record<string, string>;
   modelNames: ModelNames;
   categories: CategorySummary[];
-  cost: Cost;
-  evalCosts: Record<string, Cost>;
+  cost: CostTotal;
+  evalCosts: Record<string, CostTotal>;
   runtime: RuntimeInterval[];
 };
 export type ResultSummary = {
@@ -305,7 +325,10 @@ export type JudgeAudit = {
     Pick<JudgeRun, "id" | "state" | "startedAt"> & { mode: "monitor" | "final" }
   >;
 };
-export type EvalRunSummary = LiveEvalRun & { cost: Cost; elapsedMs: number };
+export type EvalRunSummary = LiveEvalRun & {
+  cost: CostTotal;
+  elapsedMs: number;
+};
 export type EvalRunIndex = {
   entry: ResultEntry;
   eval: string;
@@ -316,7 +339,7 @@ export type ActivityRun = {
   id: string;
   benchmarkId: string;
   resultId: string;
-  cost: Cost;
+  cost: CostTotal;
   state: {
     id: string;
     eval: string;

package/viewer/assets/{abnfDiagram-VCTEODGH-DheibVLk.js → abnfDiagram-VCTEODGH-ug3iRKuR.js}

@@ -1 +1 @@
-import{m as e,t}from"./mermaid-parser.core-CiGT3tRM.js";import{n}from"./chunk-Y2CYZVJY-DsF7k-Jl.js";import{m as r}from"./src-BnSwVg9U.js";import"./chunk-DU6HZSFF-Cty-qidK.js";import{n as i,r as a,t as o}from"./chunk-SVP7TREG-BEMJeRHW.js";import{t as s}from"./chunk-JWPE2WC7-DVXcaiue.js";import"./mermaid.core-BfFqBx2V.js";var c=e().RailroadAbnf.parser.LangiumParser,l=n(e=>{let t=e.alternatives.map(u);return t.length===1?t[0]:{type:`choice`,alternatives:t}},`transformAlternation`),u=n(e=>{let t=e.elements.map(f);return t.length===1?t[0]:{type:`sequence`,elements:t}},`transformConcatenation`),d=n(e=>{if(e.includes(`*`)){let[t,n]=e.split(`*`);return{min:t?parseInt(t,10):0,max:n?parseInt(n,10):1/0}}let t=parseInt(e,10);return{min:t,max:t}},`parseRepeat`),f=n(e=>{let t=p(e.primary);if(!e.repeat)return t;let{min:n,max:r}=d(e.repeat);return n===0&&r===1?{type:`optional`,element:t}:{type:`repetition`,element:t,min:n,max:r}},`transformElement`),p=n(e=>{switch(e.$type){case`AbnfStringLiteral`:return{type:`terminal`,value:e.value};case`AbnfNumVal`:return{type:`terminal`,value:e.value};case`AbnfRuleName`:return{type:`nonterminal`,name:e.name};case`AbnfGroup`:return l(e.element);case`AbnfOptionalGroup`:return{type:`optional`,element:l(e.element)};default:throw Error(`Unsupported ABNF primary node: ${e.$type}`)}},`transformPrimary`),m=n(e=>({name:e.name,definition:l(e.definition)}),`transformRule`),h=n(e=>{s(e,o),e.title&&o.setTitle(e.title),e.rules.map(e=>o.addRule(m(e)))},`populateDb`),g={parser:{parse:n(e=>{o.clear(),r.debug(`[ABNF Parser] Starting Langium parse`);let n=c.parse(e);if(n.lexerErrors.length>0||n.parserErrors.length>0)throw new t(n);let i=n.value;r.debug(`[ABNF Parser] Parsed rules:`,i.rules.length),h(i),r.debug(`[ABNF Parser] Parse complete`)},`parse`),parser:{yy:o}},db:o,renderer:a,styles:i};export{g as diagram};
+import{m as e,t}from"./mermaid-parser.core-VTfqiyyK.js";import{n}from"./chunk-Y2CYZVJY-DsF7k-Jl.js";import{m as r}from"./src-C171e0nu.js";import"./chunk-DU6HZSFF-BI4SVpji.js";import{n as i,r as a,t as o}from"./chunk-SVP7TREG-CGuEYo-A.js";import{t as s}from"./chunk-JWPE2WC7-DVXcaiue.js";import"./mermaid.core-eoOgKmWC.js";var c=e().RailroadAbnf.parser.LangiumParser,l=n(e=>{let t=e.alternatives.map(u);return t.length===1?t[0]:{type:`choice`,alternatives:t}},`transformAlternation`),u=n(e=>{let t=e.elements.map(f);return t.length===1?t[0]:{type:`sequence`,elements:t}},`transformConcatenation`),d=n(e=>{if(e.includes(`*`)){let[t,n]=e.split(`*`);return{min:t?parseInt(t,10):0,max:n?parseInt(n,10):1/0}}let t=parseInt(e,10);return{min:t,max:t}},`parseRepeat`),f=n(e=>{let t=p(e.primary);if(!e.repeat)return t;let{min:n,max:r}=d(e.repeat);return n===0&&r===1?{type:`optional`,element:t}:{type:`repetition`,element:t,min:n,max:r}},`transformElement`),p=n(e=>{switch(e.$type){case`AbnfStringLiteral`:return{type:`terminal`,value:e.value};case`AbnfNumVal`:return{type:`terminal`,value:e.value};case`AbnfRuleName`:return{type:`nonterminal`,name:e.name};case`AbnfGroup`:return l(e.element);case`AbnfOptionalGroup`:return{type:`optional`,element:l(e.element)};default:throw Error(`Unsupported ABNF primary node: ${e.$type}`)}},`transformPrimary`),m=n(e=>({name:e.name,definition:l(e.definition)}),`transformRule`),h=n(e=>{s(e,o),e.title&&o.setTitle(e.title),e.rules.map(e=>o.addRule(m(e)))},`populateDb`),g={parser:{parse:n(e=>{o.clear(),r.debug(`[ABNF Parser] Starting Langium parse`);let n=c.parse(e);if(n.lexerErrors.length>0||n.parserErrors.length>0)throw new t(n);let i=n.value;r.debug(`[ABNF Parser] Parsed rules:`,i.rules.length),h(i),r.debug(`[ABNF Parser] Parse complete`)},`parse`),parser:{yy:o}},db:o,renderer:a,styles:i};export{g as diagram};