@hona/openeval 0.5.9 → 0.5.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.5.9",
+  "version": "0.5.11",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/plan-retries.ts

@@ -0,0 +1,47 @@
+import type { BenchmarkDefinition, BenchmarkRun, PlanItem } from "../types";
+import type { Results } from "../infra/sqlite";
+import { candidateFingerprint, judgeFingerprint } from "./input-fingerprints";
+
+/** Explicit IDs only: do not recollect fully scored or superseded recordings. */
+export function retryPlan(
+  definition: BenchmarkDefinition,
+  runtime: BenchmarkRun["runtime"],
+  results: Results,
+  evalRunIds: readonly string[],
+): PlanItem[] {
+  if (!evalRunIds.length || new Set(evalRunIds).size !== evalRunIds.length)
+    throw new Error("Select unique EvalRun IDs to retry");
+  return evalRunIds.map(id => {
+    const previous = results.evalRun(id);
+    const slot = previous && results.slot(previous.slotId);
+    if (!previous || !slot?.active || slot.evalRunId !== previous.id)
+      throw new Error("Select the active EvalRun for each repetition");
+    const judge = slot.judgeRunId && results.judgeRun(slot.judgeRunId);
+    const unresolved = judge && judge.state === "completed" &&
+      Object.values(judge.judgment?.scores ?? {}).some(score => score.value === null);
+    if (previous.state === "running" || (previous.state === "completed" && !unresolved))
+      throw new Error("Select failed, stopped, timed-out, or completed-but-unresolved work");
+    const item = definition.evals.find(item => item.id === slot.evalId);
+    if (!item || item.prompt !== previous.input.prompt || item.sourceHash !== previous.input.sourceHash ||
+        runtime.imageId !== previous.input.imageId)
+      throw new Error("Task inputs or image changed; use scoped run for changed work");
+    // The current authored time limit may replace an older limit. Other inputs must match.
+    const recordedLimits = {
+      ...definition,
+      evals: definition.evals.map(value => value.id === item.id ? {
+        ...value, settings: { ...value.settings,
+          candidate: { ...value.settings.candidate, timeoutMs: previous.input.timeoutMs } },
+      } : value),
+    };
+    if (candidateFingerprint(recordedLimits, slot.evalId, slot.model, runtime) !== previous.input.candidateHash)
+      throw new Error("Candidate inputs other than the time limit changed");
+    return {
+      action: "candidate",
+      reason: previous.state === "completed" ? "Explicit retry of unresolved recorded work" : "Explicit retry of interrupted or failed work",
+      slot: { ...slot,
+        candidateHash: candidateFingerprint(definition, slot.evalId, slot.model, runtime),
+        judgeHash: judgeFingerprint(definition, slot.evalId),
+      },
+    };
+  });
+}

package/src/app/read-results.ts

@@ -43,10 +43,17 @@ const costOf = (run: EvalRun | JudgeRun, results?: Results): Cost => {
     complete: usd !== undefined,
   };
 };
-// Execution IDs are unique, including after merging runs.
+// Restored records represent the same original execution, not additional model spend.
 const runCosts = (runs: (EvalRun | JudgeRun)[], results: Results) => {
   const unique = new Map(runs.map((run) => [run.id, run]));
-  return sumCosts([...unique.values()].map((run) => costOf(run, results)));
+  const executions = new Map<string, EvalRun | JudgeRun>();
+  for (const run of unique.values()) {
+    const restoration = "restoration" in run ? run.restoration : undefined;
+    const key = restoration?.evalRunId ?? run.id;
+    const previous = executions.get(key);
+    if (!previous || restoration) executions.set(key, run);
+  }
+  return sumCosts([...executions.values()].map((run) => costOf(run, results)));
 };
 const entryOf = (
   run: BenchmarkRun,

package/src/app/restore-eval-run.ts

@@ -18,26 +18,41 @@ export async function restoreEvalRun(
   using results = new Results(resolve(root, "runner.db"));
   const previous = results.evalRun(evalRunId);
   const slot = previous && results.slot(previous.slotId);
+  const original = previous?.restoration
+    ? results.evalRun(previous.restoration.evalRunId)
+    : previous;
   if (!previous || !slot || !slot.active || slot.evalRunId !== previous.id ||
-      previous.state !== "failed" || !previous.interrupted || previous.evidence)
-    throw new Error("Select an active interrupted EvalRun without finalized evidence");
+      !original || original.state !== "failed" || !original.interrupted || original.evidence ||
+      (previous !== original && (previous.state !== "completed" || !previous.restoration)))
+    throw new Error("Select an active interrupted EvalRun or its restored recording");
+  if (previous.restoration && (options.databaseHash !== previous.restoration.databaseHash ||
+      options.workspaceArchiveHash !== previous.restoration.workspaceArchiveHash))
+    throw new Error("A restored recording must retain the same original backup hashes");
   if (results.benchmark!.state === "running" || results.benchmark!.mergedInto)
     throw new Error("Wait for the active BenchmarkRun and use its aggregate directory");
-  const events = (await readFile(resolve(root, "eval-runs", previous.id, "evidence/events.jsonl"), "utf8"))
+  const events = (await readFile(resolve(root, "eval-runs", original.id, "evidence/events.jsonl"), "utf8"))
     .trim().split("\n").filter(Boolean).map(line => JSON.parse(line).event);
   const creation = events.find(event => event.type === "session.created" && !event.data.parentID);
   if (!creation?.data.sessionID)
     throw new Error("The original recording does not identify its root session");
-  const initial = resolve(root, "inputs", previous.input.evalId, previous.input.sourceHash, "workspace");
-  if (!(await Bun.file(resolve(initial, "..", "ready")).exists()))
+  const frozen = resolve(root, "inputs", original.input.evalId, original.input.sourceHash, "workspace");
+  if (!(await Bun.file(resolve(frozen, "..", "ready")).exists()))
     throw new Error("The frozen starting workspace is missing");
+  const item = results.benchmark!.definition.evals.find(item => item.id === original.input.evalId);
+  if (!item || item.sourceHash !== original.input.sourceHash)
+    throw new Error("The frozen preparation no longer matches the original inputs");
   const staging = resolve(root, "eval-runs", `.restored-${randomUUID()}`);
   await mkdir(staging, { recursive: false });
   try {
+    const preparation = resolve(staging, "preparation");
+    const initial = await archive.prepareRestoredInitial(
+      results.benchmark!.definition, item, original.input.imageId, frozen, preparation,
+    );
     const restored = await archive.restoreArchivedCandidate(
-      previous.input, creation.data.sessionID, initial, options, staging,
+      original.input, creation.data.sessionID, initial, options, staging,
     );
-    const elapsedMs = Date.parse(restored.receipt.completedAt) - Date.parse(previous.startedAt);
+    await rm(preparation, { recursive: true, force: true });
+    const elapsedMs = Date.parse(restored.receipt.completedAt) - Date.parse(original.startedAt);
     if (!Number.isFinite(elapsedMs) || elapsedMs < 0)
       throw new Error("Native completion predates the original EvalRun");
     const evidence = { ...restored.evidence };
@@ -46,19 +61,19 @@ export async function restoreEvalRun(
       if (results.benchmark!.state === "running" ||
           results.slot(slot.id)?.evalRunId !== previous.id)
         throw new Error("The BenchmarkRun or selected execution changed during restoration");
-      const run = results.startEval(slot, previous.input);
+      const run = results.startEval(slot, original.input);
       const destination = resolve(root, "eval-runs", run.id);
       renameSync(staging, destination);
       evidence.directory = relative(root, resolve(destination, restored.evidence.directory));
       const completed: EvalRun = {
         ...run,
         state: "completed",
-        startedAt: previous.startedAt,
+        startedAt: original.startedAt,
         completedAt: restored.receipt.completedAt,
         elapsedMs,
         evidence,
         session: { ...restored.session, database: relative(root, resolve(destination, restored.session.database)) },
-        restoration: { ...restored.receipt, evalRunId: previous.id },
+        restoration: { ...restored.receipt, evalRunId: original.id },
         metrics: measureRecording(restored.events.map((event, sequence) => ({
           event, sequence, time: new Date("created" in event ? event.created : Date.now()).toISOString(),
         })), restored.tools, evidence),

package/src/app/retry-run.ts

@@ -1,18 +1,40 @@
 import { resolve } from "node:path";
-import type { ExecutionObserver } from "../types";
+import type { EvalRun, ExecutionObserver } from "../types";
 import { Results } from "../infra/sqlite";
 import { runtimeFingerprint } from "../infra/containers/oci";
-import { candidateFingerprint } from "./plan-benchmark";
 import { runEvalPipeline } from "./run-eval-pipeline";
 import { ExecutionBudget } from "./execution-budget";
-import { finishBenchmark } from "./run-benchmark";
+import { executeQueue, finishBenchmark } from "./run-benchmark";
 import { recoverStoppedRunner, runnerStopped } from "./recover-run";
+import { retryPlan } from "./plan-retries";
+import { CostBudget, estimateWork } from "./cost-plan";
 
 export async function retryEvalRun(
   directory: string,
   evalRunId: string,
   onEvent?: ExecutionObserver,
 ) {
+  return (await retryEvalRuns(directory, [evalRunId], onEvent))[0]!;
+}
+
+/** Read-only preflight for a precise retry scope. No model calls or selection updates. */
+export async function planEvalRunRetries(directory: string, evalRunIds: readonly string[]) {
+  using results = new Results(resolve(directory, "runner.db"), true);
+  const benchmark = results.benchmark!;
+  if (benchmark.mergedInto || benchmark.state === "running")
+    throw new Error("Wait for active work and select the aggregate run");
+  const runtime = await runtimeFingerprint(benchmark.definition.container, benchmark.definition.judge.verification);
+  const plan = retryPlan(benchmark.definition, runtime, results, evalRunIds);
+  return { plan, estimate: estimateWork(benchmark.definition, results, plan) };
+}
+
+/** A single retry round; original evidence and judgments remain immutable. */
+export async function retryEvalRuns(
+  directory: string,
+  evalRunIds: readonly string[],
+  onEvent?: ExecutionObserver,
+  options: { maxCostUSD?: number } = {},
+): Promise<EvalRun[]> {
   using results = new Results(resolve(directory, "runner.db"));
   if (results.benchmark!.mergedInto)
     throw new Error(
@@ -24,41 +46,26 @@ export async function retryEvalRun(
     recoverStoppedRunner(results);
   }
   const benchmark = results.benchmark!;
-  const previous = results.evalRun(evalRunId);
-  if (!previous || previous.state === "completed")
-    throw new Error(
-      "Select a failed eval run; completed evidence can be rejudged",
-    );
-  const slot = results.slot(previous.slotId)!;
-  if (slot.evalRunId !== previous.id)
-    throw new Error("Select the current eval run for this repetition");
   const runtime = await runtimeFingerprint(
     benchmark.definition.container,
     benchmark.definition.judge.verification,
   );
-  if (
-    candidateFingerprint(
-      benchmark.definition,
-      slot.evalId,
-      slot.model,
-      runtime,
-    ) !== slot.candidateHash
-  )
-    throw new Error(
-      "Declared candidate inputs changed; use run to schedule the affected work",
-    );
-  const definition = benchmark.definition.evals.find(
-    (evalDefinition) => evalDefinition.id === slot.evalId,
-  )!;
-  const workspace = resolve(
-    directory,
-    "inputs",
-    definition.id,
-    definition.sourceHash,
-    "workspace",
-  );
-  if (!(await Bun.file(resolve(workspace, "..", "ready")).exists()))
-    throw new Error("The saved starting workspace is missing");
+  const plan = retryPlan(benchmark.definition, runtime, results, evalRunIds);
+  const workspaces = new Map<string, string>();
+  for (const item of plan) {
+    const definition = benchmark.definition.evals.find(value => value.id === item.slot.evalId)!;
+    const workspace = resolve(directory, "inputs", definition.id, definition.sourceHash, "workspace");
+    if (!(await Bun.file(resolve(workspace, "..", "ready")).exists()))
+      throw new Error("The saved starting workspace is missing");
+    workspaces.set(item.slot.evalId, workspace);
+  }
+  const estimate = estimateWork(benchmark.definition, results, plan);
+  const existing = new Set([...results.evalRuns(), ...results.judgeRuns()].map(run => run.id));
+  const spent = () => [...results.evalRuns(), ...results.judgeRuns()]
+    .filter(run => !existing.has(run.id))
+    .reduce((sum, run) => sum + (run.session?.accounting?.costUSD ?? results.recordedCost(run.id) ?? 0), 0);
+  const costBudget = new CostBudget(options.maxCostUSD, spent);
+  const estimates = new Map(estimate.items.map(item => [item.slotId, item.estimatedUSD]));
   const context = {
     directory: resolve(directory),
     definition: benchmark.definition,
@@ -66,29 +73,58 @@ export async function retryEvalRun(
     results,
     onEvent,
   };
-  results.saveBenchmark({
-    ...benchmark,
-    state: "running",
-    scheduledSlotIds: [slot.id],
-    updatedAt: new Date().toISOString(),
+  results.transaction(() => {
+    if (results.benchmark!.state === "running") throw new Error("Another operation claimed this run");
+    for (const item of plan) {
+      const selected = results.slot(item.slot.id);
+      if (!selected?.active || selected.evalRunId !== item.slot.evalRunId ||
+          selected.judgeRunId !== item.slot.judgeRunId)
+        throw new Error("The retry selection changed during preflight");
+    }
+    results.saveBenchmark({ ...benchmark, runtime, state: "running",
+      scheduledSlotIds: plan.map(item => item.slot.id), updatedAt: new Date().toISOString(),
+      execution: { startedAt: new Date().toISOString(),
+        onlyModels: [...new Set(plan.map(item => item.slot.model))],
+        onlyEvals: [...new Set(plan.map(item => item.slot.evalId))],
+        onlyRepetitions: [...new Set(plan.map(item => item.slot.repetition))],
+        budgetUSD: options.maxCostUSD, estimatedUSD: estimate.estimatedUSD,
+        spentUSD: 0, deferred: 0 } });
   });
   const timer = setInterval(
     () =>
       results.saveBenchmark({
         ...results.benchmark!,
         updatedAt: new Date().toISOString(),
+        scheduledSlotIds: plan.filter(item => item.action !== "deferred").map(item => item.slot.id),
+        execution: { ...results.benchmark!.execution, spentUSD: spent(),
+          deferred: plan.filter(item => item.action === "deferred").length },
       }),
     5000,
   );
   try {
-    return await runEvalPipeline(
-      context,
-      slot,
-      workspace,
-      new ExecutionBudget(benchmark.definition.concurrency),
-    );
+    const budget = new ExecutionBudget(benchmark.definition.concurrency);
+    const completed = new Map<string, EvalRun>();
+    const errors: unknown[] = [];
+    await executeQueue(plan, benchmark.definition.concurrency, async item => {
+      const release = costBudget.reserve(estimates.get(item.slot.id) ?? null);
+      if (!release) {
+        item.action = "deferred";
+        item.reason = "Retry scheduling budget reached or no estimate is available";
+        return;
+      }
+      try {
+        const run = await runEvalPipeline(context, item.slot, workspaces.get(item.slot.evalId)!, budget);
+        completed.set(item.slot.id, run);
+      } catch (error) {
+        errors.push(error);
+      } finally { release(); }
+    });
+    if (errors.length) throw new AggregateError(errors, "One or more retry operations failed");
+    return plan.flatMap(item => completed.has(item.slot.id) ? [completed.get(item.slot.id)!] : []);
   } finally {
     clearInterval(timer);
+    results.saveBenchmark({ ...results.benchmark!, execution: { ...results.benchmark!.execution,
+      spentUSD: spent(), deferred: plan.filter(item => item.action === "deferred").length } });
     finishBenchmark(context);
   }
 }

package/src/cli.ts

@@ -7,7 +7,8 @@ import {
   buildImage,
   addModels,
   removeModels,
-  retryEvalRun,
+  retryEvalRuns,
+  planEvalRunRetries,
   restoreEvalRun,
   judgeRun,
   serveResults,
@@ -48,7 +49,7 @@ Commands:
   add-models <run>       Add models with repeated --model flags
   remove-models <run>    Remove active models while retaining their evidence
   refresh-model-names <run> Refresh reporting names without running evals
-  retry <run> <eval-run> Retry a candidate execution
+  retry <run> <eval-run>... Retry failed or unresolved candidate work
   rejudge <run> <eval-run> Judge the saved evidence again
   restore <run> <eval-run> Restore completed orphan work from hashed backups
 
@@ -63,6 +64,7 @@ Options:
   --only-repetition <n>  Execute only this repetition (repeatable)
   --max-cost <usd>       Scheduling budget for this invocation
   --final-only          Judge only after candidates finish
+  --dry-run             Plan explicit retries without running them
   --verification        Build only the standard verification image (image command)
   --output <dir>         New prepared-input directory (prepare command)
   --port <port>         Viewer port (default: 4173)
@@ -184,15 +186,22 @@ Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
     console.log(JSON.stringify(await restoreEvalRun(resolve(args[1]), args[2], {
       database: resolve(database), databaseHash, workspaceArchive: resolve(workspaceArchive), workspaceArchiveHash,
     }), null, 2));
-  } else if (command === "retry" || command === "rejudge") {
+  } else if (command === "retry") {
+    const flags = args.findIndex((arg, index) => index >= 2 && arg.startsWith("--"));
+    const ids = args.slice(2, flags < 0 ? args.length : flags);
+    if (!args[1] || !ids.length) throw new Error("retry requires a run directory and explicit EvalRun IDs");
+    const result = args.includes("--dry-run")
+      ? await planEvalRunRetries(resolve(args[1]), ids)
+      : await retryEvalRuns(resolve(args[1]), ids, undefined,
+        { maxCostUSD: option("--max-cost") === undefined ? undefined : Number(option("--max-cost")) });
+    console.log(JSON.stringify(!args.includes("--dry-run") && ids.length === 1 && Array.isArray(result)
+      ? result[0] : result, null, 2));
+  } else if (command === "rejudge") {
     if (!args[1] || !args[2])
       throw new Error(
         `${command} requires a benchmark run directory and eval run ID`,
       );
-    const result =
-      command === "retry"
-        ? await retryEvalRun(resolve(args[1]), args[2])
-        : await judgeRun(resolve(args[1]), args[2]);
+    const result = await judgeRun(resolve(args[1]), args[2]);
     console.log(JSON.stringify(result, null, 2));
   } else
     throw new Error(

package/src/index.ts

@@ -67,7 +67,7 @@ export { removeModels } from "./app/remove-models";
 export { refreshModelNames } from "./app/refresh-model-names";
 export type { CostEstimate } from "./app/cost-plan";
 export { mergeBenchmarkRuns } from "./app/merge-runs";
-export { retryEvalRun } from "./app/retry-run";
+export { retryEvalRun, retryEvalRuns, planEvalRunRetries } from "./app/retry-run";
 export { restoreEvalRun } from "./app/restore-eval-run";
 export { judgeRun, judgeRuns } from "./app/rejudge";
 export { judgeEvidence, recordEvidence } from "./app/judge-evidence";

package/src/infra/restore-archive.ts

@@ -1,13 +1,15 @@
 import { Database } from "bun:sqlite";
 import { mkdir, mkdtemp, rm } from "node:fs/promises";
 import { resolve, dirname } from "node:path";
-import type { EvalRunInput, OpenCodeStreamEvent } from "../types";
+import type { BenchmarkDefinition, EvalDefinition, EvalRunInput, OpenCodeStreamEvent } from "../types";
 import { EvidenceCapture } from "./evidence";
 import { extractWorkspaceArchive } from "./containers/transfer";
 import { removeCredentials } from "./opencode/auth";
 import { readArchivedSession } from "./opencode/archive";
 import { parseModel, type SessionResult } from "./opencode/session";
 import { hash, writeJson } from "./files";
+import { CandidateContainer } from "./containers/oci";
+import { createSessionDatabase } from "./opencode/host";
 
 export type ArchiveRestore = {
   database: string;
@@ -16,6 +18,26 @@ export type ArchiveRestore = {
   workspaceArchiveHash: string;
 };
 
+/** Replay only frozen author preparation, without credentials or a model prompt. */
+export async function prepareRestoredInitial(
+  definition: BenchmarkDefinition,
+  item: EvalDefinition,
+  imageId: string,
+  workspace: string,
+  staging: string,
+) {
+  if (!item.settings.prepare?.length) return workspace;
+  await mkdir(staging, { recursive: true });
+  const database = resolve(staging, "opencode.db");
+  await createSessionDatabase(database, []);
+  await using container = await CandidateContainer.create(definition.container, imageId);
+  await container.prepare(workspace, database, definition.candidate.websearch,
+    item.settings.prepare, staging);
+  const initial = resolve(staging, "workspace");
+  await container.snapshot(initial, staging);
+  return initial;
+}
+
 /** A backup must belong to the original execution, not merely resemble its answer. */
 export function verifyArchivedInput(
   input: EvalRunInput,
@@ -88,7 +110,7 @@ export async function restoreArchivedCandidate(
       completedAt: new Date(completedAt).toISOString(),
       databaseHash: options.databaseHash,
       workspaceArchiveHash: options.workspaceArchiveHash,
-      initial: "frozen prepared input, not a newly observed candidate snapshot",
+      initial: "frozen input and author preparation replayed without model calls; not a newly observed candidate snapshot",
     };
     await writeJson(resolve(directory, "restoration.json"), receipt);
     return {