@hona/openeval 0.5.10 → 0.5.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.5.10",
+  "version": "0.5.11",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/plan-retries.ts

@@ -0,0 +1,47 @@
+import type { BenchmarkDefinition, BenchmarkRun, PlanItem } from "../types";
+import type { Results } from "../infra/sqlite";
+import { candidateFingerprint, judgeFingerprint } from "./input-fingerprints";
+
+/** Explicit IDs only: do not recollect fully scored or superseded recordings. */
+export function retryPlan(
+  definition: BenchmarkDefinition,
+  runtime: BenchmarkRun["runtime"],
+  results: Results,
+  evalRunIds: readonly string[],
+): PlanItem[] {
+  if (!evalRunIds.length || new Set(evalRunIds).size !== evalRunIds.length)
+    throw new Error("Select unique EvalRun IDs to retry");
+  return evalRunIds.map(id => {
+    const previous = results.evalRun(id);
+    const slot = previous && results.slot(previous.slotId);
+    if (!previous || !slot?.active || slot.evalRunId !== previous.id)
+      throw new Error("Select the active EvalRun for each repetition");
+    const judge = slot.judgeRunId && results.judgeRun(slot.judgeRunId);
+    const unresolved = judge && judge.state === "completed" &&
+      Object.values(judge.judgment?.scores ?? {}).some(score => score.value === null);
+    if (previous.state === "running" || (previous.state === "completed" && !unresolved))
+      throw new Error("Select failed, stopped, timed-out, or completed-but-unresolved work");
+    const item = definition.evals.find(item => item.id === slot.evalId);
+    if (!item || item.prompt !== previous.input.prompt || item.sourceHash !== previous.input.sourceHash ||
+        runtime.imageId !== previous.input.imageId)
+      throw new Error("Task inputs or image changed; use scoped run for changed work");
+    // The current authored time limit may replace an older limit. Other inputs must match.
+    const recordedLimits = {
+      ...definition,
+      evals: definition.evals.map(value => value.id === item.id ? {
+        ...value, settings: { ...value.settings,
+          candidate: { ...value.settings.candidate, timeoutMs: previous.input.timeoutMs } },
+      } : value),
+    };
+    if (candidateFingerprint(recordedLimits, slot.evalId, slot.model, runtime) !== previous.input.candidateHash)
+      throw new Error("Candidate inputs other than the time limit changed");
+    return {
+      action: "candidate",
+      reason: previous.state === "completed" ? "Explicit retry of unresolved recorded work" : "Explicit retry of interrupted or failed work",
+      slot: { ...slot,
+        candidateHash: candidateFingerprint(definition, slot.evalId, slot.model, runtime),
+        judgeHash: judgeFingerprint(definition, slot.evalId),
+      },
+    };
+  });
+}

package/src/app/retry-run.ts

@@ -1,18 +1,40 @@
 import { resolve } from "node:path";
-import type { ExecutionObserver } from "../types";
+import type { EvalRun, ExecutionObserver } from "../types";
 import { Results } from "../infra/sqlite";
 import { runtimeFingerprint } from "../infra/containers/oci";
-import { candidateFingerprint } from "./plan-benchmark";
 import { runEvalPipeline } from "./run-eval-pipeline";
 import { ExecutionBudget } from "./execution-budget";
-import { finishBenchmark } from "./run-benchmark";
+import { executeQueue, finishBenchmark } from "./run-benchmark";
 import { recoverStoppedRunner, runnerStopped } from "./recover-run";
+import { retryPlan } from "./plan-retries";
+import { CostBudget, estimateWork } from "./cost-plan";
 
 export async function retryEvalRun(
   directory: string,
   evalRunId: string,
   onEvent?: ExecutionObserver,
 ) {
+  return (await retryEvalRuns(directory, [evalRunId], onEvent))[0]!;
+}
+
+/** Read-only preflight for a precise retry scope. No model calls or selection updates. */
+export async function planEvalRunRetries(directory: string, evalRunIds: readonly string[]) {
+  using results = new Results(resolve(directory, "runner.db"), true);
+  const benchmark = results.benchmark!;
+  if (benchmark.mergedInto || benchmark.state === "running")
+    throw new Error("Wait for active work and select the aggregate run");
+  const runtime = await runtimeFingerprint(benchmark.definition.container, benchmark.definition.judge.verification);
+  const plan = retryPlan(benchmark.definition, runtime, results, evalRunIds);
+  return { plan, estimate: estimateWork(benchmark.definition, results, plan) };
+}
+
+/** A single retry round; original evidence and judgments remain immutable. */
+export async function retryEvalRuns(
+  directory: string,
+  evalRunIds: readonly string[],
+  onEvent?: ExecutionObserver,
+  options: { maxCostUSD?: number } = {},
+): Promise<EvalRun[]> {
   using results = new Results(resolve(directory, "runner.db"));
   if (results.benchmark!.mergedInto)
     throw new Error(
@@ -24,41 +46,26 @@ export async function retryEvalRun(
     recoverStoppedRunner(results);
   }
   const benchmark = results.benchmark!;
-  const previous = results.evalRun(evalRunId);
-  if (!previous || previous.state === "completed")
-    throw new Error(
-      "Select a failed eval run; completed evidence can be rejudged",
-    );
-  const slot = results.slot(previous.slotId)!;
-  if (slot.evalRunId !== previous.id)
-    throw new Error("Select the current eval run for this repetition");
   const runtime = await runtimeFingerprint(
     benchmark.definition.container,
     benchmark.definition.judge.verification,
   );
-  if (
-    candidateFingerprint(
-      benchmark.definition,
-      slot.evalId,
-      slot.model,
-      runtime,
-    ) !== slot.candidateHash
-  )
-    throw new Error(
-      "Declared candidate inputs changed; use run to schedule the affected work",
-    );
-  const definition = benchmark.definition.evals.find(
-    (evalDefinition) => evalDefinition.id === slot.evalId,
-  )!;
-  const workspace = resolve(
-    directory,
-    "inputs",
-    definition.id,
-    definition.sourceHash,
-    "workspace",
-  );
-  if (!(await Bun.file(resolve(workspace, "..", "ready")).exists()))
-    throw new Error("The saved starting workspace is missing");
+  const plan = retryPlan(benchmark.definition, runtime, results, evalRunIds);
+  const workspaces = new Map<string, string>();
+  for (const item of plan) {
+    const definition = benchmark.definition.evals.find(value => value.id === item.slot.evalId)!;
+    const workspace = resolve(directory, "inputs", definition.id, definition.sourceHash, "workspace");
+    if (!(await Bun.file(resolve(workspace, "..", "ready")).exists()))
+      throw new Error("The saved starting workspace is missing");
+    workspaces.set(item.slot.evalId, workspace);
+  }
+  const estimate = estimateWork(benchmark.definition, results, plan);
+  const existing = new Set([...results.evalRuns(), ...results.judgeRuns()].map(run => run.id));
+  const spent = () => [...results.evalRuns(), ...results.judgeRuns()]
+    .filter(run => !existing.has(run.id))
+    .reduce((sum, run) => sum + (run.session?.accounting?.costUSD ?? results.recordedCost(run.id) ?? 0), 0);
+  const costBudget = new CostBudget(options.maxCostUSD, spent);
+  const estimates = new Map(estimate.items.map(item => [item.slotId, item.estimatedUSD]));
   const context = {
     directory: resolve(directory),
     definition: benchmark.definition,
@@ -66,29 +73,58 @@ export async function retryEvalRun(
     results,
     onEvent,
   };
-  results.saveBenchmark({
-    ...benchmark,
-    state: "running",
-    scheduledSlotIds: [slot.id],
-    updatedAt: new Date().toISOString(),
+  results.transaction(() => {
+    if (results.benchmark!.state === "running") throw new Error("Another operation claimed this run");
+    for (const item of plan) {
+      const selected = results.slot(item.slot.id);
+      if (!selected?.active || selected.evalRunId !== item.slot.evalRunId ||
+          selected.judgeRunId !== item.slot.judgeRunId)
+        throw new Error("The retry selection changed during preflight");
+    }
+    results.saveBenchmark({ ...benchmark, runtime, state: "running",
+      scheduledSlotIds: plan.map(item => item.slot.id), updatedAt: new Date().toISOString(),
+      execution: { startedAt: new Date().toISOString(),
+        onlyModels: [...new Set(plan.map(item => item.slot.model))],
+        onlyEvals: [...new Set(plan.map(item => item.slot.evalId))],
+        onlyRepetitions: [...new Set(plan.map(item => item.slot.repetition))],
+        budgetUSD: options.maxCostUSD, estimatedUSD: estimate.estimatedUSD,
+        spentUSD: 0, deferred: 0 } });
   });
   const timer = setInterval(
     () =>
       results.saveBenchmark({
         ...results.benchmark!,
         updatedAt: new Date().toISOString(),
+        scheduledSlotIds: plan.filter(item => item.action !== "deferred").map(item => item.slot.id),
+        execution: { ...results.benchmark!.execution, spentUSD: spent(),
+          deferred: plan.filter(item => item.action === "deferred").length },
       }),
     5000,
   );
   try {
-    return await runEvalPipeline(
-      context,
-      slot,
-      workspace,
-      new ExecutionBudget(benchmark.definition.concurrency),
-    );
+    const budget = new ExecutionBudget(benchmark.definition.concurrency);
+    const completed = new Map<string, EvalRun>();
+    const errors: unknown[] = [];
+    await executeQueue(plan, benchmark.definition.concurrency, async item => {
+      const release = costBudget.reserve(estimates.get(item.slot.id) ?? null);
+      if (!release) {
+        item.action = "deferred";
+        item.reason = "Retry scheduling budget reached or no estimate is available";
+        return;
+      }
+      try {
+        const run = await runEvalPipeline(context, item.slot, workspaces.get(item.slot.evalId)!, budget);
+        completed.set(item.slot.id, run);
+      } catch (error) {
+        errors.push(error);
+      } finally { release(); }
+    });
+    if (errors.length) throw new AggregateError(errors, "One or more retry operations failed");
+    return plan.flatMap(item => completed.has(item.slot.id) ? [completed.get(item.slot.id)!] : []);
   } finally {
     clearInterval(timer);
+    results.saveBenchmark({ ...results.benchmark!, execution: { ...results.benchmark!.execution,
+      spentUSD: spent(), deferred: plan.filter(item => item.action === "deferred").length } });
     finishBenchmark(context);
   }
 }

package/src/cli.ts

@@ -7,7 +7,8 @@ import {
   buildImage,
   addModels,
   removeModels,
-  retryEvalRun,
+  retryEvalRuns,
+  planEvalRunRetries,
   restoreEvalRun,
   judgeRun,
   serveResults,
@@ -48,7 +49,7 @@ Commands:
   add-models <run>       Add models with repeated --model flags
   remove-models <run>    Remove active models while retaining their evidence
   refresh-model-names <run> Refresh reporting names without running evals
-  retry <run> <eval-run> Retry a candidate execution
+  retry <run> <eval-run>... Retry failed or unresolved candidate work
   rejudge <run> <eval-run> Judge the saved evidence again
   restore <run> <eval-run> Restore completed orphan work from hashed backups
 
@@ -63,6 +64,7 @@ Options:
   --only-repetition <n>  Execute only this repetition (repeatable)
   --max-cost <usd>       Scheduling budget for this invocation
   --final-only          Judge only after candidates finish
+  --dry-run             Plan explicit retries without running them
   --verification        Build only the standard verification image (image command)
   --output <dir>         New prepared-input directory (prepare command)
   --port <port>         Viewer port (default: 4173)
@@ -184,15 +186,22 @@ Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
     console.log(JSON.stringify(await restoreEvalRun(resolve(args[1]), args[2], {
       database: resolve(database), databaseHash, workspaceArchive: resolve(workspaceArchive), workspaceArchiveHash,
     }), null, 2));
-  } else if (command === "retry" || command === "rejudge") {
+  } else if (command === "retry") {
+    const flags = args.findIndex((arg, index) => index >= 2 && arg.startsWith("--"));
+    const ids = args.slice(2, flags < 0 ? args.length : flags);
+    if (!args[1] || !ids.length) throw new Error("retry requires a run directory and explicit EvalRun IDs");
+    const result = args.includes("--dry-run")
+      ? await planEvalRunRetries(resolve(args[1]), ids)
+      : await retryEvalRuns(resolve(args[1]), ids, undefined,
+        { maxCostUSD: option("--max-cost") === undefined ? undefined : Number(option("--max-cost")) });
+    console.log(JSON.stringify(!args.includes("--dry-run") && ids.length === 1 && Array.isArray(result)
+      ? result[0] : result, null, 2));
+  } else if (command === "rejudge") {
     if (!args[1] || !args[2])
       throw new Error(
         `${command} requires a benchmark run directory and eval run ID`,
       );
-    const result =
-      command === "retry"
-        ? await retryEvalRun(resolve(args[1]), args[2])
-        : await judgeRun(resolve(args[1]), args[2]);
+    const result = await judgeRun(resolve(args[1]), args[2]);
     console.log(JSON.stringify(result, null, 2));
   } else
     throw new Error(

package/src/index.ts

@@ -67,7 +67,7 @@ export { removeModels } from "./app/remove-models";
 export { refreshModelNames } from "./app/refresh-model-names";
 export type { CostEstimate } from "./app/cost-plan";
 export { mergeBenchmarkRuns } from "./app/merge-runs";
-export { retryEvalRun } from "./app/retry-run";
+export { retryEvalRun, retryEvalRuns, planEvalRunRetries } from "./app/retry-run";
 export { restoreEvalRun } from "./app/restore-eval-run";
 export { judgeRun, judgeRuns } from "./app/rejudge";
 export { judgeEvidence, recordEvidence } from "./app/judge-evidence";