@hona/openeval 0.5.9 → 0.5.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.5.9",
+  "version": "0.5.10",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/read-results.ts

@@ -43,10 +43,17 @@ const costOf = (run: EvalRun | JudgeRun, results?: Results): Cost => {
     complete: usd !== undefined,
   };
 };
-// Execution IDs are unique, including after merging runs.
+// Restored records represent the same original execution, not additional model spend.
 const runCosts = (runs: (EvalRun | JudgeRun)[], results: Results) => {
   const unique = new Map(runs.map((run) => [run.id, run]));
-  return sumCosts([...unique.values()].map((run) => costOf(run, results)));
+  const executions = new Map<string, EvalRun | JudgeRun>();
+  for (const run of unique.values()) {
+    const restoration = "restoration" in run ? run.restoration : undefined;
+    const key = restoration?.evalRunId ?? run.id;
+    const previous = executions.get(key);
+    if (!previous || restoration) executions.set(key, run);
+  }
+  return sumCosts([...executions.values()].map((run) => costOf(run, results)));
 };
 const entryOf = (
   run: BenchmarkRun,

package/src/app/restore-eval-run.ts

@@ -18,26 +18,41 @@ export async function restoreEvalRun(
   using results = new Results(resolve(root, "runner.db"));
   const previous = results.evalRun(evalRunId);
   const slot = previous && results.slot(previous.slotId);
+  const original = previous?.restoration
+    ? results.evalRun(previous.restoration.evalRunId)
+    : previous;
   if (!previous || !slot || !slot.active || slot.evalRunId !== previous.id ||
-      previous.state !== "failed" || !previous.interrupted || previous.evidence)
-    throw new Error("Select an active interrupted EvalRun without finalized evidence");
+      !original || original.state !== "failed" || !original.interrupted || original.evidence ||
+      (previous !== original && (previous.state !== "completed" || !previous.restoration)))
+    throw new Error("Select an active interrupted EvalRun or its restored recording");
+  if (previous.restoration && (options.databaseHash !== previous.restoration.databaseHash ||
+      options.workspaceArchiveHash !== previous.restoration.workspaceArchiveHash))
+    throw new Error("A restored recording must retain the same original backup hashes");
   if (results.benchmark!.state === "running" || results.benchmark!.mergedInto)
     throw new Error("Wait for the active BenchmarkRun and use its aggregate directory");
-  const events = (await readFile(resolve(root, "eval-runs", previous.id, "evidence/events.jsonl"), "utf8"))
+  const events = (await readFile(resolve(root, "eval-runs", original.id, "evidence/events.jsonl"), "utf8"))
     .trim().split("\n").filter(Boolean).map(line => JSON.parse(line).event);
   const creation = events.find(event => event.type === "session.created" && !event.data.parentID);
   if (!creation?.data.sessionID)
     throw new Error("The original recording does not identify its root session");
-  const initial = resolve(root, "inputs", previous.input.evalId, previous.input.sourceHash, "workspace");
-  if (!(await Bun.file(resolve(initial, "..", "ready")).exists()))
+  const frozen = resolve(root, "inputs", original.input.evalId, original.input.sourceHash, "workspace");
+  if (!(await Bun.file(resolve(frozen, "..", "ready")).exists()))
     throw new Error("The frozen starting workspace is missing");
+  const item = results.benchmark!.definition.evals.find(item => item.id === original.input.evalId);
+  if (!item || item.sourceHash !== original.input.sourceHash)
+    throw new Error("The frozen preparation no longer matches the original inputs");
   const staging = resolve(root, "eval-runs", `.restored-${randomUUID()}`);
   await mkdir(staging, { recursive: false });
   try {
+    const preparation = resolve(staging, "preparation");
+    const initial = await archive.prepareRestoredInitial(
+      results.benchmark!.definition, item, original.input.imageId, frozen, preparation,
+    );
     const restored = await archive.restoreArchivedCandidate(
-      previous.input, creation.data.sessionID, initial, options, staging,
+      original.input, creation.data.sessionID, initial, options, staging,
     );
-    const elapsedMs = Date.parse(restored.receipt.completedAt) - Date.parse(previous.startedAt);
+    await rm(preparation, { recursive: true, force: true });
+    const elapsedMs = Date.parse(restored.receipt.completedAt) - Date.parse(original.startedAt);
     if (!Number.isFinite(elapsedMs) || elapsedMs < 0)
       throw new Error("Native completion predates the original EvalRun");
     const evidence = { ...restored.evidence };
@@ -46,19 +61,19 @@ export async function restoreEvalRun(
       if (results.benchmark!.state === "running" ||
           results.slot(slot.id)?.evalRunId !== previous.id)
         throw new Error("The BenchmarkRun or selected execution changed during restoration");
-      const run = results.startEval(slot, previous.input);
+      const run = results.startEval(slot, original.input);
       const destination = resolve(root, "eval-runs", run.id);
       renameSync(staging, destination);
       evidence.directory = relative(root, resolve(destination, restored.evidence.directory));
       const completed: EvalRun = {
         ...run,
         state: "completed",
-        startedAt: previous.startedAt,
+        startedAt: original.startedAt,
         completedAt: restored.receipt.completedAt,
         elapsedMs,
         evidence,
         session: { ...restored.session, database: relative(root, resolve(destination, restored.session.database)) },
-        restoration: { ...restored.receipt, evalRunId: previous.id },
+        restoration: { ...restored.receipt, evalRunId: original.id },
         metrics: measureRecording(restored.events.map((event, sequence) => ({
           event, sequence, time: new Date("created" in event ? event.created : Date.now()).toISOString(),
         })), restored.tools, evidence),

package/src/infra/restore-archive.ts

@@ -1,13 +1,15 @@
 import { Database } from "bun:sqlite";
 import { mkdir, mkdtemp, rm } from "node:fs/promises";
 import { resolve, dirname } from "node:path";
-import type { EvalRunInput, OpenCodeStreamEvent } from "../types";
+import type { BenchmarkDefinition, EvalDefinition, EvalRunInput, OpenCodeStreamEvent } from "../types";
 import { EvidenceCapture } from "./evidence";
 import { extractWorkspaceArchive } from "./containers/transfer";
 import { removeCredentials } from "./opencode/auth";
 import { readArchivedSession } from "./opencode/archive";
 import { parseModel, type SessionResult } from "./opencode/session";
 import { hash, writeJson } from "./files";
+import { CandidateContainer } from "./containers/oci";
+import { createSessionDatabase } from "./opencode/host";
 
 export type ArchiveRestore = {
   database: string;
@@ -16,6 +18,26 @@ export type ArchiveRestore = {
   workspaceArchiveHash: string;
 };
 
+/** Replay only frozen author preparation, without credentials or a model prompt. */
+export async function prepareRestoredInitial(
+  definition: BenchmarkDefinition,
+  item: EvalDefinition,
+  imageId: string,
+  workspace: string,
+  staging: string,
+) {
+  if (!item.settings.prepare?.length) return workspace;
+  await mkdir(staging, { recursive: true });
+  const database = resolve(staging, "opencode.db");
+  await createSessionDatabase(database, []);
+  await using container = await CandidateContainer.create(definition.container, imageId);
+  await container.prepare(workspace, database, definition.candidate.websearch,
+    item.settings.prepare, staging);
+  const initial = resolve(staging, "workspace");
+  await container.snapshot(initial, staging);
+  return initial;
+}
+
 /** A backup must belong to the original execution, not merely resemble its answer. */
 export function verifyArchivedInput(
   input: EvalRunInput,
@@ -88,7 +110,7 @@ export async function restoreArchivedCandidate(
       completedAt: new Date(completedAt).toISOString(),
       databaseHash: options.databaseHash,
       workspaceArchiveHash: options.workspaceArchiveHash,
-      initial: "frozen prepared input, not a newly observed candidate snapshot",
+      initial: "frozen input and author preparation replayed without model calls; not a newly observed candidate snapshot",
     };
     await writeJson(resolve(directory, "restoration.json"), receipt);
     return {