@hona/openeval 0.5.8 → 0.5.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.5.8",
+  "version": "0.5.10",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/read-results.ts

@@ -43,10 +43,17 @@ const costOf = (run: EvalRun | JudgeRun, results?: Results): Cost => {
     complete: usd !== undefined,
   };
 };
-// Execution IDs are unique, including after merging runs.
+// Restored records represent the same original execution, not additional model spend.
 const runCosts = (runs: (EvalRun | JudgeRun)[], results: Results) => {
   const unique = new Map(runs.map((run) => [run.id, run]));
-  return sumCosts([...unique.values()].map((run) => costOf(run, results)));
+  const executions = new Map<string, EvalRun | JudgeRun>();
+  for (const run of unique.values()) {
+    const restoration = "restoration" in run ? run.restoration : undefined;
+    const key = restoration?.evalRunId ?? run.id;
+    const previous = executions.get(key);
+    if (!previous || restoration) executions.set(key, run);
+  }
+  return sumCosts([...executions.values()].map((run) => costOf(run, results)));
 };
 const entryOf = (
   run: BenchmarkRun,

package/src/app/restore-eval-run.ts

@@ -0,0 +1,91 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, readFile, rm } from "node:fs/promises";
+import { renameSync } from "node:fs";
+import { relative, resolve } from "node:path";
+import type { EvalRun } from "../types";
+import { Results } from "../infra/sqlite";
+import * as archive from "../infra/restore-archive";
+import type { ArchiveRestore } from "../infra/restore-archive";
+import { measureRecording } from "../infra/recording/metrics";
+
+/** Restore completed orphan work as a new record; retain the failed coordinator record. */
+export async function restoreEvalRun(
+  directory: string,
+  evalRunId: string,
+  options: ArchiveRestore,
+): Promise<EvalRun> {
+  const root = resolve(directory);
+  using results = new Results(resolve(root, "runner.db"));
+  const previous = results.evalRun(evalRunId);
+  const slot = previous && results.slot(previous.slotId);
+  const original = previous?.restoration
+    ? results.evalRun(previous.restoration.evalRunId)
+    : previous;
+  if (!previous || !slot || !slot.active || slot.evalRunId !== previous.id ||
+      !original || original.state !== "failed" || !original.interrupted || original.evidence ||
+      (previous !== original && (previous.state !== "completed" || !previous.restoration)))
+    throw new Error("Select an active interrupted EvalRun or its restored recording");
+  if (previous.restoration && (options.databaseHash !== previous.restoration.databaseHash ||
+      options.workspaceArchiveHash !== previous.restoration.workspaceArchiveHash))
+    throw new Error("A restored recording must retain the same original backup hashes");
+  if (results.benchmark!.state === "running" || results.benchmark!.mergedInto)
+    throw new Error("Wait for the active BenchmarkRun and use its aggregate directory");
+  const events = (await readFile(resolve(root, "eval-runs", original.id, "evidence/events.jsonl"), "utf8"))
+    .trim().split("\n").filter(Boolean).map(line => JSON.parse(line).event);
+  const creation = events.find(event => event.type === "session.created" && !event.data.parentID);
+  if (!creation?.data.sessionID)
+    throw new Error("The original recording does not identify its root session");
+  const frozen = resolve(root, "inputs", original.input.evalId, original.input.sourceHash, "workspace");
+  if (!(await Bun.file(resolve(frozen, "..", "ready")).exists()))
+    throw new Error("The frozen starting workspace is missing");
+  const item = results.benchmark!.definition.evals.find(item => item.id === original.input.evalId);
+  if (!item || item.sourceHash !== original.input.sourceHash)
+    throw new Error("The frozen preparation no longer matches the original inputs");
+  const staging = resolve(root, "eval-runs", `.restored-${randomUUID()}`);
+  await mkdir(staging, { recursive: false });
+  try {
+    const preparation = resolve(staging, "preparation");
+    const initial = await archive.prepareRestoredInitial(
+      results.benchmark!.definition, item, original.input.imageId, frozen, preparation,
+    );
+    const restored = await archive.restoreArchivedCandidate(
+      original.input, creation.data.sessionID, initial, options, staging,
+    );
+    await rm(preparation, { recursive: true, force: true });
+    const elapsedMs = Date.parse(restored.receipt.completedAt) - Date.parse(original.startedAt);
+    if (!Number.isFinite(elapsedMs) || elapsedMs < 0)
+      throw new Error("Native completion predates the original EvalRun");
+    const evidence = { ...restored.evidence };
+    // This operation does not claim a live candidate. Validate selection again at publication.
+    return results.transaction(() => {
+      if (results.benchmark!.state === "running" ||
+          results.slot(slot.id)?.evalRunId !== previous.id)
+        throw new Error("The BenchmarkRun or selected execution changed during restoration");
+      const run = results.startEval(slot, original.input);
+      const destination = resolve(root, "eval-runs", run.id);
+      renameSync(staging, destination);
+      evidence.directory = relative(root, resolve(destination, restored.evidence.directory));
+      const completed: EvalRun = {
+        ...run,
+        state: "completed",
+        startedAt: original.startedAt,
+        completedAt: restored.receipt.completedAt,
+        elapsedMs,
+        evidence,
+        session: { ...restored.session, database: relative(root, resolve(destination, restored.session.database)) },
+        restoration: { ...restored.receipt, evalRunId: original.id },
+        metrics: measureRecording(restored.events.map((event, sequence) => ({
+          event, sequence, time: new Date("created" in event ? event.created : Date.now()).toISOString(),
+        })), restored.tools, evidence),
+      };
+      for (const event of restored.events)
+        results.append({ executionId: run.id, stage: "candidate", time: new Date("created" in event ? event.created : Date.now()).toISOString(), event });
+      results.finishEval(completed);
+      results.saveBenchmark({ ...results.benchmark!, state: "incomplete", scheduledSlotIds: [], updatedAt: new Date().toISOString() });
+      return completed;
+    });
+  } finally {
+    // Only this operation's private staging directory; published evidence is never removed.
+    await rm(staging, { recursive: true, force: true });
+  }
+}

package/src/cli.ts

@@ -8,6 +8,7 @@ import {
   addModels,
   removeModels,
   retryEvalRun,
+  restoreEvalRun,
   judgeRun,
   serveResults,
   mergeBenchmarkRuns,
@@ -49,6 +50,7 @@ Commands:
   refresh-model-names <run> Refresh reporting names without running evals
   retry <run> <eval-run> Retry a candidate execution
   rejudge <run> <eval-run> Judge the saved evidence again
+  restore <run> <eval-run> Restore completed orphan work from hashed backups
 
 Options:
   --benchmark <dir>      Benchmark directory (default: current directory)
@@ -64,6 +66,10 @@ Options:
   --verification        Build only the standard verification image (image command)
   --output <dir>         New prepared-input directory (prepare command)
   --port <port>         Viewer port (default: 4173)
+  --database <file>      Native backup for restore
+  --database-hash <sha>  SHA-256 of the native backup
+  --workspace-archive <file> Workspace tar backup for restore
+  --workspace-archive-hash <sha> SHA-256 of the workspace tar
 
 Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
   } else if (command === "--version") {
@@ -170,6 +176,14 @@ Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
   } else if (command === "refresh-model-names") {
     if (!args[1]) throw new Error("refresh-model-names requires a benchmark run directory");
     console.log(JSON.stringify(await refreshModelNames(resolve(args[1])), null, 2));
+  } else if (command === "restore") {
+    const database = option("--database"), databaseHash = option("--database-hash");
+    const workspaceArchive = option("--workspace-archive"), workspaceArchiveHash = option("--workspace-archive-hash");
+    if (!args[1] || !args[2] || !database || !databaseHash || !workspaceArchive || !workspaceArchiveHash)
+      throw new Error("restore requires the run, interrupted EvalRun, native backup, workspace tar, and both SHA-256 hashes");
+    console.log(JSON.stringify(await restoreEvalRun(resolve(args[1]), args[2], {
+      database: resolve(database), databaseHash, workspaceArchive: resolve(workspaceArchive), workspaceArchiveHash,
+    }), null, 2));
   } else if (command === "retry" || command === "rejudge") {
     if (!args[1] || !args[2])
       throw new Error(
@@ -182,7 +196,7 @@ Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
     console.log(JSON.stringify(result, null, 2));
   } else
     throw new Error(
-      "Commands: image, plan, run, view, add-models, remove-models, refresh-model-names, merge-runs, retry, rejudge, snapshot",
+      "Commands: image, plan, run, view, add-models, remove-models, refresh-model-names, merge-runs, retry, rejudge, restore, snapshot",
     );
 } catch (error) {
   console.error(error instanceof Error ? error.message : String(error));

package/src/index.ts

@@ -68,6 +68,7 @@ export { refreshModelNames } from "./app/refresh-model-names";
 export type { CostEstimate } from "./app/cost-plan";
 export { mergeBenchmarkRuns } from "./app/merge-runs";
 export { retryEvalRun } from "./app/retry-run";
+export { restoreEvalRun } from "./app/restore-eval-run";
 export { judgeRun, judgeRuns } from "./app/rejudge";
 export { judgeEvidence, recordEvidence } from "./app/judge-evidence";
 export {

package/src/infra/restore-archive.ts

@@ -0,0 +1,132 @@
+import { Database } from "bun:sqlite";
+import { mkdir, mkdtemp, rm } from "node:fs/promises";
+import { resolve, dirname } from "node:path";
+import type { BenchmarkDefinition, EvalDefinition, EvalRunInput, OpenCodeStreamEvent } from "../types";
+import { EvidenceCapture } from "./evidence";
+import { extractWorkspaceArchive } from "./containers/transfer";
+import { removeCredentials } from "./opencode/auth";
+import { readArchivedSession } from "./opencode/archive";
+import { parseModel, type SessionResult } from "./opencode/session";
+import { hash, writeJson } from "./files";
+import { CandidateContainer } from "./containers/oci";
+import { createSessionDatabase } from "./opencode/host";
+
+export type ArchiveRestore = {
+  database: string;
+  databaseHash: string;
+  workspaceArchive: string;
+  workspaceArchiveHash: string;
+};
+
+/** Replay only frozen author preparation, without credentials or a model prompt. */
+export async function prepareRestoredInitial(
+  definition: BenchmarkDefinition,
+  item: EvalDefinition,
+  imageId: string,
+  workspace: string,
+  staging: string,
+) {
+  if (!item.settings.prepare?.length) return workspace;
+  await mkdir(staging, { recursive: true });
+  const database = resolve(staging, "opencode.db");
+  await createSessionDatabase(database, []);
+  await using container = await CandidateContainer.create(definition.container, imageId);
+  await container.prepare(workspace, database, definition.candidate.websearch,
+    item.settings.prepare, staging);
+  const initial = resolve(staging, "workspace");
+  await container.snapshot(initial, staging);
+  return initial;
+}
+
+/** A backup must belong to the original execution, not merely resemble its answer. */
+export function verifyArchivedInput(
+  input: EvalRunInput,
+  sessionId: string,
+  archived: { result: SessionResult; outcome: string | undefined },
+) {
+  const root = archived.result.sessions.find(session => session.info.id === sessionId);
+  const model = parseModel(input.model);
+  const prompt = root?.messages.find(message => message.type === "user")?.text;
+  if (!root || root.info.parentID || root.info.model?.providerID !== model.providerID ||
+      root.info.model?.id !== model.id || root.info.model?.variant !== model.variant ||
+      prompt !== input.prompt)
+    throw new Error("The native backup does not match the original session, model, and prompt");
+  if (archived.outcome !== "succeeded" || archived.result.state !== "completed")
+    throw new Error("Restore requires a naturally completed native session");
+}
+
+/** Finalize a copied native recording. No candidate program, prompt, or model runs. */
+export async function restoreArchivedCandidate(
+  input: EvalRunInput,
+  sessionId: string,
+  initialWorkspace: string,
+  options: ArchiveRestore,
+  directory: string,
+) {
+  for (const [file, expected] of [
+    [options.database, options.databaseHash],
+    [options.workspaceArchive, options.workspaceArchiveHash],
+  ]) {
+    if (!/^[a-f0-9]{64}$/.test(expected!) || hash(await Bun.file(file!).bytes()) !== expected)
+      throw new Error("The recovery backup hash does not match");
+  }
+  if (Bun.file(options.workspaceArchive).size > 256 * 1024 * 1024)
+    throw new Error("The recovery workspace archive exceeds 256 MiB");
+  const staging = await mkdtemp(resolve(dirname(directory), ".restore-"));
+  try {
+    await mkdir(directory, { recursive: true });
+    const database = resolve(directory, "opencode.db");
+    // SQLite serialization includes committed WAL pages without modifying the backup.
+    using source = new Database(options.database, { readonly: true });
+    await Bun.write(database, source.serialize());
+    removeCredentials(database);
+    const events: OpenCodeStreamEvent[] = [];
+    const archived = await readArchivedSession(database, sessionId, event => events.push(event));
+    verifyArchivedInput(input, sessionId, archived);
+    const creation = events.find((event): event is Extract<OpenCodeStreamEvent, { type: "session.created" }> =>
+      event.type === "session.created" && event.data.sessionID === sessionId);
+    if (!creation) throw new Error("The native backup has no root creation event");
+    const completedAt = archived.result.sessions.find(session => session.info.id === sessionId)!.info.time.idle;
+    if (typeof completedAt !== "number" || !Number.isFinite(completedAt))
+      throw new Error("The native session has no completion timestamp");
+    removeCredentials(database);
+    const workspace = resolve(staging, "workspace");
+    await mkdir(workspace);
+    await extractWorkspaceArchive(options.workspaceArchive, workspace, { sourceRoot: "/workspace" });
+    const capture = await EvidenceCapture.create(resolve(directory, "evidence"), initialWorkspace,
+      { excludeDirectories: ["node_modules"] });
+    for (const event of events)
+      capture.event(event, new Date("created" in event ? event.created : Date.now()).toISOString());
+    const manifest = await capture.finish({
+      prompt: input.prompt,
+      response: { text: archived.result.text },
+      tools: archived.result.tools,
+      workspace,
+    });
+    await writeJson(resolve(directory, "session.json"), archived.result.sessions);
+    const receipt = {
+      sessionId,
+      restoredAt: new Date().toISOString(),
+      completedAt: new Date(completedAt).toISOString(),
+      databaseHash: options.databaseHash,
+      workspaceArchiveHash: options.workspaceArchiveHash,
+      initial: "frozen input and author preparation replayed without model calls; not a newly observed candidate snapshot",
+    };
+    await writeJson(resolve(directory, "restoration.json"), receipt);
+    return {
+      evidence: { directory: "evidence", hash: manifest.sha256 },
+      session: {
+        sessionId,
+        database: "opencode.db",
+        databaseHash: hash(await Bun.file(database).bytes()),
+        opencodeVersion: creation.data.version,
+        accounting: archived.result.accounting,
+      },
+      events,
+      tools: archived.result.tools,
+      receipt,
+    };
+  } finally {
+    await rm(staging, { recursive: true, force: true });
+  }
+}

package/src/types.ts

@@ -232,6 +232,16 @@ export type EvalRun = {
   stop?: EvalStop;
   /** The runner stopped before this session finished; the planner collects it again. */
   interrupted?: boolean;
+  /** A restored native session; the original interrupted coordinator record is retained. */
+  restoration?: {
+    evalRunId: string;
+    sessionId: string;
+    restoredAt: string;
+    completedAt: string;
+    databaseHash: string;
+    workspaceArchiveHash: string;
+    initial: string;
+  };
 };
 export type CriterionDefinition = { id: string; name: string };
 export type EvidenceCitation = (