@hona/openeval 0.5.4 → 0.5.6

package/VERIFICATION.md

@@ -8,7 +8,7 @@ run it in a disposable OCI container rather than on the runner host.
 import { VERIFICATION_IMAGE, type Benchmark } from "@hona/openeval";
 export default {
   models: ["example/model"],
-  judge: { verification: { image: VERIFICATION_IMAGE, cpus: 2, memoryMiB: 4096, workspaceMiB: 2048 } },
+  judge: { verification: { image: VERIFICATION_IMAGE, cpus: 2, memoryMiB: 4096, workspaceMiB: 2048, inputMiB: 256 } },
 } satisfies Benchmark;
 ```
 
@@ -56,7 +56,7 @@ prefix does not necessarily establish final-task failure.
 - Commands, exit statuses, image/resource identity, logs, and requested regular
   output files are retained with the JudgeRun. Viewer previews never execute HTML
   or SVG from the delivered artifact.
-- Workspace transfer is limited to 128 MiB; each output archive is limited to
+- Workspace transfer defaults to 128 MiB; each output archive is limited to
   32 MiB; logs are limited to 1 MiB per stream and are marked when truncated.
 - Containers self-expire. The parent also removes containers bearing only its
   unique execution label after worker interruption. No shared/broad cleanup.
@@ -68,6 +68,15 @@ retained in verification receipts and code-judge identity. Omitting it preserves
 existing identities and limits. Network, credentials, privileges, transfer caps,
 and the read-only image filesystem are unchanged.
 
+For larger source snapshots, set `judge.verification.inputMiB` to a positive
+integer archive limit. This is separate from `workspaceMiB`: increasing storage
+alone does not increase the transfer limit. The configured byte count must be a
+safe integer. Non-default limits are retained in receipts and code-judge identity;
+omitting the field or setting it to 128 preserves existing identities. Output and
+log caps, filesystem permissions, network access, and container isolation stay
+unchanged. Provision enough workspace storage for the actual restored files and
+any verification-generated data.
+
 The standard image includes Bun 1.4.2, Python, Git, Node, and Playwright 1.63.0
 with Chromium. Import Playwright from
 `/opt/verify/node_modules/playwright/index.mjs`. Dependencies needed by a task
@@ -83,6 +92,13 @@ evidence. Do not put evaluator scripts in candidate workspaces.
 
 ## Readable judgments and controls
 
+Retained code judgments stay scored when a metadata-only identity change leaves
+the exact self-contained executable unchanged. The reader verifies both recorded
+identities and rejects changed source, external package imports, unknown metadata,
+or a different execution protocol. This check makes no model calls and rewrites
+no evidence. A completed code verification is still required. Package/lockfile
+metadata alone must not make an unchanged historical scorecard appear unscored.
+
 `openeval prepare --output <new-directory>` assembles real candidate inputs and
 runs declared preparation in the candidate image, then archives the prepared
 workspace. `--only-eval` scopes it. This makes zero model calls, creates no

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.5.4",
+  "version": "0.5.6",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/load-benchmark.ts

@@ -18,6 +18,7 @@ import {
 import { compileCodeJudge } from "../infra/judging/code-source";
 import { readCodeCriteria } from "../infra/judging/code-criteria";
 import { RUNTIME_IMAGE } from "../infra/opencode/version";
+import { DEFAULT_INPUT_MIB, verificationInputMiB } from "../infra/verification/limits";
 import { monitorPolicy } from "./monitor-policy";
 import {
   fingerprint,
@@ -364,7 +365,7 @@ export async function loadBenchmark(
   const verification = definition.judge?.verification;
   if (verification && (
     typeof verification !== "object" || Array.isArray(verification) ||
-    Object.keys(verification).some(key => !["image", "engine", "cpus", "memoryMiB", "workspaceMiB"].includes(key)) ||
+    Object.keys(verification).some(key => !["image", "engine", "cpus", "memoryMiB", "workspaceMiB", "inputMiB"].includes(key)) ||
     typeof verification.image !== "string" || !verification.image.trim() || /\s/.test(verification.image) ||
     !["docker", "podman"].includes(verification.engine ?? engine)
   )) throw new Error("judge.verification requires an image and valid container limits");
@@ -372,6 +373,7 @@ export async function loadBenchmark(
   if (verification?.workspaceMiB !== undefined &&
     (!Number.isSafeInteger(verification.workspaceMiB) || verification.workspaceMiB < 1 || verification.workspaceMiB > verificationMemory))
     throw new Error("Verification workspaceMiB must be a positive integer within memoryMiB");
+  const inputMiB = verificationInputMiB(verification?.inputMiB);
   for (const search of [
     definition.candidate?.websearch,
     definition.judge?.websearch,
@@ -409,6 +411,7 @@ export async function loadBenchmark(
         cpus: positive(verification.cpus, 2, "Verification CPU count"),
         memoryMiB: verificationMemory,
         ...(verification.workspaceMiB === undefined ? {} : { workspaceMiB: verification.workspaceMiB }),
+        ...(inputMiB === DEFAULT_INPUT_MIB ? {} : { inputMiB }),
       } } : {}),
     },
     container: {

package/src/app/scores.ts

@@ -1,7 +1,9 @@
 import type { BenchmarkDefinition, JudgeRun, Slot } from "../types";
+import type { CodeJudgeDefinition } from "../judge-context";
 import { modelScore } from "../view";
 import { isScored } from "../judgment";
 import { categoryKey, inCategories } from "../criterion-categories";
+import { recordedCodeMatches } from "../infra/judging/code-source";
 
 /** Scores are derived from a single selection snapshot; every eval has equal weight. */
 export function benchmarkScores(
@@ -14,12 +16,23 @@ export function benchmarkScores(
   const evals = definition.evals.filter(
     (item) => !evalId || item.id === evalId,
   );
-  // A code judgment counts only when it ran the eval's current judge.ts to completion.
+  const matches = new WeakMap<CodeJudgeDefinition, WeakMap<CodeJudgeDefinition, boolean>>();
+  const equivalent = (expected: CodeJudgeDefinition, recorded: CodeJudgeDefinition | undefined) => {
+    if (!recorded) return false;
+    let previous = matches.get(expected);
+    if (!previous) { previous = new WeakMap(); matches.set(expected, previous); }
+    const cached = previous.get(recorded);
+    if (cached !== undefined) return cached;
+    const value = recordedCodeMatches(expected, recorded);
+    previous.set(recorded, value);
+    return value;
+  };
+  // A code judgment counts only after completed verification of the matching executable.
   const current = (item: (typeof evals)[number], slot: Slot) => {
     const judge = slot.judgeRunId ? index.get(slot.judgeRunId) : undefined;
     return !item.code ||
       (judge?.code?.state === "completed" &&
-        judge.input.code?.hash === item.code.hash)
+         equivalent(item.code, judge.input.code))
       ? judge
       : undefined;
   };

package/src/infra/judging/code-source.ts

@@ -10,6 +10,37 @@ const printer = new Bun.Transpiler({
   deadCodeElimination: false,
 });
 
+function portableDirectories(source: string) {
+  return source.replace(
+    /^\s*var __(?:dirname|filename) = .*$/gm,
+    (line) => line.replace(/"(?:[^"\\]|\\.)*"/g, '""'),
+  );
+}
+
+/** An unchanged self-contained executable can retain judgments whose older
+ * identity included host package manifests and lockfiles. This is a read-only
+ * identity check, not a rewrite of recorded code or a changed scoring rule.
+ * External packages still require their exact executable identity. */
+export function recordedCodeMatches(
+  expected: CodeJudgeDefinition,
+  recorded: CodeJudgeDefinition | undefined,
+) {
+  if (!recorded) return false;
+  if (recorded.hash === expected.hash) return true;
+  if (!expected.source.trim() || expected.source !== recorded.source ||
+      Object.keys(expected.dependencies).length !== 0) return false;
+  const imports = new Bun.Transpiler({ loader: "js" }).scan(expected.source).imports;
+  if (imports.some(item => !item.path.startsWith("node:") && !item.path.startsWith("bun:"))) return false;
+  const metadata = Object.keys(recorded.dependencies);
+  if (!metadata.length || metadata.some(path =>
+    !/(?:^|[\\/])(?:package\.json|bun\.lockb?|package-lock\.json|pnpm-lock\.yaml|yarn\.lock)$/.test(path))) return false;
+  // Verify both identities rather than accepting an arbitrary mismatched hash.
+  return CODE_JUDGE_PROTOCOL === 1 &&
+    recorded.hash === fingerprint({ source: recorded.source, dependencies: recorded.dependencies }) &&
+    expected.hash === fingerprint({ protocol: CODE_JUDGE_PROTOCOL,
+      source: printer.transformSync(portableDirectories(expected.source)) });
+}
+
 /** The package that provides an external import, as `name@version/path`. */
 async function packageSpecifier(path: string, from: string) {
   for (let directory = dirname(path); ; directory = dirname(directory)) {
@@ -93,10 +124,7 @@ export async function compileCodeJudge(
     );
   }
   // Bun writes __dirname and __filename as absolute paths. Like other runtime file inputs, they are not identity.
-  portable = portable.replace(
-    /^\s*var __(?:dirname|filename) = .*$/gm,
-    (line) => line.replace(/"(?:[^"\\]|\\.)*"/g, '""'),
-  );
+  portable = portableDirectories(portable);
   return {
     file,
     source,

package/src/infra/verification/image.ts

@@ -2,6 +2,7 @@ import { fileURLToPath } from "node:url";
 import type { VerificationEnvironment, VerificationRuntime } from "../../judge-context";
 import { engineCommand } from "../containers/docker";
 import { treeHash } from "../files";
+import { DEFAULT_INPUT_MIB, verificationInputMiB } from "./limits";
 
 export const VERIFICATION_IMAGE = "openeval-verification:0.5.0";
 const directory = fileURLToPath(new URL("./runtime/", import.meta.url));
@@ -25,6 +26,7 @@ export async function verificationRuntime(environment?: VerificationEnvironment)
     (!Number.isSafeInteger(environment.workspaceMiB) || environment.workspaceMiB < 1 ||
       environment.workspaceMiB > (environment.memoryMiB ?? 4096)))
     throw new Error("Verification workspaceMiB must be a positive integer within memoryMiB");
+  const inputMiB = verificationInputMiB(environment.inputMiB);
   const output = await engineCommand(engine, [
     "image", "inspect", environment.image, "--format",
     '{{.Id}}|{{index .Config.Labels "openeval.verification"}}',
@@ -37,5 +39,6 @@ export async function verificationRuntime(environment?: VerificationEnvironment)
     engine, image: environment.image, imageId,
     cpus: environment.cpus ?? 2, memoryMiB: environment.memoryMiB ?? 4096,
     ...(environment.workspaceMiB === undefined ? {} : { workspaceMiB: environment.workspaceMiB }),
+    ...(inputMiB === DEFAULT_INPUT_MIB ? {} : { inputMiB }),
   };
 }

package/src/infra/verification/limits.ts

@@ -0,0 +1,14 @@
+export const DEFAULT_INPUT_MIB = 128;
+const MIB = 1024 * 1024;
+
+/** Restored workspace archives stay bounded, including opt-in larger inputs. */
+export function verificationInputMiB(value?: number): number {
+  const limit = value === undefined ? DEFAULT_INPUT_MIB : value;
+  if (!Number.isSafeInteger(limit) || limit < 1 || !Number.isSafeInteger(limit * MIB))
+    throw new Error("Verification inputMiB must be a positive integer with a safe byte count");
+  return limit;
+}
+
+export function verificationInputBytes(value?: number): number {
+  return verificationInputMiB(value) * MIB;
+}

package/src/infra/verification/session.ts

@@ -6,10 +6,10 @@ import type { Engine } from "../../types";
 import { engineCommand } from "../containers/docker";
 import { extractWorkspaceArchive } from "../containers/transfer";
 import { contained, hash, relativePath, writeJson } from "../files";
+import { verificationInputBytes, verificationInputMiB } from "./limits";
 
 const LOG_BYTES = 1024 * 1024;
 const OUTPUT_BYTES = 32 * 1024 * 1024;
-const INPUT_BYTES = 128 * 1024 * 1024;
 
 export function verificationPath(value: string): string {
   relativePath(value, "Verification path");
@@ -116,8 +116,8 @@ export function verificationSession(options: {
         const archive = resolve(staging, "workspace.tar");
         const pack = Bun.spawn(["tar", "-cf", archive, "-C", workspace, "."], { stdout: "ignore", stderr: "pipe" });
         const [packed, packError] = await Promise.all([pack.exited, new Response(pack.stderr).text()]);
-        if (packed || (await lstat(archive)).size > INPUT_BYTES)
-          throw new Error(packed ? `Verification transfer failed: ${packError}` : "Restored verification workspace exceeds 128 MiB");
+        if (packed || (await lstat(archive)).size > verificationInputBytes(runtime.inputMiB))
+          throw new Error(packed ? `Verification transfer failed: ${packError}` : `Restored verification workspace exceeds ${verificationInputMiB(runtime.inputMiB)} MiB`);
         const name = `openeval-verify-${randomUUID()}`;
         const startedAt = new Date().toISOString();
         const deadlineAt = Date.now() + timeoutMs;

package/src/judge-context.ts

@@ -41,6 +41,8 @@ export type VerificationEnvironment = {
   memoryMiB?: number;
   /** Workspace tmpfs capacity in MiB; default 512. A custom value must fit within memoryMiB. */
   workspaceMiB?: number;
+  /** Maximum restored workspace archive size in MiB; default 128. */
+  inputMiB?: number;
 };
 export type VerificationRuntime = {
   engine: Engine;
@@ -49,6 +51,7 @@ export type VerificationRuntime = {
   cpus: number;
   memoryMiB: number;
   workspaceMiB?: number;
+  inputMiB?: number;
 };
 export type VerificationRequest = {
   revision?: Revision;