@hona/openeval 0.5.4 → 0.5.5

package/VERIFICATION.md

@@ -8,7 +8,7 @@ run it in a disposable OCI container rather than on the runner host.
 import { VERIFICATION_IMAGE, type Benchmark } from "@hona/openeval";
 export default {
   models: ["example/model"],
-  judge: { verification: { image: VERIFICATION_IMAGE, cpus: 2, memoryMiB: 4096, workspaceMiB: 2048 } },
+  judge: { verification: { image: VERIFICATION_IMAGE, cpus: 2, memoryMiB: 4096, workspaceMiB: 2048, inputMiB: 256 } },
 } satisfies Benchmark;
 ```
 
@@ -56,7 +56,7 @@ prefix does not necessarily establish final-task failure.
 - Commands, exit statuses, image/resource identity, logs, and requested regular
   output files are retained with the JudgeRun. Viewer previews never execute HTML
   or SVG from the delivered artifact.
-- Workspace transfer is limited to 128 MiB; each output archive is limited to
+- Workspace transfer defaults to 128 MiB; each output archive is limited to
   32 MiB; logs are limited to 1 MiB per stream and are marked when truncated.
 - Containers self-expire. The parent also removes containers bearing only its
   unique execution label after worker interruption. No shared/broad cleanup.
@@ -68,6 +68,15 @@ retained in verification receipts and code-judge identity. Omitting it preserves
 existing identities and limits. Network, credentials, privileges, transfer caps,
 and the read-only image filesystem are unchanged.
 
+For larger source snapshots, set `judge.verification.inputMiB` to a positive
+integer archive limit. This is separate from `workspaceMiB`: increasing storage
+alone does not increase the transfer limit. The configured byte count must be a
+safe integer. Non-default limits are retained in receipts and code-judge identity;
+omitting the field or setting it to 128 preserves existing identities. Output and
+log caps, filesystem permissions, network access, and container isolation stay
+unchanged. Provision enough workspace storage for the actual restored files and
+any verification-generated data.
+
 The standard image includes Bun 1.4.2, Python, Git, Node, and Playwright 1.63.0
 with Chromium. Import Playwright from
 `/opt/verify/node_modules/playwright/index.mjs`. Dependencies needed by a task

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.5.4",
+  "version": "0.5.5",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/load-benchmark.ts

@@ -18,6 +18,7 @@ import {
 import { compileCodeJudge } from "../infra/judging/code-source";
 import { readCodeCriteria } from "../infra/judging/code-criteria";
 import { RUNTIME_IMAGE } from "../infra/opencode/version";
+import { DEFAULT_INPUT_MIB, verificationInputMiB } from "../infra/verification/limits";
 import { monitorPolicy } from "./monitor-policy";
 import {
   fingerprint,
@@ -364,7 +365,7 @@ export async function loadBenchmark(
   const verification = definition.judge?.verification;
   if (verification && (
     typeof verification !== "object" || Array.isArray(verification) ||
-    Object.keys(verification).some(key => !["image", "engine", "cpus", "memoryMiB", "workspaceMiB"].includes(key)) ||
+    Object.keys(verification).some(key => !["image", "engine", "cpus", "memoryMiB", "workspaceMiB", "inputMiB"].includes(key)) ||
     typeof verification.image !== "string" || !verification.image.trim() || /\s/.test(verification.image) ||
     !["docker", "podman"].includes(verification.engine ?? engine)
   )) throw new Error("judge.verification requires an image and valid container limits");
@@ -372,6 +373,7 @@ export async function loadBenchmark(
   if (verification?.workspaceMiB !== undefined &&
     (!Number.isSafeInteger(verification.workspaceMiB) || verification.workspaceMiB < 1 || verification.workspaceMiB > verificationMemory))
     throw new Error("Verification workspaceMiB must be a positive integer within memoryMiB");
+  const inputMiB = verificationInputMiB(verification?.inputMiB);
   for (const search of [
     definition.candidate?.websearch,
     definition.judge?.websearch,
@@ -409,6 +411,7 @@ export async function loadBenchmark(
         cpus: positive(verification.cpus, 2, "Verification CPU count"),
         memoryMiB: verificationMemory,
         ...(verification.workspaceMiB === undefined ? {} : { workspaceMiB: verification.workspaceMiB }),
+        ...(inputMiB === DEFAULT_INPUT_MIB ? {} : { inputMiB }),
       } } : {}),
     },
     container: {

package/src/infra/verification/image.ts

@@ -2,6 +2,7 @@ import { fileURLToPath } from "node:url";
 import type { VerificationEnvironment, VerificationRuntime } from "../../judge-context";
 import { engineCommand } from "../containers/docker";
 import { treeHash } from "../files";
+import { DEFAULT_INPUT_MIB, verificationInputMiB } from "./limits";
 
 export const VERIFICATION_IMAGE = "openeval-verification:0.5.0";
 const directory = fileURLToPath(new URL("./runtime/", import.meta.url));
@@ -25,6 +26,7 @@ export async function verificationRuntime(environment?: VerificationEnvironment)
     (!Number.isSafeInteger(environment.workspaceMiB) || environment.workspaceMiB < 1 ||
       environment.workspaceMiB > (environment.memoryMiB ?? 4096)))
     throw new Error("Verification workspaceMiB must be a positive integer within memoryMiB");
+  const inputMiB = verificationInputMiB(environment.inputMiB);
   const output = await engineCommand(engine, [
     "image", "inspect", environment.image, "--format",
     '{{.Id}}|{{index .Config.Labels "openeval.verification"}}',
@@ -37,5 +39,6 @@ export async function verificationRuntime(environment?: VerificationEnvironment)
     engine, image: environment.image, imageId,
     cpus: environment.cpus ?? 2, memoryMiB: environment.memoryMiB ?? 4096,
     ...(environment.workspaceMiB === undefined ? {} : { workspaceMiB: environment.workspaceMiB }),
+    ...(inputMiB === DEFAULT_INPUT_MIB ? {} : { inputMiB }),
   };
 }

package/src/infra/verification/limits.ts

@@ -0,0 +1,14 @@
+export const DEFAULT_INPUT_MIB = 128;
+const MIB = 1024 * 1024;
+
+/** Restored workspace archives stay bounded, including opt-in larger inputs. */
+export function verificationInputMiB(value?: number): number {
+  const limit = value === undefined ? DEFAULT_INPUT_MIB : value;
+  if (!Number.isSafeInteger(limit) || limit < 1 || !Number.isSafeInteger(limit * MIB))
+    throw new Error("Verification inputMiB must be a positive integer with a safe byte count");
+  return limit;
+}
+
+export function verificationInputBytes(value?: number): number {
+  return verificationInputMiB(value) * MIB;
+}

package/src/infra/verification/session.ts

@@ -6,10 +6,10 @@ import type { Engine } from "../../types";
 import { engineCommand } from "../containers/docker";
 import { extractWorkspaceArchive } from "../containers/transfer";
 import { contained, hash, relativePath, writeJson } from "../files";
+import { verificationInputBytes, verificationInputMiB } from "./limits";
 
 const LOG_BYTES = 1024 * 1024;
 const OUTPUT_BYTES = 32 * 1024 * 1024;
-const INPUT_BYTES = 128 * 1024 * 1024;
 
 export function verificationPath(value: string): string {
   relativePath(value, "Verification path");
@@ -116,8 +116,8 @@ export function verificationSession(options: {
         const archive = resolve(staging, "workspace.tar");
         const pack = Bun.spawn(["tar", "-cf", archive, "-C", workspace, "."], { stdout: "ignore", stderr: "pipe" });
         const [packed, packError] = await Promise.all([pack.exited, new Response(pack.stderr).text()]);
-        if (packed || (await lstat(archive)).size > INPUT_BYTES)
-          throw new Error(packed ? `Verification transfer failed: ${packError}` : "Restored verification workspace exceeds 128 MiB");
+        if (packed || (await lstat(archive)).size > verificationInputBytes(runtime.inputMiB))
+          throw new Error(packed ? `Verification transfer failed: ${packError}` : `Restored verification workspace exceeds ${verificationInputMiB(runtime.inputMiB)} MiB`);
         const name = `openeval-verify-${randomUUID()}`;
         const startedAt = new Date().toISOString();
         const deadlineAt = Date.now() + timeoutMs;

package/src/judge-context.ts

@@ -41,6 +41,8 @@ export type VerificationEnvironment = {
   memoryMiB?: number;
   /** Workspace tmpfs capacity in MiB; default 512. A custom value must fit within memoryMiB. */
   workspaceMiB?: number;
+  /** Maximum restored workspace archive size in MiB; default 128. */
+  inputMiB?: number;
 };
 export type VerificationRuntime = {
   engine: Engine;
@@ -49,6 +51,7 @@ export type VerificationRuntime = {
   cpus: number;
   memoryMiB: number;
   workspaceMiB?: number;
+  inputMiB?: number;
 };
 export type VerificationRequest = {
   revision?: Revision;