@hona/openeval 0.4.0 → 0.5.0

package/src/infra/judging/code-source.ts

@@ -15,7 +15,9 @@ export async function compileCodeJudge(
   dependencies[sdkManifest] = await Bun.file(sdkManifest).text();
   const external = new Set<string>();
   const built = await Bun.build({
-    entrypoints: [file],
+    // Import only the judge function: unused reporting metadata is tree-shaken.
+    // If grading reads metadata, it remains executable and must affect the hash.
+    entrypoints: ["openeval:judge-entry"],
     root: dirname(file),
     target: "bun",
     format: "esm",
@@ -25,6 +27,11 @@ export async function compileCodeJudge(
       {
         name: "judge-dependencies",
         setup(build) {
+          build.onResolve({ filter: /^openeval:judge-entry$/ }, () => ({ path: "judge.js", namespace: "openeval-judge" }));
+          build.onLoad({ filter: /.*/, namespace: "openeval-judge" }, () => ({
+            contents: `export { default } from ${JSON.stringify(file.replaceAll("\\", "/"))};`,
+            loader: "js", resolveDir: dirname(file),
+          }));
           build.onResolve({ filter: /^[^./]/ }, (args) => {
             if (
               args.path.startsWith("node:") ||
@@ -91,6 +98,7 @@ export async function compileCodeJudge(
     source,
     sourceMap,
     dependencies,
-    hash: fingerprint({ source, sourceMap, dependencies }),
+    // Debug mappings and original-source metadata are retained, but not executable inputs.
+    hash: fingerprint({ source, dependencies }),
   };
 }

package/src/infra/judging/code-worker.ts

@@ -2,25 +2,27 @@ import { pathToFileURL } from "node:url";
 import { openRecording, type RecordingInput } from "../recording";
 import { jsonOutput } from "./code-result";
 import { errorMessage, writeJson } from "../files";
-import type { JudgeFunction, RunMetrics } from "../../judge-context";
+import type { JudgeFunction, RunMetrics, VerificationResult } from "../../judge-context";
 
 type Job = RecordingInput & { module: string; output: string; scratch: string };
 const job: Job = JSON.parse(await Bun.stdin.text());
 let metrics: RunMetrics | undefined;
+let verifications: VerificationResult[] | undefined;
 async function grade() {
   await using context = await openRecording(job, job.scratch);
   metrics = context.metrics;
+  verifications = context.verificationResults;
   const module = await import(pathToFileURL(job.module).href);
   if (typeof module.default !== "function")
     throw new Error("judge.ts must default-export a function");
   const output = jsonOutput(await (module.default as JudgeFunction)(context));
-  return { output, metrics: context.metrics };
+  return { output, metrics: context.metrics, verifications: context.verificationResults };
 }
 try {
   await writeJson(job.output, await grade());
   process.exit(0);
 } catch (error) {
-  await writeJson(job.output, { error: errorMessage(error), metrics });
+  await writeJson(job.output, { error: errorMessage(error), metrics, verifications });
   console.error(error);
   process.exit(1);
 }

package/src/infra/judging/code.ts

@@ -1,15 +1,19 @@
-import { mkdir, rm } from "node:fs/promises";
+import { mkdir, rm, readdir } from "node:fs/promises";
 import { resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+import { randomUUID } from "node:crypto";
 import type {
   CodeJudgeDefinition,
   CodeJudgeExecution,
   JsonValue,
   RunMetrics,
+  VerificationRuntime,
+  VerificationResult,
 } from "../../judge-context";
 import type { RecordingInput } from "../recording";
 import { codeJudgment, jsonOutput } from "./code-result";
 import { errorMessage, writeJson } from "../files";
+import { closeVerification } from "../verification/session";
 
 /** A separate process bounds synchronous loops as well as asynchronous judges. */
 export async function executeCodeJudge(
@@ -17,6 +21,7 @@ export async function executeCodeJudge(
   recording: RecordingInput,
   directory: string,
   timeoutMs: number,
+  verification?: VerificationRuntime,
 ): Promise<CodeJudgeExecution> {
   directory = resolve(directory);
   await mkdir(directory, { recursive: true });
@@ -31,7 +36,8 @@ export async function executeCodeJudge(
   const startedAt = new Date().toISOString();
   let timer: ReturnType<typeof setTimeout> | undefined;
   let timedOut = false;
-  let raw: JsonValue | undefined, metrics: RunMetrics | undefined;
+  let raw: JsonValue | undefined, metrics: RunMetrics | undefined, verifications: VerificationResult[] | undefined;
+  const owner = randomUUID();
   const base = { startedAt, sourceHash: code.hash, stdout, stderr };
   try {
     const child = Bun.spawn(
@@ -52,7 +58,9 @@ export async function executeCodeJudge(
       child.kill();
     }, timeoutMs);
     await child.stdin.write(
-      JSON.stringify({ ...recording, module, output, scratch }),
+      JSON.stringify({ ...recording, module, output, scratch, verification: {
+        environment: verification, directory, owner, deadlineAt: Date.now() + timeoutMs,
+      } }),
     );
     child.stdin.end();
     const exit = await child.exited;
@@ -60,6 +68,7 @@ export async function executeCodeJudge(
       throw new Error(`judge.ts exceeded its ${timeoutMs}ms time limit`);
     const result = await Bun.file(output).json();
     metrics = result.metrics;
+    verifications = result.verifications;
     if (exit !== 0 || result.error)
       throw new Error(result.error ?? `judge.ts exited with code ${exit}`);
     raw = jsonOutput(result.output);
@@ -71,11 +80,16 @@ export async function executeCodeJudge(
       elapsedMs: Date.now() - Date.parse(startedAt),
       output: raw,
       metrics,
+      verifications,
       scores: Object.fromEntries(
         Object.entries(judgment.scores).map(([id, score]) => [id, score.value]),
       ),
     };
   } catch (error) {
+    // Completed receipts survive a killed worker or a later malformed submission.
+    verifications ??= await Promise.all((await readdir(resolve(directory, "verification")).catch(() => []))
+      .map(id => Bun.file(resolve(directory, "verification", id, "receipt.json")).json().catch(() => undefined)))
+      .then(items => items.filter((item): item is VerificationResult => !!item));
     return {
       ...base,
       state: timedOut ? "timed_out" : "failed",
@@ -84,9 +98,11 @@ export async function executeCodeJudge(
       error: errorMessage(error),
       output: raw,
       metrics,
+      verifications,
     };
   } finally {
     clearTimeout(timer);
+    if (verification) await closeVerification(verification.engine, owner).catch(() => {});
     await rm(scratch, {
       recursive: true,
       force: true,

package/src/infra/judging/contract.ts

@@ -81,10 +81,10 @@ export function validateCriteria(
   };
 }
 
-function parseCitation(value: unknown): EvidenceCitation {
+export function parseCitation(value: unknown): EvidenceCitation {
   if (
     !object(value) ||
-    !["response", "tool", "message", "event", "artifact", "metric"].includes(
+    !["response", "recording", "tool", "message", "event", "artifact", "metric", "verification"].includes(
       String(value.kind),
     )
   )
@@ -104,6 +104,12 @@ function parseCitation(value: unknown): EvidenceCitation {
     ...(value.offset !== undefined ? { offset: value.offset as number } : {}),
   };
   if (value.kind === "response") return { kind: "response", ...extra };
+  if (value.kind === "recording") return { kind: "recording", ...extra };
+  if (value.kind === "verification") {
+    if (!reason(value.id) || (value.path !== undefined && !reason(value.path)))
+      throw new Error("Verification citations require an ID and an optional output path");
+    return { kind: "verification", id: value.id, ...(value.path ? { path: value.path as string } : {}), ...extra };
+  }
   if (value.kind === "artifact") {
     if (
       !reason(value.path) ||
@@ -134,6 +140,8 @@ export async function validateCitations(
 ) {
   for (const [id, score] of Object.entries(judgment.scores))
     for (const citation of score.evidence) {
+      if (citation.kind === "verification")
+        throw new Error("Verification citations must be checked against their JudgeRun receipts, not candidate evidence");
       const query: EvidenceQuery =
         citation.kind === "recording"
           ? { action: "summary" }

package/src/infra/judging/index.ts

@@ -39,6 +39,7 @@ export const judgingFingerprint = async () =>
     composition: await Bun.file(
       fileURLToPath(new URL("../../app/grade-recording.ts", import.meta.url)),
     ).text(),
+    verification: await treeHash(fileURLToPath(new URL("../verification/", import.meta.url)), { ignore: /\.test\.[jt]sx?$/ }),
     useCase: await Bun.file(
       fileURLToPath(new URL("../../app/judge-run.ts", import.meta.url)),
     ).text(),

package/src/infra/recording/index.ts

@@ -6,24 +6,28 @@ import type {
   JudgeContext,
   RecordedMessage,
   Revision,
+  VerificationRuntime,
+  VerificationResult,
 } from "../../judge-context";
 import type { EvalRun, EvidenceRef } from "../../types";
 import { CandidateEvidence } from "../evidence";
 import { contained, hash } from "../files";
 import { OPENCODE_VERSION } from "../opencode/version";
 import { measureRecording } from "./metrics";
+import { verificationSession } from "../verification/session";
 
 export type RecordingInput = {
   evidence: EvidenceRef;
   run?: EvalRun;
   runDirectory?: string;
+  verification?: { environment?: VerificationRuntime; directory: string; owner: string; deadlineAt: number };
 };
 
 /** Own disposable native readers; the authoritative archive is only read. */
 export async function openRecording(
   input: RecordingInput,
   workDirectory: string,
-): Promise<JudgeContext & AsyncDisposable> {
+): Promise<JudgeContext & AsyncDisposable & { verificationResults: VerificationResult[] }> {
   const reference = {
     ...input.evidence,
     directory: resolve(input.evidence.directory),
@@ -86,7 +90,7 @@ export async function openRecording(
     new TextDecoder("utf-8", { fatal: true }).decode(
       await evidence.readFile(path, revision),
     );
-  return {
+  const context: Omit<JudgeContext, "verification"> & AsyncDisposable = {
     response: { text: evidence.manifest.response?.text ?? "" },
     prompt: evidence.manifest.prompt,
     run: run ? structuredClone(run) : null,
@@ -237,4 +241,12 @@ export async function openRecording(
       }
     },
   };
+  const verifier = verificationSession({
+    ...input.verification,
+    directory: input.verification?.directory ?? workDirectory,
+    owner: input.verification?.owner ?? "unconfigured",
+    deadlineAt: input.verification?.deadlineAt ?? Date.now() + 120_000,
+    materialize: context.workspace.materialize,
+  });
+  return { ...context, verification: verifier.context, verificationResults: verifier.results };
 }

package/src/infra/verification/image.ts

@@ -0,0 +1,33 @@
+import { fileURLToPath } from "node:url";
+import type { VerificationEnvironment, VerificationRuntime } from "../../judge-context";
+import { engineCommand } from "../containers/docker";
+import { treeHash } from "../files";
+
+export const VERIFICATION_IMAGE = "openeval-verification:0.5.0";
+const directory = fileURLToPath(new URL("./runtime/", import.meta.url));
+
+export async function buildVerificationImage(environment: VerificationEnvironment) {
+  if (environment.image !== VERIFICATION_IMAGE)
+    throw new Error("Build custom verification images with your container engine; image --verification builds the standard image only");
+  await engineCommand(environment.engine ?? "docker", [
+    "build", "--label", `openeval.verification=${await treeHash(directory)}`,
+    "-t", environment.image, directory,
+  ], { timeoutMs: 600_000 });
+}
+
+export async function verificationRuntime(environment?: VerificationEnvironment): Promise<VerificationRuntime | undefined> {
+  if (!environment) return undefined;
+  const engine = environment.engine ?? "docker";
+  if (!["docker", "podman"].includes(engine) || !environment.image || /\s/.test(environment.image) ||
+    [environment.cpus ?? 2, environment.memoryMiB ?? 4096].some(value => !Number.isSafeInteger(value) || value < 1))
+    throw new Error("Verification requires a valid image, engine, and positive integer resource limits");
+  const output = await engineCommand(engine, [
+    "image", "inspect", environment.image, "--format",
+    '{{.Id}}|{{index .Config.Labels "openeval.verification"}}',
+  ]);
+  const [imageId, label] = output.split("|");
+  if (!/^sha256:[a-f0-9]{64}$/.test(imageId)) throw new Error("Verification image has no immutable image ID");
+  if (environment.image === VERIFICATION_IMAGE && label !== await treeHash(directory))
+    throw new Error("Verification runtime changed. Run image --verification first.");
+  return { engine, image: environment.image, imageId, cpus: environment.cpus ?? 2, memoryMiB: environment.memoryMiB ?? 4096 };
+}

package/src/infra/verification/runtime/Dockerfile

@@ -0,0 +1,18 @@
+FROM mcr.microsoft.com/playwright:v1.63.0-noble
+COPY --from=oven/bun:1.4.2 /usr/local/bin/bun /usr/local/bin/bun
+RUN apt-get update && apt-get install -y --no-install-recommends git ripgrep python3 g++ make tar coreutils \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /opt/verify
+COPY package.json bun.lock ./
+RUN bun install --frozen-lockfile --ignore-scripts \
+    && useradd --create-home --uid 10001 verify \
+    && mkdir -p /workspace /verification \
+    && chown verify:verify /workspace \
+    && git config --system --add safe.directory '*'
+ENV HOME=/tmp/home
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
+ENV NODE_PATH=/opt/verify/node_modules
+WORKDIR /workspace
+USER 10001:10001
+ENTRYPOINT []
+CMD ["sleep", "infinity"]

package/src/infra/verification/runtime/bun.lock

@@ -0,0 +1,16 @@
+{
+  "lockfileVersion": 2,
+  "configVersion": 1,
+  "workspaces": {
+    "": {
+      "dependencies": {
+        "playwright": "1.63.0",
+      },
+    },
+  },
+  "packages": {
+    "playwright": ["playwright@1.63.0", "", { "dependencies": { "playwright-core": "1.63.0" }, "bin": { "playwright": "cli.js" } }, "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg=="],
+
+    "playwright-core": ["playwright-core@1.63.0", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg=="],
+  }
+}

package/src/infra/verification/runtime/package.json

@@ -0,0 +1,5 @@
+{
+  "private": true,
+  "type": "module",
+  "dependencies": { "playwright": "1.63.0" }
+}

package/src/infra/verification/session.ts

@@ -0,0 +1,199 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, rm, lstat } from "node:fs/promises";
+import { resolve, dirname, posix } from "node:path";
+import type { VerificationRequest, VerificationResult, VerificationRuntime } from "../../judge-context";
+import type { Engine } from "../../types";
+import { engineCommand } from "../containers/docker";
+import { extractWorkspaceArchive } from "../containers/transfer";
+import { contained, hash, relativePath, writeJson } from "../files";
+
+const LOG_BYTES = 1024 * 1024;
+const OUTPUT_BYTES = 32 * 1024 * 1024;
+const INPUT_BYTES = 128 * 1024 * 1024;
+
+export function verificationPath(value: string): string {
+  relativePath(value, "Verification path");
+  if (value.includes("\\") || value.includes("\0") || value.startsWith("/") || /^[a-z]:/i.test(value))
+    throw new Error("Verification paths must use contained POSIX-relative paths");
+  return posix.normalize(value);
+}
+
+/** No host mounts, credentials, published ports, network, or elevated capabilities. */
+export function verificationArgs(runtime: VerificationRuntime, owner: string, name: string, timeoutMs: number) {
+  return [
+    "run", "--detach", "--rm", "--init", "--name", name,
+    "--label", `openeval.verification-owner=${owner}`,
+    "--network=none", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--read-only",
+    "--pids-limit", "128", "--cpus", String(runtime.cpus), "--memory", `${runtime.memoryMiB}m`,
+    "--memory-swap", `${runtime.memoryMiB}m`, "--user", "10001:10001",
+    "--tmpfs", "/workspace:rw,exec,nosuid,nodev,mode=1777,size=512m",
+    "--tmpfs", "/verification:rw,exec,nosuid,nodev,mode=0755,size=64m",
+    "--tmpfs", "/tmp:rw,exec,nosuid,nodev,mode=1777,size=256m",
+    "--env", "HOME=/tmp/home", "--env", "CI=1",
+    "--entrypoint", "timeout", runtime.imageId,
+    "--signal=TERM", "--kill-after=5s", `${Math.ceil(timeoutMs / 1000)}s`, "sleep", "infinity",
+  ];
+}
+
+async function bounded(stream: ReadableStream<Uint8Array>, limit: number, overflow?: () => void) {
+  const chunks: Uint8Array[] = [];
+  let bytes = 0, truncated = false;
+  for await (const chunk of stream) {
+    const room = Math.max(0, limit - bytes);
+    if (room) chunks.push(chunk.slice(0, room));
+    bytes += Math.min(room, chunk.length);
+    if (chunk.length > room) {
+      truncated = true;
+      overflow?.();
+    }
+  }
+  return { bytes: Buffer.concat(chunks), truncated };
+}
+
+/** Remove only containers owned by this code-judge execution, including after worker death. */
+export async function closeVerification(engine: Engine, owner: string) {
+  const names = await engineCommand(engine, ["ps", "-aq", "--filter", `label=openeval.verification-owner=${owner}`]);
+  if (names.trim()) await engineCommand(engine, ["rm", "--force", ...names.split(/\s+/)]);
+}
+
+export function verificationSession(options: {
+  environment?: VerificationRuntime;
+  directory: string;
+  owner: string;
+  deadlineAt: number;
+  materialize: (revision: "initial" | "final") => Promise<string>;
+}) {
+  const results: VerificationResult[] = [];
+  const outputs = new Map<string, VerificationResult>();
+  const read = async (result: VerificationResult, path: string) => {
+    const retained = outputs.get(result.id);
+    if (!retained) throw new Error("Read outputs from a verification produced by this context");
+    const artifact = retained.artifacts.find(item => item.path === path);
+    const file = artifact?.file ?? (path === "stdout" ? retained.stdout : path === "stderr" ? retained.stderr : undefined);
+    if (!file) throw new Error(`Verification did not retain ${path}`);
+    const bytes = await Bun.file(contained(options.directory, file)).bytes();
+    if (artifact && hash(bytes) !== artifact.sha256) throw new Error("Verification artifact hash mismatch");
+    return bytes;
+  };
+  return {
+    results,
+    context: {
+      read,
+      async text(result: VerificationResult, path: string) {
+        return new TextDecoder("utf-8", { fatal: true }).decode(await read(result, path));
+      },
+      async run(request: VerificationRequest): Promise<VerificationResult> {
+        const runtime = options.environment;
+        if (!runtime) throw new Error("Set judge.verification and build its image before verifying artifacts");
+        if (!Array.isArray(request.commands) || !request.commands.length || request.commands.some(argv =>
+          !Array.isArray(argv) || !argv.length || argv.some(arg => typeof arg !== "string" || !arg || arg.includes("\0"))))
+          throw new Error("Verification requires non-empty command argv arrays");
+        const requested = request.timeoutMs ?? 120_000;
+        if (!Number.isSafeInteger(requested) || requested < 1) throw new Error("Verification timeout must be a positive integer");
+        const timeoutMs = Math.min(requested, options.deadlineAt - Date.now());
+        if (timeoutMs <= 0) throw new Error("Code-judge deadline was reached before verification");
+        const cwd = verificationPath(request.cwd ?? ".");
+        const artifacts = (request.artifacts ?? []).map(verificationPath);
+        if (artifacts.some(path => path === "stdout" || path === "stderr"))
+          throw new Error("stdout and stderr are reserved verification log names");
+        const revision = request.revision ?? "final";
+        if (revision !== "initial" && revision !== "final") throw new Error("Unknown verification revision");
+        const id = `verification_${randomUUID()}`;
+        const directory = resolve(options.directory, "verification", id);
+        const staging = resolve(directory, ".transfer");
+        const trusted = resolve(staging, "trusted");
+        await mkdir(trusted, { recursive: true });
+        let trustedBytes = 0;
+        for (const [path, content] of Object.entries(request.files ?? {})) {
+          if (typeof content !== "string") throw new Error("Verification file contents must be text");
+          trustedBytes += Buffer.byteLength(content);
+          if (trustedBytes > OUTPUT_BYTES) throw new Error("Trusted verification inputs exceed 32 MiB");
+          const target = contained(trusted, verificationPath(path));
+          await mkdir(dirname(target), { recursive: true });
+          await Bun.write(target, content);
+        }
+        const workspace = await options.materialize(revision);
+        const archive = resolve(staging, "workspace.tar");
+        const pack = Bun.spawn(["tar", "-cf", archive, "-C", workspace, "."], { stdout: "ignore", stderr: "pipe" });
+        const [packed, packError] = await Promise.all([pack.exited, new Response(pack.stderr).text()]);
+        if (packed || (await lstat(archive)).size > INPUT_BYTES)
+          throw new Error(packed ? `Verification transfer failed: ${packError}` : "Restored verification workspace exceeds 128 MiB");
+        const name = `openeval-verify-${randomUUID()}`;
+        const startedAt = new Date().toISOString();
+        const deadlineAt = Date.now() + timeoutMs;
+        const commands: VerificationResult["commands"] = [];
+        let stdout = "", stderr = "", logsTruncated = false, timedOut = false, exitCode: number | null = null;
+        const result: VerificationResult = {
+          id, state: "completed", revision, environment: runtime, startedAt, elapsedMs: 0,
+          commands, exitCode, stdout: "", stderr: "", logsTruncated: false, artifacts: [], missingArtifacts: [],
+        };
+        try {
+          await engineCommand(runtime.engine, verificationArgs(runtime, options.owner, name, timeoutMs));
+          await engineCommand(runtime.engine, ["exec", "--interactive", "--user", "10001:10001", name,
+            "tar", "--no-same-owner", "--no-overwrite-dir", "-xf", "-", "-C", "/workspace"], { input: archive, timeoutMs });
+          // docker cp rejects a read-only rootfs even when its target is tmpfs.
+          // Only this trusted upload runs as root; artifact commands remain non-root.
+          const trustedArchive = resolve(staging, "trusted.tar");
+          const packTrusted = Bun.spawn(["tar", "-cf", trustedArchive, "-C", trusted, "."], { stdout: "ignore", stderr: "pipe" });
+          const [trustedExit, trustedError] = await Promise.all([packTrusted.exited, new Response(packTrusted.stderr).text()]);
+          if (trustedExit) throw new Error(`Cannot transfer trusted check inputs: ${trustedError}`);
+          await engineCommand(runtime.engine, ["exec", "--interactive", "--user", "0:0", name,
+            "tar", "--no-same-owner", "--no-overwrite-dir", "-xf", "-", "-C", "/verification"], { input: trustedArchive, timeoutMs });
+          for (const argv of request.commands) {
+            const started = Date.now();
+            const remaining = deadlineAt - started;
+            if (remaining <= 0) { timedOut = true; break; }
+            const child = Bun.spawn([runtime.engine, "exec", "--user", "10001:10001", "--workdir",
+              posix.join("/workspace", cwd), name, ...argv], { stdout: "pipe", stderr: "pipe", stdin: "ignore" });
+            const timer = setTimeout(() => { timedOut = true; child.kill(); }, remaining);
+            try {
+              const [code, out, err] = await Promise.all([child.exited, bounded(child.stdout, LOG_BYTES), bounded(child.stderr, LOG_BYTES)]);
+              exitCode = timedOut ? null : code;
+              commands.push({ argv: [...argv], exitCode, elapsedMs: Date.now() - started });
+              stdout += out.bytes.toString("utf8"); stderr += err.bytes.toString("utf8");
+              logsTruncated ||= out.truncated || err.truncated;
+              if (timedOut || code !== 0) break;
+            } finally { clearTimeout(timer); }
+          }
+          if (!timedOut) for (const path of artifacts) {
+            const tar = resolve(staging, `output-${randomUUID()}.tar`);
+            // Read inside the running mount namespace: docker cp cannot reliably
+            // see files in a container's tmpfs mounts.
+            const child = Bun.spawn([runtime.engine, "exec", "--user", "10001:10001", name,
+              "tar", "-cf", "-", "-C", posix.dirname(`/workspace/${path}`), "--", posix.basename(path)], { stdout: "pipe", stderr: "pipe" });
+            const timer = setTimeout(() => child.kill(), 30_000);
+            const [code, output, error] = await Promise.all([child.exited, bounded(child.stdout, OUTPUT_BYTES, () => child.kill()), bounded(child.stderr, LOG_BYTES)]).finally(() => clearTimeout(timer));
+            if (output.truncated) throw new Error("Verification output exceeds 32 MiB");
+            if (code !== 0) {
+              if (/not found|no such file|could not find/i.test(error.bytes.toString())) { result.missingArtifacts.push(path); continue; }
+              throw new Error(`Cannot retain verification output: ${error.bytes.toString()}`);
+            }
+            await Bun.write(tar, output.bytes);
+            const extracted = resolve(staging, `output-${randomUUID()}`);
+            await mkdir(extracted);
+            await extractWorkspaceArchive(tar, extracted, { sourceRoot: "/workspace" });
+            const file = contained(extracted, posix.basename(path));
+            if (!(await lstat(file)).isFile()) throw new Error("Verification artifacts must be regular files, not links or directories");
+            const bytes = await Bun.file(file).bytes();
+            const relative = `verification/${id}/artifacts/${path}`;
+            const target = contained(options.directory, relative);
+            await mkdir(dirname(target), { recursive: true });
+            await Bun.write(target, bytes);
+            result.artifacts.push({ path, bytes: bytes.length, sha256: hash(bytes), file: relative });
+          }
+          Object.assign(result, { state: timedOut ? "timed_out" : "completed", elapsedMs: Date.now() - Date.parse(startedAt), exitCode, logsTruncated });
+          result.stdout = `verification/${id}/stdout.log`; result.stderr = `verification/${id}/stderr.log`;
+          await Bun.write(resolve(directory, "stdout.log"), stdout.slice(0, LOG_BYTES));
+          await Bun.write(resolve(directory, "stderr.log"), stderr.slice(0, LOG_BYTES));
+          result.logsTruncated ||= stdout.length > LOG_BYTES || stderr.length > LOG_BYTES;
+          await writeJson(resolve(directory, "receipt.json"), result);
+          outputs.set(id, result); results.push(result);
+          return structuredClone(result);
+        } finally {
+          await engineCommand(runtime.engine, ["rm", "--force", name]).catch(() => {});
+          await rm(staging, { recursive: true, force: true });
+        }
+      },
+    },
+  };
+}

package/src/judge-context.ts

@@ -12,6 +12,8 @@ import type {
   ToolCall,
   Cost,
   Accounting,
+  Engine,
+  EvidenceCitation,
 } from "./types";
 
 export type JsonValue =
@@ -23,6 +25,62 @@ export type JsonValue =
   | { [key: string]: JsonValue };
 export type ScoreValue = boolean | number | null;
 
+/** Scalars remain sufficient. Add explanations only when they help a reader. */
+export type CodeScore = ScoreValue | {
+  value: ScoreValue;
+  reason: string;
+  evidence?: EvidenceCitation[];
+  measurements?: Record<string, JsonValue>;
+};
+
+export type VerificationEnvironment = {
+  engine?: Engine;
+  /** A locally available OCI image; resolved to an immutable ID before collection. */
+  image: string;
+  cpus?: number;
+  memoryMiB?: number;
+};
+export type VerificationRuntime = {
+  engine: Engine;
+  image: string;
+  imageId: string;
+  cpus: number;
+  memoryMiB: number;
+};
+export type VerificationRequest = {
+  revision?: Revision;
+  cwd?: string;
+  /** Commands run in order; a nonzero exit stops this verification. No implicit shell. */
+  commands: readonly (readonly string[])[];
+  /** Trusted, frozen judge inputs, copied to /verification; never to the candidate. */
+  files?: Record<string, string>;
+  /** Relative paths under the restored workspace; missing outputs are reported. */
+  artifacts?: readonly string[];
+  timeoutMs?: number;
+};
+export type VerificationArtifact = {
+  path: string;
+  bytes: number;
+  sha256: string;
+  /** Relative to the code-judge execution directory, not the candidate workspace. */
+  file: string;
+};
+export type VerificationResult = {
+  id: string;
+  state: "completed" | "timed_out";
+  revision: Revision;
+  environment: VerificationRuntime;
+  startedAt: string;
+  elapsedMs: number;
+  commands: Array<{ argv: string[]; exitCode: number | null; elapsedMs: number }>;
+  exitCode: number | null;
+  stdout: string;
+  stderr: string;
+  logsTruncated: boolean;
+  artifacts: VerificationArtifact[];
+  missingArtifacts: string[];
+};
+
 /** Code judges are ordinary functions. Only an optional scores map has grading semantics. */
 export type JudgeFunction = (
   context: JudgeContext,
@@ -106,6 +164,13 @@ export interface JudgeContext {
     /** Restore a disposable snapshot for author-owned verification commands. */
     materialize(revision?: Revision): Promise<string>;
   };
+  readonly verification: {
+    /** Execute an artifact in a credential-free, network-isolated OCI container. */
+    run(request: VerificationRequest): Promise<VerificationResult>;
+    /** Read one retained output, checking its hash. No original runtime-state claim. */
+    read(result: VerificationResult, path: string): Promise<Uint8Array>;
+    text(result: VerificationResult, path: string): Promise<string>;
+  };
   readonly native: {
     readonly version: string | null;
     readonly readerVersion: string;
@@ -138,4 +203,5 @@ export type CodeJudgeExecution = {
   sourceHash: string;
   stdout: string;
   stderr: string;
+  verifications?: VerificationResult[];
 };