@workos/quickstudy 0.10.0 → 0.12.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workos/quickstudy",
-  "version": "0.10.0",
+  "version": "0.12.0",
   "description": "An open-source, company-agnostic eval harness for measuring whether coding agents can complete real integration tasks, across an agent x feature x framework x surface matrix.",
   "license": "MIT",
   "type": "module",

package/src/build-info.generated.ts

@@ -9,7 +9,7 @@ export const EMBEDDED_BUILD_INFO: {
   // (the `x-release-please-version` marker anchors the replacement). It is
   // restored as-is after a binary build (scripts/build-binary.ts), so the
   // committed source `--version` stays in lockstep with package.json.
-  version: "0.10.0", // x-release-please-version
+  version: "0.12.0", // x-release-please-version
   revision: null,
   dirty: null,
   sourceHash: null,

package/src/evals/types.ts

@@ -8,10 +8,12 @@
 /**
  * Closed set of eval suites. `benchmark` and `regression` are the headline
  * suites: the leaderboard aggregates count both. `smoke` holds evals every
- * arm solves — run to catch breakage, never moving the headline. `other` is
- * anything else outside the headline.
+ * arm solves — run to catch breakage, never moving the headline. `discovery`
+ * holds vendor-neutral tasks whose outcome is categorical (which product was
+ * chosen) rather than pass/fail; recorded per attempt, never headline.
+ * `other` is anything else outside the headline.
  */
-export const EVAL_SUITES = ["benchmark", "regression", "smoke", "other"] as const;
+export const EVAL_SUITES = ["benchmark", "regression", "smoke", "other", "discovery"] as const;
 export type EvalSuite = (typeof EVAL_SUITES)[number];
 
 /** The suites whose attempts count toward the leaderboard aggregates. */

package/src/execution-policy.ts

@@ -1,3 +1,4 @@
+import { workspaceFilesIdentity, type WorkspaceFiles, type WorkspaceFilesIdentity } from "./runtime/workspace-files.ts";
 import type { LoadedEval } from "./evals/types.ts";
 import type { LoadedExperiment } from "./experiments/types.ts";
 import type { RuntimeImageIdentity } from "./manifest.ts";
@@ -19,6 +20,8 @@ export const DEFAULT_BUDGETS: LifecycleBudgets = {
   cleanupMs: 30_000,
 };
 export interface PairEnvironment {
+  /** Omitted on historical/no-overlay pairs; payload bytes never enter manifests. */
+  workspaceFiles?: WorkspaceFilesIdentity;
   image: RuntimeImageIdentity | null;
   egress: { proxy: boolean; allowlist: string[] };
   webPolicy: WebPolicy;
@@ -50,11 +53,20 @@ export function resolvePairEnvironments(options: {
   images?: Record<string, RuntimeImageIdentity>;
   egressProxy?: boolean;
   adapterHosts?: Record<string, readonly string[]>;
+  /**
+   * Per experiment: the hosts of the agent's endpoint overrides. They join
+   * the allowlist only for pairs the proxy fronts, where the list is live and
+   * an IP-literal gateway is refused loudly by design; an open or unproxied
+   * pair never carries them, so such a gateway cannot stop the run.
+   */
+  adapterEndpointHosts?: Record<string, readonly string[]>;
   /** Per experiment: the agent's non-secret endpoint overrides (see PairEnvironment.endpoints). */
   adapterEndpoints?: Record<string, Record<string, string>>;
   budgets?: Partial<LifecycleBudgets>;
   mcpHashes?: Record<string, string>;
   imageTags?: Record<string, string>;
+  /** Runner-captured declarations; entries with undefined explicitly mean no overlay. */
+  workspaceFilesByExperiment?: ReadonlyMap<string, WorkspaceFiles | undefined>;
 }): Record<string, PairEnvironment> {
   const budgets = { ...DEFAULT_BUDGETS, ...options.budgets };
   for (const [name, value] of Object.entries(budgets))
@@ -63,19 +75,28 @@ export function resolvePairEnvironments(options: {
   for (const loaded of options.evals)
     for (const { experiment } of options.experiments) {
       const runtime = experiment.runtime;
+      const files = options.workspaceFilesByExperiment ? options.workspaceFilesByExperiment.get(experiment.id) : runtime.workspaceFiles;
       const tag =
         options.imageTags?.[pairKey(loaded.metadata.id, experiment.id)] ??
         runtime.image ??
         runtime.containerImage?.(loaded.metadata);
+      // With the flag, every container is proxied unless its runtime opts
+      // out with egress: "open". No runtime hosts means only the adapter's
+      // provider endpoints are permitted. An open pair still records the
+      // runtime and adapter hosts (inert) so identity holds if the run flag
+      // flips, but endpoint-override hosts join only a live allowlist: the
+      // proxy refuses IP literals, and an IP-literal gateway on an arm the
+      // proxy never fronts must not stop the run.
+      const proxied = runtime.kind === "container" && (options.egressProxy ?? false) && runtime.egress !== "open";
       pairs[pairKey(loaded.metadata.id, experiment.id)] = {
+        ...(files ? { workspaceFiles: workspaceFilesIdentity(files) } : {}),
         image: tag ? (options.images?.[tag] ?? { tag, id: "unresolved", repoDigests: [] }) : null,
-        // With the flag, every container is proxied. No runtime hosts means
-        // only the adapter's provider endpoints are permitted.
         egress: {
-          proxy: runtime.kind === "container" && (options.egressProxy ?? false),
+          proxy: proxied,
           allowlist: composeAllowlistFromHosts([
             ...(runtime.egressHosts?.(loaded.metadata) ?? []),
             ...(options.adapterHosts?.[experiment.id] ?? []),
+            ...(proxied ? (options.adapterEndpointHosts?.[experiment.id] ?? []) : []),
           ]),
         },
         webPolicy: runtime.webPolicy?.() ?? "native-web-blocked",

package/src/experiments/types.ts

@@ -6,9 +6,17 @@
  * manifest), so nothing may mutate it after definition.
  */
 
+import type { EvalMetadata } from "../evals/types.ts";
+import { validateWorkspaceFilesRuntime, type WorkspaceFiles } from "../runtime/workspace-files.ts";
 import type { SurfaceUsageConfig } from "../surface-usage.ts";
 import type { RuntimeCapabilities } from "../runtime/types.ts";
 
+/** A shared scarce resource: the scheduler lets at most `limit` attempts hold `key` at once. */
+export interface RuntimeCapacity {
+  key: string;
+  limit: number;
+}
+
 /** Which agent executes the attempt, plus the identity fields recorded in the manifest. */
 export interface AgentSpec {
   /** Adapter name, e.g. "echo" (the built-in no-op pipeline smoke agent). */
@@ -35,13 +43,29 @@ export interface Runtime extends RuntimeCapabilities {
   kind: "host" | "container";
   /** Shared observation targets, independent of which surfaces this arm offers. */
   observationTargets?: SurfaceUsageConfig;
-  /** Shared scarce resource pool; scheduler waits instead of starting doomed jobs. */
-  capacity?: { key: string; limit: number };
+  /**
+   * Shared scarce resource pool; the scheduler waits instead of starting
+   * doomed jobs. A resolver decides per eval, so an eval that consumes no
+   * pooled resource can answer `undefined` and never hold a slot. The static
+   * form is identity; a resolver, like every capability function, is not.
+   */
+  capacity?: RuntimeCapacity | ((metadata: EvalMetadata) => RuntimeCapacity | undefined);
   /** Explicit environment fields the declared treatment may vary in a comparison.
    * Images and lifecycle budgets can never be exempted. All group members must
    * declare the same fields. Source/config identity still changes per attempt.
    */
-  treatmentPolicyFields?: Array<"egress" | "webPolicy" | "pathTreatment" | "mcp" | "promptPreamble">;
+  treatmentPolicyFields?: Array<"egress" | "webPolicy" | "pathTreatment" | "mcp" | "promptPreamble" | "workspaceFiles">;
+  /**
+   * Whether this experiment's attempts go through the run's egress proxy.
+   * "open" skips the proxy sidecar and network for these attempts even when
+   * the run was started with --egress-proxy, so an arm that must reach the
+   * open internet can share a run with proxied arms. The choice is recorded
+   * in each pair environment (identity); exempt it in a comparison group with
+   * treatmentPolicyFields: ["egress"]. Container runtimes only.
+   */
+  egress?: "proxied" | "open";
+  /** Attempt-local non-secret files, staged without replacing the normal adapter. */
+  workspaceFiles?: WorkspaceFiles;
   /** Container image tag (container runtimes; alternative to `containerImage`). */
   image?: string;
   /** Scripted command run in /workspace (container runtimes, this phase). */
@@ -112,5 +136,11 @@ export function defineExperiment(experiment: Experiment): Experiment {
   if (experiment.comparisonGroup !== undefined && experiment.comparisonGroup.trim() === "") {
     throw new Error(`defineExperiment: experiment "${experiment.id}" comparisonGroup must not be empty`);
   }
+  if (kind === "host" && experiment.runtime.egress === "open") {
+    throw new Error(
+      `defineExperiment: experiment "${experiment.id}" runtime.egress "open" is only meaningful on container runtimes`,
+    );
+  }
+  validateWorkspaceFilesRuntime(experiment.runtime);
   return deepFreeze(experiment);
 }

package/src/index.ts

@@ -17,8 +17,11 @@ export type {
   LoadedEval,
 } from "./evals/types.ts";
 export { EVAL_SUITES } from "./evals/types.ts";
+export { defineWorkspaceFiles, WORKSPACE_FILES_LIMITS } from "./runtime/workspace-files.ts";
+export type { WorkspaceFile, WorkspaceFiles, WorkspaceFilesIdentity } from "./runtime/workspace-files.ts";
+export type { WorkspaceFilesEvidence } from "./isolation/workspace-files.ts";
 export { defineExperiment } from "./experiments/types.ts";
-export type { AgentSpec, Experiment, LoadedExperiment, Runtime } from "./experiments/types.ts";
+export type { AgentSpec, Experiment, LoadedExperiment, Runtime, RuntimeCapacity } from "./experiments/types.ts";
 export type {
   ProvisionedEnvironment,
   RuntimeCapabilities,

package/src/isolation/docker.ts

@@ -21,6 +21,8 @@ import { mkdir, mkdtemp, rm, writeFile, cp, readdir, lstat, readlink, realpath }
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 import { sanitizeServerName, type McpUpstream } from "./mcp-proxy/server.ts";
+import { WORKSPACE_FILES_PROGRAM, type WorkspaceFilesRequest } from "./workspace-files.ts";
+import type { WorkspaceFilesIdentity } from "../runtime/workspace-files.ts";
 import { EXCLUDED_WORKSPACE_DIRS } from "../store/artifacts.ts";
 
 /** The label key every attempt container carries. */
@@ -66,14 +68,17 @@ export interface ExecResult {
  */
 async function runDocker(
   args: string[],
-  opts: { timeoutMs?: number; stdin?: string; env?: Record<string, string> } = {},
+  opts: { timeoutMs?: number; stdin?: string; env?: Record<string, string>; signal?: AbortSignal } = {},
 ): Promise<ExecResult> {
+  opts.signal?.throwIfAborted();
   const proc = Bun.spawn(["docker", ...args], {
     stdin: opts.stdin === undefined ? "ignore" : Buffer.from(opts.stdin),
     stdout: "pipe",
     stderr: "pipe",
     ...(opts.env !== undefined ? { env: { ...process.env, ...opts.env } } : {}),
   });
+  const abort = () => proc.kill("SIGKILL");
+  opts.signal?.addEventListener("abort", abort, { once: true });
   let timedOut = false;
   let timer: ReturnType<typeof setTimeout> | undefined;
   {
@@ -88,13 +93,14 @@ async function runDocker(
     proc.exited,
   ]);
   if (timer !== undefined) clearTimeout(timer);
+  opts.signal?.removeEventListener("abort", abort);
   return { exitCode, stdout, stderr, timedOut };
 }
 
 /** Run `docker <args>` and throw a DockerError on non-zero exit. */
 async function mustDocker(
   args: string[],
-  opts: { timeoutMs?: number; stdin?: string; env?: Record<string, string> } = {},
+  opts: { timeoutMs?: number; stdin?: string; env?: Record<string, string>; signal?: AbortSignal } = {},
 ): Promise<ExecResult> {
   const result = await runDocker(args, opts);
   if (result.exitCode !== 0) throw new DockerError(args, result.exitCode, result.stderr);
@@ -193,7 +199,9 @@ export interface AttemptContainer {
   /** Run a command in /workspace inside the container. Never throws for non-zero exits. */
   execInWorkspace(cmd: string[], opts?: { timeoutMs?: number }): Promise<ExecResult>;
   /** Write a file inside the container (parent directories created). */
-  writeFile(path: string, content: string): Promise<void>;
+  writeFile(path: string, content: string, opts?: { signal?: AbortSignal }): Promise<void>;
+  /** Bounded create-only overlay transport; input rides stdin, never argv. */
+  workspaceFiles(request: WorkspaceFilesRequest, signal: AbortSignal): Promise<WorkspaceFilesIdentity>;
   /** Copy /workspace to a host directory, excluding node_modules/vendor/.venv. */
   exportWorkspace(toDir: string): Promise<void>;
   /** `docker rm -f` — kill and remove. Idempotent. */
@@ -282,10 +290,22 @@ export async function createAttemptContainer(opts: CreateAttemptContainerOptions
       }
       return result;
     },
-    writeFile: async (path, content) => {
+    workspaceFiles: async (request, signal) => {
+      try {
+        const result = await runDocker(["exec", "-i", "-w", CONTAINER_WORKSPACE, id, "/usr/local/bin/node", "-e", WORKSPACE_FILES_PROGRAM, CONTAINER_WORKSPACE], {
+          stdin: JSON.stringify(request), signal,
+        });
+        if (signal.aborted || result.timedOut || result.exitCode !== 0) throw new Error("staging failed");
+        return JSON.parse(result.stdout) as WorkspaceFilesIdentity;
+      } catch {
+        // No raw stdout/stderr, command, or payload may enter diagnostics.
+        throw new Error("workspaceFiles: filesystem operation failed or was cancelled/timed out");
+      }
+    },
+    writeFile: async (path, content, writeOpts = {}) => {
       // `$1` positional keeps the target path out of shell interpolation.
       await mustDocker(["exec", "-i", id, "sh", "-c", 'mkdir -p "$(dirname "$1")" && cat > "$1"', "sh", path], {
-        stdin: content,
+        stdin: content, signal: writeOpts.signal,
       });
     },
     exportWorkspace: async (toDir) => {

package/src/isolation/workspace-files.ts

@@ -0,0 +1,92 @@
+import type { WorkspaceFiles, WorkspaceFilesIdentity } from "../runtime/workspace-files.ts";
+
+/** The same program runs in the container and offline filesystem tests. No shell or payload in argv. */
+export const WORKSPACE_FILES_PROGRAM = String.raw`
+const fs = require('node:fs');
+const crypto = require('node:crypto');
+const path = require('node:path');
+function apply(root, request) {
+  const { files, absent } = request.declaration;
+  const fail = () => { throw new Error('unsafe workspace state'); };
+  const stat = (p) => { try { return fs.lstatSync(p); } catch (e) { if (e.code === 'ENOENT') return null; throw e; } };
+  if (!stat(root)?.isDirectory() || stat(root).isSymbolicLink()) fail();
+  // No processes controlled by the agent run during preparation. On Linux,
+  // directory descriptors also anchor traversal against concurrent renames.
+  const descriptors = [];
+  const directory = (p) => {
+    const s = stat(p);
+    if (!s?.isDirectory() || s.isSymbolicLink()) fail();
+    const fd = fs.openSync(p, fs.constants.O_RDONLY | fs.constants.O_DIRECTORY | fs.constants.O_NOFOLLOW);
+    descriptors.push(fd);
+    return process.platform === 'linux' ? '/proc/self/fd/' + fd : p;
+  };
+  const base = directory(root);
+  const leaf = (relative, create = false) => {
+    while (descriptors.length > 1) fs.closeSync(descriptors.pop());
+    const parts = relative.split('/');
+    let parent = base;
+    for (const part of parts.slice(0, -1)) {
+      const next = parent + '/' + part;
+      if (!stat(next)) {
+        if (!create) return null;
+        fs.mkdirSync(next, { mode: 0o755 });
+      }
+      parent = directory(next);
+    }
+    return parent + '/' + parts.at(-1);
+  };
+  const checkAbsent = (relative) => { const p = leaf(relative); if (p !== null && stat(p)) fail(); };
+  try {
+    // Setup paths may be outside the workspace, but cannot alias it through
+    // symlinks. Lexical overlap is separately rejected before any write.
+    for (const target of request.setupPaths) {
+      const absolute = path.resolve(root, target);
+      let current = '/';
+      for (const part of absolute.split('/').filter(Boolean)) {
+        current = path.join(current, part);
+        const s = stat(current);
+        if (!s) break;
+        if (s.isSymbolicLink()) fail();
+      }
+    }
+    for (const relative of absent) checkAbsent(relative);
+    if (request.phase === 'stage') {
+      // Validate the entire inventory before writing even the first file.
+      for (const file of files) checkAbsent(file.path);
+      for (const file of files) {
+        const p = leaf(file.path, true);
+        const fd = fs.openSync(p, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, 0o644);
+        try { fs.writeFileSync(fd, Buffer.from(file.content, 'base64')); fs.fchmodSync(fd, 0o644); }
+        finally { fs.closeSync(fd); }
+      }
+    }
+    const inventory = files.map((file) => {
+      const p = leaf(file.path);
+      if (!p) fail();
+      const fd = fs.openSync(p, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK);
+      try {
+        const s = fs.fstatSync(fd);
+        if (!s.isFile() || s.nlink !== 1 || (s.mode & 0o777) !== 0o644 || s.size > 262144) fail();
+        const bytes = fs.readFileSync(fd);
+        return { path: file.path, size: bytes.length, sha256: crypto.createHash('sha256').update(bytes).digest('hex'), source: file.source };
+      } finally { fs.closeSync(fd); }
+    });
+    return { version: 1, files: inventory, absent };
+  } finally { for (const fd of descriptors.reverse()) fs.closeSync(fd); }
+}
+try {
+  const request = JSON.parse(fs.readFileSync(0, 'utf8'));
+  process.stdout.write(JSON.stringify(apply(process.argv[1], request)));
+} catch { process.stderr.write('workspaceFiles: filesystem validation/write/read-back failed'); process.exitCode = 1; }
+`;
+
+export interface WorkspaceFilesRequest {
+  declaration: WorkspaceFiles;
+  phase: "stage" | "verify";
+  setupPaths: string[];
+}
+export interface WorkspaceFilesEvidence {
+  version: 1;
+  verified: WorkspaceFilesIdentity;
+  verifiedAt: number;
+}

package/src/manifest.ts

@@ -10,6 +10,7 @@
  * adding or removing a component is a deliberate, test-visible act.
  */
 
+import { validateWorkspaceFilesRuntime, workspaceFilesIdentity, type WorkspaceFiles } from "./runtime/workspace-files.ts";
 import { readFileSync } from "node:fs";
 import { join, resolve } from "node:path";
 import { EMBEDDED_BUILD_INFO } from "./build-info.generated.ts";
@@ -151,6 +152,8 @@ export interface BuildRunManifestOptions {
   harness?: HarnessIdentity;
   pairs?: Record<string, PairEnvironment>;
   scorerIdentities?: Record<string, Record<string, unknown>>;
+  /** Share the runner's immutable payload snapshot without copying runtime instances. */
+  workspaceFilesByExperiment?: ReadonlyMap<string, WorkspaceFiles | undefined>;
 }
 
 /** Build before the first paid attempt; all reads are deterministic and local. */
@@ -178,6 +181,8 @@ export function buildRunManifest(options: BuildRunManifestOptions): RunManifest
 
   const experiments: Record<string, ExperimentIdentity> = {};
   for (const { experiment, path } of options.experiments) {
+    const files = options.workspaceFilesByExperiment ? options.workspaceFilesByExperiment.get(experiment.id) : experiment.runtime.workspaceFiles;
+    validateWorkspaceFilesRuntime({ kind: experiment.runtime.kind, command: experiment.runtime.command, workspaceFiles: files });
     const imageTag = experiment.runtime.image;
     const components: ExperimentIdentityComponents = {
       source: sourceIdentity(path),
@@ -190,7 +195,9 @@ export function buildRunManifest(options: BuildRunManifestOptions): RunManifest
       },
       runtime: contentHash({
         kind: experiment.runtime.kind,
-        capacity: experiment.runtime.capacity ?? null,
+        ...(files ? { workspaceFiles: workspaceFilesIdentity(files) } : {}),
+        // A capacity resolver is a capability function: per-eval, never identity.
+        capacity: typeof experiment.runtime.capacity === "function" ? null : (experiment.runtime.capacity ?? null),
         observationTargets: experiment.runtime.observationTargets ?? null,
         treatmentPolicyFields: [...(experiment.runtime.treatmentPolicyFields ?? [])].sort(),
         image: experiment.runtime.image ?? null,

package/src/report/aggregate.ts

@@ -3,8 +3,8 @@
  * the same attempts and (banded) pairs as the rest of the report.
  *
  * Every number is restricted to headline-suite attempts (HEADLINE_SUITES:
- * benchmark and regression), so smoke and other suites never move the
- * headline row. Telemetry follows the report's rule: sums and means cover
+ * benchmark and regression), so the smoke, discovery and other suites never
+ * move the headline row. Telemetry follows the report's rule: sums and means cover
  * only attempts that reported a value, and are null (never 0) when none did.
  */
 import { isHeadlineSuite } from "../evals/types.ts";

package/src/runner/capacity.ts

@@ -0,0 +1,17 @@
+import type { EvalMetadata } from "../evals/types.ts";
+import type { Runtime, RuntimeCapacity } from "../experiments/types.ts";
+
+/**
+ * The capacity an attempt of `metadata` holds under `runtime`: the static
+ * pool, the resolver's answer for that eval, or none. The scheduler validates
+ * the shape of whatever comes back.
+ */
+export function resolveRuntimeCapacity(
+  runtime: Runtime | undefined,
+  metadata: EvalMetadata | undefined,
+): RuntimeCapacity | undefined {
+  const capacity = runtime?.capacity;
+  if (capacity === undefined) return undefined;
+  if (typeof capacity === "function") return metadata === undefined ? undefined : capacity(metadata);
+  return capacity;
+}

package/src/runner/execute.ts

@@ -21,6 +21,8 @@
  * and never allowed to mask the attempt's result.
  */
 
+import { stageWorkspaceFiles } from "./workspace-files.ts";
+import { defineWorkspaceFiles, validateWorkspaceFilesRuntime, workspaceFilesIdentity, type WorkspaceFiles } from "../runtime/workspace-files.ts";
 import { mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -36,6 +38,7 @@ import { EchoAdapter } from "../adapters/echo.ts";
 import { OracleAdapter, OracleContainerAdapter } from "../adapters/oracle.ts";
 import { validEvalResult } from "../evals/result.ts";
 import { importEvalScorer } from "../evals/discovery.ts";
+import { resolveRuntimeCapacity } from "./capacity.ts";
 import { deadline, StageTimeoutError } from "./deadline.ts";
 import { scoreInWorker } from "./score-worker.ts";
 import { prepWorkspaceSource } from "./workspace-source.ts";
@@ -102,6 +105,8 @@ export interface ExecuteRunV3Options {
   config?: Partial<RunConfig>;
   /** Host adapters by name. Defaults to the built-in echo adapter. */
   hostAdapters?: Record<string, AgentAdapter>;
+  /** Internal offline-test seam; not part of the public consumer API. */
+  createContainer?: typeof createAttemptContainer;
   /**
    * Container agent adapter factories by name, constructed per experiment
    * from its `agent` spec. Defaults to the built-in claude/codex adapters.
@@ -314,6 +319,13 @@ export function collectOfferedSurfaces(options: {
 }
 
 export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSummaryV3> {
+  // Snapshot only the payload: runtime methods must keep their original receiver,
+  // including prototype methods and private fields. Capture before the first await.
+  const workspaceFilesByExperiment = new Map(options.experiments.map(({ experiment }) => {
+    validateWorkspaceFilesRuntime(experiment.runtime);
+    const files = experiment.runtime.workspaceFiles;
+    return [experiment.id, files === undefined ? undefined : defineWorkspaceFiles(files)] as const;
+  }));
   const { store, artifacts, plan } = options;
   const log = options.log ?? ((line: string) => console.log(line));
   const attemptTimeoutMs = options.attemptTimeoutMs ?? options.budgets?.agentMs ?? DEFAULT_ATTEMPT_TIMEOUT_MS;
@@ -459,25 +471,33 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
       if (scorer.identity) scorerIdentities[loaded.metadata.id] = scorer.identity;
     }
     // Each agent's endpoint overrides set on the host: the non-secret base
-    // URLs join the pair environment (identity), their hosts join the egress
-    // allowlist — but only under the proxy, where the allowlist is live and an
-    // IP-literal gateway is refused loudly by design; without it the list is
-    // inert and an IP-literal override must not stop the run.
+    // URLs join the pair environment (identity); their hosts reach the egress
+    // allowlist only for pairs the proxy fronts (pair resolution decides: the
+    // run flag on and the runtime not opted out), where the list is live and
+    // an IP-literal gateway is refused loudly by design. Elsewhere the list
+    // is inert and an IP-literal override must not stop the run.
     const overridesByExperiment = new Map([...containerAgents].map(([id, adapter]) => [id, collectOptionalHostEnv([adapter])] as const));
     const adapterEndpoints = Object.fromEntries([...overridesByExperiment].flatMap(([id, overrides]) => {
       const endpoints = Object.fromEntries(Object.entries(overrides).filter(([name]) => !isSecretEnvName(name)));
       return Object.keys(endpoints).length > 0 ? [[id, endpoints] as const] : [];
     }));
-    const pairs = resolvePairEnvironments({ ...options, images, imageTags: imageTagsByPair,
-      adapterHosts: Object.fromEntries([...containerAgents].map(([id, adapter]) => [id, [
-        ...(adapter.egressHosts ?? []),
-        ...(options.egressProxy ? endpointHosts(overridesByExperiment.get(id) ?? {}) : []),
-      ]])),
+    const pairs = resolvePairEnvironments({ ...options, workspaceFilesByExperiment, images, imageTags: imageTagsByPair,
+      adapterHosts: Object.fromEntries([...containerAgents].map(([id, adapter]) => [id, [...(adapter.egressHosts ?? [])]])),
+      adapterEndpointHosts: Object.fromEntries([...overridesByExperiment].map(([id, overrides]) => [id, endpointHosts(overrides)])),
       adapterEndpoints,
       budgets: { ...options.budgets, agentMs: options.attemptTimeoutMs ?? options.budgets?.agentMs ?? attemptTimeoutMs },
       mcpHashes: Object.fromEntries([...declaredByExperiment].map(([id, servers]) => [id, hashString(canonicalJson(servers))])),
     });
-    manifest = buildRunManifest({ evals: options.evals, experiments: options.experiments, images, cliVersions, pairs, scorerIdentities });
+    manifest = buildRunManifest({ evals: options.evals, experiments: options.experiments, workspaceFilesByExperiment, images, cliVersions, pairs, scorerIdentities });
+  }
+  // Injected/stale manifests must not authorize a different payload (or silently
+  // omit one). Historical no-overlay manifests retain their original shape.
+  for (const entry of plan) {
+    const files = workspaceFilesByExperiment.get(entry.experimentId);
+    const intended = files ? workspaceFilesIdentity(files) : undefined;
+    if (canonicalJson(intended ?? null) !== canonicalJson(manifest.pairs?.[pairKey(entry.evalId, entry.experimentId)]?.workspaceFiles ?? null)) {
+      throw new Error("workspaceFiles: manifest payload identity mismatch");
+    }
   }
   const recorded = options.resumeRunId ? store.getRun(options.resumeRunId) : null;
   if (options.resumeRunId && !recorded) throw new Error(`cannot resume unknown run ${options.resumeRunId}`);
@@ -582,7 +602,7 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
                 loadedEval,
                 experiment,
                 entry,
-                { runId, attemptId, attemptTimeoutMs, persisted, environment: manifest.pairs?.[pairKey(entry.evalId, entry.experimentId)] },
+                { runId, attemptId, attemptTimeoutMs, persisted, environment: manifest.pairs?.[pairKey(entry.evalId, entry.experimentId)], workspaceFiles: workspaceFilesByExperiment.get(entry.experimentId) },
                 treatments,
                 containerAgents.get(experiment.id),
               );
@@ -616,7 +636,10 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
         `attempt ${attemptId} (${entry.evalId} × ${entry.experimentId} trial ${entry.trialIndex + 1}): ` +
           `${disposition.status}${disposition.result ? ` — ${disposition.result.passed ? "passed" : "failed"}` : ""}`,
       );
-    }, { ...(options.signal ? { signal: options.signal } : {}), capacity: (entry) => experimentsById.get(entry.experimentId)?.runtime.capacity });
+    }, {
+      ...(options.signal ? { signal: options.signal } : {}),
+      capacity: (entry) => resolveRuntimeCapacity(experimentsById.get(entry.experimentId)?.runtime, evalsById.get(entry.evalId)?.metadata),
+    });
 
     const current = activeAttempts(store.listAttempts(runId));
     summary.passed = current.filter((a) => a.status === "completed" && a.result?.passed).length;
@@ -649,6 +672,7 @@ interface AttemptArgs {
   attemptTimeoutMs: number;
   persisted: NewAttempt;
   environment?: PairEnvironment;
+  workspaceFiles?: WorkspaceFiles;
 }
 
 async function runHostAttemptV3(
@@ -745,6 +769,7 @@ async function runContainerAttemptV3(
   const { artifacts } = options;
   const { runId, attemptId, persisted } = args;
   const runtime = experiment.runtime;
+  const workspaceFiles = args.workspaceFiles;
   const environment = args.environment;
   let egress: { networkName: string; sidecar: EgressSidecar } | undefined;
   let mcpIp: string | undefined;
@@ -855,7 +880,7 @@ async function runContainerAttemptV3(
 
       const paths = await inStage("environment", () => artifacts.prepareAttempt(runId, attemptId));
       const fixtureDir = sourceDir;
-      const container = await inStage("environment", () => createAttemptContainer({
+      const container = await inStage("environment", () => (options.createContainer ?? createAttemptContainer)({
         image,
         env,
         resultsDir: paths.attemptDir,
@@ -886,6 +911,7 @@ async function runContainerAttemptV3(
               return (await mustExec(container, ["git", "rev-parse", "HEAD"])).stdout.trim();
             });
           } else {
+            if (workspaceFiles) throw new StageFailure("environment", new Error("workspaceFiles: git baseline is required"));
             (options.log ?? console.log)(
               `attempt ${attemptId}: git is unavailable in image "${image}" — diff.patch will be empty ` +
                 `(the agent-runtime image ships git; only bare test images lack it)`,
@@ -908,8 +934,13 @@ async function runContainerAttemptV3(
           };
           // Treatment config files (e.g. the MCP treatment's vendor config).
           await inStage("environment", async () => {
-            for (const [path, content] of Object.entries(agentAdapter.setupFiles(promptCtx))) {
-              await container.writeFile(path, content);
+            const setup = agentAdapter.setupFiles(promptCtx);
+            if (workspaceFiles) {
+              const evidence = await stageWorkspaceFiles(container, workspaceFiles, environment!.workspaceFiles!, setup, options.signal);
+              await artifacts.writeWorkspaceFilesEvidence(runId, attemptId, evidence);
+              if (options.signal?.aborted) throw new Error("workspaceFiles: staging cancelled");
+            } else {
+              for (const [path, content] of Object.entries(setup)) await container.writeFile(path, content);
             }
           });
           command = agentAdapter.command(promptCtx);
@@ -917,6 +948,7 @@ async function runContainerAttemptV3(
           command = runtime.command as string[];
         }
 
+        if (workspaceFiles && options.signal?.aborted) throw new StageFailure("environment", new Error("workspaceFiles: staging cancelled"));
         persisted.agentStartedAt = Date.now();
         let agentResult: Awaited<ReturnType<typeof container.execInWorkspace>>;
         try {
@@ -1039,7 +1071,10 @@ async function runContainerAttemptV3(
       try { await egress.sidecar.teardown(); } catch { containerTeardownFailure = "egress sidecar teardown failed"; }
       try { await removeRunNetwork(egress.networkName); } catch { containerTeardownFailure = "egress network teardown failed"; }
     }
-    if (sourceDir) await rm(sourceDir, { recursive: true, force: true });
+    if (sourceDir) {
+      try { await rm(sourceDir, { recursive: true, force: true }); }
+      catch { containerTeardownFailure = [containerTeardownFailure, "workspace source cleanup failed"].filter(Boolean).join("; "); }
+    }
   }
 
   // Explicit teardown accounting: the attempt keeps its status and result;

package/src/runner/workspace-files.ts

@@ -0,0 +1,52 @@
+import { posix } from "node:path";
+import { canonicalJson } from "../identity-diff.ts";
+import type { AttemptContainer } from "../isolation/docker.ts";
+import type { WorkspaceFilesEvidence } from "../isolation/workspace-files.ts";
+import { pathsOverlap, WORKSPACE_FILES_LIMITS, type WorkspaceFiles, type WorkspaceFilesIdentity } from "../runtime/workspace-files.ts";
+
+/** No race-to-timeout: await cancelled transport, then let the attempt's finally tear down. */
+export async function stageWorkspaceFiles(
+  container: AttemptContainer,
+  declaration: WorkspaceFiles,
+  intended: WorkspaceFilesIdentity,
+  setup: Record<string, string>,
+  parentSignal?: AbortSignal,
+): Promise<WorkspaceFilesEvidence> {
+  const controller = new AbortController();
+  const abort = () => controller.abort();
+  const timer = setTimeout(abort, WORKSPACE_FILES_LIMITS.timeoutMs);
+  parentSignal?.addEventListener("abort", abort, { once: true });
+  if (parentSignal?.aborted) abort();
+  const signal = controller.signal;
+  try {
+    const setupPaths = Object.keys(setup);
+    for (const path of setupPaths) {
+      const relative = posix.relative("/workspace", posix.resolve("/workspace", path));
+      if ([...intended.files.map((file) => file.path), ...intended.absent].some((target) => relative === "" || pathsOverlap(relative, target))) {
+        throw new Error("workspaceFiles: adapter setup collision");
+      }
+    }
+    const check = async (phase: "stage" | "verify") => {
+      signal.throwIfAborted();
+      const actual = await container.workspaceFiles({ declaration, phase, setupPaths }, signal);
+      signal.throwIfAborted();
+      if (canonicalJson(actual) !== canonicalJson(intended)) throw new Error("workspaceFiles: read-back/absence verification mismatch");
+      return actual;
+    };
+    await check("stage");
+    for (const [path, content] of Object.entries(setup)) {
+      signal.throwIfAborted();
+      await container.writeFile(path, content, { signal });
+    }
+    const verified = await check("verify");
+    return { version: 1, verified, verifiedAt: Date.now() };
+  } catch (err) {
+    if (signal.aborted) throw new Error("workspaceFiles: staging cancelled or timed out");
+    // Only our own fixed diagnostics survive; transport/setup errors may carry payloads.
+    if (err instanceof Error && ["workspaceFiles: adapter setup collision", "workspaceFiles: read-back/absence verification mismatch"].includes(err.message)) throw err;
+    throw new Error("workspaceFiles: staging filesystem/setup operation failed");
+  } finally {
+    clearTimeout(timer);
+    parentSignal?.removeEventListener("abort", abort);
+  }
+}