@workos/quickstudy 0.9.0 → 0.11.0

package/README.md

@@ -130,10 +130,11 @@ preflight key probe at it:
 | Agent  | Overrides                                        | How the CLI gets them                                                                  |
 | ------ | ------------------------------------------------ | -------------------------------------------------------------------------------------- |
 | claude | `ANTHROPIC_BASE_URL`, `ANTHROPIC_CUSTOM_HEADERS` | env, read natively by Claude Code (and by the `diagnose` analyst)                      |
-| codex  | `OPENAI_BASE_URL`                                | rendered into `config.toml` as a custom model provider (codex ignores the env var)     |
+| codex  | `OPENAI_BASE_URL`, `OPENAI_CUSTOM_HEADERS`       | rendered into `config.toml` as a custom model provider (`base_url`, `http_headers`); codex has no env channel for either |
 
-`ANTHROPIC_CUSTOM_HEADERS` is `Name: Value` lines and is redacted from
-artifacts like a key. With Cloudflare AI Gateway holding the provider keys,
+The `*_CUSTOM_HEADERS` bundles are `Name: Value` lines and are redacted from
+artifacts like a key; a `cf-aig-metadata` line is how a gateway attributes
+the traffic per app. With Cloudflare AI Gateway holding the provider keys,
 the gateway token *is* the API key:
 
 ```bash
@@ -142,8 +143,12 @@ export ANTHROPIC_API_KEY=<gateway token>
 export ANTHROPIC_CUSTOM_HEADERS="cf-aig-authorization: Bearer <gateway token>"
 export OPENAI_BASE_URL=https://gateway.ai.cloudflare.com/v1/<account>/<gateway>/openai
 export OPENAI_API_KEY=<gateway token>
+export OPENAI_CUSTOM_HEADERS='cf-aig-metadata: {"app":"my-evals"}'
 ```
 
+Leave every override unset and both agents run vendor-direct against
+api.anthropic.com and api.openai.com — nothing is rendered, nothing changes.
+
 ## How it works
 
 Every attempt is one point in a 2-D plan:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workos/quickstudy",
-  "version": "0.9.0",
+  "version": "0.11.0",
   "description": "An open-source, company-agnostic eval harness for measuring whether coding agents can complete real integration tasks, across an agent x feature x framework x surface matrix.",
   "license": "MIT",
   "type": "module",

package/src/adapters/codex.ts

@@ -10,38 +10,55 @@
  * command wrapper pipes it into `codex login --with-api-key` (stdin, never
  * argv) to write the auth.json the CLI requires.
  *
- * Endpoint override: an OPENAI_BASE_URL set on the host (an LLM gateway) is
+ * Endpoint overrides: an OPENAI_BASE_URL set on the host (an LLM gateway) is
  * rendered into the same config.toml as a custom model provider — codex
  * 0.149 ignores the env var for its built-in provider (verified live: it
  * still dialed api.openai.com), while a `[model_providers.*]` entry with
  * `env_key = "OPENAI_API_KEY"` routes every request through the override.
+ * OPENAI_CUSTOM_HEADERS (`Name: Value` lines, the Codex counterpart of Claude
+ * Code's ANTHROPIC_CUSTOM_HEADERS) becomes that provider's `http_headers` —
+ * e.g. `cf-aig-metadata` so a gateway can attribute the traffic. Codex has
+ * no env-var channel for extra headers, so the values land in config.toml
+ * inside the attempt container: the same trust boundary as the env that
+ * already carries them, written over stdin, destroyed with the container.
+ * With neither override set nothing is rendered and the built-in provider
+ * talks to api.openai.com — vendor-direct stays the default.
  */
 
 import type { McpServerConfig } from "../isolation/mcp.ts";
-import { endpointUrl } from "../secrets.ts";
+import { endpointUrl, parseHeaderLines } from "../secrets.ts";
 import { parseCodexStream } from "./parse.ts";
 import type { AttemptMetrics, ContainerAgentAdapter, ContainerAttemptContext } from "./types.ts";
 
 /** Container path of the Codex config (containers run as root). */
 export const CODEX_CONFIG_PATH = "/root/.codex/config.toml";
 
-/** The config.toml model provider id an OPENAI_BASE_URL override is rendered as. */
+/** The config.toml model provider id the OPENAI_* overrides are rendered as. */
 export const CODEX_GATEWAY_PROVIDER = "quickstudy-gateway";
 
+/** Codex's own default for the built-in provider; the probe and a headers-only override use it. */
+const OPENAI_DEFAULT_BASE_URL = "https://api.openai.com/v1";
+
 /**
  * A custom model provider pointing at `baseUrl` (the Responses API, which
- * codex requires), selected as the default. Top-level keys must precede any
- * table, so this block always renders first in the file.
+ * codex requires), carrying `headers` on every request, selected as the
+ * default. Top-level keys must precede any table, so this block always
+ * renders first in the file. JSON string escapes are valid TOML basic-string
+ * escapes, which keeps header values such as JSON metadata intact.
  */
-function renderCodexProviderToml(baseUrl: string): string {
+function renderCodexProviderToml(baseUrl: string, headers: Readonly<Record<string, string>>): string {
+  const entries = Object.entries(headers);
   return [
     `model_provider = ${JSON.stringify(CODEX_GATEWAY_PROVIDER)}`,
     "",
     `[model_providers.${CODEX_GATEWAY_PROVIDER}]`,
-    'name = "OpenAI (via OPENAI_BASE_URL)"',
+    'name = "OpenAI (quickstudy endpoint overrides)"',
     `base_url = ${JSON.stringify(baseUrl)}`,
     'env_key = "OPENAI_API_KEY"',
     'wire_api = "responses"',
+    ...(entries.length > 0
+      ? [`http_headers = { ${entries.map(([name, value]) => `${JSON.stringify(name)} = ${JSON.stringify(value)}`).join(", ")} }`]
+      : []),
     "",
   ].join("\n");
 }
@@ -66,13 +83,15 @@ export class CodexAdapter implements ContainerAgentAdapter {
   readonly versionCommand = ["codex", "--version"] as const;
   readonly requiredHostEnv = ["OPENAI_API_KEY"] as const;
   /**
-   * Endpoint override, forwarded only when set: OPENAI_BASE_URL (vendor
+   * Endpoint overrides, forwarded only when set: OPENAI_BASE_URL (vendor
    * default https://api.openai.com/v1) — e.g. Cloudflare AI Gateway's
-   * `/openai` endpoint, where the gateway token doubles as OPENAI_API_KEY.
-   * Rendered into config.toml by setupFiles (see the module comment); the
-   * key probe follows it.
+   * `/openai` endpoint, where the gateway token doubles as OPENAI_API_KEY —
+   * and OPENAI_CUSTOM_HEADERS (`Name: Value` lines) for whatever else the
+   * endpoint wants on each request, such as `cf-aig-metadata`. Both are
+   * rendered into config.toml by setupFiles (see the module comment); the
+   * key probe follows them.
    */
-  readonly optionalHostEnv = ["OPENAI_BASE_URL"] as const;
+  readonly optionalHostEnv = ["OPENAI_BASE_URL", "OPENAI_CUSTOM_HEADERS"] as const;
   /**
    * Egress-proxy allowlist (harness-default layer). Codex is reqwest-based;
    * reqwest honors HTTP_PROXY/HTTPS_PROXY env by default (system proxies) —
@@ -103,15 +122,19 @@ export class CodexAdapter implements ContainerAgentAdapter {
 
   keyProbe(env: Record<string, string>): { url: string; headers: Record<string, string> } {
     return {
-      url: endpointUrl(env["OPENAI_BASE_URL"], "https://api.openai.com/v1", "models"),
-      headers: { Authorization: `Bearer ${env["OPENAI_API_KEY"] ?? ""}` },
+      url: endpointUrl(env["OPENAI_BASE_URL"], OPENAI_DEFAULT_BASE_URL, "models"),
+      headers: { ...parseHeaderLines(env["OPENAI_CUSTOM_HEADERS"]), Authorization: `Bearer ${env["OPENAI_API_KEY"] ?? ""}` },
     };
   }
 
   setupFiles(ctx: ContainerAttemptContext): Record<string, string> {
     const sections: string[] = [];
     const baseUrl = ctx.overrides?.["OPENAI_BASE_URL"];
-    if (baseUrl !== undefined) sections.push(renderCodexProviderToml(baseUrl));
+    const headers = parseHeaderLines(ctx.overrides?.["OPENAI_CUSTOM_HEADERS"]);
+    // Either override needs the custom provider; headers alone keep the vendor endpoint.
+    if (baseUrl !== undefined || Object.keys(headers).length > 0) {
+      sections.push(renderCodexProviderToml(baseUrl ?? OPENAI_DEFAULT_BASE_URL, headers));
+    }
     if (ctx.mcpServers !== undefined) sections.push(renderCodexMcpToml(ctx.mcpServers));
     if (sections.length === 0) return {};
     return { [CODEX_CONFIG_PATH]: sections.join("\n") };

package/src/adapters/types.ts

@@ -133,10 +133,12 @@ export interface ContainerAttemptContext {
   /** The runtime's native web-tool policy (default: blocked). */
   webPolicy: WebPolicy;
   /**
-   * The adapter's endpoint overrides that are set on the host (its
-   * `optionalHostEnv` minus secret-named entries — never provider keys or
-   * header bundles), for a CLI that takes a config-file entry rather than an
-   * env var. Absent when none are set.
+   * The adapter's own endpoint overrides that are set on the host (every
+   * `optionalHostEnv` value, header bundles included — never the provider
+   * keys), for a CLI that takes a config-file entry rather than an env var.
+   * Anything rendered from these is written inside the attempt container
+   * only, the same boundary the env already crosses. Absent when none are
+   * set.
    */
   overrides?: Readonly<Record<string, string>>;
 }

package/src/build-info.generated.ts

@@ -9,7 +9,7 @@ export const EMBEDDED_BUILD_INFO: {
   // (the `x-release-please-version` marker anchors the replacement). It is
   // restored as-is after a binary build (scripts/build-binary.ts), so the
   // committed source `--version` stays in lockstep with package.json.
-  version: "0.9.0", // x-release-please-version
+  version: "0.11.0", // x-release-please-version
   revision: null,
   dirty: null,
   sourceHash: null,

package/src/execution-policy.ts

@@ -1,3 +1,4 @@
+import { workspaceFilesIdentity, type WorkspaceFiles, type WorkspaceFilesIdentity } from "./runtime/workspace-files.ts";
 import type { LoadedEval } from "./evals/types.ts";
 import type { LoadedExperiment } from "./experiments/types.ts";
 import type { RuntimeImageIdentity } from "./manifest.ts";
@@ -19,6 +20,8 @@ export const DEFAULT_BUDGETS: LifecycleBudgets = {
   cleanupMs: 30_000,
 };
 export interface PairEnvironment {
+  /** Omitted on historical/no-overlay pairs; payload bytes never enter manifests. */
+  workspaceFiles?: WorkspaceFilesIdentity;
   image: RuntimeImageIdentity | null;
   egress: { proxy: boolean; allowlist: string[] };
   webPolicy: WebPolicy;
@@ -55,6 +58,8 @@ export function resolvePairEnvironments(options: {
   budgets?: Partial<LifecycleBudgets>;
   mcpHashes?: Record<string, string>;
   imageTags?: Record<string, string>;
+  /** Runner-captured declarations; entries with undefined explicitly mean no overlay. */
+  workspaceFilesByExperiment?: ReadonlyMap<string, WorkspaceFiles | undefined>;
 }): Record<string, PairEnvironment> {
   const budgets = { ...DEFAULT_BUDGETS, ...options.budgets };
   for (const [name, value] of Object.entries(budgets))
@@ -63,11 +68,13 @@ export function resolvePairEnvironments(options: {
   for (const loaded of options.evals)
     for (const { experiment } of options.experiments) {
       const runtime = experiment.runtime;
+      const files = options.workspaceFilesByExperiment ? options.workspaceFilesByExperiment.get(experiment.id) : runtime.workspaceFiles;
       const tag =
         options.imageTags?.[pairKey(loaded.metadata.id, experiment.id)] ??
         runtime.image ??
         runtime.containerImage?.(loaded.metadata);
       pairs[pairKey(loaded.metadata.id, experiment.id)] = {
+        ...(files ? { workspaceFiles: workspaceFilesIdentity(files) } : {}),
         image: tag ? (options.images?.[tag] ?? { tag, id: "unresolved", repoDigests: [] }) : null,
         // With the flag, every container is proxied. No runtime hosts means
         // only the adapter's provider endpoints are permitted.

package/src/experiments/types.ts

@@ -6,9 +6,17 @@
  * manifest), so nothing may mutate it after definition.
  */
 
+import type { EvalMetadata } from "../evals/types.ts";
+import { validateWorkspaceFilesRuntime, type WorkspaceFiles } from "../runtime/workspace-files.ts";
 import type { SurfaceUsageConfig } from "../surface-usage.ts";
 import type { RuntimeCapabilities } from "../runtime/types.ts";
 
+/** A shared scarce resource: the scheduler lets at most `limit` attempts hold `key` at once. */
+export interface RuntimeCapacity {
+  key: string;
+  limit: number;
+}
+
 /** Which agent executes the attempt, plus the identity fields recorded in the manifest. */
 export interface AgentSpec {
   /** Adapter name, e.g. "echo" (the built-in no-op pipeline smoke agent). */
@@ -35,13 +43,20 @@ export interface Runtime extends RuntimeCapabilities {
   kind: "host" | "container";
   /** Shared observation targets, independent of which surfaces this arm offers. */
   observationTargets?: SurfaceUsageConfig;
-  /** Shared scarce resource pool; scheduler waits instead of starting doomed jobs. */
-  capacity?: { key: string; limit: number };
+  /**
+   * Shared scarce resource pool; the scheduler waits instead of starting
+   * doomed jobs. A resolver decides per eval, so an eval that consumes no
+   * pooled resource can answer `undefined` and never hold a slot. The static
+   * form is identity; a resolver, like every capability function, is not.
+   */
+  capacity?: RuntimeCapacity | ((metadata: EvalMetadata) => RuntimeCapacity | undefined);
   /** Explicit environment fields the declared treatment may vary in a comparison.
    * Images and lifecycle budgets can never be exempted. All group members must
    * declare the same fields. Source/config identity still changes per attempt.
    */
-  treatmentPolicyFields?: Array<"egress" | "webPolicy" | "pathTreatment" | "mcp" | "promptPreamble">;
+  treatmentPolicyFields?: Array<"egress" | "webPolicy" | "pathTreatment" | "mcp" | "promptPreamble" | "workspaceFiles">;
+  /** Attempt-local non-secret files, staged without replacing the normal adapter. */
+  workspaceFiles?: WorkspaceFiles;
   /** Container image tag (container runtimes; alternative to `containerImage`). */
   image?: string;
   /** Scripted command run in /workspace (container runtimes, this phase). */
@@ -112,5 +127,6 @@ export function defineExperiment(experiment: Experiment): Experiment {
   if (experiment.comparisonGroup !== undefined && experiment.comparisonGroup.trim() === "") {
     throw new Error(`defineExperiment: experiment "${experiment.id}" comparisonGroup must not be empty`);
   }
+  validateWorkspaceFilesRuntime(experiment.runtime);
   return deepFreeze(experiment);
 }

package/src/index.ts

@@ -17,8 +17,11 @@ export type {
   LoadedEval,
 } from "./evals/types.ts";
 export { EVAL_SUITES } from "./evals/types.ts";
+export { defineWorkspaceFiles, WORKSPACE_FILES_LIMITS } from "./runtime/workspace-files.ts";
+export type { WorkspaceFile, WorkspaceFiles, WorkspaceFilesIdentity } from "./runtime/workspace-files.ts";
+export type { WorkspaceFilesEvidence } from "./isolation/workspace-files.ts";
 export { defineExperiment } from "./experiments/types.ts";
-export type { AgentSpec, Experiment, LoadedExperiment, Runtime } from "./experiments/types.ts";
+export type { AgentSpec, Experiment, LoadedExperiment, Runtime, RuntimeCapacity } from "./experiments/types.ts";
 export type {
   ProvisionedEnvironment,
   RuntimeCapabilities,

package/src/isolation/docker.ts

@@ -21,6 +21,8 @@ import { mkdir, mkdtemp, rm, writeFile, cp, readdir, lstat, readlink, realpath }
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 import { sanitizeServerName, type McpUpstream } from "./mcp-proxy/server.ts";
+import { WORKSPACE_FILES_PROGRAM, type WorkspaceFilesRequest } from "./workspace-files.ts";
+import type { WorkspaceFilesIdentity } from "../runtime/workspace-files.ts";
 import { EXCLUDED_WORKSPACE_DIRS } from "../store/artifacts.ts";
 
 /** The label key every attempt container carries. */
@@ -66,14 +68,17 @@ export interface ExecResult {
  */
 async function runDocker(
   args: string[],
-  opts: { timeoutMs?: number; stdin?: string; env?: Record<string, string> } = {},
+  opts: { timeoutMs?: number; stdin?: string; env?: Record<string, string>; signal?: AbortSignal } = {},
 ): Promise<ExecResult> {
+  opts.signal?.throwIfAborted();
   const proc = Bun.spawn(["docker", ...args], {
     stdin: opts.stdin === undefined ? "ignore" : Buffer.from(opts.stdin),
     stdout: "pipe",
     stderr: "pipe",
     ...(opts.env !== undefined ? { env: { ...process.env, ...opts.env } } : {}),
   });
+  const abort = () => proc.kill("SIGKILL");
+  opts.signal?.addEventListener("abort", abort, { once: true });
   let timedOut = false;
   let timer: ReturnType<typeof setTimeout> | undefined;
   {
@@ -88,13 +93,14 @@ async function runDocker(
     proc.exited,
   ]);
   if (timer !== undefined) clearTimeout(timer);
+  opts.signal?.removeEventListener("abort", abort);
   return { exitCode, stdout, stderr, timedOut };
 }
 
 /** Run `docker <args>` and throw a DockerError on non-zero exit. */
 async function mustDocker(
   args: string[],
-  opts: { timeoutMs?: number; stdin?: string; env?: Record<string, string> } = {},
+  opts: { timeoutMs?: number; stdin?: string; env?: Record<string, string>; signal?: AbortSignal } = {},
 ): Promise<ExecResult> {
   const result = await runDocker(args, opts);
   if (result.exitCode !== 0) throw new DockerError(args, result.exitCode, result.stderr);
@@ -193,7 +199,9 @@ export interface AttemptContainer {
   /** Run a command in /workspace inside the container. Never throws for non-zero exits. */
   execInWorkspace(cmd: string[], opts?: { timeoutMs?: number }): Promise<ExecResult>;
   /** Write a file inside the container (parent directories created). */
-  writeFile(path: string, content: string): Promise<void>;
+  writeFile(path: string, content: string, opts?: { signal?: AbortSignal }): Promise<void>;
+  /** Bounded create-only overlay transport; input rides stdin, never argv. */
+  workspaceFiles(request: WorkspaceFilesRequest, signal: AbortSignal): Promise<WorkspaceFilesIdentity>;
   /** Copy /workspace to a host directory, excluding node_modules/vendor/.venv. */
   exportWorkspace(toDir: string): Promise<void>;
   /** `docker rm -f` — kill and remove. Idempotent. */
@@ -282,10 +290,22 @@ export async function createAttemptContainer(opts: CreateAttemptContainerOptions
       }
       return result;
     },
-    writeFile: async (path, content) => {
+    workspaceFiles: async (request, signal) => {
+      try {
+        const result = await runDocker(["exec", "-i", "-w", CONTAINER_WORKSPACE, id, "/usr/local/bin/node", "-e", WORKSPACE_FILES_PROGRAM, CONTAINER_WORKSPACE], {
+          stdin: JSON.stringify(request), signal,
+        });
+        if (signal.aborted || result.timedOut || result.exitCode !== 0) throw new Error("staging failed");
+        return JSON.parse(result.stdout) as WorkspaceFilesIdentity;
+      } catch {
+        // No raw stdout/stderr, command, or payload may enter diagnostics.
+        throw new Error("workspaceFiles: filesystem operation failed or was cancelled/timed out");
+      }
+    },
+    writeFile: async (path, content, writeOpts = {}) => {
       // `$1` positional keeps the target path out of shell interpolation.
       await mustDocker(["exec", "-i", id, "sh", "-c", 'mkdir -p "$(dirname "$1")" && cat > "$1"', "sh", path], {
-        stdin: content,
+        stdin: content, signal: writeOpts.signal,
       });
     },
     exportWorkspace: async (toDir) => {

package/src/isolation/workspace-files.ts

@@ -0,0 +1,92 @@
+import type { WorkspaceFiles, WorkspaceFilesIdentity } from "../runtime/workspace-files.ts";
+
+/** The same program runs in the container and offline filesystem tests. No shell or payload in argv. */
+export const WORKSPACE_FILES_PROGRAM = String.raw`
+const fs = require('node:fs');
+const crypto = require('node:crypto');
+const path = require('node:path');
+function apply(root, request) {
+  const { files, absent } = request.declaration;
+  const fail = () => { throw new Error('unsafe workspace state'); };
+  const stat = (p) => { try { return fs.lstatSync(p); } catch (e) { if (e.code === 'ENOENT') return null; throw e; } };
+  if (!stat(root)?.isDirectory() || stat(root).isSymbolicLink()) fail();
+  // No processes controlled by the agent run during preparation. On Linux,
+  // directory descriptors also anchor traversal against concurrent renames.
+  const descriptors = [];
+  const directory = (p) => {
+    const s = stat(p);
+    if (!s?.isDirectory() || s.isSymbolicLink()) fail();
+    const fd = fs.openSync(p, fs.constants.O_RDONLY | fs.constants.O_DIRECTORY | fs.constants.O_NOFOLLOW);
+    descriptors.push(fd);
+    return process.platform === 'linux' ? '/proc/self/fd/' + fd : p;
+  };
+  const base = directory(root);
+  const leaf = (relative, create = false) => {
+    while (descriptors.length > 1) fs.closeSync(descriptors.pop());
+    const parts = relative.split('/');
+    let parent = base;
+    for (const part of parts.slice(0, -1)) {
+      const next = parent + '/' + part;
+      if (!stat(next)) {
+        if (!create) return null;
+        fs.mkdirSync(next, { mode: 0o755 });
+      }
+      parent = directory(next);
+    }
+    return parent + '/' + parts.at(-1);
+  };
+  const checkAbsent = (relative) => { const p = leaf(relative); if (p !== null && stat(p)) fail(); };
+  try {
+    // Setup paths may be outside the workspace, but cannot alias it through
+    // symlinks. Lexical overlap is separately rejected before any write.
+    for (const target of request.setupPaths) {
+      const absolute = path.resolve(root, target);
+      let current = '/';
+      for (const part of absolute.split('/').filter(Boolean)) {
+        current = path.join(current, part);
+        const s = stat(current);
+        if (!s) break;
+        if (s.isSymbolicLink()) fail();
+      }
+    }
+    for (const relative of absent) checkAbsent(relative);
+    if (request.phase === 'stage') {
+      // Validate the entire inventory before writing even the first file.
+      for (const file of files) checkAbsent(file.path);
+      for (const file of files) {
+        const p = leaf(file.path, true);
+        const fd = fs.openSync(p, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, 0o644);
+        try { fs.writeFileSync(fd, Buffer.from(file.content, 'base64')); fs.fchmodSync(fd, 0o644); }
+        finally { fs.closeSync(fd); }
+      }
+    }
+    const inventory = files.map((file) => {
+      const p = leaf(file.path);
+      if (!p) fail();
+      const fd = fs.openSync(p, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK);
+      try {
+        const s = fs.fstatSync(fd);
+        if (!s.isFile() || s.nlink !== 1 || (s.mode & 0o777) !== 0o644 || s.size > 262144) fail();
+        const bytes = fs.readFileSync(fd);
+        return { path: file.path, size: bytes.length, sha256: crypto.createHash('sha256').update(bytes).digest('hex'), source: file.source };
+      } finally { fs.closeSync(fd); }
+    });
+    return { version: 1, files: inventory, absent };
+  } finally { for (const fd of descriptors.reverse()) fs.closeSync(fd); }
+}
+try {
+  const request = JSON.parse(fs.readFileSync(0, 'utf8'));
+  process.stdout.write(JSON.stringify(apply(process.argv[1], request)));
+} catch { process.stderr.write('workspaceFiles: filesystem validation/write/read-back failed'); process.exitCode = 1; }
+`;
+
+export interface WorkspaceFilesRequest {
+  declaration: WorkspaceFiles;
+  phase: "stage" | "verify";
+  setupPaths: string[];
+}
+export interface WorkspaceFilesEvidence {
+  version: 1;
+  verified: WorkspaceFilesIdentity;
+  verifiedAt: number;
+}

package/src/manifest.ts

@@ -10,6 +10,7 @@
  * adding or removing a component is a deliberate, test-visible act.
  */
 
+import { validateWorkspaceFilesRuntime, workspaceFilesIdentity, type WorkspaceFiles } from "./runtime/workspace-files.ts";
 import { readFileSync } from "node:fs";
 import { join, resolve } from "node:path";
 import { EMBEDDED_BUILD_INFO } from "./build-info.generated.ts";
@@ -151,6 +152,8 @@ export interface BuildRunManifestOptions {
   harness?: HarnessIdentity;
   pairs?: Record<string, PairEnvironment>;
   scorerIdentities?: Record<string, Record<string, unknown>>;
+  /** Share the runner's immutable payload snapshot without copying runtime instances. */
+  workspaceFilesByExperiment?: ReadonlyMap<string, WorkspaceFiles | undefined>;
 }
 
 /** Build before the first paid attempt; all reads are deterministic and local. */
@@ -178,6 +181,8 @@ export function buildRunManifest(options: BuildRunManifestOptions): RunManifest
 
   const experiments: Record<string, ExperimentIdentity> = {};
   for (const { experiment, path } of options.experiments) {
+    const files = options.workspaceFilesByExperiment ? options.workspaceFilesByExperiment.get(experiment.id) : experiment.runtime.workspaceFiles;
+    validateWorkspaceFilesRuntime({ kind: experiment.runtime.kind, command: experiment.runtime.command, workspaceFiles: files });
     const imageTag = experiment.runtime.image;
     const components: ExperimentIdentityComponents = {
       source: sourceIdentity(path),
@@ -190,7 +195,9 @@ export function buildRunManifest(options: BuildRunManifestOptions): RunManifest
       },
       runtime: contentHash({
         kind: experiment.runtime.kind,
-        capacity: experiment.runtime.capacity ?? null,
+        ...(files ? { workspaceFiles: workspaceFilesIdentity(files) } : {}),
+        // A capacity resolver is a capability function: per-eval, never identity.
+        capacity: typeof experiment.runtime.capacity === "function" ? null : (experiment.runtime.capacity ?? null),
         observationTargets: experiment.runtime.observationTargets ?? null,
         treatmentPolicyFields: [...(experiment.runtime.treatmentPolicyFields ?? [])].sort(),
         image: experiment.runtime.image ?? null,

package/src/runner/capacity.ts

@@ -0,0 +1,17 @@
+import type { EvalMetadata } from "../evals/types.ts";
+import type { Runtime, RuntimeCapacity } from "../experiments/types.ts";
+
+/**
+ * The capacity an attempt of `metadata` holds under `runtime`: the static
+ * pool, the resolver's answer for that eval, or none. The scheduler validates
+ * the shape of whatever comes back.
+ */
+export function resolveRuntimeCapacity(
+  runtime: Runtime | undefined,
+  metadata: EvalMetadata | undefined,
+): RuntimeCapacity | undefined {
+  const capacity = runtime?.capacity;
+  if (capacity === undefined) return undefined;
+  if (typeof capacity === "function") return metadata === undefined ? undefined : capacity(metadata);
+  return capacity;
+}

package/src/runner/execute.ts

@@ -21,6 +21,8 @@
  * and never allowed to mask the attempt's result.
  */
 
+import { stageWorkspaceFiles } from "./workspace-files.ts";
+import { defineWorkspaceFiles, validateWorkspaceFilesRuntime, workspaceFilesIdentity, type WorkspaceFiles } from "../runtime/workspace-files.ts";
 import { mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -36,6 +38,7 @@ import { EchoAdapter } from "../adapters/echo.ts";
 import { OracleAdapter, OracleContainerAdapter } from "../adapters/oracle.ts";
 import { validEvalResult } from "../evals/result.ts";
 import { importEvalScorer } from "../evals/discovery.ts";
+import { resolveRuntimeCapacity } from "./capacity.ts";
 import { deadline, StageTimeoutError } from "./deadline.ts";
 import { scoreInWorker } from "./score-worker.ts";
 import { prepWorkspaceSource } from "./workspace-source.ts";
@@ -102,6 +105,8 @@ export interface ExecuteRunV3Options {
   config?: Partial<RunConfig>;
   /** Host adapters by name. Defaults to the built-in echo adapter. */
   hostAdapters?: Record<string, AgentAdapter>;
+  /** Internal offline-test seam; not part of the public consumer API. */
+  createContainer?: typeof createAttemptContainer;
   /**
    * Container agent adapter factories by name, constructed per experiment
    * from its `agent` spec. Defaults to the built-in claude/codex adapters.
@@ -314,6 +319,13 @@ export function collectOfferedSurfaces(options: {
 }
 
 export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSummaryV3> {
+  // Snapshot only the payload: runtime methods must keep their original receiver,
+  // including prototype methods and private fields. Capture before the first await.
+  const workspaceFilesByExperiment = new Map(options.experiments.map(({ experiment }) => {
+    validateWorkspaceFilesRuntime(experiment.runtime);
+    const files = experiment.runtime.workspaceFiles;
+    return [experiment.id, files === undefined ? undefined : defineWorkspaceFiles(files)] as const;
+  }));
   const { store, artifacts, plan } = options;
   const log = options.log ?? ((line: string) => console.log(line));
   const attemptTimeoutMs = options.attemptTimeoutMs ?? options.budgets?.agentMs ?? DEFAULT_ATTEMPT_TIMEOUT_MS;
@@ -468,7 +480,7 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
       const endpoints = Object.fromEntries(Object.entries(overrides).filter(([name]) => !isSecretEnvName(name)));
       return Object.keys(endpoints).length > 0 ? [[id, endpoints] as const] : [];
     }));
-    const pairs = resolvePairEnvironments({ ...options, images, imageTags: imageTagsByPair,
+    const pairs = resolvePairEnvironments({ ...options, workspaceFilesByExperiment, images, imageTags: imageTagsByPair,
       adapterHosts: Object.fromEntries([...containerAgents].map(([id, adapter]) => [id, [
         ...(adapter.egressHosts ?? []),
         ...(options.egressProxy ? endpointHosts(overridesByExperiment.get(id) ?? {}) : []),
@@ -477,7 +489,16 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
       budgets: { ...options.budgets, agentMs: options.attemptTimeoutMs ?? options.budgets?.agentMs ?? attemptTimeoutMs },
       mcpHashes: Object.fromEntries([...declaredByExperiment].map(([id, servers]) => [id, hashString(canonicalJson(servers))])),
     });
-    manifest = buildRunManifest({ evals: options.evals, experiments: options.experiments, images, cliVersions, pairs, scorerIdentities });
+    manifest = buildRunManifest({ evals: options.evals, experiments: options.experiments, workspaceFilesByExperiment, images, cliVersions, pairs, scorerIdentities });
+  }
+  // Injected/stale manifests must not authorize a different payload (or silently
+  // omit one). Historical no-overlay manifests retain their original shape.
+  for (const entry of plan) {
+    const files = workspaceFilesByExperiment.get(entry.experimentId);
+    const intended = files ? workspaceFilesIdentity(files) : undefined;
+    if (canonicalJson(intended ?? null) !== canonicalJson(manifest.pairs?.[pairKey(entry.evalId, entry.experimentId)]?.workspaceFiles ?? null)) {
+      throw new Error("workspaceFiles: manifest payload identity mismatch");
+    }
   }
   const recorded = options.resumeRunId ? store.getRun(options.resumeRunId) : null;
   if (options.resumeRunId && !recorded) throw new Error(`cannot resume unknown run ${options.resumeRunId}`);
@@ -582,7 +603,7 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
                 loadedEval,
                 experiment,
                 entry,
-                { runId, attemptId, attemptTimeoutMs, persisted, environment: manifest.pairs?.[pairKey(entry.evalId, entry.experimentId)] },
+                { runId, attemptId, attemptTimeoutMs, persisted, environment: manifest.pairs?.[pairKey(entry.evalId, entry.experimentId)], workspaceFiles: workspaceFilesByExperiment.get(entry.experimentId) },
                 treatments,
                 containerAgents.get(experiment.id),
               );
@@ -616,7 +637,10 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
         `attempt ${attemptId} (${entry.evalId} × ${entry.experimentId} trial ${entry.trialIndex + 1}): ` +
           `${disposition.status}${disposition.result ? ` — ${disposition.result.passed ? "passed" : "failed"}` : ""}`,
       );
-    }, { ...(options.signal ? { signal: options.signal } : {}), capacity: (entry) => experimentsById.get(entry.experimentId)?.runtime.capacity });
+    }, {
+      ...(options.signal ? { signal: options.signal } : {}),
+      capacity: (entry) => resolveRuntimeCapacity(experimentsById.get(entry.experimentId)?.runtime, evalsById.get(entry.evalId)?.metadata),
+    });
 
     const current = activeAttempts(store.listAttempts(runId));
     summary.passed = current.filter((a) => a.status === "completed" && a.result?.passed).length;
@@ -649,6 +673,7 @@ interface AttemptArgs {
   attemptTimeoutMs: number;
   persisted: NewAttempt;
   environment?: PairEnvironment;
+  workspaceFiles?: WorkspaceFiles;
 }
 
 async function runHostAttemptV3(
@@ -745,6 +770,7 @@ async function runContainerAttemptV3(
   const { artifacts } = options;
   const { runId, attemptId, persisted } = args;
   const runtime = experiment.runtime;
+  const workspaceFiles = args.workspaceFiles;
   const environment = args.environment;
   let egress: { networkName: string; sidecar: EgressSidecar } | undefined;
   let mcpIp: string | undefined;
@@ -823,7 +849,6 @@ async function runContainerAttemptV3(
       // Endpoint overrides set on the host for this agent (gateway base URLs
       // and the headers they need) — see ContainerAgentAdapter.optionalHostEnv.
       const overrides = agentAdapter !== undefined ? collectOptionalHostEnv([agentAdapter]) : {};
-      const publicOverrides = Object.fromEntries(Object.entries(overrides).filter(([name]) => !isSecretEnvName(name)));
       if (agentAdapter !== undefined) {
         // The agent's provider keys, validated at run preflight — injected
         // at create via the 0600 env-file channel, never argv.
@@ -856,7 +881,7 @@ async function runContainerAttemptV3(
 
       const paths = await inStage("environment", () => artifacts.prepareAttempt(runId, attemptId));
       const fixtureDir = sourceDir;
-      const container = await inStage("environment", () => createAttemptContainer({
+      const container = await inStage("environment", () => (options.createContainer ?? createAttemptContainer)({
         image,
         env,
         resultsDir: paths.attemptDir,
@@ -887,6 +912,7 @@ async function runContainerAttemptV3(
               return (await mustExec(container, ["git", "rev-parse", "HEAD"])).stdout.trim();
             });
           } else {
+            if (workspaceFiles) throw new StageFailure("environment", new Error("workspaceFiles: git baseline is required"));
             (options.log ?? console.log)(
               `attempt ${attemptId}: git is unavailable in image "${image}" — diff.patch will be empty ` +
                 `(the agent-runtime image ships git; only bare test images lack it)`,
@@ -902,14 +928,20 @@ async function runContainerAttemptV3(
             trialIndex: entry.trialIndex,
             ...(rewrittenServers !== undefined ? { mcpServers: rewrittenServers } : {}),
             webPolicy: environment?.webPolicy ?? "native-web-blocked",
-            // Never the keys or header bundles: adapters whose CLI wants a
-            // config-file entry (codex's model provider) render from here.
-            ...(Object.keys(publicOverrides).length > 0 ? { overrides: publicOverrides } : {}),
+            // The adapter's own overrides (never the provider keys): a CLI
+            // that wants a config-file entry (codex's model provider and its
+            // http_headers) renders them from here, inside the container.
+            ...(Object.keys(overrides).length > 0 ? { overrides } : {}),
           };
           // Treatment config files (e.g. the MCP treatment's vendor config).
           await inStage("environment", async () => {
-            for (const [path, content] of Object.entries(agentAdapter.setupFiles(promptCtx))) {
-              await container.writeFile(path, content);
+            const setup = agentAdapter.setupFiles(promptCtx);
+            if (workspaceFiles) {
+              const evidence = await stageWorkspaceFiles(container, workspaceFiles, environment!.workspaceFiles!, setup, options.signal);
+              await artifacts.writeWorkspaceFilesEvidence(runId, attemptId, evidence);
+              if (options.signal?.aborted) throw new Error("workspaceFiles: staging cancelled");
+            } else {
+              for (const [path, content] of Object.entries(setup)) await container.writeFile(path, content);
             }
           });
           command = agentAdapter.command(promptCtx);
@@ -917,6 +949,7 @@ async function runContainerAttemptV3(
           command = runtime.command as string[];
         }
 
+        if (workspaceFiles && options.signal?.aborted) throw new StageFailure("environment", new Error("workspaceFiles: staging cancelled"));
         persisted.agentStartedAt = Date.now();
         let agentResult: Awaited<ReturnType<typeof container.execInWorkspace>>;
         try {
@@ -1039,7 +1072,10 @@ async function runContainerAttemptV3(
       try { await egress.sidecar.teardown(); } catch { containerTeardownFailure = "egress sidecar teardown failed"; }
       try { await removeRunNetwork(egress.networkName); } catch { containerTeardownFailure = "egress network teardown failed"; }
     }
-    if (sourceDir) await rm(sourceDir, { recursive: true, force: true });
+    if (sourceDir) {
+      try { await rm(sourceDir, { recursive: true, force: true }); }
+      catch { containerTeardownFailure = [containerTeardownFailure, "workspace source cleanup failed"].filter(Boolean).join("; "); }
+    }
   }
 
   // Explicit teardown accounting: the attempt keeps its status and result;

package/src/runner/workspace-files.ts

@@ -0,0 +1,52 @@
+import { posix } from "node:path";
+import { canonicalJson } from "../identity-diff.ts";
+import type { AttemptContainer } from "../isolation/docker.ts";
+import type { WorkspaceFilesEvidence } from "../isolation/workspace-files.ts";
+import { pathsOverlap, WORKSPACE_FILES_LIMITS, type WorkspaceFiles, type WorkspaceFilesIdentity } from "../runtime/workspace-files.ts";
+
+/** No race-to-timeout: await cancelled transport, then let the attempt's finally tear down. */
+export async function stageWorkspaceFiles(
+  container: AttemptContainer,
+  declaration: WorkspaceFiles,
+  intended: WorkspaceFilesIdentity,
+  setup: Record<string, string>,
+  parentSignal?: AbortSignal,
+): Promise<WorkspaceFilesEvidence> {
+  const controller = new AbortController();
+  const abort = () => controller.abort();
+  const timer = setTimeout(abort, WORKSPACE_FILES_LIMITS.timeoutMs);
+  parentSignal?.addEventListener("abort", abort, { once: true });
+  if (parentSignal?.aborted) abort();
+  const signal = controller.signal;
+  try {
+    const setupPaths = Object.keys(setup);
+    for (const path of setupPaths) {
+      const relative = posix.relative("/workspace", posix.resolve("/workspace", path));
+      if ([...intended.files.map((file) => file.path), ...intended.absent].some((target) => relative === "" || pathsOverlap(relative, target))) {
+        throw new Error("workspaceFiles: adapter setup collision");
+      }
+    }
+    const check = async (phase: "stage" | "verify") => {
+      signal.throwIfAborted();
+      const actual = await container.workspaceFiles({ declaration, phase, setupPaths }, signal);
+      signal.throwIfAborted();
+      if (canonicalJson(actual) !== canonicalJson(intended)) throw new Error("workspaceFiles: read-back/absence verification mismatch");
+      return actual;
+    };
+    await check("stage");
+    for (const [path, content] of Object.entries(setup)) {
+      signal.throwIfAborted();
+      await container.writeFile(path, content, { signal });
+    }
+    const verified = await check("verify");
+    return { version: 1, verified, verifiedAt: Date.now() };
+  } catch (err) {
+    if (signal.aborted) throw new Error("workspaceFiles: staging cancelled or timed out");
+    // Only our own fixed diagnostics survive; transport/setup errors may carry payloads.
+    if (err instanceof Error && ["workspaceFiles: adapter setup collision", "workspaceFiles: read-back/absence verification mismatch"].includes(err.message)) throw err;
+    throw new Error("workspaceFiles: staging filesystem/setup operation failed");
+  } finally {
+    clearTimeout(timer);
+    parentSignal?.removeEventListener("abort", abort);
+  }
+}

package/src/runtime/workspace-files.ts

@@ -0,0 +1,91 @@
+import { hashString, SECRET_FILE } from "../hash.ts";
+import { EXCLUDED_WORKSPACE_DIRS } from "../store/artifacts.ts";
+
+/** Non-secret, regular, non-executable files. Source labels are declarations, not attestations. */
+export interface WorkspaceFile {
+  path: string;
+  content: string;
+  encoding: "utf8" | "base64";
+  source: { id: string; version: string };
+}
+
+/** v1: create-only, no symlinks/hardlinks, 0644 files, final read-back, no deletion. */
+export interface WorkspaceFiles {
+  version: 1;
+  files: readonly WorkspaceFile[];
+  absent: readonly string[];
+}
+
+export interface WorkspaceFilesIdentity {
+  version: 1;
+  files: readonly { path: string; size: number; sha256: string; source: { id: string; version: string } }[];
+  absent: readonly string[];
+}
+
+export const WORKSPACE_FILES_LIMITS = Object.freeze({ files: 128, absent: 128, fileBytes: 262144, totalBytes: 1048576, pathBytes: 240, depth: 16, timeoutMs: 30000 });
+
+function invalid(reason: string): never {
+  // Never interpolate consumer data: even malformed metadata may contain secrets.
+  throw new Error(`workspaceFiles: ${reason}`);
+}
+
+function pathCheck(path: unknown): asserts path is string {
+  if (typeof path !== "string" || path.length > WORKSPACE_FILES_LIMITS.pathBytes) invalid("invalid relative path");
+  const parts = path.split("/");
+  if (parts.length > WORKSPACE_FILES_LIMITS.depth || parts.some((part) =>
+    !/^[A-Za-z0-9_.-]+$/.test(part) || part === "." || part === ".." || part.endsWith(".") ||
+    /^(con|prn|aux|nul|com[0-9]|lpt[0-9])(?:\.|$)/i.test(part) ||
+    EXCLUDED_WORKSPACE_DIRS.includes(part.toLowerCase()) || SECRET_FILE.test(part) ||
+    /^\.(?:git|hg|svn)/i.test(part) || /^\.quickstudy/i.test(part)
+  )) invalid("unsafe or reserved relative path");
+}
+
+export function pathsOverlap(a: string, b: string): boolean {
+  a = a.toLowerCase(); b = b.toLowerCase();
+  return a === b || a.startsWith(`${b}/`) || b.startsWith(`${a}/`);
+}
+
+/** Validate, copy, canonicalize to base64, and freeze without retaining mutable caller state. */
+export function defineWorkspaceFiles(input: WorkspaceFiles): WorkspaceFiles {
+  if (!input || input.version !== 1) invalid("unsupported staging version");
+  if (!Array.isArray(input.files) || !Array.isArray(input.absent) ||
+      input.files.length > WORKSPACE_FILES_LIMITS.files || input.absent.length > WORKSPACE_FILES_LIMITS.absent) invalid("invalid inventory or count limit exceeded");
+  let total = 0;
+  const files = input.files.map((file) => {
+    if (!file || typeof file.content !== "string" || file.content.length > WORKSPACE_FILES_LIMITS.fileBytes * 4) invalid("invalid or oversized payload");
+    pathCheck(file.path);
+    if (!file.source || ![file.source.id, file.source.version].every((v) => typeof v === "string" && /^[A-Za-z0-9][A-Za-z0-9_.:@+-]{0,159}$/.test(v))) invalid("invalid source id/version (use opaque labels, not paths or URLs)");
+    if (file.encoding !== "utf8" && file.encoding !== "base64") invalid("unsupported encoding");
+    const bytes = Buffer.from(file.content, file.encoding === "utf8" ? "utf8" : "base64");
+    if (file.encoding === "base64" && bytes.toString("base64") !== file.content) invalid("base64 must be canonical padded RFC 4648");
+    if (file.encoding === "utf8" && bytes.toString("utf8") !== file.content) invalid("utf8 must not contain unpaired surrogates");
+    total += bytes.length;
+    if (bytes.length > WORKSPACE_FILES_LIMITS.fileBytes || total > WORKSPACE_FILES_LIMITS.totalBytes) invalid("payload size limit exceeded");
+    return Object.freeze({ path: file.path, content: bytes.toString("base64"), encoding: "base64" as const, source: Object.freeze({ id: file.source.id, version: file.source.version }) });
+  }).sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
+  const absent = [...input.absent];
+  for (const path of absent) pathCheck(path);
+  const paths = [...files.map((file) => file.path), ...absent];
+  for (let i = 0; i < paths.length; i++) for (let j = i + 1; j < paths.length; j++) {
+    if (pathsOverlap(paths[i]!, paths[j]!)) invalid("duplicate or overlapping destinations/absence assertions");
+  }
+  return Object.freeze({ version: 1, files: Object.freeze(files), absent: Object.freeze(absent.sort()) });
+}
+
+export function workspaceFilesIdentity(input: WorkspaceFiles): WorkspaceFilesIdentity {
+  const declaration = defineWorkspaceFiles(input);
+  return {
+    version: declaration.version,
+    files: declaration.files.map((file) => {
+      const bytes = Buffer.from(file.content, "base64");
+      return { path: file.path, size: bytes.length, sha256: hashString(bytes), source: file.source };
+    }),
+    absent: declaration.absent,
+  };
+}
+
+export function validateWorkspaceFilesRuntime(runtime: { kind: string; command?: string[]; workspaceFiles?: WorkspaceFiles }): void {
+  if (runtime.workspaceFiles === undefined) return;
+  defineWorkspaceFiles(runtime.workspaceFiles);
+  if (runtime.kind !== "container" || (runtime.command?.length ?? 0) > 0) invalid("requires a container adapter; host and scripted commands are unsupported");
+}

package/src/store/artifacts.ts

@@ -22,6 +22,7 @@
 
 import { cp, mkdir, writeFile, lstat, realpath, readlink } from "node:fs/promises";
 import { basename, join, resolve } from "node:path";
+import type { WorkspaceFilesEvidence } from "../isolation/workspace-files.ts";
 import type { TranscriptEvent } from "../adapters/types.ts";
 
 /** Dependency directories never copied into an exported workspace. */
@@ -73,6 +74,12 @@ export class ArtifactsStore {
     return paths;
   }
 
+  /** Placement evidence, distinct from intended identity; proves no model use. */
+  async writeWorkspaceFilesEvidence(runId: string, attemptId: string, evidence: WorkspaceFilesEvidence): Promise<void> {
+    const paths = await this.prepareAttempt(runId, attemptId);
+    await writeFile(join(paths.attemptDir, "workspace-files.json"), `${JSON.stringify(evidence, null, 2)}\n`, "utf8");
+  }
+
   async writeTranscript(runId: string, attemptId: string, events: TranscriptEvent[]): Promise<string> {
     const paths = await this.prepareAttempt(runId, attemptId);
     const jsonl = events.map((event) => JSON.stringify(event)).join("\n");