@workos/quickstudy 0.8.1 → 0.10.0

package/README.md

@@ -118,6 +118,37 @@ experiment module — changing them changes the experiment's identity, by
 design. See the benchmark repository's `examples/weekly-sweep.yml` for what
 a full scheduled sweep costs and looks like in CI.
 
+### Routing through an LLM gateway
+
+The vendor CLIs and the diagnosis analyst all honor endpoint overrides, so a
+gateway or proxy (Cloudflare AI Gateway, LiteLLM, a corporate egress) is an
+env change, not a code change. Each adapter declares its overrides
+(`optionalHostEnv`); the harness forwards them into attempt containers only
+when set, adds a base URL's host to the egress allowlist, and points the
+preflight key probe at it:
+
+| Agent  | Overrides                                        | How the CLI gets them                                                                  |
+| ------ | ------------------------------------------------ | -------------------------------------------------------------------------------------- |
+| claude | `ANTHROPIC_BASE_URL`, `ANTHROPIC_CUSTOM_HEADERS` | env, read natively by Claude Code (and by the `diagnose` analyst)                      |
+| codex  | `OPENAI_BASE_URL`, `OPENAI_CUSTOM_HEADERS`       | rendered into `config.toml` as a custom model provider (`base_url`, `http_headers`); codex has no env channel for either |
+
+The `*_CUSTOM_HEADERS` bundles are `Name: Value` lines and are redacted from
+artifacts like a key; a `cf-aig-metadata` line is how a gateway attributes
+the traffic per app. With Cloudflare AI Gateway holding the provider keys,
+the gateway token *is* the API key:
+
+```bash
+export ANTHROPIC_BASE_URL=https://gateway.ai.cloudflare.com/v1/<account>/<gateway>/anthropic
+export ANTHROPIC_API_KEY=<gateway token>
+export ANTHROPIC_CUSTOM_HEADERS="cf-aig-authorization: Bearer <gateway token>"
+export OPENAI_BASE_URL=https://gateway.ai.cloudflare.com/v1/<account>/<gateway>/openai
+export OPENAI_API_KEY=<gateway token>
+export OPENAI_CUSTOM_HEADERS='cf-aig-metadata: {"app":"my-evals"}'
+```
+
+Leave every override unset and both agents run vendor-direct against
+api.anthropic.com and api.openai.com — nothing is rendered, nothing changes.
+
 ## How it works
 
 Every attempt is one point in a 2-D plan:
@@ -181,7 +212,9 @@ failing pair's harness-selected evidence (per-check tallies, transcript
 excerpts, egress denials) to a structured LLM call and writes `diagnosis.json`
 beside `report.json` — findings are evidence-linked hypotheses, never
 verdicts, keeping the report itself attribution-free. Requires
-`ANTHROPIC_API_KEY`; the UI renders the result as a Diagnosis view.
+`ANTHROPIC_API_KEY` (and follows the claude adapter's gateway overrides, see
+[Routing through an LLM gateway](#routing-through-an-llm-gateway)); the UI
+renders the result as a Diagnosis view.
 
 ## Writing evals
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workos/quickstudy",
-  "version": "0.8.1",
+  "version": "0.10.0",
   "description": "An open-source, company-agnostic eval harness for measuring whether coding agents can complete real integration tasks, across an agent x feature x framework x surface matrix.",
   "license": "MIT",
   "type": "module",

package/src/adapters/claude.ts

@@ -14,6 +14,7 @@
  */
 
 import type { McpServerConfig } from "../isolation/mcp.ts";
+import { endpointUrl, parseHeaderLines } from "../secrets.ts";
 import { parseClaudeStream } from "./parse.ts";
 import type { AttemptMetrics, ContainerAgentAdapter, ContainerAttemptContext } from "./types.ts";
 
@@ -38,10 +39,21 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter {
   readonly identity;
   readonly versionCommand = ["claude", "--version"] as const;
   readonly requiredHostEnv = ["ANTHROPIC_API_KEY"] as const;
+  /**
+   * Endpoint overrides, forwarded only when set. ANTHROPIC_BASE_URL routes
+   * the CLI (and the key probe) through an Anthropic-compatible gateway —
+   * e.g. Cloudflare AI Gateway's `/anthropic` endpoint, where the gateway
+   * token doubles as ANTHROPIC_API_KEY; ANTHROPIC_CUSTOM_HEADERS (`Name:
+   * Value` lines) carries whatever else that gateway wants, such as
+   * `cf-aig-authorization` or `cf-aig-metadata`. Claude Code reads both
+   * natively, so no flag changes.
+   */
+  readonly optionalHostEnv = ["ANTHROPIC_BASE_URL", "ANTHROPIC_CUSTOM_HEADERS"] as const;
   /**
    * Egress-proxy allowlist (harness-default layer). Claude Code documents
    * corporate-proxy support via HTTPS_PROXY/HTTP_PROXY; api.anthropic.com is
-   * the LLM endpoint, statsig/sentry are its telemetry (allowed so denial
+   * the default LLM endpoint (an ANTHROPIC_BASE_URL override adds its own
+   * host at run time), statsig/sentry are its telemetry (allowed so denial
    * logs stay signal, not noise). Verified live by the egress-proxy regression
    * check (`bun scripts/dev-attempt.ts --experiment <id> --egress-proxy`).
    */
@@ -66,8 +78,12 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter {
 
   keyProbe(env: Record<string, string>): { url: string; headers: Record<string, string> } {
     return {
-      url: "https://api.anthropic.com/v1/models",
-      headers: { "x-api-key": env["ANTHROPIC_API_KEY"] ?? "", "anthropic-version": "2023-06-01" },
+      url: endpointUrl(env["ANTHROPIC_BASE_URL"], "https://api.anthropic.com", "v1/models"),
+      headers: {
+        ...parseHeaderLines(env["ANTHROPIC_CUSTOM_HEADERS"]),
+        "x-api-key": env["ANTHROPIC_API_KEY"] ?? "",
+        "anthropic-version": "2023-06-01",
+      },
     };
   }
 

package/src/adapters/codex.ts

@@ -9,15 +9,60 @@
  * container create, but codex >= 0.144 no longer reads it directly — the
  * command wrapper pipes it into `codex login --with-api-key` (stdin, never
  * argv) to write the auth.json the CLI requires.
+ *
+ * Endpoint overrides: an OPENAI_BASE_URL set on the host (an LLM gateway) is
+ * rendered into the same config.toml as a custom model provider — codex
+ * 0.149 ignores the env var for its built-in provider (verified live: it
+ * still dialed api.openai.com), while a `[model_providers.*]` entry with
+ * `env_key = "OPENAI_API_KEY"` routes every request through the override.
+ * OPENAI_CUSTOM_HEADERS (`Name: Value` lines, the Codex counterpart of Claude
+ * Code's ANTHROPIC_CUSTOM_HEADERS) becomes that provider's `http_headers` —
+ * e.g. `cf-aig-metadata` so a gateway can attribute the traffic. Codex has
+ * no env-var channel for extra headers, so the values land in config.toml
+ * inside the attempt container: the same trust boundary as the env that
+ * already carries them, written over stdin, destroyed with the container.
+ * With neither override set nothing is rendered and the built-in provider
+ * talks to api.openai.com — vendor-direct stays the default.
  */
 
 import type { McpServerConfig } from "../isolation/mcp.ts";
+import { endpointUrl, parseHeaderLines } from "../secrets.ts";
 import { parseCodexStream } from "./parse.ts";
 import type { AttemptMetrics, ContainerAgentAdapter, ContainerAttemptContext } from "./types.ts";
 
 /** Container path of the Codex config (containers run as root). */
 export const CODEX_CONFIG_PATH = "/root/.codex/config.toml";
 
+/** The config.toml model provider id the OPENAI_* overrides are rendered as. */
+export const CODEX_GATEWAY_PROVIDER = "quickstudy-gateway";
+
+/** Codex's own default for the built-in provider; the probe and a headers-only override use it. */
+const OPENAI_DEFAULT_BASE_URL = "https://api.openai.com/v1";
+
+/**
+ * A custom model provider pointing at `baseUrl` (the Responses API, which
+ * codex requires), carrying `headers` on every request, selected as the
+ * default. Top-level keys must precede any table, so this block always
+ * renders first in the file. JSON string escapes are valid TOML basic-string
+ * escapes, which keeps header values such as JSON metadata intact.
+ */
+function renderCodexProviderToml(baseUrl: string, headers: Readonly<Record<string, string>>): string {
+  const entries = Object.entries(headers);
+  return [
+    `model_provider = ${JSON.stringify(CODEX_GATEWAY_PROVIDER)}`,
+    "",
+    `[model_providers.${CODEX_GATEWAY_PROVIDER}]`,
+    'name = "OpenAI (quickstudy endpoint overrides)"',
+    `base_url = ${JSON.stringify(baseUrl)}`,
+    'env_key = "OPENAI_API_KEY"',
+    'wire_api = "responses"',
+    ...(entries.length > 0
+      ? [`http_headers = { ${entries.map(([name, value]) => `${JSON.stringify(name)} = ${JSON.stringify(value)}`).join(", ")} }`]
+      : []),
+    "",
+  ].join("\n");
+}
+
 /**
  * `config.toml` MCP table rendered from the runtime's MCP servers (url-based,
  * credential-free — authed servers already point at the token-proxy sidecar).
@@ -37,13 +82,24 @@ export class CodexAdapter implements ContainerAgentAdapter {
   readonly identity;
   readonly versionCommand = ["codex", "--version"] as const;
   readonly requiredHostEnv = ["OPENAI_API_KEY"] as const;
+  /**
+   * Endpoint overrides, forwarded only when set: OPENAI_BASE_URL (vendor
+   * default https://api.openai.com/v1) — e.g. Cloudflare AI Gateway's
+   * `/openai` endpoint, where the gateway token doubles as OPENAI_API_KEY —
+   * and OPENAI_CUSTOM_HEADERS (`Name: Value` lines) for whatever else the
+   * endpoint wants on each request, such as `cf-aig-metadata`. Both are
+   * rendered into config.toml by setupFiles (see the module comment); the
+   * key probe follows them.
+   */
+  readonly optionalHostEnv = ["OPENAI_BASE_URL", "OPENAI_CUSTOM_HEADERS"] as const;
   /**
    * Egress-proxy allowlist (harness-default layer). Codex is reqwest-based;
    * reqwest honors HTTP_PROXY/HTTPS_PROXY env by default (system proxies) —
    * verified live by the egress-proxy regression check (the CONNECT tunnel
    * to api.openai.com round-tripped). chatgpt.com came from that run's
    * denial log: codex probes its ChatGPT-auth endpoints even under key auth.
-   * (Its github.com update-check denials are left denied on purpose.)
+   * (Its github.com update-check denials are left denied on purpose.) An
+   * OPENAI_BASE_URL override adds its own host at run time.
    */
   readonly egressHosts = ["api.openai.com", "auth.openai.com", "chatgpt.com", "*.chatgpt.com"] as const;
 
@@ -66,14 +122,22 @@ export class CodexAdapter implements ContainerAgentAdapter {
 
   keyProbe(env: Record<string, string>): { url: string; headers: Record<string, string> } {
     return {
-      url: "https://api.openai.com/v1/models",
-      headers: { Authorization: `Bearer ${env["OPENAI_API_KEY"] ?? ""}` },
+      url: endpointUrl(env["OPENAI_BASE_URL"], OPENAI_DEFAULT_BASE_URL, "models"),
+      headers: { ...parseHeaderLines(env["OPENAI_CUSTOM_HEADERS"]), Authorization: `Bearer ${env["OPENAI_API_KEY"] ?? ""}` },
     };
   }
 
   setupFiles(ctx: ContainerAttemptContext): Record<string, string> {
-    if (ctx.mcpServers === undefined) return {};
-    return { [CODEX_CONFIG_PATH]: renderCodexMcpToml(ctx.mcpServers) };
+    const sections: string[] = [];
+    const baseUrl = ctx.overrides?.["OPENAI_BASE_URL"];
+    const headers = parseHeaderLines(ctx.overrides?.["OPENAI_CUSTOM_HEADERS"]);
+    // Either override needs the custom provider; headers alone keep the vendor endpoint.
+    if (baseUrl !== undefined || Object.keys(headers).length > 0) {
+      sections.push(renderCodexProviderToml(baseUrl ?? OPENAI_DEFAULT_BASE_URL, headers));
+    }
+    if (ctx.mcpServers !== undefined) sections.push(renderCodexMcpToml(ctx.mcpServers));
+    if (sections.length === 0) return {};
+    return { [CODEX_CONFIG_PATH]: sections.join("\n") };
   }
 
   command(ctx: ContainerAttemptContext): string[] {

package/src/adapters/types.ts

@@ -132,6 +132,15 @@ export interface ContainerAttemptContext {
   mcpServers?: Readonly<Record<string, McpServerConfig>>;
   /** The runtime's native web-tool policy (default: blocked). */
   webPolicy: WebPolicy;
+  /**
+   * The adapter's own endpoint overrides that are set on the host (every
+   * `optionalHostEnv` value, header bundles included — never the provider
+   * keys), for a CLI that takes a config-file entry rather than an env var.
+   * Anything rendered from these is written inside the attempt container
+   * only, the same boundary the env already crosses. Absent when none are
+   * set.
+   */
+  overrides?: Readonly<Record<string, string>>;
 }
 
 /**
@@ -153,10 +162,11 @@ export interface ContainerAgentAdapter {
   /** Command used during preflight to record the executable's exact version. */
   versionCommand?: readonly string[];
   /**
-   * Cheap authenticated request composed from the collected provider keys,
-   * probed at run preflight so a present-but-rejected key aborts BEFORE any
-   * container spend. A 401/403 response stops the run; network failures only
-   * warn — an unreachable probe must not gate a run the agent might complete.
+   * Cheap authenticated request composed from the collected provider keys
+   * (plus any `optionalHostEnv` overrides that are set), probed at run
+   * preflight so a present-but-rejected key aborts BEFORE any container
+   * spend. A 401/403 response stops the run; network failures only warn — an
+   * unreachable probe must not gate a run the agent might complete.
    */
   keyProbe?(env: Record<string, string>): { url: string; headers: Record<string, string> };
   /**
@@ -164,6 +174,15 @@ export interface ContainerAgentAdapter {
    * (shared provider keys — see src/secrets.ts).
    */
   requiredHostEnv: readonly string[];
+  /**
+   * Host env vars forwarded into the container when set, never required:
+   * endpoint overrides such as a vendor base URL that routes the CLI through
+   * an LLM gateway or proxy, and the custom headers that gateway needs. A
+   * value that is an absolute URL also adds its host to the egress allowlist
+   * and retargets the key probe; secret-named values (headers carrying a
+   * bearer token) are redacted from artifacts like provider keys.
+   */
+  optionalHostEnv?: readonly string[];
   /**
    * Hosts this agent's CLI must reach (its LLM/vendor endpoints) — the
    * harness-default layer of the egress allowlist when the run uses the

package/src/build-info.generated.ts

@@ -9,7 +9,7 @@ export const EMBEDDED_BUILD_INFO: {
   // (the `x-release-please-version` marker anchors the replacement). It is
   // restored as-is after a binary build (scripts/build-binary.ts), so the
   // committed source `--version` stays in lockstep with package.json.
-  version: "0.8.1", // x-release-please-version
+  version: "0.10.0", // x-release-please-version
   revision: null,
   dirty: null,
   sourceHash: null,

package/src/execution-policy.ts

@@ -27,6 +27,13 @@ export interface PairEnvironment {
   /** Runtime text prepended to the eval prompt for this pair (null = prompt verbatim). */
   promptPreamble: string | null;
   budgets: LifecycleBudgets;
+  /**
+   * The agent's endpoint overrides in force (non-secret base URLs, e.g.
+   * OPENAI_BASE_URL → an LLM gateway route), recorded in full so a changed
+   * route changes identity and a resume cannot mix routes. Absent when the
+   * agent runs vendor-direct — such pairs hash exactly as before.
+   */
+  endpoints?: Record<string, string>;
 }
 
 /** The prompt an agent receives: the runtime's preamble (if any), a blank line, the eval's verbatim body. */
@@ -43,6 +50,8 @@ export function resolvePairEnvironments(options: {
   images?: Record<string, RuntimeImageIdentity>;
   egressProxy?: boolean;
   adapterHosts?: Record<string, readonly string[]>;
+  /** Per experiment: the agent's non-secret endpoint overrides (see PairEnvironment.endpoints). */
+  adapterEndpoints?: Record<string, Record<string, string>>;
   budgets?: Partial<LifecycleBudgets>;
   mcpHashes?: Record<string, string>;
   imageTags?: Record<string, string>;
@@ -77,6 +86,10 @@ export function resolvePairEnvironments(options: {
           return text === "" ? null : text;
         })(),
         budgets: { ...budgets },
+        ...(() => {
+          const endpoints = options.adapterEndpoints?.[experiment.id];
+          return endpoints !== undefined && Object.keys(endpoints).length > 0 ? { endpoints: { ...endpoints } } : {};
+        })(),
       };
     }
   return pairs;

package/src/isolation/docker.ts

@@ -59,15 +59,20 @@ export interface ExecResult {
   timedOut: boolean;
 }
 
-/** Run `docker <args>`, capturing output. Never throws for non-zero exits. */
+/**
+ * Run `docker <args>`, capturing output. Never throws for non-zero exits.
+ * `env` is added to the docker CLI's own environment — the channel for
+ * values named on argv by `-e KEY` alone.
+ */
 async function runDocker(
   args: string[],
-  opts: { timeoutMs?: number; stdin?: string } = {},
+  opts: { timeoutMs?: number; stdin?: string; env?: Record<string, string> } = {},
 ): Promise<ExecResult> {
   const proc = Bun.spawn(["docker", ...args], {
     stdin: opts.stdin === undefined ? "ignore" : Buffer.from(opts.stdin),
     stdout: "pipe",
     stderr: "pipe",
+    ...(opts.env !== undefined ? { env: { ...process.env, ...opts.env } } : {}),
   });
   let timedOut = false;
   let timer: ReturnType<typeof setTimeout> | undefined;
@@ -87,7 +92,10 @@ async function runDocker(
 }
 
 /** Run `docker <args>` and throw a DockerError on non-zero exit. */
-async function mustDocker(args: string[], opts: { timeoutMs?: number; stdin?: string } = {}): Promise<ExecResult> {
+async function mustDocker(
+  args: string[],
+  opts: { timeoutMs?: number; stdin?: string; env?: Record<string, string> } = {},
+): Promise<ExecResult> {
   const result = await runDocker(args, opts);
   if (result.exitCode !== 0) throw new DockerError(args, result.exitCode, result.stderr);
   return result;
@@ -198,36 +206,55 @@ export interface AttemptContainer {
  * fixture into /workspace. On any failure the partial container is removed
  * before the error propagates — a failed create never leaks a container.
  */
+/**
+ * Route an attempt's env between its two argv-free channels. Single-line
+ * values become env-file lines; a value that itself contains newlines (a
+ * `Name: Value` header bundle) cannot be expressed in a line-based env-file,
+ * so it rides the docker CLI's own environment instead, named on argv by
+ * `-e KEY` alone — the CLI forwards the value intact and argv never shows it.
+ */
+export function splitAttemptEnv(env: Record<string, string>): { envFileLines: string[]; inherited: Record<string, string> } {
+  const envFileLines: string[] = [];
+  const inherited: Record<string, string> = {};
+  for (const [key, value] of Object.entries(env)) {
+    if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) throw new Error(`invalid environment variable name "${key}"`);
+    if (value.includes("\n")) inherited[key] = value;
+    else envFileLines.push(`${key}=${value}`);
+  }
+  return { envFileLines, inherited };
+}
+
 export async function createAttemptContainer(opts: CreateAttemptContainerOptions): Promise<AttemptContainer> {
   // Secrets travel via a transient env-file, not argv (argv is visible in
-  // host process listings while `docker run` executes).
+  // host process listings while `docker run` executes); multi-line values
+  // take the CLI-environment channel (see splitAttemptEnv).
   const envDir = await mkdtemp(join(tmpdir(), "quickstudy-env-"));
   const envFile = join(envDir, "attempt.env");
-  const lines = Object.entries(opts.env).map(([key, value]) => {
-    if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) throw new Error(`invalid environment variable name "${key}"`);
-    if (value.includes("\n")) throw new Error(`environment variable "${key}" contains a newline`);
-    return `${key}=${value}`;
-  });
-  await writeFile(envFile, lines.join("\n") + (lines.length > 0 ? "\n" : ""), { mode: 0o600 });
+  const { envFileLines, inherited } = splitAttemptEnv(opts.env);
+  await writeFile(envFile, envFileLines.join("\n") + (envFileLines.length > 0 ? "\n" : ""), { mode: 0o600 });
 
   let id: string;
   try {
-    const created = await mustDocker([
-      "run",
-      "-d",
-      "--label",
-      `${ATTEMPT_LABEL}=${opts.attemptId}`,
-      "--env-file",
-      envFile,
-      "-v",
-      `${resolve(opts.resultsDir)}:${CONTAINER_RESULTS}`,
-      "-w",
-      CONTAINER_WORKSPACE,
-      ...(opts.network !== undefined ? ["--network", opts.network] : []),
-      opts.image,
-      "sleep",
-      "infinity",
-    ]);
+    const created = await mustDocker(
+      [
+        "run",
+        "-d",
+        "--label",
+        `${ATTEMPT_LABEL}=${opts.attemptId}`,
+        "--env-file",
+        envFile,
+        ...Object.keys(inherited).flatMap((key) => ["-e", key]),
+        "-v",
+        `${resolve(opts.resultsDir)}:${CONTAINER_RESULTS}`,
+        "-w",
+        CONTAINER_WORKSPACE,
+        ...(opts.network !== undefined ? ["--network", opts.network] : []),
+        opts.image,
+        "sleep",
+        "infinity",
+      ],
+      { env: inherited },
+    );
     id = created.stdout.trim();
   } catch (err) {
     // `docker run` can fail AFTER creating the container (start failure) —

package/src/llm.ts

@@ -14,6 +14,7 @@
 
 import Anthropic from "@anthropic-ai/sdk";
 import type { z } from "zod";
+import { parseHeaderLines } from "./secrets.ts";
 
 /** One structured-output request. `schema` is a JSON Schema object. */
 export interface StructuredRequest {
@@ -47,16 +48,23 @@ export class ModelOutputError extends Error {
 const API_MAX_RETRIES = 3;
 
 /**
- * Anthropic-backed client. Reuses ANTHROPIC_API_KEY from the environment;
- * transport-level retries are the SDK's (429/5xx/connection errors).
+ * Anthropic-backed client. Reuses ANTHROPIC_API_KEY and ANTHROPIC_BASE_URL
+ * from the environment (the SDK reads both), plus ANTHROPIC_CUSTOM_HEADERS
+ * (`Name: Value` lines — Claude Code's convention) as default headers, so the
+ * analyst takes the same route as the agent CLI when both are pointed at an
+ * LLM gateway. Transport-level retries are the SDK's (429/5xx/connection
+ * errors).
  */
 export class AnthropicModelClient implements StructuredModelClient {
   private readonly client: Anthropic;
   private lastResolvedModel: string | null = null;
 
-  constructor(options: { apiKey?: string } = {}) {
+  constructor(options: { apiKey?: string; baseURL?: string; defaultHeaders?: Record<string, string> } = {}) {
+    const defaultHeaders = options.defaultHeaders ?? parseHeaderLines(process.env["ANTHROPIC_CUSTOM_HEADERS"]);
     this.client = new Anthropic({
       ...(options.apiKey !== undefined ? { apiKey: options.apiKey } : {}),
+      ...(options.baseURL !== undefined ? { baseURL: options.baseURL } : {}),
+      ...(Object.keys(defaultHeaders).length > 0 ? { defaultHeaders } : {}),
       maxRetries: API_MAX_RETRIES,
     });
   }

package/src/runner/execute.ts

@@ -70,7 +70,7 @@ import { attemptIdentityHash, buildRunManifest, type RunManifest, type RuntimeIm
 import { isMovingModelAlias } from "../model-identity.ts";
 import type { AttemptPlan } from "../plan.ts";
 import type { ProvisionedEnvironment, RuntimeProvisionContext } from "../runtime/types.ts";
-import { collectProviderKeys, hostSecretEnv, redactSecrets, validateProviderKeys, type ProviderKeyProbe } from "../secrets.ts";
+import { collectOptionalHostEnv, collectProviderKeys, endpointHosts, headerBundleSecrets, hostSecretEnv, isSecretEnvName, redactSecrets, validateProviderKeys, type ProviderKeyProbe } from "../secrets.ts";
 import { ArtifactsStore } from "../store/artifacts.ts";
 import { activeAttempts, type ErrorKind, type NewAttempt, type ResultsStore, type RunConfig } from "../store/db.ts";
 import { acquireRunLock } from "./run-lock.ts";
@@ -402,7 +402,7 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
     probesByAdapter.set(adapter.name, {
       name: adapter.name,
       envVars: adapter.requiredHostEnv,
-      ...adapter.keyProbe(collectProviderKeys([adapter])),
+      ...adapter.keyProbe({ ...collectProviderKeys([adapter]), ...collectOptionalHostEnv([adapter]) }),
     });
   }
   await validateProviderKeys([...probesByAdapter.values()], { log });
@@ -458,8 +458,22 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
       const scorer = await importEvalScorer(loaded.scorerPath);
       if (scorer.identity) scorerIdentities[loaded.metadata.id] = scorer.identity;
     }
+    // Each agent's endpoint overrides set on the host: the non-secret base
+    // URLs join the pair environment (identity), their hosts join the egress
+    // allowlist — but only under the proxy, where the allowlist is live and an
+    // IP-literal gateway is refused loudly by design; without it the list is
+    // inert and an IP-literal override must not stop the run.
+    const overridesByExperiment = new Map([...containerAgents].map(([id, adapter]) => [id, collectOptionalHostEnv([adapter])] as const));
+    const adapterEndpoints = Object.fromEntries([...overridesByExperiment].flatMap(([id, overrides]) => {
+      const endpoints = Object.fromEntries(Object.entries(overrides).filter(([name]) => !isSecretEnvName(name)));
+      return Object.keys(endpoints).length > 0 ? [[id, endpoints] as const] : [];
+    }));
     const pairs = resolvePairEnvironments({ ...options, images, imageTags: imageTagsByPair,
-      adapterHosts: Object.fromEntries([...containerAgents].map(([id, adapter]) => [id, adapter.egressHosts ?? []])),
+      adapterHosts: Object.fromEntries([...containerAgents].map(([id, adapter]) => [id, [
+        ...(adapter.egressHosts ?? []),
+        ...(options.egressProxy ? endpointHosts(overridesByExperiment.get(id) ?? {}) : []),
+      ]])),
+      adapterEndpoints,
       budgets: { ...options.budgets, agentMs: options.attemptTimeoutMs ?? options.budgets?.agentMs ?? attemptTimeoutMs },
       mcpHashes: Object.fromEntries([...declaredByExperiment].map(([id, servers]) => [id, hashString(canonicalJson(servers))])),
     });
@@ -806,12 +820,25 @@ async function runContainerAttemptV3(
         // every auth block and pointed authed servers at the proxy sidecar.
         env["QUICKSTUDY_MCP_SERVERS"] = JSON.stringify(rewrittenServers);
       }
+      // Endpoint overrides set on the host for this agent (gateway base URLs
+      // and the headers they need) — see ContainerAgentAdapter.optionalHostEnv.
+      const overrides = agentAdapter !== undefined ? collectOptionalHostEnv([agentAdapter]) : {};
       if (agentAdapter !== undefined) {
         // The agent's provider keys, validated at run preflight — injected
         // at create via the 0600 env-file channel, never argv.
         const providerKeys = collectProviderKeys([agentAdapter]);
         Object.assign(env, providerKeys);
         Object.assign(secretEnv, providerKeys);
+        // The overrides ride the same channel; only the secret-named ones
+        // feed redaction — a base URL in a transcript is diagnostic, a
+        // bearer header is not. A header bundle's credentials are targets of
+        // their own: a transcript that prints just the token is still scrubbed.
+        Object.assign(env, overrides);
+        for (const [name, value] of Object.entries(overrides)) {
+          if (!isSecretEnvName(name)) continue;
+          secretEnv[name] = value;
+          Object.assign(secretEnv, headerBundleSecrets(name, value));
+        }
       }
       if (egress) {
         const proxyEnv = egress.sidecar.proxyEnv();
@@ -874,6 +901,10 @@ async function runContainerAttemptV3(
             trialIndex: entry.trialIndex,
             ...(rewrittenServers !== undefined ? { mcpServers: rewrittenServers } : {}),
             webPolicy: environment?.webPolicy ?? "native-web-blocked",
+            // The adapter's own overrides (never the provider keys): a CLI
+            // that wants a config-file entry (codex's model provider and its
+            // http_headers) renders them from here, inside the container.
+            ...(Object.keys(overrides).length > 0 ? { overrides } : {}),
           };
           // Treatment config files (e.g. the MCP treatment's vendor config).
           await inStage("environment", async () => {

package/src/secrets.ts

@@ -48,6 +48,12 @@ export interface ProviderKeyRequirement {
   name: string;
   /** Host env var names the adapter requires. */
   requiredHostEnv: readonly string[];
+  /**
+   * Host env var names forwarded when set and never required: endpoint
+   * overrides (a vendor base URL that routes the CLI through an LLM gateway
+   * or proxy) and the custom headers such a gateway needs.
+   */
+  optionalHostEnv?: readonly string[];
 }
 
 /** One preflight probe: a cheap authenticated GET a provider will 401 on a bad key. */
@@ -128,6 +134,111 @@ export function collectProviderKeys(
   return keys;
 }
 
+/**
+ * Read every OPTIONAL host env var the adapters declare (see
+ * `ProviderKeyRequirement.optionalHostEnv`). Absent or empty variables are
+ * simply not forwarded — an empty `ANTHROPIC_BASE_URL=` from a CI template
+ * must not reach a CLI that would treat "" as a base URL.
+ */
+export function collectOptionalHostEnv(
+  requirements: readonly ProviderKeyRequirement[],
+  hostEnv: Record<string, string | undefined> = process.env,
+): Record<string, string> {
+  const env: Record<string, string> = {};
+  for (const requirement of requirements) {
+    for (const envVar of requirement.optionalHostEnv ?? []) {
+      const value = hostEnv[envVar];
+      if (value !== undefined && value !== "") env[envVar] = value;
+    }
+  }
+  return env;
+}
+
+/**
+ * Hostnames of every forwarded value that is an absolute http(s) URL — the
+ * endpoints an override points at, which the egress allowlist must admit
+ * alongside the adapter's vendor defaults. Non-URL values (header lines)
+ * contribute nothing.
+ */
+export function endpointHosts(env: Record<string, string>): string[] {
+  const hosts: string[] = [];
+  for (const value of Object.values(env)) {
+    let url: URL;
+    try {
+      url = new URL(value);
+    } catch {
+      continue;
+    }
+    if ((url.protocol === "https:" || url.protocol === "http:") && url.hostname !== "") hosts.push(url.hostname);
+  }
+  return hosts;
+}
+
+/**
+ * `<base>/<path>` for a probe: the override when set (trailing slashes
+ * tolerated — gateways document both forms), else the vendor's own base.
+ */
+export function endpointUrl(override: string | undefined, vendorBase: string, path: string): string {
+  const base = override !== undefined && override !== "" ? override : vendorBase;
+  return `${base.replace(/\/+$/, "")}/${path.replace(/^\/+/, "")}`;
+}
+
+/**
+ * Parse `Name: Value` header lines (newline-separated — the format Claude
+ * Code reads from ANTHROPIC_CUSTOM_HEADERS). Blank lines and lines without a
+ * colon are skipped; the first colon splits name from value.
+ */
+export function parseHeaderLines(text: string | undefined): Record<string, string> {
+  const headers: Record<string, string> = {};
+  for (const line of (text ?? "").split(/\r?\n/)) {
+    const colon = line.indexOf(":");
+    if (colon <= 0) continue;
+    const name = line.slice(0, colon).trim();
+    const value = line.slice(colon + 1).trim();
+    if (name !== "") headers[name] = value;
+  }
+  return headers;
+}
+
+/**
+ * Does an env var NAME look secret-bearing? Keys, tokens, passwords,
+ * credentials — and header bundles, which carry a gateway's bearer token.
+ * Endpoint URLs deliberately do not match: they are diagnostic, not secret.
+ */
+export function isSecretEnvName(name: string): boolean {
+  return /key|token|password|secret|credential|header/i.test(name);
+}
+
+/** Header names whose values are credentials (`cf-aig-authorization`, `x-api-key`, …). */
+const SECRET_HEADER = /auth|key|token|secret|credential|password/i;
+
+/**
+ * The redaction targets inside one header bundle (an env var holding `Name:
+ * Value` lines): each credential header's full value, plus the bare
+ * credential behind a `Bearer`/`Basic`/`Token` scheme — a transcript that
+ * prints only the token, not the whole bundle, must still be scrubbed. Keys
+ * name the bundle and header so the redaction marker says what was removed.
+ */
+export function headerBundleSecrets(bundleName: string, bundle: string): Record<string, string> {
+  const derived: Record<string, string> = {};
+  for (const [header, value] of Object.entries(parseHeaderLines(bundle))) {
+    if (!SECRET_HEADER.test(header)) continue;
+    derived[`${bundleName}[${header}]`] = value;
+    const scheme = /^(?:Bearer|Basic|Token)\s+(\S+)$/i.exec(value);
+    if (scheme?.[1] !== undefined) derived[`${bundleName}[${header}]:credential`] = scheme[1];
+  }
+  return derived;
+}
+
+/** A secret env map plus the credentials carried inside any header bundle it holds. */
+export function expandSecretEnv(env: Record<string, string>): Record<string, string> {
+  const expanded = { ...env };
+  for (const [name, value] of Object.entries(env)) {
+    if (/header/i.test(name)) Object.assign(expanded, headerBundleSecrets(name, value));
+  }
+  return expanded;
+}
+
 /** Values shorter than this are never redacted (they would shred ordinary text). */
 const MIN_REDACTABLE_LENGTH = 6;
 
@@ -145,7 +256,9 @@ export function redactSecrets(text: string, env: Record<string, string>): string
   return redacted;
 }
 
-/** Provider/judge secret env only; ordinary PATH and workspace values are evidence. */
+/** Provider/judge secret env only (header-bundle credentials included); ordinary PATH and workspace values are evidence. */
 export function hostSecretEnv(env: Record<string, string | undefined> = process.env): Record<string, string> {
-  return Object.fromEntries(Object.entries(env).filter(([name, value]) => /key|token|password|secret|credential/i.test(name) && typeof value === "string")) as Record<string, string>;
+  return expandSecretEnv(
+    Object.fromEntries(Object.entries(env).filter(([name, value]) => isSecretEnvName(name) && typeof value === "string")) as Record<string, string>,
+  );
 }