@workos/quickstudy 0.12.1 → 0.13.0

package/README.md

@@ -149,6 +149,50 @@ export OPENAI_CUSTOM_HEADERS='cf-aig-metadata: {"app":"my-evals"}'
 Leave every override unset and both agents run vendor-direct against
 api.anthropic.com and api.openai.com — nothing is rendered, nothing changes.
 
+#### Per-experiment routing (`agent.endpoint`)
+
+The variables above are per adapter: every codex arm in a run reads the
+same `OPENAI_BASE_URL`. To route ONE experiment elsewhere — a second
+gateway, or a gateway's unified OpenAI-compatible API serving a third-party
+model — the experiment names the host variables that carry its key, base
+URL and headers, and the host sets them next to the vendor ones:
+
+```ts
+export default defineExperiment({
+  id: "acme-gateway-mcp",
+  comparisonGroup: "acme-gateway",
+  treatment: "mcp",
+  agent: {
+    adapter: "codex",
+    provider: "acme",                 // recorded as-is in the manifest and report
+    model: "acme/coder-9",            // whatever the endpoint expects in `model`
+    reasoning: "high",
+    endpoint: {
+      apiKeyEnv: "GATEWAY_API_TOKEN",       // secret-named: redacted like a key
+      baseUrlEnv: "GATEWAY_BASE_URL",       // not secret-named: identity + egress
+      headersEnv: "GATEWAY_HEADERS",        // secret-named; `Name: Value` lines
+      wireApi: "chat",                      // codex only: chat completions
+    },
+  },
+  runtime: ...,
+});
+```
+
+Unset fields keep the vendor defaults. The names must be POSIX env var
+names; `defineExperiment` refuses a key or headers variable that is not
+secret-named, a base URL variable that is, and `wireApi` on anything but
+codex. The forwarded values travel exactly as before: the key as a provider
+key, the base URL into the pair environment and the egress allowlist, the
+headers redacted from artifacts. Codex renders them into its custom model
+provider (`env_key`, `base_url`, `http_headers`, `wire_api`); Claude Code
+only reads the vendor names, so the adapter re-exports custom ones under
+them inside the container before exec — a custom key as both
+`ANTHROPIC_API_KEY` (`x-api-key`) and `ANTHROPIC_AUTH_TOKEN`
+(`Authorization: Bearer`), since a provider-native Anthropic route reads
+the former and a gateway's unified API the latter. The preflight key probe
+follows the experiment's variables, so a bad gateway token still fails
+before any container starts.
+
 ## How it works
 
 Every attempt is one point in a 2-D plan:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workos/quickstudy",
-  "version": "0.12.1",
+  "version": "0.13.0",
   "description": "An open-source, company-agnostic eval harness for measuring whether coding agents can complete real integration tasks, across an agent x feature x framework x surface matrix.",
   "license": "MIT",
   "type": "module",

package/src/adapters/claude.ts

@@ -13,6 +13,8 @@
  * across experiments.
  */
 
+import { type ResolvedEndpoint, resolveEndpoint } from "../experiments/endpoint.ts";
+import type { AgentEndpointSpec } from "../experiments/types.ts";
 import type { McpServerConfig } from "../isolation/mcp.ts";
 import { endpointUrl, parseHeaderLines } from "../secrets.ts";
 import { parseClaudeStream } from "./parse.ts";
@@ -21,6 +23,18 @@ import type { AttemptMetrics, ContainerAgentAdapter, ContainerAttemptContext } f
 /** Container path of the MCP config Claude Code auto-discovers. */
 export const CLAUDE_MCP_CONFIG_PATH = "/workspace/.mcp.json";
 
+/**
+ * The vendor's env names: what every claude experiment gets unless its
+ * `agent.endpoint` names something else (experiments/endpoint.ts). The wire
+ * is fixed — Claude Code speaks the Messages API and nothing else.
+ */
+export const CLAUDE_DEFAULT_ENDPOINT: ResolvedEndpoint = {
+  apiKeyEnv: "ANTHROPIC_API_KEY",
+  baseUrlEnv: "ANTHROPIC_BASE_URL",
+  headersEnv: "ANTHROPIC_CUSTOM_HEADERS",
+  wireApi: "responses",
+};
+
 /**
  * Project-scope `.mcp.json` rendered from the runtime's MCP servers
  * (streamable HTTP). The configs arrive credential-free — authed servers
@@ -38,17 +52,26 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter {
   readonly name: string;
   readonly identity;
   readonly versionCommand = ["claude", "--version"] as const;
-  readonly requiredHostEnv = ["ANTHROPIC_API_KEY"] as const;
   /**
-   * Endpoint overrides, forwarded only when set. ANTHROPIC_BASE_URL routes
+   * The env names this experiment's traffic is routed by: the vendor's
+   * (ANTHROPIC_API_KEY / ANTHROPIC_BASE_URL / ANTHROPIC_CUSTOM_HEADERS)
+   * unless `agent.endpoint` named others. Claude Code only ever reads the
+   * vendor names, so custom names are re-exported under them by a shell shim
+   * in `command` — inside the container, from env the harness already placed.
+   */
+  readonly endpoint: ResolvedEndpoint;
+  /** The key variable: required on the host, forwarded into the container. */
+  readonly requiredHostEnv: readonly string[];
+  /**
+   * Endpoint overrides, forwarded only when set. The base URL variable routes
    * the CLI (and the key probe) through an Anthropic-compatible gateway —
    * e.g. Cloudflare AI Gateway's `/anthropic` endpoint, where the gateway
-   * token doubles as ANTHROPIC_API_KEY; ANTHROPIC_CUSTOM_HEADERS (`Name:
-   * Value` lines) carries whatever else that gateway wants, such as
+   * token doubles as the API key; the headers variable (`Name: Value`
+   * lines) carries whatever else that gateway wants, such as
    * `cf-aig-authorization` or `cf-aig-metadata`. Claude Code reads both
-   * natively, so no flag changes.
+   * natively under the vendor names, so no flag changes.
    */
-  readonly optionalHostEnv = ["ANTHROPIC_BASE_URL", "ANTHROPIC_CUSTOM_HEADERS"] as const;
+  readonly optionalHostEnv: readonly string[];
   /**
    * Egress-proxy allowlist (harness-default layer). Claude Code documents
    * corporate-proxy support via HTTPS_PROXY/HTTP_PROXY; api.anthropic.com is
@@ -59,10 +82,16 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter {
    */
   readonly egressHosts = ["api.anthropic.com", "statsig.anthropic.com", "*.sentry.io"] as const;
 
-  constructor(name = "claude", config: { model?: string; reasoning?: string } = {}) {
+  constructor(
+    name = "claude",
+    config: { model?: string; reasoning?: string; provider?: string; endpoint?: AgentEndpointSpec } = {},
+  ) {
     this.name = name;
+    this.endpoint = resolveEndpoint(name, CLAUDE_DEFAULT_ENDPOINT, config.endpoint);
+    this.requiredHostEnv = [this.endpoint.apiKeyEnv];
+    this.optionalHostEnv = [this.endpoint.baseUrlEnv, this.endpoint.headersEnv];
     this.identity = {
-      provider: "anthropic",
+      provider: config.provider ?? "anthropic",
       model: config.model ?? "UNPINNED",
       reasoning: config.reasoning ?? "UNPINNED",
       cliVersion: "unknown",
@@ -78,15 +107,43 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter {
 
   keyProbe(env: Record<string, string>): { url: string; headers: Record<string, string> } {
     return {
-      url: endpointUrl(env["ANTHROPIC_BASE_URL"], "https://api.anthropic.com", "v1/models"),
+      url: endpointUrl(env[this.endpoint.baseUrlEnv], "https://api.anthropic.com", "v1/models"),
       headers: {
-        ...parseHeaderLines(env["ANTHROPIC_CUSTOM_HEADERS"]),
-        "x-api-key": env["ANTHROPIC_API_KEY"] ?? "",
+        ...parseHeaderLines(env[this.endpoint.headersEnv]),
+        "x-api-key": env[this.endpoint.apiKeyEnv] ?? "",
         "anthropic-version": "2023-06-01",
       },
     };
   }
 
+  /**
+   * `export VENDOR="$CUSTOM"` lines for every endpoint variable whose name
+   * differs from the vendor's — empty when nothing differs, so the default
+   * invocation stays the bare `claude` argv. Optional variables are only
+   * re-exported when set: an empty ANTHROPIC_BASE_URL would be a base URL.
+   *
+   * A custom key variable is presented both ways Claude Code can send a
+   * credential — `x-api-key` (ANTHROPIC_API_KEY) and `Authorization:
+   * Bearer` (ANTHROPIC_AUTH_TOKEN) — because gateways differ on which one
+   * they read: a provider-native Anthropic route takes the key header, a
+   * unified API takes the bearer. The vendor's own route only ever sees
+   * the vendor variable, so this changes nothing for it.
+   */
+  private endpointExports(): string[] {
+    const lines: string[] = [];
+    const { apiKeyEnv, baseUrlEnv, headersEnv } = this.endpoint;
+    if (apiKeyEnv !== CLAUDE_DEFAULT_ENDPOINT.apiKeyEnv) {
+      lines.push(`export ANTHROPIC_API_KEY="$${apiKeyEnv}"`, `export ANTHROPIC_AUTH_TOKEN="$${apiKeyEnv}"`);
+    }
+    if (baseUrlEnv !== CLAUDE_DEFAULT_ENDPOINT.baseUrlEnv) {
+      lines.push(`[ -n "\${${baseUrlEnv}:-}" ] && export ANTHROPIC_BASE_URL="$${baseUrlEnv}"`);
+    }
+    if (headersEnv !== CLAUDE_DEFAULT_ENDPOINT.headersEnv) {
+      lines.push(`[ -n "\${${headersEnv}:-}" ] && export ANTHROPIC_CUSTOM_HEADERS="$${headersEnv}"`);
+    }
+    return lines;
+  }
+
   setupFiles(ctx: ContainerAttemptContext): Record<string, string> {
     if (ctx.mcpServers === undefined) return {};
     return { [CLAUDE_MCP_CONFIG_PATH]: renderMcpJson(ctx.mcpServers) };
@@ -114,7 +171,12 @@ export class ClaudeCodeAdapter implements ContainerAgentAdapter {
       args.push("--mcp-config", CLAUDE_MCP_CONFIG_PATH);
     }
     if (ctx.webPolicy !== "native-web-allowed") args.push("--disallowedTools", "WebSearch,WebFetch");
-    return args;
+    const exports = this.endpointExports();
+    if (exports.length === 0) return args;
+    // Custom endpoint names: re-export them under the names Claude Code reads,
+    // then exec the same argv. Names only reach the shell — resolveEndpoint
+    // admits POSIX env names and nothing else; values stay in the environment.
+    return ["sh", "-c", `${exports.join("; ")}; exec "$@"`, "sh", ...args];
   }
 
   parseStream(raw: string): AttemptMetrics {

package/src/adapters/codex.ts

@@ -25,6 +25,8 @@
  * talks to api.openai.com — vendor-direct stays the default.
  */
 
+import { type ResolvedEndpoint, type WireApi, resolveEndpoint } from "../experiments/endpoint.ts";
+import type { AgentEndpointSpec } from "../experiments/types.ts";
 import type { McpServerConfig } from "../isolation/mcp.ts";
 import { endpointUrl, parseHeaderLines } from "../secrets.ts";
 import { parseCodexStream } from "./parse.ts";
@@ -40,13 +42,31 @@ export const CODEX_GATEWAY_PROVIDER = "quickstudy-gateway";
 const OPENAI_DEFAULT_BASE_URL = "https://api.openai.com/v1";
 
 /**
- * A custom model provider pointing at `baseUrl` (the Responses API, which
- * codex requires), carrying `headers` on every request, selected as the
- * default. Top-level keys must precede any table, so this block always
- * renders first in the file. JSON string escapes are valid TOML basic-string
- * escapes, which keeps header values such as JSON metadata intact.
+ * The vendor's env names and wire: what every codex experiment gets unless
+ * its `agent.endpoint` names something else (experiments/endpoint.ts).
  */
-function renderCodexProviderToml(baseUrl: string, headers: Readonly<Record<string, string>>): string {
+export const CODEX_DEFAULT_ENDPOINT: ResolvedEndpoint = {
+  apiKeyEnv: "OPENAI_API_KEY",
+  baseUrlEnv: "OPENAI_BASE_URL",
+  headersEnv: "OPENAI_CUSTOM_HEADERS",
+  wireApi: "responses",
+};
+
+/**
+ * A custom model provider pointing at `baseUrl`, speaking `wireApi` (the
+ * Responses API codex prefers, or chat completions for endpoints that only
+ * offer that), reading its key from `envKey`, carrying `headers` on every
+ * request, selected as the default. Top-level keys must precede any table,
+ * so this block always renders first in the file. JSON string escapes are
+ * valid TOML basic-string escapes, which keeps header values such as JSON
+ * metadata intact.
+ */
+function renderCodexProviderToml(
+  baseUrl: string,
+  headers: Readonly<Record<string, string>>,
+  envKey: string,
+  wireApi: WireApi,
+): string {
   const entries = Object.entries(headers);
   return [
     `model_provider = ${JSON.stringify(CODEX_GATEWAY_PROVIDER)}`,
@@ -54,8 +74,8 @@ function renderCodexProviderToml(baseUrl: string, headers: Readonly<Record<strin
     `[model_providers.${CODEX_GATEWAY_PROVIDER}]`,
     'name = "OpenAI (quickstudy endpoint overrides)"',
     `base_url = ${JSON.stringify(baseUrl)}`,
-    'env_key = "OPENAI_API_KEY"',
-    'wire_api = "responses"',
+    `env_key = ${JSON.stringify(envKey)}`,
+    `wire_api = ${JSON.stringify(wireApi)}`,
     ...(entries.length > 0
       ? [`http_headers = { ${entries.map(([name, value]) => `${JSON.stringify(name)} = ${JSON.stringify(value)}`).join(", ")} }`]
       : []),
@@ -81,17 +101,25 @@ export class CodexAdapter implements ContainerAgentAdapter {
   readonly name: string;
   readonly identity;
   readonly versionCommand = ["codex", "--version"] as const;
-  readonly requiredHostEnv = ["OPENAI_API_KEY"] as const;
   /**
-   * Endpoint overrides, forwarded only when set: OPENAI_BASE_URL (vendor
-   * default https://api.openai.com/v1) — e.g. Cloudflare AI Gateway's
-   * `/openai` endpoint, where the gateway token doubles as OPENAI_API_KEY —
-   * and OPENAI_CUSTOM_HEADERS (`Name: Value` lines) for whatever else the
+   * The env names this experiment's traffic is routed by: the vendor's
+   * (OPENAI_API_KEY / OPENAI_BASE_URL / OPENAI_CUSTOM_HEADERS) unless
+   * `agent.endpoint` named others — the way two codex arms in one run reach
+   * two gateways, or one gateway's unified API serving a third-party model.
+   */
+  readonly endpoint: ResolvedEndpoint;
+  /** The key variable: required on the host, forwarded into the container. */
+  readonly requiredHostEnv: readonly string[];
+  /**
+   * Endpoint overrides, forwarded only when set: the base URL variable
+   * (vendor default https://api.openai.com/v1) — e.g. Cloudflare AI Gateway's
+   * `/openai` endpoint, where the gateway token doubles as the API key —
+   * and the headers variable (`Name: Value` lines) for whatever else the
    * endpoint wants on each request, such as `cf-aig-metadata`. Both are
    * rendered into config.toml by setupFiles (see the module comment); the
    * key probe follows them.
    */
-  readonly optionalHostEnv = ["OPENAI_BASE_URL", "OPENAI_CUSTOM_HEADERS"] as const;
+  readonly optionalHostEnv: readonly string[];
   /**
    * Egress-proxy allowlist (harness-default layer). Codex is reqwest-based;
    * reqwest honors HTTP_PROXY/HTTPS_PROXY env by default (system proxies) —
@@ -103,10 +131,16 @@ export class CodexAdapter implements ContainerAgentAdapter {
    */
   readonly egressHosts = ["api.openai.com", "auth.openai.com", "chatgpt.com", "*.chatgpt.com"] as const;
 
-  constructor(name = "codex", config: { model?: string; reasoning?: string } = {}) {
+  constructor(
+    name = "codex",
+    config: { model?: string; reasoning?: string; provider?: string; endpoint?: AgentEndpointSpec } = {},
+  ) {
     this.name = name;
+    this.endpoint = resolveEndpoint(name, CODEX_DEFAULT_ENDPOINT, config.endpoint);
+    this.requiredHostEnv = [this.endpoint.apiKeyEnv];
+    this.optionalHostEnv = [this.endpoint.baseUrlEnv, this.endpoint.headersEnv];
     this.identity = {
-      provider: "openai",
+      provider: config.provider ?? "openai",
       model: config.model ?? "UNPINNED",
       reasoning: config.reasoning ?? "UNPINNED",
       cliVersion: "unknown",
@@ -122,18 +156,30 @@ export class CodexAdapter implements ContainerAgentAdapter {
 
   keyProbe(env: Record<string, string>): { url: string; headers: Record<string, string> } {
     return {
-      url: endpointUrl(env["OPENAI_BASE_URL"], OPENAI_DEFAULT_BASE_URL, "models"),
-      headers: { ...parseHeaderLines(env["OPENAI_CUSTOM_HEADERS"]), Authorization: `Bearer ${env["OPENAI_API_KEY"] ?? ""}` },
+      url: endpointUrl(env[this.endpoint.baseUrlEnv], OPENAI_DEFAULT_BASE_URL, "models"),
+      headers: {
+        ...parseHeaderLines(env[this.endpoint.headersEnv]),
+        Authorization: `Bearer ${env[this.endpoint.apiKeyEnv] ?? ""}`,
+      },
     };
   }
 
   setupFiles(ctx: ContainerAttemptContext): Record<string, string> {
     const sections: string[] = [];
-    const baseUrl = ctx.overrides?.["OPENAI_BASE_URL"];
-    const headers = parseHeaderLines(ctx.overrides?.["OPENAI_CUSTOM_HEADERS"]);
-    // Either override needs the custom provider; headers alone keep the vendor endpoint.
-    if (baseUrl !== undefined || Object.keys(headers).length > 0) {
-      sections.push(renderCodexProviderToml(baseUrl ?? OPENAI_DEFAULT_BASE_URL, headers));
+    const baseUrl = ctx.overrides?.[this.endpoint.baseUrlEnv];
+    const headers = parseHeaderLines(ctx.overrides?.[this.endpoint.headersEnv]);
+    // Either override needs the custom provider (headers alone keep the
+    // vendor endpoint); so does a non-vendor key variable or wire, which the
+    // built-in provider cannot express.
+    const custom =
+      baseUrl !== undefined ||
+      Object.keys(headers).length > 0 ||
+      this.endpoint.apiKeyEnv !== CODEX_DEFAULT_ENDPOINT.apiKeyEnv ||
+      this.endpoint.wireApi !== CODEX_DEFAULT_ENDPOINT.wireApi;
+    if (custom) {
+      sections.push(
+        renderCodexProviderToml(baseUrl ?? OPENAI_DEFAULT_BASE_URL, headers, this.endpoint.apiKeyEnv, this.endpoint.wireApi),
+      );
     }
     if (ctx.mcpServers !== undefined) sections.push(renderCodexMcpToml(ctx.mcpServers));
     if (sections.length === 0) return {};
@@ -145,9 +191,11 @@ export class CodexAdapter implements ContainerAgentAdapter {
       "sh",
       "-c",
       // codex 0.144+ ignores the OPENAI_API_KEY env var; mint auth.json from
-      // it in-container. Login chatter is silenced so stdout stays pure JSONL
-      // for parseStream; the key reaches login via stdin, never argv.
-      'printenv OPENAI_API_KEY | codex login --with-api-key >/dev/null 2>&1 && exec "$@"',
+      // the key variable in-container. Login chatter is silenced so stdout
+      // stays pure JSONL for parseStream; the key reaches login via stdin,
+      // never argv. The variable NAME is interpolated — resolveEndpoint
+      // admits only POSIX env names, so nothing else can reach the shell.
+      `printenv ${this.endpoint.apiKeyEnv} | codex login --with-api-key >/dev/null 2>&1 && exec "$@"`,
       "sh",
       "codex",
       "exec",

package/src/build-info.generated.ts

@@ -9,7 +9,7 @@ export const EMBEDDED_BUILD_INFO: {
   // (the `x-release-please-version` marker anchors the replacement). It is
   // restored as-is after a binary build (scripts/build-binary.ts), so the
   // committed source `--version` stays in lockstep with package.json.
-  version: "0.12.1", // x-release-please-version
+  version: "0.13.0", // x-release-please-version
   revision: null,
   dirty: null,
   sourceHash: null,

package/src/experiments/endpoint.ts

@@ -0,0 +1,138 @@
+/**
+ * `agent.endpoint` resolution: the experiment names the HOST ENV VARS that
+ * carry an agent's API key, base URL and extra headers; the adapter fills in
+ * its vendor defaults for whatever is unset and checks the names before any
+ * container starts. Names, never values, live here — the values stay on the
+ * host and ride the same channels the vendor defaults always have.
+ */
+
+import { isSecretEnvName } from "../secrets.ts";
+import type { AgentEndpointSpec } from "./types.ts";
+
+/** POSIX env var names the shell shims interpolate verbatim; nothing else is accepted. */
+const ENV_NAME = /^[A-Z][A-Z0-9_]*$/;
+
+export const WIRE_APIS = ["responses", "chat"] as const;
+export type WireApi = (typeof WIRE_APIS)[number];
+
+/** An adapter's endpoint with every field decided. */
+export interface ResolvedEndpoint {
+  apiKeyEnv: string;
+  baseUrlEnv: string;
+  headersEnv: string;
+  wireApi: WireApi;
+}
+
+/** Thrown for an `agent.endpoint` the harness refuses; the message names the field. */
+export class InvalidAgentEndpointError extends Error {
+  constructor(detail: string) {
+    super(`agent.endpoint: ${detail}`);
+    this.name = "InvalidAgentEndpointError";
+  }
+}
+
+/**
+ * Check an experiment's `agent.endpoint` without an adapter: every name must
+ * be a POSIX env var name, the key and headers variables must be
+ * secret-named (so they are redacted from artifacts like a key), the base
+ * URL variable must not be (its value is identity and joins the egress
+ * allowlist), and the three must differ. `wireApi` is codex's alone.
+ */
+export function validateAgentEndpoint(spec: AgentEndpointSpec | undefined, adapter: string): void {
+  if (spec === undefined) return;
+  if (spec === null || typeof spec !== "object" || Array.isArray(spec)) {
+    throw new InvalidAgentEndpointError("must be an object");
+  }
+  const names = new Map<string, string>();
+  for (const field of ["apiKeyEnv", "baseUrlEnv", "headersEnv"] as const) {
+    const value = spec[field];
+    if (value === undefined) continue;
+    if (typeof value !== "string" || !ENV_NAME.test(value)) {
+      throw new InvalidAgentEndpointError(
+        `${field} must be an env var name matching ${ENV_NAME} (got ${JSON.stringify(value)})`,
+      );
+    }
+    const clash = names.get(value);
+    if (clash !== undefined) throw new InvalidAgentEndpointError(`${field} and ${clash} both name ${value}`);
+    names.set(value, field);
+  }
+  if (spec.apiKeyEnv !== undefined && !isSecretEnvName(spec.apiKeyEnv)) {
+    throw new InvalidAgentEndpointError(
+      `apiKeyEnv ${spec.apiKeyEnv} must be secret-named (contain KEY, TOKEN, SECRET, PASSWORD or CREDENTIAL) so its value is redacted`,
+    );
+  }
+  if (spec.headersEnv !== undefined && !isSecretEnvName(spec.headersEnv)) {
+    throw new InvalidAgentEndpointError(
+      `headersEnv ${spec.headersEnv} must be secret-named (a HEADERS suffix will do) so its value is redacted`,
+    );
+  }
+  if (spec.baseUrlEnv !== undefined && isSecretEnvName(spec.baseUrlEnv)) {
+    throw new InvalidAgentEndpointError(
+      `baseUrlEnv ${spec.baseUrlEnv} must not be secret-named: a base URL is identity and must survive redaction`,
+    );
+  }
+  if (spec.wireApi !== undefined) {
+    if (!(WIRE_APIS as readonly string[]).includes(spec.wireApi)) {
+      throw new InvalidAgentEndpointError(`wireApi must be one of ${WIRE_APIS.join(", ")} (got ${JSON.stringify(spec.wireApi)})`);
+    }
+    if (adapter !== "codex") {
+      throw new InvalidAgentEndpointError(`wireApi applies to the codex adapter only (this experiment uses "${adapter}")`);
+    }
+  }
+}
+
+const NAME_FIELDS = ["apiKeyEnv", "baseUrlEnv", "headersEnv"] as const;
+
+/**
+ * Validate `spec` for `adapter` and fill the unset fields from `defaults`.
+ * A declared name may not be one of the adapter's vendor variables for a
+ * different role: `apiKeyEnv: "ANTHROPIC_CUSTOM_HEADERS"` would make the
+ * shim read the header bundle as the key and then overwrite it, and the
+ * three resolved names must stay distinct for the same reason.
+ */
+export function resolveEndpoint(adapter: string, defaults: ResolvedEndpoint, spec: AgentEndpointSpec = {}): ResolvedEndpoint {
+  validateAgentEndpoint(spec, adapter);
+  for (const field of NAME_FIELDS) {
+    const value = spec[field];
+    if (value === undefined) continue;
+    for (const role of NAME_FIELDS) {
+      if (role !== field && value === defaults[role]) {
+        throw new InvalidAgentEndpointError(
+          `${field} ${value} is the ${adapter} adapter's ${role} variable; a name cannot serve two roles`,
+        );
+      }
+    }
+  }
+  const resolved: ResolvedEndpoint = {
+    apiKeyEnv: spec.apiKeyEnv ?? defaults.apiKeyEnv,
+    baseUrlEnv: spec.baseUrlEnv ?? defaults.baseUrlEnv,
+    headersEnv: spec.headersEnv ?? defaults.headersEnv,
+    wireApi: spec.wireApi ?? defaults.wireApi,
+  };
+  const seen = new Map<string, string>();
+  for (const field of NAME_FIELDS) {
+    const clash = seen.get(resolved[field]);
+    if (clash !== undefined) {
+      throw new InvalidAgentEndpointError(`${field} and ${clash} both resolve to ${resolved[field]}`);
+    }
+    seen.set(resolved[field], field);
+  }
+  return resolved;
+}
+
+/**
+ * The endpoint as identity: the declared fields, verbatim, in a fixed key
+ * order — what the run manifest records and comparison groups diff. Two
+ * arms that differ only here are two configurations, never one treatment.
+ * `undefined` when nothing is declared, so an experiment without an
+ * endpoint keeps the identity it had before the field existed.
+ */
+export function endpointIdentity(spec: AgentEndpointSpec | undefined): Record<string, string> | undefined {
+  if (spec === undefined) return undefined;
+  const identity: Record<string, string> = {};
+  for (const field of [...NAME_FIELDS, "wireApi"] as const) {
+    const value = spec[field];
+    if (value !== undefined) identity[field] = value;
+  }
+  return Object.keys(identity).length === 0 ? undefined : identity;
+}

package/src/experiments/groups.ts

@@ -24,7 +24,7 @@
 
 import { pairKey } from "../execution-policy.ts";
 import { diffIdentity, type IdentityMismatch } from "../identity-diff.ts";
-import type { RunManifest } from "../manifest.ts";
+import { agentEndpointComponent, type RunManifest } from "../manifest.ts";
 import type { Experiment } from "./types.ts";
 
 export interface ComparisonGroup {
@@ -62,6 +62,7 @@ function comparableView(experiment: Experiment, manifest: RunManifest): Record<s
       model: experiment.agent.model ?? "unspecified",
       reasoning: experiment.agent.reasoning ?? "unspecified",
       cliVersion: experiment.agent.cliVersion ?? "unspecified",
+      ...agentEndpointComponent(experiment.agent.endpoint),
     },
     image: components?.image ?? null,
     runtime: {

package/src/experiments/types.ts

@@ -8,6 +8,7 @@
 
 import type { EvalMetadata } from "../evals/types.ts";
 import { validateWorkspaceFilesRuntime, type WorkspaceFiles } from "../runtime/workspace-files.ts";
+import { validateAgentEndpoint } from "./endpoint.ts";
 import type { SurfaceUsageConfig } from "../surface-usage.ts";
 import type { RuntimeCapabilities } from "../runtime/types.ts";
 
@@ -17,6 +18,36 @@ export interface RuntimeCapacity {
   limit: number;
 }
 
+/**
+ * Where an agent's LLM traffic goes, named by HOST ENV VARS rather than
+ * values: the experiment says which variables carry its API key, base URL
+ * and extra headers, and the host supplies them at run time. Every field is
+ * optional; unset ones keep the adapter's vendor defaults (`OPENAI_API_KEY`,
+ * `OPENAI_BASE_URL`, … for codex; `ANTHROPIC_*` for claude).
+ *
+ * This is what lets two experiments of one adapter reach different
+ * endpoints in the same run — one vendor-direct, one through a gateway's
+ * unified API serving a third-party model — without the adapter growing a
+ * vendor table. The values stay secrets or identity exactly as before: the
+ * key variable must be secret-named (`isSecretEnvName`), the base-URL
+ * variable must not be (its value joins the pair environment and the
+ * egress allowlist), and the headers variable is redacted like a key.
+ */
+export interface AgentEndpointSpec {
+  /** Host env var holding the API key the CLI authenticates with. Secret-named. */
+  apiKeyEnv?: string;
+  /** Host env var holding the base URL override. Non-secret name; forwarded only when set. */
+  baseUrlEnv?: string;
+  /** Host env var holding `Name: Value` header lines. Secret-named; forwarded only when set. */
+  headersEnv?: string;
+  /**
+   * codex only: the custom provider's wire protocol. `responses` (default)
+   * is codex's native Responses API; `chat` is OpenAI chat completions, the
+   * shape most OpenAI-compatible gateways and third-party models accept.
+   */
+  wireApi?: "responses" | "chat";
+}
+
 /** Which agent executes the attempt, plus the identity fields recorded in the manifest. */
 export interface AgentSpec {
   /** Adapter name, e.g. "echo" (the built-in no-op pipeline smoke agent). */
@@ -25,6 +56,8 @@ export interface AgentSpec {
   model?: string;
   reasoning?: string;
   cliVersion?: string;
+  /** Endpoint routing by host env var name; see AgentEndpointSpec. Hashed into identity like every other field. */
+  endpoint?: AgentEndpointSpec;
 }
 
 /**
@@ -142,5 +175,10 @@ export function defineExperiment(experiment: Experiment): Experiment {
     );
   }
   validateWorkspaceFilesRuntime(experiment.runtime);
+  try {
+    validateAgentEndpoint(experiment.agent.endpoint, experiment.agent.adapter);
+  } catch (err) {
+    throw new Error(`defineExperiment: experiment "${experiment.id}" ${err instanceof Error ? err.message : String(err)}`);
+  }
   return deepFreeze(experiment);
 }

package/src/index.ts

@@ -21,7 +21,14 @@ export { defineWorkspaceFiles, WORKSPACE_FILES_LIMITS } from "./runtime/workspac
 export type { WorkspaceFile, WorkspaceFiles, WorkspaceFilesIdentity } from "./runtime/workspace-files.ts";
 export type { WorkspaceFilesEvidence } from "./isolation/workspace-files.ts";
 export { defineExperiment } from "./experiments/types.ts";
-export type { AgentSpec, Experiment, LoadedExperiment, Runtime, RuntimeCapacity } from "./experiments/types.ts";
+export type {
+  AgentEndpointSpec,
+  AgentSpec,
+  Experiment,
+  LoadedExperiment,
+  Runtime,
+  RuntimeCapacity,
+} from "./experiments/types.ts";
 export type {
   ProvisionedEnvironment,
   RuntimeCapabilities,

package/src/manifest.ts

@@ -19,7 +19,8 @@ import { sourceIdentity } from "./source-identity.ts";
 import { pairKey, resolvePairEnvironments, type PairEnvironment } from "./execution-policy.ts";
 import { canonicalJson } from "./identity-diff.ts";
 import type { EvalMetadata, LoadedEval } from "./evals/types.ts";
-import type { LoadedExperiment } from "./experiments/types.ts";
+import { endpointIdentity } from "./experiments/endpoint.ts";
+import type { AgentEndpointSpec, LoadedExperiment } from "./experiments/types.ts";
 import type { AttemptPlan } from "./plan.ts";
 
 export const MANIFEST_VERSION = "v4";
@@ -65,6 +66,17 @@ export interface EvalIdentity {
   solution: string | null;
 }
 
+/**
+ * The agent's `endpoint` component, spread into the recorded agent fields:
+ * `{ endpoint }` when the experiment declares one, nothing otherwise, so an
+ * experiment without one keeps the exact identity it had before the field
+ * existed. Shared with the comparison-group view so both read one shape.
+ */
+export function agentEndpointComponent(spec: AgentEndpointSpec | undefined): { endpoint?: Record<string, string> } {
+  const endpoint = endpointIdentity(spec);
+  return endpoint === undefined ? {} : { endpoint };
+}
+
 /** Per-experiment identity components — phase 2's comparison-group diff reads these. */
 export interface ExperimentIdentityComponents {
   /** Hash of the experiment's defining module source. */
@@ -76,6 +88,8 @@ export interface ExperimentIdentityComponents {
     model: string;
     reasoning: string;
     cliVersion: string;
+    /** The declared `agent.endpoint` (experiments/endpoint.ts endpointIdentity); absent when none. */
+    endpoint?: Record<string, string>;
   };
   /** Hash of the canonicalized runtime configuration. */
   runtime: string;
@@ -192,6 +206,7 @@ export function buildRunManifest(options: BuildRunManifestOptions): RunManifest
         model: experiment.agent.model ?? "unspecified",
         reasoning: experiment.agent.reasoning ?? "unspecified",
         cliVersion: options.cliVersions?.[experiment.id] ?? experiment.agent.cliVersion ?? "unspecified",
+        ...agentEndpointComponent(experiment.agent.endpoint),
       },
       runtime: contentHash({
         kind: experiment.runtime.kind,

package/src/runner/execute.ts

@@ -172,6 +172,16 @@ async function inStage<T>(kind: ErrorKind, work: () => Promise<T>): Promise<T> {
   }
 }
 
+/**
+ * What makes one key probe distinct from another: the adapter and the host
+ * variables it reads. Same adapter, same variables, same values — one probe
+ * covers every experiment that shares them; a different `agent.endpoint`
+ * names different variables and gets its own.
+ */
+export function keyProbeIdentity(adapter: Pick<ContainerAgentAdapter, "name" | "requiredHostEnv" | "optionalHostEnv">): string {
+  return [adapter.name, [...adapter.requiredHostEnv].join(","), [...(adapter.optionalHostEnv ?? [])].join(",")].join("|");
+}
+
 function defaultHostAdapters(): Record<string, AgentAdapter> {
   const echo = new EchoAdapter();
   const oracle = new OracleAdapter();
@@ -180,8 +190,20 @@ function defaultHostAdapters(): Record<string, AgentAdapter> {
 
 function defaultContainerAdapters(): Record<string, (agent: AgentSpec) => ContainerAgentAdapter> {
   return {
-    claude: (agent) => new ClaudeCodeAdapter("claude", { model: agent.model, reasoning: agent.reasoning }),
-    codex: (agent) => new CodexAdapter("codex", { model: agent.model, reasoning: agent.reasoning }),
+    claude: (agent) =>
+      new ClaudeCodeAdapter("claude", {
+        model: agent.model,
+        reasoning: agent.reasoning,
+        provider: agent.provider,
+        endpoint: agent.endpoint,
+      }),
+    codex: (agent) =>
+      new CodexAdapter("codex", {
+        model: agent.model,
+        reasoning: agent.reasoning,
+        provider: agent.provider,
+        endpoint: agent.endpoint,
+      }),
     oracle: () => new OracleContainerAdapter(),
   };
 }
@@ -405,19 +427,24 @@ export async function executeRunV3(options: ExecuteRunV3Options): Promise<RunSum
 
   // Live key preflight: a key that is present but rejected by the provider
   // would burn a container per attempt just to 401 inside it. Probe each
-  // unique agent's endpoint once — before any docker, sidecar, or run-row
-  // work — and say plainly whether the run starts. Adapters without a probe
-  // (tests, custom adapters) skip it.
-  const probesByAdapter = new Map<string, ProviderKeyProbe>();
+  // unique endpoint once — before any docker, sidecar, or run-row work —
+  // and say plainly whether the run starts. "Unique" is the adapter plus
+  // the host variables it reads (keyProbeIdentity): two codex arms routed
+  // by different `agent.endpoint` variables are two probes, while arms
+  // sharing the vendor defaults stay one. Adapters without a probe (tests,
+  // custom adapters) skip it.
+  const probesByEndpoint = new Map<string, ProviderKeyProbe>();
   for (const adapter of containerAgents.values()) {
-    if (adapter.keyProbe === undefined || probesByAdapter.has(adapter.name)) continue;
-    probesByAdapter.set(adapter.name, {
+    if (adapter.keyProbe === undefined) continue;
+    const identity = keyProbeIdentity(adapter);
+    if (probesByEndpoint.has(identity)) continue;
+    probesByEndpoint.set(identity, {
       name: adapter.name,
       envVars: adapter.requiredHostEnv,
       ...adapter.keyProbe({ ...collectProviderKeys([adapter]), ...collectOptionalHostEnv([adapter]) }),
     });
   }
-  await validateProviderKeys([...probesByAdapter.values()], { log });
+  await validateProviderKeys([...probesByEndpoint.values()], { log });
   log(`run starting: ${plan.length} attempt(s)`);
 
   // Resolve image digests and agent CLI versions BEFORE building the manifest: