@workos/quickstudy 0.9.0 → 0.10.0

package/README.md

@@ -130,10 +130,11 @@ preflight key probe at it:
 | Agent  | Overrides                                        | How the CLI gets them                                                                  |
 | ------ | ------------------------------------------------ | -------------------------------------------------------------------------------------- |
 | claude | `ANTHROPIC_BASE_URL`, `ANTHROPIC_CUSTOM_HEADERS` | env, read natively by Claude Code (and by the `diagnose` analyst)                      |
-| codex  | `OPENAI_BASE_URL`                                | rendered into `config.toml` as a custom model provider (codex ignores the env var)     |
+| codex  | `OPENAI_BASE_URL`, `OPENAI_CUSTOM_HEADERS`       | rendered into `config.toml` as a custom model provider (`base_url`, `http_headers`); codex has no env channel for either |
 
-`ANTHROPIC_CUSTOM_HEADERS` is `Name: Value` lines and is redacted from
-artifacts like a key. With Cloudflare AI Gateway holding the provider keys,
+The `*_CUSTOM_HEADERS` bundles are `Name: Value` lines and are redacted from
+artifacts like a key; a `cf-aig-metadata` line is how a gateway attributes
+the traffic per app. With Cloudflare AI Gateway holding the provider keys,
 the gateway token *is* the API key:
 
 ```bash
@@ -142,8 +143,12 @@ export ANTHROPIC_API_KEY=<gateway token>
 export ANTHROPIC_CUSTOM_HEADERS="cf-aig-authorization: Bearer <gateway token>"
 export OPENAI_BASE_URL=https://gateway.ai.cloudflare.com/v1/<account>/<gateway>/openai
 export OPENAI_API_KEY=<gateway token>
+export OPENAI_CUSTOM_HEADERS='cf-aig-metadata: {"app":"my-evals"}'
 ```
 
+Leave every override unset and both agents run vendor-direct against
+api.anthropic.com and api.openai.com — nothing is rendered, nothing changes.
+
 ## How it works
 
 Every attempt is one point in a 2-D plan:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workos/quickstudy",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "description": "An open-source, company-agnostic eval harness for measuring whether coding agents can complete real integration tasks, across an agent x feature x framework x surface matrix.",
   "license": "MIT",
   "type": "module",

package/src/adapters/codex.ts

@@ -10,38 +10,55 @@
  * command wrapper pipes it into `codex login --with-api-key` (stdin, never
  * argv) to write the auth.json the CLI requires.
  *
- * Endpoint override: an OPENAI_BASE_URL set on the host (an LLM gateway) is
+ * Endpoint overrides: an OPENAI_BASE_URL set on the host (an LLM gateway) is
  * rendered into the same config.toml as a custom model provider — codex
  * 0.149 ignores the env var for its built-in provider (verified live: it
  * still dialed api.openai.com), while a `[model_providers.*]` entry with
  * `env_key = "OPENAI_API_KEY"` routes every request through the override.
+ * OPENAI_CUSTOM_HEADERS (`Name: Value` lines, the Codex counterpart of Claude
+ * Code's ANTHROPIC_CUSTOM_HEADERS) becomes that provider's `http_headers` —
+ * e.g. `cf-aig-metadata` so a gateway can attribute the traffic. Codex has
+ * no env-var channel for extra headers, so the values land in config.toml
+ * inside the attempt container: the same trust boundary as the env that
+ * already carries them, written over stdin, destroyed with the container.
+ * With neither override set nothing is rendered and the built-in provider
+ * talks to api.openai.com — vendor-direct stays the default.
  */
 
 import type { McpServerConfig } from "../isolation/mcp.ts";
-import { endpointUrl } from "../secrets.ts";
+import { endpointUrl, parseHeaderLines } from "../secrets.ts";
 import { parseCodexStream } from "./parse.ts";
 import type { AttemptMetrics, ContainerAgentAdapter, ContainerAttemptContext } from "./types.ts";
 
 /** Container path of the Codex config (containers run as root). */
 export const CODEX_CONFIG_PATH = "/root/.codex/config.toml";
 
-/** The config.toml model provider id an OPENAI_BASE_URL override is rendered as. */
+/** The config.toml model provider id the OPENAI_* overrides are rendered as. */
 export const CODEX_GATEWAY_PROVIDER = "quickstudy-gateway";
 
+/** Codex's own default for the built-in provider; the probe and a headers-only override use it. */
+const OPENAI_DEFAULT_BASE_URL = "https://api.openai.com/v1";
+
 /**
  * A custom model provider pointing at `baseUrl` (the Responses API, which
- * codex requires), selected as the default. Top-level keys must precede any
- * table, so this block always renders first in the file.
+ * codex requires), carrying `headers` on every request, selected as the
+ * default. Top-level keys must precede any table, so this block always
+ * renders first in the file. JSON string escapes are valid TOML basic-string
+ * escapes, which keeps header values such as JSON metadata intact.
  */
-function renderCodexProviderToml(baseUrl: string): string {
+function renderCodexProviderToml(baseUrl: string, headers: Readonly<Record<string, string>>): string {
+  const entries = Object.entries(headers);
   return [
     `model_provider = ${JSON.stringify(CODEX_GATEWAY_PROVIDER)}`,
     "",
     `[model_providers.${CODEX_GATEWAY_PROVIDER}]`,
-    'name = "OpenAI (via OPENAI_BASE_URL)"',
+    'name = "OpenAI (quickstudy endpoint overrides)"',
     `base_url = ${JSON.stringify(baseUrl)}`,
     'env_key = "OPENAI_API_KEY"',
     'wire_api = "responses"',
+    ...(entries.length > 0
+      ? [`http_headers = { ${entries.map(([name, value]) => `${JSON.stringify(name)} = ${JSON.stringify(value)}`).join(", ")} }`]
+      : []),
     "",
   ].join("\n");
 }
@@ -66,13 +83,15 @@ export class CodexAdapter implements ContainerAgentAdapter {
   readonly versionCommand = ["codex", "--version"] as const;
   readonly requiredHostEnv = ["OPENAI_API_KEY"] as const;
   /**
-   * Endpoint override, forwarded only when set: OPENAI_BASE_URL (vendor
+   * Endpoint overrides, forwarded only when set: OPENAI_BASE_URL (vendor
    * default https://api.openai.com/v1) — e.g. Cloudflare AI Gateway's
-   * `/openai` endpoint, where the gateway token doubles as OPENAI_API_KEY.
-   * Rendered into config.toml by setupFiles (see the module comment); the
-   * key probe follows it.
+   * `/openai` endpoint, where the gateway token doubles as OPENAI_API_KEY —
+   * and OPENAI_CUSTOM_HEADERS (`Name: Value` lines) for whatever else the
+   * endpoint wants on each request, such as `cf-aig-metadata`. Both are
+   * rendered into config.toml by setupFiles (see the module comment); the
+   * key probe follows them.
    */
-  readonly optionalHostEnv = ["OPENAI_BASE_URL"] as const;
+  readonly optionalHostEnv = ["OPENAI_BASE_URL", "OPENAI_CUSTOM_HEADERS"] as const;
   /**
    * Egress-proxy allowlist (harness-default layer). Codex is reqwest-based;
    * reqwest honors HTTP_PROXY/HTTPS_PROXY env by default (system proxies) —
@@ -103,15 +122,19 @@ export class CodexAdapter implements ContainerAgentAdapter {
 
   keyProbe(env: Record<string, string>): { url: string; headers: Record<string, string> } {
     return {
-      url: endpointUrl(env["OPENAI_BASE_URL"], "https://api.openai.com/v1", "models"),
-      headers: { Authorization: `Bearer ${env["OPENAI_API_KEY"] ?? ""}` },
+      url: endpointUrl(env["OPENAI_BASE_URL"], OPENAI_DEFAULT_BASE_URL, "models"),
+      headers: { ...parseHeaderLines(env["OPENAI_CUSTOM_HEADERS"]), Authorization: `Bearer ${env["OPENAI_API_KEY"] ?? ""}` },
     };
   }
 
   setupFiles(ctx: ContainerAttemptContext): Record<string, string> {
     const sections: string[] = [];
     const baseUrl = ctx.overrides?.["OPENAI_BASE_URL"];
-    if (baseUrl !== undefined) sections.push(renderCodexProviderToml(baseUrl));
+    const headers = parseHeaderLines(ctx.overrides?.["OPENAI_CUSTOM_HEADERS"]);
+    // Either override needs the custom provider; headers alone keep the vendor endpoint.
+    if (baseUrl !== undefined || Object.keys(headers).length > 0) {
+      sections.push(renderCodexProviderToml(baseUrl ?? OPENAI_DEFAULT_BASE_URL, headers));
+    }
     if (ctx.mcpServers !== undefined) sections.push(renderCodexMcpToml(ctx.mcpServers));
     if (sections.length === 0) return {};
     return { [CODEX_CONFIG_PATH]: sections.join("\n") };

package/src/adapters/types.ts

@@ -133,10 +133,12 @@ export interface ContainerAttemptContext {
   /** The runtime's native web-tool policy (default: blocked). */
   webPolicy: WebPolicy;
   /**
-   * The adapter's endpoint overrides that are set on the host (its
-   * `optionalHostEnv` minus secret-named entries — never provider keys or
-   * header bundles), for a CLI that takes a config-file entry rather than an
-   * env var. Absent when none are set.
+   * The adapter's own endpoint overrides that are set on the host (every
+   * `optionalHostEnv` value, header bundles included — never the provider
+   * keys), for a CLI that takes a config-file entry rather than an env var.
+   * Anything rendered from these is written inside the attempt container
+   * only, the same boundary the env already crosses. Absent when none are
+   * set.
    */
   overrides?: Readonly<Record<string, string>>;
 }

package/src/build-info.generated.ts

@@ -9,7 +9,7 @@ export const EMBEDDED_BUILD_INFO: {
   // (the `x-release-please-version` marker anchors the replacement). It is
   // restored as-is after a binary build (scripts/build-binary.ts), so the
   // committed source `--version` stays in lockstep with package.json.
-  version: "0.9.0", // x-release-please-version
+  version: "0.10.0", // x-release-please-version
   revision: null,
   dirty: null,
   sourceHash: null,

package/src/runner/execute.ts

@@ -823,7 +823,6 @@ async function runContainerAttemptV3(
       // Endpoint overrides set on the host for this agent (gateway base URLs
       // and the headers they need) — see ContainerAgentAdapter.optionalHostEnv.
       const overrides = agentAdapter !== undefined ? collectOptionalHostEnv([agentAdapter]) : {};
-      const publicOverrides = Object.fromEntries(Object.entries(overrides).filter(([name]) => !isSecretEnvName(name)));
       if (agentAdapter !== undefined) {
         // The agent's provider keys, validated at run preflight — injected
         // at create via the 0600 env-file channel, never argv.
@@ -902,9 +901,10 @@ async function runContainerAttemptV3(
             trialIndex: entry.trialIndex,
             ...(rewrittenServers !== undefined ? { mcpServers: rewrittenServers } : {}),
             webPolicy: environment?.webPolicy ?? "native-web-blocked",
-            // Never the keys or header bundles: adapters whose CLI wants a
-            // config-file entry (codex's model provider) render from here.
-            ...(Object.keys(publicOverrides).length > 0 ? { overrides: publicOverrides } : {}),
+            // The adapter's own overrides (never the provider keys): a CLI
+            // that wants a config-file entry (codex's model provider and its
+            // http_headers) renders them from here, inside the container.
+            ...(Object.keys(overrides).length > 0 ? { overrides } : {}),
           };
           // Treatment config files (e.g. the MCP treatment's vendor config).
           await inStage("environment", async () => {