@workos/quickstudy 0.0.1

package/src/adapters/claude.ts

@@ -0,0 +1,107 @@
+/**
+ * Claude Code adapter: drives the `claude` CLI headless inside the attempt
+ * container.
+ *
+ * `--dangerously-skip-permissions` is acceptable here ONLY because the
+ * container is the security boundary — a fresh filesystem, one write-only
+ * results mount, and force-removal after grading.
+ *
+ * Treatment differences are environmental, never prompt-level: the MCP
+ * treatment places `.mcp.json` into /workspace before start; the CLI
+ * treatment relies on the runtime-exposed PATH directory; the web treatment
+ * leaves native web tools enabled. The task prompt stays byte-identical
+ * across experiments.
+ */
+
+import type { McpServerConfig } from "../isolation/mcp.ts";
+import { parseClaudeStream } from "./parse.ts";
+import type { AttemptMetrics, ContainerAgentAdapter, ContainerAttemptContext } from "./types.ts";
+
+/** Container path of the MCP config Claude Code auto-discovers. */
+export const CLAUDE_MCP_CONFIG_PATH = "/workspace/.mcp.json";
+
+/**
+ * Project-scope `.mcp.json` rendered from the runtime's MCP servers
+ * (streamable HTTP). The configs arrive credential-free — authed servers
+ * already point at the token-proxy sidecar — so only the URL is rendered.
+ */
+function renderMcpJson(servers: Readonly<Record<string, McpServerConfig>> = {}): string {
+  const mcpServers = Object.fromEntries(
+    Object.entries(servers).map(([name, spec]) => [name, { type: "http", url: spec.url }]),
+  );
+  return `${JSON.stringify({ mcpServers }, null, 2)}\n`;
+}
+
+export class ClaudeCodeAdapter implements ContainerAgentAdapter {
+  readonly kind = "container" as const;
+  readonly name: string;
+  readonly identity;
+  readonly versionCommand = ["claude", "--version"] as const;
+  readonly requiredHostEnv = ["ANTHROPIC_API_KEY"] as const;
+  /**
+   * Egress-proxy allowlist (harness-default layer). Claude Code documents
+   * corporate-proxy support via HTTPS_PROXY/HTTP_PROXY; api.anthropic.com is
+   * the LLM endpoint, statsig/sentry are its telemetry (allowed so denial
+   * logs stay signal, not noise). Verified live by the egress-proxy regression
+   * check (`bun scripts/dev-attempt.ts --experiment <id> --egress-proxy`).
+   */
+  readonly egressHosts = ["api.anthropic.com", "statsig.anthropic.com", "*.sentry.io"] as const;
+
+  constructor(name = "claude", config: { model?: string; reasoning?: string } = {}) {
+    this.name = name;
+    this.identity = {
+      provider: "anthropic",
+      model: config.model ?? "UNPINNED",
+      reasoning: config.reasoning ?? "UNPINNED",
+      cliVersion: "unknown",
+      surfacePolicies: {
+        docs: "native WebSearch/WebFetch allowed",
+        mcp: "--disallowedTools WebSearch,WebFetch; MCP configured",
+        cli: "--disallowedTools WebSearch,WebFetch",
+      },
+      nativeWebControl: "controlled" as const,
+      reproducible: true,
+    };
+  }
+
+  keyProbe(env: Record<string, string>): { url: string; headers: Record<string, string> } {
+    return {
+      url: "https://api.anthropic.com/v1/models",
+      headers: { "x-api-key": env["ANTHROPIC_API_KEY"] ?? "", "anthropic-version": "2023-06-01" },
+    };
+  }
+
+  setupFiles(ctx: ContainerAttemptContext): Record<string, string> {
+    if (ctx.mcpServers === undefined) return {};
+    return { [CLAUDE_MCP_CONFIG_PATH]: renderMcpJson(ctx.mcpServers) };
+  }
+
+  command(ctx: ContainerAttemptContext): string[] {
+    const args = [
+      "claude",
+      "-p",
+      ctx.prompt,
+      // stream-json is the parseable telemetry source; it requires --verbose
+      // in print mode.
+      "--output-format",
+      "stream-json",
+      "--verbose",
+      "--dangerously-skip-permissions",
+      "--model",
+      this.identity.model,
+      "--effort",
+      this.identity.reasoning,
+    ];
+    if (ctx.mcpServers !== undefined) {
+      // Explicit config beats auto-discovery: headless runs must not depend
+      // on project-trust prompts.
+      args.push("--mcp-config", CLAUDE_MCP_CONFIG_PATH);
+    }
+    if (ctx.webPolicy !== "native-web-allowed") args.push("--disallowedTools", "WebSearch,WebFetch");
+    return args;
+  }
+
+  parseStream(raw: string): AttemptMetrics {
+    return parseClaudeStream(raw);
+  }
+}

package/src/adapters/codex.ts

@@ -0,0 +1,107 @@
+/**
+ * Codex CLI adapter: drives `codex exec --json` headless inside the attempt
+ * container.
+ *
+ * Sandbox/approval bypass is acceptable ONLY because the container is the
+ * security boundary (same rationale as the Claude adapter). MCP treatment
+ * config lives in `~/.codex/config.toml` in-container — Codex has no
+ * per-project MCP file. Auth: the OPENAI_API_KEY env var is injected at
+ * container create, but codex >= 0.144 no longer reads it directly — the
+ * command wrapper pipes it into `codex login --with-api-key` (stdin, never
+ * argv) to write the auth.json the CLI requires.
+ */
+
+import type { McpServerConfig } from "../isolation/mcp.ts";
+import { parseCodexStream } from "./parse.ts";
+import type { AttemptMetrics, ContainerAgentAdapter, ContainerAttemptContext } from "./types.ts";
+
+/** Container path of the Codex config (containers run as root). */
+export const CODEX_CONFIG_PATH = "/root/.codex/config.toml";
+
+/**
+ * `config.toml` MCP table rendered from the runtime's MCP servers (url-based,
+ * credential-free — authed servers already point at the token-proxy sidecar).
+ * Server names outside TOML's bare-key charset get quoted.
+ */
+function renderCodexMcpToml(servers: Readonly<Record<string, McpServerConfig>> = {}): string {
+  const blocks = Object.entries(servers).map(([name, spec]) => {
+    const key = /^[A-Za-z0-9_-]+$/.test(name) ? name : JSON.stringify(name);
+    return `[mcp_servers.${key}]\nurl = ${JSON.stringify(spec.url)}\n`;
+  });
+  return blocks.length > 0 ? blocks.join("\n") : "[mcp_servers]\n";
+}
+
+export class CodexAdapter implements ContainerAgentAdapter {
+  readonly kind = "container" as const;
+  readonly name: string;
+  readonly identity;
+  readonly versionCommand = ["codex", "--version"] as const;
+  readonly requiredHostEnv = ["OPENAI_API_KEY"] as const;
+  /**
+   * Egress-proxy allowlist (harness-default layer). Codex is reqwest-based;
+   * reqwest honors HTTP_PROXY/HTTPS_PROXY env by default (system proxies) —
+   * verified live by the egress-proxy regression check (the CONNECT tunnel
+   * to api.openai.com round-tripped). chatgpt.com came from that run's
+   * denial log: codex probes its ChatGPT-auth endpoints even under key auth.
+   * (Its github.com update-check denials are left denied on purpose.)
+   */
+  readonly egressHosts = ["api.openai.com", "auth.openai.com", "chatgpt.com", "*.chatgpt.com"] as const;
+
+  constructor(name = "codex", config: { model?: string; reasoning?: string } = {}) {
+    this.name = name;
+    this.identity = {
+      provider: "openai",
+      model: config.model ?? "UNPINNED",
+      reasoning: config.reasoning ?? "UNPINNED",
+      cliVersion: "unknown",
+      surfacePolicies: {
+        docs: 'web_search="live"',
+        mcp: 'web_search="disabled"; MCP configured',
+        cli: 'web_search="disabled"',
+      },
+      nativeWebControl: "controlled" as const,
+      reproducible: true,
+    };
+  }
+
+  keyProbe(env: Record<string, string>): { url: string; headers: Record<string, string> } {
+    return {
+      url: "https://api.openai.com/v1/models",
+      headers: { Authorization: `Bearer ${env["OPENAI_API_KEY"] ?? ""}` },
+    };
+  }
+
+  setupFiles(ctx: ContainerAttemptContext): Record<string, string> {
+    if (ctx.mcpServers === undefined) return {};
+    return { [CODEX_CONFIG_PATH]: renderCodexMcpToml(ctx.mcpServers) };
+  }
+
+  command(ctx: ContainerAttemptContext): string[] {
+    return [
+      "sh",
+      "-c",
+      // codex 0.144+ ignores the OPENAI_API_KEY env var; mint auth.json from
+      // it in-container. Login chatter is silenced so stdout stays pure JSONL
+      // for parseStream; the key reaches login via stdin, never argv.
+      'printenv OPENAI_API_KEY | codex login --with-api-key >/dev/null 2>&1 && exec "$@"',
+      "sh",
+      "codex",
+      "exec",
+      "--json",
+      "--model",
+      this.identity.model,
+      "--config",
+      `model_reasoning_effort=${JSON.stringify(this.identity.reasoning)}`,
+      "--config",
+      `web_search=${JSON.stringify(ctx.webPolicy === "native-web-allowed" ? "live" : "disabled")}`,
+      // The container is the boundary; codex's own sandbox would fight the
+      // fixture's build tooling for no additional isolation.
+      "--dangerously-bypass-approvals-and-sandbox",
+      ctx.prompt,
+    ];
+  }
+
+  parseStream(raw: string): AttemptMetrics {
+    return parseCodexStream(raw);
+  }
+}

package/src/adapters/echo.ts

@@ -0,0 +1,57 @@
+/**
+ * The echo adapter: a no-op agent for pipeline testing.
+ *
+ * It copies the fixture into the workspace, writes a canned transcript, and
+ * makes exactly one trivial file edit (NOTES.md). This proves the full
+ * run -> grade -> persist pipeline end-to-end without Docker or LLM spend,
+ * and gives every later phase a fast integration-test target.
+ */
+
+import { cp, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { AgentAdapter, AttemptContext, AttemptOutcome, TranscriptEvent } from "./types.ts";
+
+export class EchoAdapter implements AgentAdapter {
+  readonly name: string;
+  readonly identity = {
+    provider: "local", model: "echo-v1", reasoning: "none", cliVersion: "built-in",
+    surfacePolicies: { docs: "no-network", mcp: "no-network", cli: "no-network" },
+    nativeWebControl: "controlled" as const,
+    reproducible: true,
+  };
+
+  constructor(name = "echo") {
+    this.name = name;
+  }
+
+  async runAttempt(ctx: AttemptContext): Promise<AttemptOutcome> {
+    await cp(ctx.fixtureDir, ctx.workspaceDir, { recursive: true });
+
+    const notes =
+      `# Notes\n\n` +
+      `Echo adapter placeholder: no-op pipeline smoke for attempt ${ctx.attemptId} ` +
+      `(trial ${ctx.trialIndex}).\n`;
+    await writeFile(join(ctx.workspaceDir, "NOTES.md"), notes, "utf8");
+
+    const transcript: TranscriptEvent[] = [
+      { role: "user", content: ctx.prompt, at: new Date().toISOString() },
+      {
+        role: "assistant",
+        content: `Copied the fixture and wrote NOTES.md (echo pipeline smoke).`,
+        at: new Date().toISOString(),
+      },
+    ];
+
+    const noteLines = notes.trimEnd().split("\n");
+    const diff =
+      `--- /dev/null\n+++ b/NOTES.md\n@@ -0,0 +1,${noteLines.length} @@\n` +
+      noteLines.map((line) => `+${line}`).join("\n") +
+      "\n";
+
+    return {
+      transcript,
+      diff,
+      metrics: { tokensIn: ctx.prompt.length, tokensOut: notes.length, turns: 1, costUsd: 0 },
+    };
+  }
+}

package/src/adapters/parse.ts

@@ -0,0 +1,117 @@
+/**
+ * Shared parsers for the agents' headless output streams, reducing each raw
+ * JSONL stream to telemetry: {tokensIn, tokensOut, turns, costUsd}.
+ *
+ * Tolerance is the design constraint: vendor CLIs drift between releases
+ * (versions are pinned in images/agent-runtime/Dockerfile, and these parsers
+ * are tested against checked-in fixture streams), so a malformed or
+ * unrecognized stream must degrade gracefully — never a crash, and never a
+ * lost transcript (the raw stream is persisted before parsing).
+ *
+ * Degradation records NULL token counts, not zeros: a stream that carried no
+ * usage data is "telemetry unavailable", and zeros would poison cost stats
+ * (the store's telemetry columns are nullable for exactly this). Turns stay
+ * numeric — visible turns are countable even from a truncated stream.
+ */
+
+import type { AttemptMetrics } from "./types.ts";
+
+/** Parse a JSONL stream leniently: non-JSON lines are skipped, never fatal. */
+export function parseJsonlLines(raw: string): unknown[] {
+  const objects: unknown[] = [];
+  for (const line of raw.split("\n")) {
+    const trimmed = line.trim();
+    if (trimmed === "" || !trimmed.startsWith("{")) continue;
+    try {
+      objects.push(JSON.parse(trimmed));
+    } catch {
+      // Tolerated: agents interleave non-JSON noise (progress bars, warnings).
+    }
+  }
+  return objects;
+}
+
+function num(value: unknown): number {
+  return typeof value === "number" && Number.isFinite(value) ? value : 0;
+}
+
+/** A token field that is absent is "unavailable" (null) — never coerced to 0. */
+function numOrNull(value: unknown): number | null {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+function record(value: unknown): Record<string, unknown> {
+  return value !== null && typeof value === "object" ? (value as Record<string, unknown>) : {};
+}
+
+/**
+ * Claude Code `--output-format stream-json`: one JSON event per line, ending
+ * (on success) with a `type: "result"` event that carries authoritative
+ * usage, num_turns, and total_cost_usd. Cache reads/writes count as input —
+ * they are billed tokens the attempt consumed. A stream that died before its
+ * result event has no usage data: token counts degrade to null.
+ */
+export function parseClaudeStream(raw: string): AttemptMetrics {
+  const metrics: AttemptMetrics = { tokensIn: null, tokensOut: null, turns: 0 };
+  let assistantEvents = 0;
+  for (const event of parseJsonlLines(raw)) {
+    const obj = record(event);
+    if (obj["type"] === "assistant") assistantEvents += 1;
+    if (obj["type"] !== "result") continue;
+    const usage = record(obj["usage"]);
+    metrics.tokensIn =
+      num(usage["input_tokens"]) + num(usage["cache_creation_input_tokens"]) + num(usage["cache_read_input_tokens"]);
+    metrics.tokensOut = num(usage["output_tokens"]);
+    metrics.turns = num(obj["num_turns"]);
+    const cost = obj["total_cost_usd"];
+    if (typeof cost === "number" && Number.isFinite(cost)) metrics.costUsd = cost;
+  }
+  // A stream that died before its result event still counts visible turns.
+  if (metrics.turns === 0) metrics.turns = assistantEvents;
+  return metrics;
+}
+
+/**
+ * Codex `exec --json`: one JSON event per line. Two shapes exist across
+ * pinned-version history, and both are handled:
+ *   - thread events: `{"type":"turn.completed","usage":{...}}` per turn;
+ *   - legacy protocol events: `{"msg":{"type":"token_count",...}}` with
+ *     cumulative totals (last one wins) and `agent_message` turns.
+ *
+ * Unlike Anthropic usage (where cache tokens are separate counters), OpenAI
+ * usage reports `input_tokens` as the total with cached tokens as a SUBSET —
+ * adding `cached_input_tokens` on top would double-count. A stream with no
+ * usage events at all degrades to null token counts.
+ */
+export function parseCodexStream(raw: string): AttemptMetrics {
+  const metrics: AttemptMetrics = { tokensIn: null, tokensOut: null, turns: 0 };
+  let legacyTurns = 0;
+  for (const event of parseJsonlLines(raw)) {
+    const obj = record(event);
+
+    if (obj["type"] === "turn.completed") {
+      const usage = record(obj["usage"]);
+      const input = numOrNull(usage["input_tokens"]);
+      const output = numOrNull(usage["output_tokens"]);
+      if (input !== null) metrics.tokensIn = (metrics.tokensIn ?? 0) + input;
+      if (output !== null) metrics.tokensOut = (metrics.tokensOut ?? 0) + output;
+      metrics.turns += 1;
+      continue;
+    }
+
+    const msg = record(obj["msg"]);
+    if (msg["type"] === "agent_message") legacyTurns += 1;
+    if (msg["type"] === "token_count") {
+      // Cumulative totals: the last token_count event is authoritative.
+      const info = record(msg["info"]);
+      const total = record(info["total_token_usage"]);
+      const tokensIn = num(total["input_tokens"]) || num(msg["input_tokens"]);
+      const tokensOut = num(total["output_tokens"]) || num(msg["output_tokens"]);
+      if (tokensIn > 0) metrics.tokensIn = tokensIn;
+      if (tokensOut > 0) metrics.tokensOut = tokensOut;
+    }
+  }
+  if (metrics.turns === 0) metrics.turns = legacyTurns;
+  return metrics;
+}
+

package/src/adapters/types.ts

@@ -0,0 +1,152 @@
+/**
+ * The agent adapter contract.
+ *
+ * The v3 runner depends only on these interfaces: it prepares the attempt's
+ * starting state from the eval's `local/`, hands the adapter the eval's
+ * prompt plus the experiment's treatment context, and expects back a
+ * transcript, a diff, and metrics. Host adapters run in a scratch directory
+ * (the echo pipeline smoke); container adapters compose a headless CLI
+ * invocation the runner executes inside the attempt container.
+ */
+
+import type { McpServerConfig } from "../isolation/mcp.ts";
+import type { WebPolicy } from "../runtime/types.ts";
+
+/** Immutable agent configuration recorded in every experiment manifest. */
+export interface AdapterIdentityConfig {
+  provider: string;
+  model: string;
+  reasoning: string;
+  /** Filled from the executable in the selected image when possible. */
+  cliVersion: string;
+  surfacePolicies: Record<string, string>;
+  /** Whether non-web treatments can disable native/vendor web tools. This is not network isolation. */
+  nativeWebControl: "controlled" | "uncontrolled";
+  reproducible: boolean;
+}
+
+/** One event in an attempt's transcript (persisted as JSONL). */
+export interface TranscriptEvent {
+  role: "system" | "user" | "assistant" | "tool";
+  content: string;
+  /** ISO-8601 timestamp. */
+  at: string;
+}
+
+export interface AttemptMetrics {
+  /**
+   * `null` means telemetry unavailable — the vendor stream carried no usage
+   * data (or died before it). Never conflate with 0: zeros poison cost
+   * comparisons, nulls render as "unavailable". The store's telemetry
+   * columns have been nullable from the start for exactly this case.
+   */
+  tokensIn: number | null;
+  tokensOut: number | null;
+  turns: number;
+  costUsd?: number;
+}
+
+/** Everything a host adapter gets for one attempt. */
+export interface AttemptContext {
+  /** Aborted when the current stage expires. Implementations must stop work. */
+  signal?: AbortSignal;
+  runId: string;
+  attemptId: string;
+  /** Absolute path to the directory holding the attempt's starting state. */
+  fixtureDir: string;
+  /**
+   * Absolute path to an empty scratch directory. The adapter populates it
+   * from the fixture and performs its edits there; the runner exports
+   * it into the artifacts store afterwards.
+   */
+  workspaceDir: string;
+  /** The eval's task prompt, byte-identical across experiments. */
+  prompt: string;
+  trialIndex: number;
+}
+
+/** What an adapter returns for a completed attempt. */
+export interface AttemptOutcome {
+  transcript: TranscriptEvent[];
+  /** Unified diff of the workspace against the fixture. */
+  diff: string;
+  metrics: AttemptMetrics;
+}
+
+export interface AgentAdapter {
+  /** The adapter name experiments select via `agent.adapter` (e.g. "echo"). */
+  name: string;
+  identity?: AdapterIdentityConfig;
+  runAttempt(ctx: AttemptContext): Promise<AttemptOutcome>;
+}
+
+/**
+ * What a container adapter gets when composing its invocation. Host paths are
+ * deliberately absent — a container adapter never touches the host
+ * filesystem; the runner owns the container lifecycle so scoring can reach
+ * the live sandbox before teardown.
+ */
+export interface ContainerAttemptContext {
+  runId: string;
+  attemptId: string;
+  /** The eval's task prompt, byte-identical across experiments. */
+  prompt: string;
+  trialIndex: number;
+  /**
+   * The experiment runtime's MCP servers, credential-free (authed entries
+   * already point at the token-proxy sidecar) — present only when the
+   * runtime declares servers. The adapter renders these into its vendor's
+   * config format via setupFiles.
+   */
+  mcpServers?: Readonly<Record<string, McpServerConfig>>;
+  /** The runtime's native web-tool policy (default: blocked). */
+  webPolicy: WebPolicy;
+}
+
+/**
+ * A real agent driven headless inside the attempt container. The adapter is
+ * pure composition + parsing: which files to place (treatment config),
+ * which command to run in /workspace, which host env vars must exist, and
+ * how to turn the raw output stream into telemetry. The raw stream itself is
+ * the transcript artifact — parsing failures can never lose evidence.
+ */
+export interface ContainerAgentAdapter {
+  kind: "container";
+  /** The adapter name experiments select via `agent.adapter` (e.g. "claude", "codex"). */
+  name: string;
+  identity?: AdapterIdentityConfig;
+  /** Command used during preflight to record the executable's exact version. */
+  versionCommand?: readonly string[];
+  /**
+   * Cheap authenticated request composed from the collected provider keys,
+   * probed at run preflight so a present-but-rejected key aborts BEFORE any
+   * container spend. A 401/403 response stops the run; network failures only
+   * warn — an unreachable probe must not gate a run the agent might complete.
+   */
+  keyProbe?(env: Record<string, string>): { url: string; headers: Record<string, string> };
+  /**
+   * Host env vars that must be present before any container is created
+   * (shared provider keys — see src/secrets.ts).
+   */
+  requiredHostEnv: readonly string[];
+  /**
+   * Hosts this agent's CLI must reach (its LLM/vendor endpoints) — the
+   * harness-default layer of the egress allowlist when the run uses the
+   * egress proxy (see isolation/proxy/allowlist.ts). Adapters that omit it
+   * contribute nothing, which under the proxy means the agent cannot reach
+   * any endpoint the runtime's own egress hosts don't list.
+   */
+  egressHosts?: readonly string[];
+  /**
+   * Files to write inside the container before the agent starts, keyed by
+   * absolute container path. This is where treatments differ: the MCP
+   * treatment places MCP config, others place nothing.
+   */
+  setupFiles(ctx: ContainerAttemptContext): Record<string, string>;
+  /** The headless invocation, executed in /workspace inside the container. */
+  command(ctx: ContainerAttemptContext): string[];
+  /** Parse the raw output stream into telemetry. Must tolerate malformed streams. */
+  parseStream(raw: string): AttemptMetrics;
+}
+
+export type AnyAgentAdapter = AgentAdapter | ContainerAgentAdapter;

package/src/build-info.generated.ts

@@ -0,0 +1,12 @@
+/** Populated temporarily by scripts/build-binary.ts for compiled releases. */
+export const EMBEDDED_BUILD_INFO: {
+  version: string;
+  revision: string | null;
+  dirty: boolean | null;
+  sourceHash: string | null;
+} = {
+  version: "0.1.0",
+  revision: null,
+  dirty: null,
+  sourceHash: null,
+};