firegoat 0.24.0 → 0.26.0

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.24.0",
+  version: "0.26.0",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -6976,7 +6976,7 @@ var package_default = {
   },
   scripts: {
     build: "bun build src/cli.ts --target node --outfile dist/cli.js",
-    test: "bun test",
+    test: "bun test --timeout 30000",
     typecheck: "bun check",
     lint: "biome check .",
     format: "biome check --write .",
@@ -6987,7 +6987,7 @@ var package_default = {
 };
 
 // src/config.ts
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync as existsSync4, readFileSync as readFileSync3 } from "node:fs";
 
 // node_modules/yaml/dist/index.js
 var composer = require_composer();
@@ -7035,6 +7035,830 @@ var $stringify = publicApi.stringify;
 var $visit = visit.visit;
 var $visitAsync = visit.visitAsync;
 
+// src/harnesses/claude.ts
+import { spawnSync } from "node:child_process";
+import { existsSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+var FLAGS = [
+  "--output-format",
+  "stream-json",
+  "--verbose",
+  "--dangerously-skip-permissions",
+  "--strict-mcp-config",
+  "--setting-sources",
+  "",
+  "--disable-slash-commands"
+];
+var SET = "firegoat sets it for every run";
+function parse(line) {
+  try {
+    const e = JSON.parse(line);
+    return typeof e === "object" && e !== null ? e : undefined;
+  } catch {
+    return;
+  }
+}
+function content(e) {
+  const message = e?.message;
+  return Array.isArray(message?.content) ? message.content : [];
+}
+function resultText(content) {
+  if (typeof content === "string")
+    return content;
+  if (Array.isArray(content))
+    return content.map((c) => typeof c.text === "string" ? c.text : `[${c.type}]`).join(`
+`);
+  return "";
+}
+function steps(events) {
+  const out = [];
+  for (const e of events) {
+    if (e?.type === "assistant")
+      for (const b of content(e)) {
+        if (b.type === "tool_use")
+          out.push({
+            kind: "tool_call",
+            name: typeof b.name === "string" ? b.name : "",
+            input: typeof b.input === "object" && b.input !== null ? b.input : null
+          });
+        else if (b.type === "text" && typeof b.text === "string")
+          out.push({ kind: "text", text: b.text });
+      }
+    else if (e?.type === "user") {
+      for (const b of content(e))
+        if (b.type === "tool_result")
+          out.push({
+            kind: "tool_result",
+            isError: b.is_error === true,
+            text: resultText(b.content).trimEnd()
+          });
+    }
+  }
+  return out;
+}
+function read(stdout, exited, timeoutMs) {
+  const events = stdout.split(`
+`).filter((l) => l.trim() !== "").map(parse);
+  let final = "";
+  for (const e of events)
+    if (e?.type === "result" && typeof e.result === "string")
+      final = e.result;
+  const result = events.findLast((e) => e?.type === "result");
+  const trace = {
+    steps: steps(events),
+    final,
+    turns: result?.num_turns ?? null,
+    cost: result?.total_cost_usd ?? null
+  };
+  const init = events.find((e) => e?.type === "system" && e.subtype === "init");
+  if (!init) {
+    const said = stdout.trim().split(`
+`)[0]?.slice(0, 300);
+    return {
+      ...trace,
+      reportedModel: null,
+      version: null,
+      ok: false,
+      reason: `claude did not start a session (exit ${exited.code ?? "by signal"})${said ? `: ${said}` : ", see stderr.txt"}`
+    };
+  }
+  const reportedModel = typeof init.model === "string" ? init.model : null;
+  const version = typeof init.claude_code_version === "string" ? init.claude_code_version : null;
+  if (typeof init.session_id !== "string" || init.session_id === "")
+    return {
+      ...trace,
+      reportedModel,
+      version,
+      ok: false,
+      reason: "claude's init line has no session_id"
+    };
+  const sessionId = init.session_id;
+  if (exited.timedOut)
+    return {
+      ...trace,
+      reportedModel,
+      version,
+      ok: false,
+      reason: `timed out after ${timeoutMs / 1000}s`,
+      sessionId
+    };
+  if (!result)
+    return {
+      ...trace,
+      reportedModel,
+      version,
+      ok: false,
+      reason: `claude exited ${exited.code ?? "by signal"} without a result, see stderr.txt`,
+      sessionId
+    };
+  if (result.is_error === true || result.subtype !== "success")
+    return {
+      ...trace,
+      reportedModel,
+      version,
+      ok: false,
+      reason: `claude result is an error (subtype=${result.subtype})${result.result ? `: ${result.result.slice(0, 300)}` : ""}`,
+      sessionId
+    };
+  return { ...trace, reportedModel, version, ok: true, sessionId };
+}
+function progress(line) {
+  const e = parse(line);
+  if (e?.type !== "assistant")
+    return;
+  const p = {};
+  for (const b of content(e)) {
+    const cmd = b.input?.command;
+    if (b.type === "tool_use" && typeof cmd === "string")
+      p.command = cmd.split(`
+`)[0];
+  }
+  return p;
+}
+function checkLogin(opts) {
+  const r = spawnSync(opts.bin, ["auth", "status"], {
+    cwd: opts.cwd,
+    env: opts.env,
+    encoding: "utf8",
+    timeout: 30000
+  });
+  if (r.error)
+    return {
+      loggedOut: false,
+      reason: `could not run ${opts.bin}: ${r.error.message}`
+    };
+  let status;
+  try {
+    status = JSON.parse(r.stdout);
+  } catch {}
+  if (status?.loggedIn === true)
+    return;
+  if (status?.loggedIn === false)
+    return {
+      loggedOut: true,
+      reason: "claude is not logged in. Run `claude` once and log in (or set ANTHROPIC_API_KEY), then run eval again"
+    };
+  return {
+    loggedOut: false,
+    reason: `claude auth status gave no login state (exit ${r.status}): ${(r.stdout + r.stderr).trim().slice(0, 300)}`
+  };
+}
+var claude = {
+  name: "claude-code",
+  bin: "claude",
+  installHint: "Install Claude Code (https://claude.com/claude-code) or pass --harness-bin PATH.",
+  configDirName: ".claude",
+  reservedArgs: [
+    [["-p", "--print", ...FLAGS.filter((f) => f.startsWith("-"))], SET],
+    [["--model"], "choose the model with top-level model: (or eval --model)"],
+    [
+      ["--session-id", "--resume", "-r", "--continue", "-c"],
+      "every run is a fresh session"
+    ]
+  ],
+  uploads: true,
+  args: ({ prompt, model, extraArgs }) => [
+    "-p",
+    prompt,
+    ...FLAGS,
+    ...model === undefined ? [] : ["--model", model],
+    ...extraArgs
+  ],
+  env: ({ configDir, run }) => ({
+    CLAUDE_CONFIG_DIR: configDir,
+    CLAUDE_SECURESTORAGE_CONFIG_DIR: "",
+    ...run === undefined ? {} : { CLAUDE_ENV_FILE: run.homeEnvFile }
+  }),
+  authLabel: () => process.env.ANTHROPIC_API_KEY ? "ANTHROPIC_API_KEY (claude bills the API key; unset it to use your claude login)" : "your claude login",
+  checkLogin,
+  progress,
+  read,
+  sessionsDir: (configDir) => join(configDir, "projects"),
+  findSession(configDir, sessionId) {
+    const projects = join(configDir, "projects");
+    if (!existsSync(projects))
+      return null;
+    for (const d of readdirSync(projects)) {
+      const f = join(projects, d, `${sessionId}.jsonl`);
+      if (existsSync(f))
+        return f;
+    }
+    return null;
+  }
+};
+
+// src/harnesses/codex.ts
+import { spawnSync as spawnSync2 } from "node:child_process";
+import {
+  existsSync as existsSync2,
+  lstatSync,
+  readdirSync as readdirSync2,
+  readFileSync,
+  symlinkSync,
+  unlinkSync
+} from "node:fs";
+import { homedir } from "node:os";
+import { join as join3 } from "node:path";
+
+// src/home.ts
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join as join2 } from "node:path";
+function homeEnv(home) {
+  return {
+    HOME: home,
+    XDG_CONFIG_HOME: join2(home, ".config"),
+    XDG_CACHE_HOME: join2(home, ".cache"),
+    XDG_DATA_HOME: join2(home, ".local", "share"),
+    XDG_STATE_HOME: join2(home, ".local", "state")
+  };
+}
+var shellQuote = (s) => `'${s.replaceAll("'", `'\\''`)}'`;
+function makeRunHome(dir) {
+  const home = join2(dir, "home");
+  const env = homeEnv(home);
+  for (const d of Object.values(env))
+    mkdirSync(d, { recursive: true });
+  const envFile = join2(dir, "home.env");
+  writeFileSync(envFile, Object.entries(env).map(([k, v]) => `export ${k}=${shellQuote(v)}
+`).join(""));
+  return { home, envFile };
+}
+
+// src/harnesses/codex.ts
+var FLAGS2 = [
+  "--json",
+  "--skip-git-repo-check",
+  "--dangerously-bypass-approvals-and-sandbox"
+];
+var NO_PROJECT_DOC = ["-c", "project_doc_max_bytes=0"];
+function parse2(line) {
+  try {
+    const e = JSON.parse(line);
+    return typeof e === "object" && e !== null ? e : undefined;
+  } catch {
+    return;
+  }
+}
+function unwrapShell(command) {
+  const m = /^\/bin\/(?:ba|z)?sh -l?c (.+)$/s.exec(command);
+  if (!m)
+    return command;
+  const word = m[1];
+  let out = "";
+  for (let i = 0;i < word.length; i++) {
+    const c = word[i];
+    if (c === "'") {
+      const end = word.indexOf("'", i + 1);
+      if (end < 0)
+        return command;
+      out += word.slice(i + 1, end);
+      i = end;
+    } else if (c === "\\" && i + 1 < word.length)
+      out += word[++i];
+    else if (c === '"') {
+      let j = i + 1;
+      for (;j < word.length && word[j] !== '"'; j++) {
+        if (word[j] === "\\" && '"\\$`'.includes(word[j + 1] ?? ""))
+          j++;
+        out += word[j];
+      }
+      if (j >= word.length)
+        return command;
+      i = j;
+    } else if (/\s/.test(c))
+      return command;
+    else
+      out += c;
+  }
+  return out;
+}
+function str(v) {
+  return typeof v === "string" ? v : "";
+}
+function itemInput(item) {
+  const { id: _id, type: _type, status: _status, ...rest } = item;
+  return rest;
+}
+function steps2(events) {
+  const out = [];
+  const started = new Set;
+  for (const e of events) {
+    const item = e?.item;
+    if (!item)
+      continue;
+    if (item.type === "command_execution") {
+      const call = {
+        kind: "tool_call",
+        name: "shell",
+        input: { command: unwrapShell(str(item.command)) }
+      };
+      if (e?.type === "item.started") {
+        started.add(item.id);
+        out.push(call);
+      } else if (e?.type === "item.completed") {
+        if (item.id === undefined || !started.has(item.id))
+          out.push(call);
+        out.push({
+          kind: "tool_result",
+          isError: item.exit_code !== 0,
+          text: str(item.aggregated_output).trimEnd()
+        });
+      }
+      continue;
+    }
+    if (e?.type !== "item.completed")
+      continue;
+    if (item.type === "agent_message") {
+      if (str(item.text).trim() !== "")
+        out.push({ kind: "text", text: str(item.text) });
+    } else if (item.type !== "reasoning" && item.type !== "error")
+      out.push({
+        kind: "tool_call",
+        name: str(item.type),
+        input: itemInput(item)
+      });
+  }
+  return out;
+}
+function errorMessage(raw) {
+  try {
+    const inner = JSON.parse(raw)?.error?.message;
+    if (typeof inner === "string")
+      return inner;
+  } catch {}
+  return raw;
+}
+function read2(stdout, exited, timeoutMs) {
+  const events = stdout.split(`
+`).filter((l) => l.trim() !== "").map(parse2);
+  const trace = {
+    steps: steps2(events),
+    final: events.filter((e) => e?.type === "item.completed" && e.item?.type === "agent_message").map((e) => str(e?.item?.text)).at(-1) ?? "",
+    reportedModel: null,
+    version: null,
+    turns: null,
+    cost: null
+  };
+  const thread = events.find((e) => e?.type === "thread.started");
+  if (!thread) {
+    const said = stdout.trim().split(`
+`)[0]?.slice(0, 300);
+    return {
+      ...trace,
+      ok: false,
+      reason: `codex did not start a thread (exit ${exited.code ?? "by signal"})${said ? `: ${said}` : ", see stderr.txt"}`
+    };
+  }
+  const sessionId = str(thread.thread_id);
+  if (sessionId === "")
+    return {
+      ...trace,
+      ok: false,
+      reason: "codex's thread.started line has no thread_id"
+    };
+  if (exited.timedOut)
+    return {
+      ...trace,
+      ok: false,
+      reason: `timed out after ${timeoutMs / 1000}s`,
+      sessionId
+    };
+  const failed = events.find((e) => e?.type === "turn.failed");
+  if (failed) {
+    const message = str(failed.error?.message);
+    return {
+      ...trace,
+      ok: false,
+      reason: `codex turn failed${message ? `: ${errorMessage(message).slice(0, 300)}` : ""}`,
+      sessionId
+    };
+  }
+  if (!events.some((e) => e?.type === "turn.completed"))
+    return {
+      ...trace,
+      ok: false,
+      reason: `codex exited ${exited.code ?? "by signal"} without completing its turn, see stderr.txt`,
+      sessionId
+    };
+  return { ...trace, ok: true, sessionId };
+}
+function progress2(line) {
+  const e = parse2(line);
+  const item = e?.item;
+  if (e?.type === "item.started" && item?.type === "command_execution")
+    return { command: unwrapShell(str(item.command)).split(`
+`)[0] };
+  if (e?.type === "item.completed" && item?.type === "agent_message")
+    return {};
+  return;
+}
+function userCodexHome() {
+  return process.env.CODEX_HOME || join3(homedir(), ".codex");
+}
+function checkLogin2(opts) {
+  const source = join3(userCodexHome(), "auth.json");
+  if (!existsSync2(source))
+    return {
+      loggedOut: true,
+      reason: `codex is not logged in (no ${source}). Run \`codex login\`, then run eval again`
+    };
+  const link = join3(opts.configDir, "auth.json");
+  let current;
+  try {
+    current = lstatSync(link);
+  } catch {}
+  if (current && !current.isSymbolicLink())
+    return {
+      loggedOut: false,
+      reason: `${link} is a file, not firegoat's link to ${source}: codex may have refreshed its login into it. Move it over ${source} if it is newer, else delete it, then run eval again`
+    };
+  if (current)
+    unlinkSync(link);
+  symlinkSync(source, link);
+  const r = spawnSync2(opts.bin, ["login", "status"], {
+    cwd: opts.cwd,
+    env: opts.env,
+    encoding: "utf8",
+    timeout: 30000
+  });
+  if (r.error)
+    return {
+      loggedOut: false,
+      reason: `could not run ${opts.bin}: ${r.error.message}`
+    };
+  if (r.status === 0)
+    return;
+  return {
+    loggedOut: true,
+    reason: `codex is not logged in (codex login status exit ${r.status}: ${(r.stdout + r.stderr).trim().slice(0, 300)}). Run \`codex login\`, then run eval again`
+  };
+}
+function findSession(configDir, sessionId) {
+  const sessions = join3(configDir, "sessions");
+  if (!existsSync2(sessions))
+    return null;
+  const suffix = `-${sessionId}.jsonl`;
+  for (const y of readdirSync2(sessions))
+    for (const m of readdirSync2(join3(sessions, y)))
+      for (const d of readdirSync2(join3(sessions, y, m)))
+        for (const f of readdirSync2(join3(sessions, y, m, d)))
+          if (f.startsWith("rollout-") && f.endsWith(suffix))
+            return join3(sessions, y, m, d, f);
+  return null;
+}
+function sessionModel(sessionFile) {
+  let model = null;
+  for (const line of readFileSync(sessionFile, "utf8").split(`
+`)) {
+    const e = parse2(line);
+    const m = e?.payload?.model;
+    if (e?.type === "turn_context" && typeof m === "string")
+      model = m;
+  }
+  return model;
+}
+var codex = {
+  name: "codex-cli",
+  bin: "codex",
+  installHint: "Install Codex CLI (https://github.com/openai/codex) or pass --harness-bin PATH.",
+  configDirName: ".codex",
+  reservedArgs: [
+    [[...FLAGS2, "-C", "--cd"], "firegoat sets it for every run"],
+    [
+      ["-m", "--model"],
+      "choose the model with top-level model: (or eval --model)"
+    ],
+    [["resume", "fork"], "every run is a fresh session"],
+    [["--ephemeral"], "firegoat reads the session file codex saves"]
+  ],
+  uploads: false,
+  args: ({ prompt, model, extraArgs, cwd }) => [
+    "exec",
+    ...FLAGS2,
+    "-C",
+    cwd,
+    ...model === undefined ? [] : ["-m", model],
+    ...NO_PROJECT_DOC,
+    ...extraArgs,
+    "--",
+    prompt
+  ],
+  env: ({ configDir, run }) => ({
+    CODEX_HOME: configDir,
+    ...run === undefined ? {} : homeEnv(run.home)
+  }),
+  authLabel: () => `your codex login (${join3(userCodexHome(), "auth.json")})${process.env.CODEX_API_KEY ? "; CODEX_API_KEY is set and passes through" : ""}`,
+  checkLogin: checkLogin2,
+  progress: progress2,
+  read: read2,
+  sessionsDir: (configDir) => join3(configDir, "sessions"),
+  findSession,
+  sessionModel
+};
+
+// src/harnesses/grok.ts
+import {
+  closeSync,
+  existsSync as existsSync3,
+  lstatSync as lstatSync2,
+  openSync,
+  readdirSync as readdirSync3,
+  readFileSync as readFileSync2,
+  readlinkSync,
+  readSync,
+  statSync,
+  symlinkSync as symlinkSync2,
+  unlinkSync as unlinkSync2
+} from "node:fs";
+import { homedir as homedir2 } from "node:os";
+import { join as join4 } from "node:path";
+var BIN = join4(homedir2(), ".grok", "bin", "grok");
+var USER_AUTH = join4(process.env.GROK_HOME || join4(homedir2(), ".grok"), "auth.json");
+var REFRESH_WINDOW_MS = 5 * 60000;
+var FLAGS3 = [
+  "--output-format",
+  "streaming-messages-json",
+  "--always-approve",
+  "--no-auto-update"
+];
+function parse3(line) {
+  try {
+    const e = JSON.parse(line);
+    return typeof e === "object" && e !== null ? e : undefined;
+  } catch {
+    return;
+  }
+}
+var isObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
+function content2(e) {
+  const message = e?.message;
+  return Array.isArray(message?.content) ? message.content : [];
+}
+var PROMPT_KEYS = ["output_for_prompt", "tool_output_for_prompt", "content"];
+function promptText(v) {
+  if (typeof v === "string")
+    return v;
+  if (!isObject(v))
+    return;
+  for (const k of PROMPT_KEYS)
+    if (typeof v[k] === "string")
+      return v[k];
+  for (const [k, x] of Object.entries(v)) {
+    if (k === "type")
+      continue;
+    const t = promptText(x);
+    if (t !== undefined)
+      return t;
+  }
+  return;
+}
+function toolResult(b) {
+  const raw = b.content;
+  let json;
+  if (typeof raw === "string")
+    try {
+      json = JSON.parse(raw);
+    } catch {}
+  const failed = isObject(json) && (typeof json.exit_code === "number" && json.exit_code !== 0 || json.timed_out === true);
+  const text = isObject(json) ? promptText(json) ?? raw : typeof raw === "string" ? raw : Array.isArray(raw) ? raw.map((c) => typeof c.text === "string" ? c.text : `[${c.type}]`).join(`
+`) : "";
+  return {
+    kind: "tool_result",
+    isError: b.is_error === true || failed,
+    text: text.trimEnd()
+  };
+}
+function steps3(events) {
+  const out = [];
+  for (const e of events) {
+    if (e?.type === "assistant")
+      for (const b of content2(e)) {
+        if (b.type === "tool_use")
+          out.push({
+            kind: "tool_call",
+            name: typeof b.name === "string" ? b.name : "",
+            input: isObject(b.input) ? b.input : null
+          });
+        else if (b.type === "text" && typeof b.text === "string")
+          out.push({ kind: "text", text: b.text });
+      }
+    else if (e?.type === "user") {
+      for (const b of content2(e))
+        if (b.type === "tool_result")
+          out.push(toolResult(b));
+    }
+  }
+  return out;
+}
+function resultError(result) {
+  const errors = Array.isArray(result.errors) ? result.errors.filter((x) => typeof x === "string").join("; ") : "";
+  const said = errors || result.result || "";
+  return `grok result is an error (subtype=${result.subtype})${said ? `: ${said.slice(0, 300)}` : ""}`;
+}
+function read3(stdout, exited, timeoutMs) {
+  const events = stdout.split(`
+`).filter((l) => l.trim() !== "").map(parse3);
+  const result = events.findLast((e) => e?.type === "result");
+  const trace = {
+    steps: steps3(events),
+    final: typeof result?.result === "string" ? result.result : "",
+    turns: result?.num_turns ?? null,
+    cost: result?.total_cost_usd ?? null
+  };
+  const init = events.find((e) => e?.type === "system" && e.subtype === "init");
+  const reportedModel = typeof init?.model === "string" && init.model !== "unknown" ? init.model : null;
+  const sessionId = typeof init?.session_id === "string" && init.session_id !== "" ? init.session_id : undefined;
+  const fail = (reason) => ({
+    ...trace,
+    reportedModel,
+    version: null,
+    ok: false,
+    reason,
+    ...sessionId === undefined ? {} : { sessionId }
+  });
+  if (exited.timedOut)
+    return fail(`timed out after ${timeoutMs / 1000}s`);
+  if (!init) {
+    const said = stdout.trim().split(`
+`)[0]?.slice(0, 300);
+    return fail(`grok did not start a session (exit ${exited.code ?? "by signal"})${said ? `: ${said}` : ", see stderr.txt"}`);
+  }
+  if (result && (result.is_error === true || result.subtype !== "success"))
+    return fail(resultError(result));
+  if (sessionId === undefined)
+    return fail("grok's init line has no session_id");
+  if (!result)
+    return fail(`grok exited ${exited.code ?? "by signal"} without a result, see stderr.txt`);
+  return { ...trace, reportedModel, version: null, ok: true, sessionId };
+}
+function progress3(line) {
+  const e = parse3(line);
+  if (e?.type !== "assistant")
+    return;
+  const p = {};
+  for (const b of content2(e)) {
+    const cmd = b.input?.command;
+    if (b.type === "tool_use" && typeof cmd === "string")
+      p.command = cmd.split(`
+`)[0];
+  }
+  return p;
+}
+function isScript(bin) {
+  const fd = openSync(bin, "r");
+  try {
+    const head = Buffer.alloc(2);
+    return readSync(fd, head, 0, 2, 0) === 2 && head.toString() === "#!";
+  } finally {
+    closeSync(fd);
+  }
+}
+function loginProblem() {
+  const relogin = "Run `grok login`, then run eval again";
+  if (!existsSync3(USER_AUTH))
+    return {
+      loggedOut: true,
+      reason: `grok is not logged in (no ${USER_AUTH}). ${relogin}. Headless grok needs that login: XAI_API_KEY alone is not enough`
+    };
+  let auth;
+  try {
+    auth = JSON.parse(readFileSync2(USER_AUTH, "utf8"));
+  } catch {
+    return {
+      loggedOut: false,
+      reason: `${USER_AUTH} is not valid JSON. ${relogin}`
+    };
+  }
+  const logins = isObject(auth) ? Object.values(auth).filter((v) => isObject(v) && typeof v.key === "string" && v.key !== "") : [];
+  if (logins.length === 0)
+    return {
+      loggedOut: true,
+      reason: `grok is not logged in (${USER_AUTH} holds no login). ${relogin}`
+    };
+  const expiries = logins.map((l) => l.expires_at);
+  if (expiries.some((x) => x === undefined))
+    return;
+  const times = expiries.map((x) => Date.parse(String(x)));
+  if (times.some(Number.isNaN))
+    return {
+      loggedOut: false,
+      reason: `${USER_AUTH} has an expires_at that is not a date (${expiries.join(", ")}). ${relogin}`
+    };
+  const latest = Math.max(...times);
+  if (latest - Date.now() > REFRESH_WINDOW_MS)
+    return;
+  return {
+    loggedOut: false,
+    reason: `grok's login ${latest < Date.now() ? "expired" : "expires"} at ${new Date(latest).toISOString()}. Run grok once (e.g. \`grok -p hi\`) so it refreshes the login in ${USER_AUTH}, then run eval again: eval's runs must not refresh it themselves`
+  };
+}
+function linkAuth(configDir) {
+  const link = join4(configDir, "auth.json");
+  let st;
+  try {
+    st = lstatSync2(link);
+  } catch (e) {
+    if (e.code !== "ENOENT")
+      throw e;
+  }
+  if (st && !st.isSymbolicLink())
+    return {
+      loggedOut: false,
+      reason: `${link} is a file, not a link to ${USER_AUTH}: a grok run refreshed its login there, so it is newer than ${USER_AUTH}. Move it back with \`mv ${link} ${USER_AUTH}\`, then run eval again`
+    };
+  if (st && readlinkSync(link) === USER_AUTH)
+    return;
+  if (st)
+    unlinkSync2(link);
+  symlinkSync2(USER_AUTH, link);
+  return;
+}
+function checkLogin3(opts) {
+  if (isScript(opts.bin))
+    return {
+      loggedOut: false,
+      reason: `${opts.bin} is a script, not the grok executable (cmux, for one, puts a bash wrapper named grok on PATH). Pass grok's own binary with --harness-bin, ${BIN} by default`
+    };
+  return loginProblem() ?? linkAuth(opts.configDir);
+}
+var grok = {
+  name: "grok-cli",
+  bin: BIN,
+  installHint: "Install grok (curl -fsSL https://x.ai/cli/install.sh | bash) or pass --harness-bin PATH.",
+  configDirName: ".grok",
+  reservedArgs: [
+    [
+      [
+        "-p",
+        "--single",
+        "--prompt-file",
+        "--prompt-json",
+        "--output-format",
+        "--json-schema",
+        "--always-approve",
+        "--permission-mode",
+        "--no-auto-update",
+        "--cwd",
+        "-w",
+        "--worktree"
+      ],
+      "firegoat sets it for every run"
+    ],
+    [
+      ["-m", "--model"],
+      "choose the model with top-level model: (or eval --model)"
+    ],
+    [
+      [
+        "-r",
+        "--resume",
+        "-c",
+        "--continue",
+        "-s",
+        "--session-id",
+        "--fork-session"
+      ],
+      "every run is a fresh session"
+    ]
+  ],
+  uploads: false,
+  args: ({ prompt, model, extraArgs }) => [
+    "-p",
+    prompt,
+    ...FLAGS3,
+    ...model === undefined ? [] : ["-m", model],
+    ...extraArgs
+  ],
+  env: ({ configDir, run }) => ({
+    GROK_HOME: configDir,
+    GROK_DISABLE_AUTOUPDATER: "1",
+    ...run === undefined ? {} : homeEnv(run.home)
+  }),
+  authLabel: () => `your grok login (${USER_AUTH})`,
+  checkLogin: checkLogin3,
+  progress: progress3,
+  read: read3,
+  sessionsDir: (configDir) => join4(configDir, "sessions"),
+  findSession(configDir, sessionId) {
+    const sessions = join4(configDir, "sessions");
+    if (!existsSync3(sessions))
+      return null;
+    for (const d of readdirSync3(sessions)) {
+      const dir = join4(sessions, d, sessionId);
+      if (existsSync3(dir) && statSync(dir).isDirectory())
+        return dir;
+    }
+    return null;
+  }
+};
+
+// src/harnesses/index.ts
+var HARNESSES = new Map([claude, codex, grok].map((h) => [h.name, h]));
+var DEFAULT_HARNESS = claude.name;
+function validHarnesses() {
+  return `(valid: ${[...HARNESSES.keys()].join(", ")})`;
+}
+
 // src/config.ts
 var CONFIG_FILE = "firegoat.yaml";
 var DEFAULT_TIMEOUT = 600;
@@ -7047,7 +7871,7 @@ function fail(path, msg) {
 function isRecord(v) {
   return typeof v === "object" && v !== null && !Array.isArray(v);
 }
-function str(v, path) {
+function str2(v, path) {
   if (typeof v !== "string" || v.trim() === "")
     fail(path, "must be a non-empty string");
   return v;
@@ -7075,23 +7899,29 @@ function parseConfig(text) {
     fail("(root)", "must be a mapping");
   onlyKeys(raw, "(root)", [
     "runs",
+    "harness",
     "model",
     "build",
     "install",
     "cleanup",
     "vars",
     "env",
-    "claude_args",
+    "harness_args",
     "tests"
   ]);
   const runs = posInt(raw.runs, "runs", 1);
-  const models = parseModels(raw.model);
+  const harnesses = raw.harness === undefined ? [DEFAULT_HARNESS] : nameList(raw.harness, "harness", "a harness name");
+  harnesses.forEach((h, i) => {
+    if (!HARNESSES.has(h))
+      fail(Array.isArray(raw.harness) ? `harness[${i}]` : "harness", `unknown harness ${JSON.stringify(h)} ${validHarnesses()}`);
+  });
+  const models = raw.model === undefined ? [] : nameList(raw.model, "model", "a model name");
   const build = commands(raw.build, "build", EVAL_PLACEHOLDERS);
   const install = commands(raw.install, "install", RUN_PLACEHOLDERS);
   const cleanup = commands(raw.cleanup, "cleanup", RUN_PLACEHOLDERS);
   const vars = parseVars(raw.vars);
   const env = parseEnv(raw.env, "env");
-  const claudeArgs = parseClaudeArgs(raw.claude_args, "claude_args");
+  const harnessArgs = parseHarnessArgs(raw.harness_args, "harness_args");
   if (!Array.isArray(raw.tests) || raw.tests.length === 0)
     fail("tests", "must be a non-empty list");
   const names = new Set;
@@ -7108,9 +7938,9 @@ function parseConfig(text) {
       "env",
       "install",
       "cleanup",
-      "claude_args"
+      "harness_args"
     ]);
-    const name = str(t.name, `${p}.name`);
+    const name = str2(t.name, `${p}.name`);
     if (!NAME.test(name))
       fail(`${p}.name`, "may only contain letters, digits, '.', '_' and '-'");
     if (names.has(name))
@@ -7125,27 +7955,28 @@ function parseConfig(text) {
       if (!isRecord(c))
         fail(cp, "must be a mapping");
       onlyKeys(c, cp, ["run"]);
-      const run = str(c.run, `${cp}.run`);
+      const run = str2(c.run, `${cp}.run`);
       checkPlaceholders(run, `${cp}.run`, RUN_PLACEHOLDERS);
       checks.push({ run });
     });
     return {
       name,
-      prompt: str(t.prompt, `${p}.prompt`),
-      end_goal: str(t.end_goal, `${p}.end_goal`),
+      prompt: str2(t.prompt, `${p}.prompt`),
+      end_goal: str2(t.end_goal, `${p}.end_goal`),
       checks,
       timeout: posInt(t.timeout, `${p}.timeout`, DEFAULT_TIMEOUT),
       env: { ...env, ...parseEnv(t.env, `${p}.env`) },
       install: commands(t.install, `${p}.install`, RUN_PLACEHOLDERS),
       cleanup: commands(t.cleanup, `${p}.cleanup`, RUN_PLACEHOLDERS),
-      claudeArgs: [
-        ...claudeArgs,
-        ...parseClaudeArgs(t.claude_args, `${p}.claude_args`)
+      harnessArgs: [
+        ...harnessArgs,
+        ...parseHarnessArgs(t.harness_args, `${p}.harness_args`)
       ]
     };
   });
   return {
     runs,
+    harnesses,
     models,
     build,
     install,
@@ -7164,63 +7995,45 @@ function commands(raw, path, allowed) {
   const list = typeof raw === "string" ? [raw] : raw;
   return list.map((c, i) => {
     const p = typeof raw === "string" ? path : `${path}[${i}]`;
-    const cmd = str(c, p);
+    const cmd = str2(c, p);
     checkPlaceholders(cmd, p, allowed);
     return cmd;
   });
 }
-function parseModels(raw) {
-  if (raw === undefined)
-    return [];
+function nameList(raw, key, what) {
   if (typeof raw === "string")
-    return [str(raw, "model")];
+    return [str2(raw, key)];
   if (!Array.isArray(raw) || raw.length === 0)
-    fail("model", "must be a model name or a non-empty list of them");
+    fail(key, `must be ${what} or a non-empty list of them`);
   const seen = new Set;
   return raw.map((m, i) => {
-    const name = str(m, `model[${i}]`);
+    const name = str2(m, `${key}[${i}]`);
     if (seen.has(name))
-      fail(`model[${i}]`, `duplicate model ${JSON.stringify(name)}`);
+      fail(`${key}[${i}]`, `duplicate ${key} ${JSON.stringify(name)}`);
     seen.add(name);
     return name;
   });
 }
-var OWNED_FLAGS = [
-  [
-    [
-      "-p",
-      "--print",
-      "--output-format",
-      "--verbose",
-      "--setting-sources",
-      "--strict-mcp-config",
-      "--disable-slash-commands",
-      "--dangerously-skip-permissions"
-    ],
-    "firegoat sets it for every run"
-  ],
-  [["--model"], "choose the model with top-level model: (or eval --model)"],
-  [
-    ["--session-id", "--resume", "-r", "--continue", "-c"],
-    "every run is a fresh session"
-  ]
-];
-function parseClaudeArgs(raw, path) {
+function parseHarnessArgs(raw, path) {
   if (raw === undefined)
     return [];
   if (!Array.isArray(raw))
-    fail(path, "must be a list of claude arguments, one per entry, e.g. [--chrome]");
+    fail(path, "must be a list of harness arguments, one per entry, e.g. [--chrome]");
   return raw.map((a, i) => {
     const p = `${path}[${i}]`;
-    const arg = str(a, p);
-    for (const [flags, why] of OWNED_FLAGS)
-      for (const f of flags)
-        if (arg === f || arg.startsWith(`${f}=`))
-          fail(p, `${f} is not allowed: ${why}`);
+    const arg = str2(a, p);
     checkPlaceholders(arg, p, RUN_PLACEHOLDERS);
-    return arg;
+    return { arg, path: p };
   });
 }
+function checkHarnessArgs(tests, harness) {
+  for (const t of tests)
+    for (const { arg, path } of t.harnessArgs)
+      for (const [flags, why] of harness.reservedArgs)
+        for (const f of flags)
+          if (arg === f || arg.startsWith(`${f}=`))
+            fail(path, `${f} is not allowed for ${harness.name}: ${why}`);
+}
 var VAR_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/;
 function parseVars(raw) {
   if (raw === undefined)
@@ -7240,7 +8053,10 @@ function parseVars(raw) {
 var RESERVED_ENV = new Set([
   "PATH",
   "CLAUDE_CONFIG_DIR",
-  "CLAUDE_SECURESTORAGE_CONFIG_DIR"
+  "CLAUDE_SECURESTORAGE_CONFIG_DIR",
+  "CODEX_HOME",
+  "GROK_HOME",
+  "GROK_DISABLE_AUTOUPDATER"
 ]);
 var HOME_ENV = new Set([
   "HOME",
@@ -7270,7 +8086,7 @@ function parseEnv(raw, path) {
     if (HOME_ENV.has(k))
       fail(p, "Reserved: every run gets its own home ($FIREGOAT_HOME)");
     if (RESERVED_ENV.has(k) || k.startsWith("FIREGOAT_"))
-      fail(p, "is set by firegoat and cannot be overridden (reserved: PATH, CLAUDE_CONFIG_DIR, CLAUDE_SECURESTORAGE_CONFIG_DIR, FIREGOAT_*)");
+      fail(p, "is set by firegoat and cannot be overridden (reserved: PATH, CLAUDE_CONFIG_DIR, CLAUDE_SECURESTORAGE_CONFIG_DIR, CODEX_HOME, GROK_HOME, GROK_DISABLE_AUTOUPDATER, FIREGOAT_*)");
     if (typeof v !== "string")
       fail(p, "must be a string (quote numbers and booleans)");
     checkPlaceholders(v, p, RUN_PLACEHOLDERS);
@@ -7324,9 +8140,9 @@ function render(text, vars) {
 }
 function loadConfig(dir) {
   const file = `${dir}/${CONFIG_FILE}`;
-  if (!existsSync(file))
+  if (!existsSync4(file))
     throw new ConfigError(`${CONFIG_FILE} not found in ${dir}. Run: npx firegoat init`);
-  return parseConfig(readFileSync(file, "utf8"));
+  return parseConfig(readFileSync3(file, "utf8"));
 }
 
 // src/eval.ts
@@ -7334,27 +8150,71 @@ import { randomBytes as randomBytes2 } from "node:crypto";
 import {
   appendFileSync,
   createWriteStream as createWriteStream2,
-  mkdirSync as mkdirSync4,
+  mkdirSync as mkdirSync5,
   writeFileSync as writeFileSync4
 } from "node:fs";
-import { basename as basename3, delimiter as delimiter2, join as join7 } from "node:path";
+import { basename as basename3, delimiter as delimiter2, join as join10 } from "node:path";
+
+// src/agent-dirs.ts
+import { createHash, randomInt } from "node:crypto";
+import { mkdirSync as mkdirSync2, realpathSync, symlinkSync as symlinkSync3 } from "node:fs";
+import { homedir as homedir3 } from "node:os";
+import { join as join5 } from "node:path";
+var ID_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789";
+var ID_LENGTH = 8;
+var MAX_TRIES = 10;
+function projectAgentDir(projectRoot) {
+  const hash = createHash("sha256").update(realpathSync(projectRoot)).digest("hex").slice(0, 12);
+  return join5(firegoatHome(), "runs", hash);
+}
+function firegoatHome() {
+  return join5(realpathSync(homedir3()), ".firegoat");
+}
+function randomId() {
+  let id = "";
+  for (let i = 0;i < ID_LENGTH; i++)
+    id += ID_CHARS[randomInt(ID_CHARS.length)];
+  return id;
+}
+function ensureProjectAgentDir(projectRoot) {
+  mkdirSync2(firegoatHome(), { recursive: true, mode: 448 });
+  const dir = projectAgentDir(projectRoot);
+  mkdirSync2(dir, { recursive: true });
+  return dir;
+}
+function makeAgentDir(projectDir) {
+  for (let i = 0;i < MAX_TRIES; i++) {
+    const dir = join5(projectDir, randomId());
+    try {
+      mkdirSync2(dir);
+      return dir;
+    } catch (e) {
+      if (e.code !== "EEXIST")
+        throw e;
+    }
+  }
+  throw new Error(`Could not create a new run folder in ${projectDir}: ${MAX_TRIES} random names were taken`);
+}
+function linkDir(target, link) {
+  symlinkSync3(target, link, "dir");
+}
 
 // src/credentials.ts
 import { randomBytes } from "node:crypto";
 import {
   chmodSync,
-  closeSync,
+  closeSync as closeSync2,
   fsyncSync,
-  lstatSync,
-  mkdirSync,
-  openSync,
-  readFileSync as readFileSync2,
+  lstatSync as lstatSync3,
+  mkdirSync as mkdirSync3,
+  openSync as openSync2,
+  readFileSync as readFileSync4,
   renameSync,
-  unlinkSync,
+  unlinkSync as unlinkSync3,
   writeSync
 } from "node:fs";
-import { homedir } from "node:os";
-import { join } from "node:path";
+import { homedir as homedir4 } from "node:os";
+import { join as join6 } from "node:path";
 
 // src/firegoat-api.ts
 var DEFAULT_BASE_URL = "https://firegoat.ai/evals";
@@ -7506,10 +8366,10 @@ async function whoami(base, key) {
 // src/credentials.ts
 var KEY = /^fire_[\x21-\x7e]+$/;
 function credentialsDir() {
-  return join(homedir(), ".firegoat");
+  return join6(homedir4(), ".firegoat");
 }
 function credentialsPath() {
-  return join(credentialsDir(), "credentials.json");
+  return join6(credentialsDir(), "credentials.json");
 }
 function uid() {
   if (!process.getuid)
@@ -7518,7 +8378,7 @@ function uid() {
 }
 function lstatOrNull(p) {
   try {
-    return lstatSync(p);
+    return lstatSync3(p);
   } catch (e) {
     if (e.code === "ENOENT")
       return null;
@@ -7548,7 +8408,7 @@ function loadCredentials() {
     throw new ConfigError(`${path} is readable by others (mode ${(st.mode & 511).toString(8)}): chmod 600 it or run firegoat login again`);
   let c;
   try {
-    c = JSON.parse(readFileSync2(path, "utf8"));
+    c = JSON.parse(readFileSync4(path, "utf8"));
   } catch {
     throw new ConfigError(`${path} is not valid JSON: run firegoat logout, then firegoat login`);
   }
@@ -7572,20 +8432,20 @@ function saveCredentials(c) {
   if (st)
     assertOwned(dir, st, "dir");
   else
-    mkdirSync(dir, { mode: 448 });
+    mkdirSync3(dir, { mode: 448 });
   chmodSync(dir, 448);
   const tmp = `${credentialsPath()}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`;
-  const fd = openSync(tmp, "wx", 384);
+  const fd = openSync2(tmp, "wx", 384);
   try {
     writeSync(fd, `${JSON.stringify(c, null, 2)}
 `);
     fsyncSync(fd);
   } catch (e) {
-    closeSync(fd);
-    unlinkSync(tmp);
+    closeSync2(fd);
+    unlinkSync3(tmp);
     throw e;
   }
-  closeSync(fd);
+  closeSync2(fd);
   renameSync(tmp, credentialsPath());
 }
 function removeCredentials() {
@@ -7606,7 +8466,7 @@ function removeCredentials() {
     if (!(e instanceof ConfigError))
       throw e;
   }
-  unlinkSync(path);
+  unlinkSync3(path);
   return { saved };
 }
 function resolveKey(base) {
@@ -7628,15 +8488,15 @@ function resolveKey(base) {
 var EXIT = { pass: 0, fail: 1, notGraded: 2, usage: 3 };
 
 // src/firegoat-dir.ts
-import { mkdirSync as mkdirSync2, writeFileSync } from "node:fs";
-import { join as join2 } from "node:path";
+import { mkdirSync as mkdirSync4, writeFileSync as writeFileSync2 } from "node:fs";
+import { join as join7 } from "node:path";
 var GITIGNORE = `*
 `;
 function ensureFiregoatDir(root) {
-  const dir = join2(root, ".firegoat");
-  mkdirSync2(dir, { recursive: true });
+  const dir = join7(root, ".firegoat");
+  mkdirSync4(dir, { recursive: true });
   try {
-    writeFileSync(join2(dir, ".gitignore"), GITIGNORE, { flag: "wx" });
+    writeFileSync2(join7(dir, ".gitignore"), GITIGNORE, { flag: "wx" });
   } catch (e) {
     if (e.code !== "EEXIST")
       throw e;
@@ -7645,17 +8505,48 @@ function ensureFiregoatDir(root) {
 }
 
 // src/git.ts
-import { spawnSync } from "node:child_process";
+import { spawnSync as spawnSync3 } from "node:child_process";
 import { basename, dirname } from "node:path";
 
 // src/process.ts
-import { spawn } from "node:child_process";
+import { execFileSync, spawn } from "node:child_process";
 var NPM_INJECTED = /^(npm_config_|npm_lifecycle_|npm_package_)|^(npm_execpath|npm_node_execpath|npm_command|INIT_CWD)$/;
 var CLAUDE_SESSION = /^(CLAUDECODE|CLAUDE_CODE_.+|CLAUDE_PID|CLAUDE_ENV_FILE|CLAUDE_SESSION_ID|CLAUDE_PROJECT_DIR|CLAUDE_EFFORT|CLAUDE_AGENT_SDK_.+|CLAUDE_PLUGIN_ROOT|CLAUDE_PLUGIN_DATA|CLAUDE_SKILL_DIR)$/;
 var CLAUDE_AUTH = /^CLAUDE_CODE_(OAUTH_TOKEN|OAUTH_REFRESH_TOKEN|USE_[A-Z_]+|SKIP_[A-Z]+_AUTH|CLIENT_CERT|CLIENT_KEY|CLIENT_KEY_PASSPHRASE)$/;
 function hostEnv() {
   return Object.fromEntries(Object.entries(process.env).filter(([k]) => !NPM_INJECTED.test(k) && !k.startsWith("FIREGOAT_") && (!CLAUDE_SESSION.test(k) || CLAUDE_AUTH.test(k))));
 }
+function processTree(pid) {
+  const children = new Map;
+  const ps = execFileSync("ps", ["-A", "-o", "pid=,ppid="], {
+    encoding: "utf8"
+  });
+  for (const line of ps.split(`
+`)) {
+    const [child, parent] = line.trim().split(/\s+/).map(Number);
+    if (!child || parent === undefined)
+      continue;
+    children.set(parent, [...children.get(parent) ?? [], child]);
+  }
+  const tree = [pid];
+  for (let i = 0;i < tree.length; i++)
+    tree.push(...children.get(tree[i]) ?? []);
+  return tree;
+}
+function killTree(pid) {
+  let tree = [pid];
+  let problem;
+  try {
+    tree = processTree(pid);
+  } catch (e) {
+    problem = e.message;
+  }
+  for (const p of tree)
+    try {
+      process.kill(p, "SIGKILL");
+    } catch {}
+  return problem;
+}
 function run(opts) {
   return new Promise((resolve) => {
     const child = spawn(opts.cmd, opts.args, {
@@ -7667,7 +8558,12 @@ function run(opts) {
     let settled = false;
     const timer = setTimeout(() => {
       timedOut = true;
-      child.kill("SIGKILL");
+      if (child.pid === undefined)
+        return;
+      const problem = killTree(child.pid);
+      if (problem)
+        opts.onStderr(Buffer.from(`firegoat: timed out; could not list ${opts.cmd}'s child processes (${problem}), killed only ${opts.cmd}
+`));
     }, opts.timeoutMs);
     const done = (r) => {
       if (settled)
@@ -7714,7 +8610,7 @@ async function runShell(opts) {
 
 // src/git.ts
 function git(dir, args) {
-  const r = spawnSync("git", args, {
+  const r = spawnSync3("git", args, {
     cwd: dir,
     env: hostEnv(),
     encoding: "utf8"
@@ -7724,7 +8620,7 @@ function git(dir, args) {
   return { code: r.status ?? -1, stdout: r.stdout.trim() };
 }
 function gitRepo(dir) {
-  const r = spawnSync("git", [
+  const r = spawnSync3("git", [
     "rev-parse",
     "--path-format=absolute",
     "--git-dir",
@@ -7807,213 +8703,11 @@ async function runAgent(harness, opts) {
   return harness.read(stdout, exited, opts.timeoutMs);
 }
 
-// src/harnesses/claude.ts
-import { spawnSync as spawnSync2 } from "node:child_process";
-import { existsSync as existsSync2, readdirSync } from "node:fs";
-import { join as join3 } from "node:path";
-var FLAGS = [
-  "--output-format",
-  "stream-json",
-  "--verbose",
-  "--dangerously-skip-permissions",
-  "--strict-mcp-config",
-  "--setting-sources",
-  "",
-  "--disable-slash-commands"
-];
-function parse(line) {
-  try {
-    const e = JSON.parse(line);
-    return typeof e === "object" && e !== null ? e : undefined;
-  } catch {
-    return;
-  }
-}
-function content(e) {
-  const message = e?.message;
-  return Array.isArray(message?.content) ? message.content : [];
-}
-function resultText(content) {
-  if (typeof content === "string")
-    return content;
-  if (Array.isArray(content))
-    return content.map((c) => typeof c.text === "string" ? c.text : `[${c.type}]`).join(`
-`);
-  return "";
-}
-function steps(events) {
-  const out = [];
-  for (const e of events) {
-    if (e?.type === "assistant")
-      for (const b of content(e)) {
-        if (b.type === "tool_use")
-          out.push({
-            kind: "tool_call",
-            name: typeof b.name === "string" ? b.name : "",
-            input: typeof b.input === "object" && b.input !== null ? b.input : null
-          });
-        else if (b.type === "text" && typeof b.text === "string")
-          out.push({ kind: "text", text: b.text });
-      }
-    else if (e?.type === "user") {
-      for (const b of content(e))
-        if (b.type === "tool_result")
-          out.push({
-            kind: "tool_result",
-            isError: b.is_error === true,
-            text: resultText(b.content).trimEnd()
-          });
-    }
-  }
-  return out;
-}
-function read(stdout, exited, timeoutMs) {
-  const events = stdout.split(`
-`).filter((l) => l.trim() !== "").map(parse);
-  let final = "";
-  for (const e of events)
-    if (e?.type === "result" && typeof e.result === "string")
-      final = e.result;
-  const result = events.findLast((e) => e?.type === "result");
-  const trace = {
-    steps: steps(events),
-    final,
-    turns: result?.num_turns ?? null,
-    cost: result?.total_cost_usd ?? null
-  };
-  const init = events.find((e) => e?.type === "system" && e.subtype === "init");
-  if (!init) {
-    const said = stdout.trim().split(`
-`)[0]?.slice(0, 300);
-    return {
-      ...trace,
-      reportedModel: null,
-      version: null,
-      ok: false,
-      reason: `claude did not start a session (exit ${exited.code ?? "by signal"})${said ? `: ${said}` : ", see stderr.txt"}`
-    };
-  }
-  const reportedModel = typeof init.model === "string" ? init.model : null;
-  const version = typeof init.claude_code_version === "string" ? init.claude_code_version : null;
-  if (typeof init.session_id !== "string" || init.session_id === "")
-    return {
-      ...trace,
-      reportedModel,
-      version,
-      ok: false,
-      reason: "claude's init line has no session_id"
-    };
-  const sessionId = init.session_id;
-  if (exited.timedOut)
-    return {
-      ...trace,
-      reportedModel,
-      version,
-      ok: false,
-      reason: `timed out after ${timeoutMs / 1000}s`,
-      sessionId
-    };
-  if (!result)
-    return {
-      ...trace,
-      reportedModel,
-      version,
-      ok: false,
-      reason: `claude exited ${exited.code ?? "by signal"} without a result, see stderr.txt`,
-      sessionId
-    };
-  if (result.is_error === true || result.subtype !== "success")
-    return {
-      ...trace,
-      reportedModel,
-      version,
-      ok: false,
-      reason: `claude result is an error (subtype=${result.subtype})${result.result ? `: ${result.result.slice(0, 300)}` : ""}`,
-      sessionId
-    };
-  return { ...trace, reportedModel, version, ok: true, sessionId };
-}
-function progress(line) {
-  const e = parse(line);
-  if (e?.type !== "assistant")
-    return;
-  const p = {};
-  for (const b of content(e)) {
-    const cmd = b.input?.command;
-    if (b.type === "tool_use" && typeof cmd === "string")
-      p.command = cmd.split(`
-`)[0];
-  }
-  return p;
-}
-function checkLogin(opts) {
-  const r = spawnSync2(opts.bin, ["auth", "status"], {
-    cwd: opts.cwd,
-    env: opts.env,
-    encoding: "utf8",
-    timeout: 30000
-  });
-  if (r.error)
-    return {
-      loggedOut: false,
-      reason: `could not run ${opts.bin}: ${r.error.message}`
-    };
-  let status;
-  try {
-    status = JSON.parse(r.stdout);
-  } catch {}
-  if (status?.loggedIn === true)
-    return;
-  if (status?.loggedIn === false)
-    return {
-      loggedOut: true,
-      reason: "claude is not logged in. Run `claude` once and log in (or set ANTHROPIC_API_KEY), then run eval again"
-    };
-  return {
-    loggedOut: false,
-    reason: `claude auth status gave no login state (exit ${r.status}): ${(r.stdout + r.stderr).trim().slice(0, 300)}`
-  };
-}
-var claude = {
-  name: "claude",
-  bin: "claude",
-  installHint: "Install Claude Code (https://claude.com/claude-code) or pass --claude-bin PATH.",
-  configDirName: ".claude",
-  args: ({ prompt, model, extraArgs }) => [
-    "-p",
-    prompt,
-    ...FLAGS,
-    ...model === undefined ? [] : ["--model", model],
-    ...extraArgs
-  ],
-  env: ({ configDir, homeEnvFile }) => ({
-    CLAUDE_CONFIG_DIR: configDir,
-    CLAUDE_SECURESTORAGE_CONFIG_DIR: "",
-    ...homeEnvFile === undefined ? {} : { CLAUDE_ENV_FILE: homeEnvFile }
-  }),
-  authLabel: () => process.env.ANTHROPIC_API_KEY ? "ANTHROPIC_API_KEY (claude bills the API key; unset it to use your claude login)" : "your claude login",
-  checkLogin,
-  progress,
-  read,
-  sessionsDir: (configDir) => join3(configDir, "projects"),
-  findSession(configDir, sessionId) {
-    const projects = join3(configDir, "projects");
-    if (!existsSync2(projects))
-      return null;
-    for (const d of readdirSync(projects)) {
-      const f = join3(projects, d, `${sessionId}.jsonl`);
-      if (existsSync2(f))
-        return f;
-    }
-    return null;
-  }
-};
-
 // src/help.ts
 var EXIT_CODES = `Exit codes:
   0  Every run's script checks passed
   1  A script check failed
-  2  At least one run was not graded (claude missing or logged out, crash, timeout,
+  2  At least one run was not graded (the harness missing or logged out, crash, timeout,
      build failed); wins over 1
   3  Usage or config error (bad flag, unknown test, invalid or missing firegoat.yaml,
      firegoat.ai answered 401/403 for the upload key); any other failure of
@@ -8067,10 +8761,18 @@ firegoat.yaml, committed.
 
 firegoat.yaml reference:
   runs: 1                     Runs per test
+  harness: <name> | [<name>, ...]
+                              The agent CLI: claude-code (default), codex-cli
+                              or grok-cli.
+                              A list runs every test runs times per harness;
+                              eval --harness overrides
   model: <name> | [<name>, ...]
-                              claude --model; a list runs every test runs times
-                              per model (n counts 1..runs x models). Unset: no
-                              --model, claude's default. eval --model overrides
+                              The harness's --model (-m for codex, grok); a list
+                              runs
+                              every test runs times per model, on each harness
+                              (n counts 1..runs x harnesses x models, harness by
+                              harness, then model). Unset: no --model, the
+                              harness's default. eval --model overrides
   build: <sh> | [<sh>, ...]   Once per eval, in this folder; what it writes to
                               $FIREGOAT_OUT is first on the agent's PATH
   install: <sh> | [<sh>, ...] In each fresh folder, before the agent starts; a
@@ -8090,7 +8792,8 @@ firegoat.yaml reference:
                               $FIREGOAT_HOME (no other $ expansion: $HOME stays
                               literal). The build step gets only the entries that
                               use only {{vars.*}}, {{run.id}} and $FIREGOAT_OUT.
-                              PATH, CLAUDE_CONFIG_DIR, FIREGOAT_* and
+                              PATH, CLAUDE_CONFIG_DIR, CODEX_HOME, GROK_HOME,
+                              GROK_DISABLE_AUTOUPDATER, FIREGOAT_* and
                               ANTHROPIC_MODEL (use model:) are reserved, and so
                               are HOME, XDG_CONFIG_HOME, XDG_CACHE_HOME,
                               XDG_DATA_HOME, XDG_STATE_HOME and CLAUDE_ENV_FILE:
@@ -8098,20 +8801,33 @@ firegoat.yaml reference:
                               tool that keeps state in ~ starts fresh in each
                               run. Only a tool that ignores HOME needs its own
                               var, e.g. MYTOOL_HOME: "$FIREGOAT_HOME/.mytool"
-  claude_args: [<arg>, ...]   Extra claude arguments, after firegoat's own flags
-                              and --model. Each entry is one argument, passed
-                              as-is (no shell splitting): [--chrome] (Claude in
-                              Chrome, for tests that need a browser),
-                              [--plugin-dir, ./plugins/x], --mcp-config, --agents,
-                              --append-system-prompt. Entries expand like env:
-                              values ($FIREGOAT_WORKDIR, $FIREGOAT_HOME,
-                              $FIREGOAT_OUT, {{vars.KEY}}, {{run.*}}); run.json
-                              claudeArgs has the result. Flags firegoat sets are
-                              errors, also as --flag=value: -p, --print,
+  harness_args: [<arg>, ...]  Extra harness arguments, after firegoat's own flags
+                              and --model (for codex, before the prompt), to
+                              every selected harness. Each entry is one
+                              argument, passed as-is (no shell splitting):
+                              claude: [--chrome] (Claude in Chrome, for tests
+                              that need a browser), [--plugin-dir, ./plugins/x],
+                              --mcp-config, --agents, --append-system-prompt;
+                              codex: [-c, key=value], --enable, --add-dir.
+                              Entries expand like env: values
+                              ($FIREGOAT_WORKDIR, $FIREGOAT_HOME, $FIREGOAT_OUT,
+                              {{vars.KEY}}, {{run.*}}); run.json harnessArgs has
+                              the result. Flags firegoat sets are errors, also
+                              as --flag=value. claude-code: -p, --print,
                               --output-format, --verbose, --setting-sources,
                               --strict-mcp-config, --disable-slash-commands,
                               --dangerously-skip-permissions, --model (use
-                              model:), --session-id, --resume, -r, --continue, -c
+                              model:), --session-id, --resume, -r, --continue,
+                              -c. codex-cli: --json, --skip-git-repo-check,
+                              --dangerously-bypass-approvals-and-sandbox, -C,
+                              --cd, -m, --model (use model:), resume, fork,
+                              --ephemeral. grok-cli: -p, --single,
+                              --prompt-file, --prompt-json, --output-format,
+                              --json-schema, --always-approve,
+                              --permission-mode, --no-auto-update, --cwd, -w,
+                              --worktree, -m, --model (use model:), -r,
+                              --resume, -c, --continue, -s, --session-id,
+                              --fork-session
   tests:
     - name: quickstart        Letters, digits, '.', '_', '-'; unique
       prompt: ...             What a user would ask the agent; one concrete outcome
@@ -8123,7 +8839,7 @@ firegoat.yaml reference:
       env: {NAME: value}      Optional, on top of the top-level env:
       install: <sh> | [...]   Optional, runs after the top-level install
       cleanup: <sh> | [...]   Optional, runs before the top-level cleanup
-      claude_args: [...]      Optional, after the top-level claude_args
+      harness_args: [...]     Optional, after the top-level harness_args
 
   Step order: build once per eval; then per run: install (top-level, then the
   test's), the agent, checks, cleanup (the test's, then top-level).
@@ -8140,11 +8856,15 @@ firegoat.yaml reference:
   for any value.
   The build, install and cleanup commands and the checks get the vars as env
   vars, plus $FIREGOAT_OUT and $FIREGOAT_WORKDIR; install, checks and cleanup also
-  get $FIREGOAT_HOME, the run's own home, as HOME (build keeps your real HOME).
+  get $FIREGOAT_HOME, the run's own home, as HOME (build keeps your real HOME),
+  $FIREGOAT_RUN_DIR (the run's .firegoat/runs/<runId>/<test>/<n>/, where its logs
+  go) and $FIREGOAT_REPO (the folder holding firegoat.yaml). The agent gets
+  neither: keep values it must not find in $FIREGOAT_RUN_DIR, e.g. install:
+  echo "$EXPECTED" > "$FIREGOAT_RUN_DIR/expected", and a check reads it back.
   Checks also get $FIREGOAT_TRANSCRIPT, a short plain-text
   transcript, greppable line by line (format: eval --help, Transcript). So
   '^\\[[0-9]+\\] \\$ .*foo' matches only commands. Checks also get
-  $FIREGOAT_SESSION (claude's full session file, JSON lines).
+  $FIREGOAT_SESSION (the harness's full session file, JSON lines).
 
   The agent never gets vars, only env:. The vars npm/npx set for firegoat itself
   (npm_config_*, npm_lifecycle_*, npm_package_*, npm_execpath, npm_node_execpath,
@@ -8155,7 +8875,7 @@ firegoat.yaml reference:
   so the agent's commands see them too.
 
 Testing an MCP server: commit its config as fixtures/mcp.json, copy it in build:
-and pass it with claude_args. Only the configs you pass load: --strict-mcp-config
+and pass it with harness_args (claude-code). Only the configs you pass load: --strict-mcp-config
 is always on. An MCP server inherits claude's env, which keeps your real HOME
 (claude's login is there), so set the server's HOME to the run's home, or it
 writes to yours. claude gets $FIREGOAT_HOME, $FIREGOAT_WORKDIR and $FIREGOAT_OUT
@@ -8166,7 +8886,7 @@ relative paths in it are from the run's folder, claude's cwd:
       "env": {"HOME": "\${FIREGOAT_HOME}"}}}}
   firegoat.yaml:
     build: cp -R fixtures/. "$FIREGOAT_OUT"
-    claude_args: [--mcp-config, $FIREGOAT_OUT/mcp.json]
+    harness_args: [--mcp-config, $FIREGOAT_OUT/mcp.json]
 
 Example:
   build: npm pack --pack-destination "$FIREGOAT_OUT"
@@ -8190,7 +8910,7 @@ Example:
         Get an acme API key from the acme dashboard (I'm logged in in Chrome) and put
         it in .env as ACME_API_KEY.
       end_goal: .env holds a key copied from the dashboard, not a made-up one.
-      claude_args: [--chrome]
+      harness_args: [--chrome]
 
 init also says that firegoat sends anonymous usage stats; FIREGOAT_TELEMETRY=0
 (or DO_NOT_TRACK=1) turns them off. Fields: firegoat --help, Telemetry.
@@ -8235,11 +8955,17 @@ Exit codes: 0 logged out (or was not logged in), 3 the file or ~/.firegoat is a
 symlink, not a regular file/directory, or someone else's (left alone).`;
 var EVAL_HELP = `Usage: firegoat eval [test..] [--runs N] [--model NAME].. [--parallel N]
                      [--var KEY=value].. [--json] [--no-upload] [--dry-run]
-                     [--claude-bin PATH]
-
-Runs each test in firegoat.yaml (in the current folder): your claude gets the prompt
-in a fresh, empty folder, .firegoat/runs/<runId>/<test>/<n>/work, with build:'s
-outputs on PATH. Then the test's checks run in that folder.
+                     [--harness NAME].. [--harness-bin PATH]
+
+Runs each test in firegoat.yaml (in the current folder): your agent harness
+(claude-code by default, codex-cli or grok-cli; harness: in firegoat.yaml) gets the prompt
+in a fresh, empty folder outside the repo, ~/.firegoat/runs/<hash>/<random>/work,
+with build:'s outputs on PATH. Then the test's checks run in that folder. <hash>
+is a hash of the repo's path (a linked worktree uses its main checkout's) and
+<random> is new for each run, so the agent's paths name neither the project nor
+the test, and git there sees no repo. The run's logs stay in the repo, in
+.firegoat/runs/<runId>/<test>/<n>/, whose work and home are symlinks to the
+run's folders.
 
 Step order: build: once, in this folder; then per run, in its folder: install:
 (top-level, then the test's), the agent, checks, cleanup: (the test's, then
@@ -8248,16 +8974,39 @@ a claude error or a timeout; a failing cleanup step prints a warning and never
 changes the run's status or the exit code.
 
 claude runs on its own login (or ANTHROPIC_API_KEY if set, as claude always does)
-with a fresh config dir, .firegoat/.claude: none of your settings, CLAUDE.md,
-plugins, skills or MCP servers (claude_args: in firegoat.yaml adds claude flags,
-e.g. --chrome or --plugin-dir). claude saves its session there. There is no
-sandbox: the agent can reach the rest of your machine, including the repo.
-
-Every run gets its own HOME, .firegoat/runs/<runId>/<test>/<n>/home (with
-XDG_CONFIG_HOME, XDG_CACHE_HOME, XDG_DATA_HOME, XDG_STATE_HOME inside it), as
+with a fresh config dir, ~/.firegoat/runs/<hash>/.claude: none of your settings,
+CLAUDE.md, plugins, skills or MCP servers (harness_args: in firegoat.yaml adds
+claude flags, e.g. --chrome or --plugin-dir). claude saves its session there. There
+is no sandbox: the agent can still reach the rest of your machine, including the
+repo, if it goes looking.
+
+codex-cli runs codex exec --json --dangerously-bypass-approvals-and-sandbox with a
+fresh CODEX_HOME, ~/.firegoat/runs/<hash>/.codex, holding a link to your codex login
+(~/.codex/auth.json, or $CODEX_HOME/auth.json; run codex login first), and with
+-c project_doc_max_bytes=0: no AGENTS.md, config.toml, skills or plugins of
+yours. codex's own HOME is the run's home (below), also for its Read/Write. It
+saves its session (rollout) under that CODEX_HOME's sessions/. firegoat.ai doesn't
+read codex sessions yet: those runs are not uploaded. codex reports no turns or
+cost; its model comes from the session file.
+
+grok-cli runs ~/.grok/bin/grok (where grok's installer puts it; never a grok on
+PATH, which may be a wrapper such as cmux's: a #! script is refused) as grok -p
+--output-format streaming-messages-json --always-approve --no-auto-update, with a
+fresh GROK_HOME, ~/.firegoat/runs/<hash>/.grok, holding a link to your grok login
+($GROK_HOME/auth.json or ~/.grok/auth.json; run grok login first; XAI_API_KEY alone
+does not log headless grok in). grok's own HOME is the run's home (below): with
+your real one it loads Claude Code's skills, plugins, MCP servers and hooks. eval
+refuses a login that expires within 5 minutes (run grok -p hi once to refresh it):
+a refresh inside a run would replace the link with a file, which the next eval
+asks you to move back over ~/.grok/auth.json. The session is a directory,
+$GROK_HOME/sessions/<cwd>/<id>/ ($FIREGOAT_SESSION). firegoat.ai doesn't read
+grok sessions yet: those runs are not uploaded.
+
+Every run gets its own HOME, ~/.firegoat/runs/<hash>/<random>/home next to work
+(with XDG_CONFIG_HOME, XDG_CACHE_HOME, XDG_DATA_HOME, XDG_STATE_HOME inside it), as
 $FIREGOAT_HOME: tools that keep state in ~ start fresh and parallel runs don't
 collide. claude itself keeps your real HOME (its login is there) and sources
-<n>/home.env (CLAUDE_ENV_FILE) before each Bash command the agent runs; install,
+home.env next to it (CLAUDE_ENV_FILE) before each Bash command the agent runs; install,
 checks and cleanup get the run's HOME directly; build keeps yours. Limits: claude's
 own Read/Write tools expand ~ to your real home; getpwuid lookups (os.userInfo,
 python pwd) still see it; keychain logins (gh, docker, git credential helpers) look
@@ -8273,14 +9022,18 @@ nested session. Login and provider vars stay: ANTHROPIC_*, CLAUDE_CODE_OAUTH_TOK
 CLAUDE_CODE_OAUTH_REFRESH_TOKEN, CLAUDE_CODE_USE_*, CLAUDE_CODE_SKIP_*_AUTH,
 CLAUDE_CODE_CLIENT_CERT, CLAUDE_CODE_CLIENT_KEY, CLAUDE_CODE_CLIENT_KEY_PASSPHRASE.
 
-In a linked git worktree (git worktree add), .firegoat/runs/ and .firegoat/.claude
-live in the main checkout instead, so runs outlive the worktree; firegoat.yaml,
-fixtures and build: stay in the current folder.
+In a linked git worktree (git worktree add), .firegoat/runs/ lives in the main
+checkout instead, and <hash> is the main checkout's, so runs outlive the worktree;
+firegoat.yaml, fixtures and build: stay in the current folder.
 
   test..             Only these tests (by name)
   --runs N           Runs per test (overrides runs: in firegoat.yaml)
-  --model NAME       claude --model, replacing model: in firegoat.yaml; repeat it
-                     (--model a --model b) to run each test runs times per model
+  --harness NAME     claude-code, codex-cli or grok-cli, replacing harness: in
+                     firegoat.yaml;
+                     repeat it to run each test on each harness
+  --model NAME       The harness's --model, replacing model: in firegoat.yaml;
+                     repeat it (--model a --model b) to run each test runs times
+                     per model
   --parallel N       Runs at once (default 4; 1 runs them one after another).
                      Each run has its own HOME; state outside it (a fixed port,
                      /tmp paths) can still collide: use 1 then
@@ -8292,19 +9045,25 @@ fixtures and build: stay in the current folder.
                      ("pass", "fail" or "error" = not graded) and exitCode
   --no-upload        Keep runs local even when logged in to firegoat.ai
   --dry-run          Check firegoat.yaml as eval would (test names, --var,
-                     --model, missing vars), print each planned run
-                     ("<test>#<n> [model]" with its install, checks and cleanup
-                     counts and rendered claude_args, paths under
-                     runs/<runId>/) and exit 0. Builds nothing, starts no
-                     claude, uploads nothing, writes no files. Config errors
+                     --model, --harness, missing vars), print each planned
+                     run ("<test>#<n> [harness model]" with its install, checks
+                     and cleanup counts and rendered harness_args, paths under
+                     ~/.firegoat/runs/<hash>/<random>/) and exit 0. Builds
+                     nothing, starts no agent, uploads nothing, writes no
+                     files. Config errors
                      exit 3. With --json: {dryRun, build, runs: [{test, n,
-                     model, claudeArgs, install, checks, cleanup}]}, steps
-                     and checks with their placeholders filled
-  --claude-bin PATH  Path to claude (default: claude on PATH)
-
-Upload: after firegoat login (or with FIREGOAT_API_KEY), each run's claude session,
-checks.txt and stderr.txt go to firegoat.ai as a private trace, and its URL is
-printed ("✓ <test>#<n> <url>", or "✗ ... Upload failed: <why>") and saved as
+                     harness, model, harnessArgs, install, checks,
+                     cleanup}]}, steps and checks with their placeholders
+                     filled
+  --harness-bin PATH Path to the harness binary (default: claude or codex on
+                     PATH, ~/.grok/bin/grok); only with exactly one harness
+
+Run labels: "PASS <test>#<n> [<harness> <model>]": the harness shows when the eval
+runs more than one, or not claude-code; the model when one is set.
+
+Upload: after firegoat login (or with FIREGOAT_API_KEY), each claude-code run's
+session, checks.txt and stderr.txt go to firegoat.ai as a private trace, and its
+URL is printed ("✓ <test>#<n> <url>", or "✗ ... Upload failed: <why>") and saved as
 upload in run.json. Before any run, eval checks the key: a 401/403 exits 3, any
 other failure (unreachable, 429, 5xx, wrong FIREGOAT_BASE_URL) exits 2 (fix it, or
 --no-upload). A failed upload never changes a run's status or the exit code; eval
@@ -8313,27 +9072,32 @@ Runs without a session file are not uploaded. FIREGOAT_BASE_URL points elsewhere
 (default https://firegoat.ai/evals). See firegoat login --help.
 
 Output: .firegoat/runs/<runId>/
+  out                     A symlink to $FIREGOAT_OUT,
+                          ~/.firegoat/runs/<hash>/<random>/out
   summary.json            firegoatVersion, runId, runsDir, dir (where eval
                           ran), git {branch, commit, dirty} (no git outside a
-                          repo), upload {origin, email} (null when not
-                          uploading), runs (every run.json), then the verdict:
-                          totals, status and exitCode
+                          repo), harnesses, upload {origin, email} (null when
+                          not uploading), runs (every run.json), then the
+                          verdict: totals, status and exitCode
   ../index.jsonl          One line per eval: runId, startedAt, dir, branch,
-                          commit, passed, failed, notGraded, cost (null when
+                          commit, harnesses, passed, failed, notGraded, cost (null when
                           any run's cost is unknown), uploaded, uploadFailed
-  <test>/<n>/run.json     One run: status and reason (from the checks), model
-                          (asked for, or null) and claudeModel (what claude's
-                          init line reported), claudeArgs (claude_args
-                          passed), prompt, end_goal,
+  <test>/<n>/run.json     One run: status and reason (from the checks), harness,
+                          model (asked for, or null) and reportedModel (what
+                          the harness reported running, or null), harnessArgs
+                          (harness_args passed), prompt, end_goal,
                           checks [{run, code}], cleanup [{run, code}],
-                          and paths: workdir, home (the run's HOME),
-                          transcript, session (claude's session file), stderr,
+                          and paths: dir (this folder), workdir, home (the
+                          run's HOME; both real paths, under ~/.firegoat),
+                          transcript, session (the harness's session file), stderr,
                           checksLog, cleanupLog; startedAt; upload ({status: "uploaded",
                           url} | {status: "failed", error} | {status:
                           "skipped", reason}, null when not uploading)
   <test>/<n>/transcript.txt  What the agent did, below
-  <test>/<n>/home/, home.env  The run's HOME, and the file claude sources
-                          before the agent's Bash commands
+  <test>/<n>/work, home   Symlinks to the agent's folder and the run's HOME
+                          (home.env, which claude sources before the agent's
+                          Bash commands, is next to them)
+
   build.txt, <test>/<n>/install.txt, checks.txt, cleanup.txt   Logs
 
 Transcript ($FIREGOAT_TRANSCRIPT), greppable line by line; n counts tool calls:
@@ -8359,6 +9123,8 @@ Examples:
   npx firegoat eval quickstart --runs 3     One test, three runs
   npx firegoat eval --model claude-haiku-5-5 --model claude-sonnet-5-5
                                             Each test on both models
+  npx firegoat eval --harness claude-code --harness codex-cli
+                                            Each test on claude and on codex
   npx firegoat eval --var SHA=$(git rev-parse HEAD)
   npx firegoat eval --dry-run               Validate firegoat.yaml, list the runs
 
@@ -8387,30 +9153,6 @@ Example:
 
 Exit codes: 0 sent, 2 the service failed (or rate limit), 3 usage error.`;
 
-// src/home.ts
-import { mkdirSync as mkdirSync3, writeFileSync as writeFileSync2 } from "node:fs";
-import { join as join4 } from "node:path";
-function homeEnv(home) {
-  return {
-    HOME: home,
-    XDG_CONFIG_HOME: join4(home, ".config"),
-    XDG_CACHE_HOME: join4(home, ".cache"),
-    XDG_DATA_HOME: join4(home, ".local", "share"),
-    XDG_STATE_HOME: join4(home, ".local", "state")
-  };
-}
-var shellQuote = (s) => `'${s.replaceAll("'", `'\\''`)}'`;
-function makeRunHome(dir) {
-  const home = join4(dir, "home");
-  const env = homeEnv(home);
-  for (const d of Object.values(env))
-    mkdirSync3(d, { recursive: true });
-  const envFile = join4(dir, "home.env");
-  writeFileSync2(envFile, Object.entries(env).map(([k, v]) => `export ${k}=${shellQuote(v)}
-`).join(""));
-  return { home, envFile };
-}
-
 // src/steps.ts
 import { createWriteStream } from "node:fs";
 var STEP_TIMEOUT_MS = 600000;
@@ -8545,8 +9287,8 @@ async function sendTelemetry(event, exitCode, startedAt) {
 }
 
 // src/transcript.ts
-import { readdirSync as readdirSync2, statSync, writeFileSync as writeFileSync3 } from "node:fs";
-import { join as join5, relative } from "node:path";
+import { readdirSync as readdirSync4, statSync as statSync2, writeFileSync as writeFileSync3 } from "node:fs";
+import { join as join8, relative } from "node:path";
 var OUTPUT_LINES = 15;
 var OUTPUT_CHARS = 1500;
 var TEXT_CHARS = 1500;
@@ -8572,7 +9314,15 @@ var prose = (text) => prefixLines("agent: ", text);
 function describeCall(step, name, input) {
   if (typeof input.command === "string")
     return prefixLines(`[${step}] $ `, input.command);
-  for (const key of ["file_path", "path", "url", "pattern", "query"])
+  for (const key of [
+    "file_path",
+    "path",
+    "target_file",
+    "target_directory",
+    "url",
+    "pattern",
+    "query"
+  ])
     if (typeof input[key] === "string")
       return `[${step}] ${name} ${input[key]}`;
   return `[${step}] ${name} ${truncate(JSON.stringify(input), 1, INPUT_CHARS)}`;
@@ -8587,11 +9337,11 @@ function describeToolCall(step, c) {
 function workdirFiles(dir) {
   const out = [];
   const walk = (d) => {
-    for (const name of readdirSync2(d)) {
+    for (const name of readdirSync4(d)) {
       if (SKIP_DIRS.has(name))
         continue;
-      const p = join5(d, name);
-      if (statSync(p).isDirectory())
+      const p = join8(d, name);
+      if (statSync2(p).isDirectory())
         walk(p);
       else
         out.push(relative(dir, p));
@@ -8665,6 +9415,10 @@ function tier(rows) {
   const share = graded.filter((r) => r.state === "pass").length / graded.length;
   return share === 1 ? "S" : share >= 0.75 ? "A" : share >= 0.5 ? "B" : "C";
 }
+function runTag(harness, model) {
+  const parts = [harness, model].filter((x) => x !== null);
+  return parts.length > 0 ? ` [${parts.join(" ")}]` : "";
+}
 function notGradedWhy(reason) {
   const said = /claude result is an error \(subtype=[^)]*\): (.*)$/s.exec(reason);
   if (said)
@@ -8750,8 +9504,7 @@ function plainUI(write) {
         printed.add(r);
         const label = r.state === "error" ? "ERROR" : r.state.toUpperCase();
         const u = usageParts(r).join(", ");
-        const model = r.model === null ? "" : ` [${r.model}]`;
-        write(`${label} ${r.test}#${r.n}${model} ${r.reason}${u ? ` (${u})` : ""}`);
+        write(`${label} ${r.test}#${r.n}${runTag(r.harness, r.model)} ${r.reason}${u ? ` (${u})` : ""}`);
       }
       for (const r of rows) {
         if (uploadPrinted.has(r) || !r.upload || r.upload === "uploading")
@@ -8826,11 +9579,14 @@ function ttyUI(out, opts) {
       const mark = build.state === "ok" ? p(C.green, "✓") : build.state === "fail" ? p(C.red, "✗") : p(C.ember, SPIN[frame % SPIN.length]);
       out.push(`  ${build.state === "running" ? p(C.ember, "◐") : p(C.green, "●")}  Build${" ".repeat(Math.max(1, w - 20))}${mark}${build.seconds !== undefined ? ` ${build.seconds}s` : ""}`);
     }
-    const modelCols = Math.min(24, Math.max(0, ...rows.map((r) => r.model?.length ?? 0)));
+    const column = (width) => Math.min(24, Math.max(0, ...rows.map(width)));
+    const harnessCols = column((r) => r.harness?.length ?? 0);
+    const modelCols = column((r) => r.model?.length ?? 0);
     rows.forEach((r, i) => {
       out.push(p(C.gray, "  │"));
       const num = String(i + 1).padStart(2, "0");
       const name = cut(`${r.test}${r.n > 1 ? `#${r.n}` : ""}`, 16).padEnd(16);
+      const harness = harnessCols > 0 ? ` ${p(C.cyan, cut(r.harness ?? "", harnessCols).padEnd(harnessCols))}` : "";
       const model = modelCols > 0 ? ` ${p(C.cyan, cut(r.model ?? "", modelCols).padEnd(modelCols))}` : "";
       let detail = "";
       if (r.state === "running")
@@ -8839,7 +9595,7 @@ function ttyUI(out, opts) {
         detail = `${checksPassed(r).padEnd(11)} ${p(C.gray, usage(r))}`;
       else if (r.state === "error")
         detail = p(C.gray, usage(r));
-      out.push(`  ${glyph(r)}  ${p(C.gray, num)}  ${p(C.bold, name)}${model} ${p(C.ember, ramp(r))} ${statusWord(r)}  ${detail}`);
+      out.push(`  ${glyph(r)}  ${p(C.gray, num)}  ${p(C.bold, name)}${harness}${model} ${p(C.ember, ramp(r))} ${statusWord(r)}  ${detail}`);
       if (r.state === "running" && r.command)
         out.push(p(C.gray, `  ┊        └ $ ${cut(r.command, w - 16)}`));
       if (r.state === "fail")
@@ -8919,7 +9675,7 @@ function ttyUI(out, opts) {
 }
 
 // src/upload.ts
-import { existsSync as existsSync3, readFileSync as readFileSync3 } from "node:fs";
+import { existsSync as existsSync5, readFileSync as readFileSync5 } from "node:fs";
 import { basename as basename2 } from "node:path";
 var MAX_FILE = 20 * 1024 * 1024;
 var MAX_CHECKS = 200;
@@ -8947,7 +9703,7 @@ function projectName(v) {
 function runModel(r) {
   return {
     requested: label(r.model, "Model"),
-    reported: label(r.claudeModel, "Claude's reported model")
+    reported: label(r.reportedModel, "Reported model")
   };
 }
 function runGit(g) {
@@ -9001,7 +9757,7 @@ async function uploadRun(t, r) {
   };
   const bytes = {};
   for (const name of FILES) {
-    const b = existsSync3(paths[name]) ? readFileSync3(paths[name]) : Buffer.alloc(0);
+    const b = existsSync5(paths[name]) ? readFileSync5(paths[name]) : Buffer.alloc(0);
     if (b.byteLength > MAX_FILE)
       return {
         status: "failed",
@@ -9083,13 +9839,13 @@ async function uploadRun(t, r) {
 }
 
 // src/which.ts
-import { accessSync, constants, statSync as statSync2 } from "node:fs";
-import { delimiter, join as join6, resolve } from "node:path";
+import { accessSync, constants, statSync as statSync3 } from "node:fs";
+import { delimiter, join as join9, resolve } from "node:path";
 function resolveBin(bin) {
   const executable = (p) => {
     try {
       accessSync(p, constants.X_OK);
-      return statSync2(p).isFile();
+      return statSync3(p).isFile();
     } catch {
       return false;
     }
@@ -9097,8 +9853,8 @@ function resolveBin(bin) {
   if (bin.includes("/"))
     return executable(bin) ? resolve(bin) : undefined;
   for (const dir of (process.env.PATH ?? "").split(delimiter)) {
-    if (dir && executable(join6(dir, bin)))
-      return join6(dir, bin);
+    if (dir && executable(join9(dir, bin)))
+      return join9(dir, bin);
   }
   return;
 }
@@ -9112,12 +9868,12 @@ function parseArgs(args) {
   const opts = {
     names: [],
     models: [],
+    harnesses: [],
     parallel: 4,
     json: false,
     noUpload: false,
     dryRun: false,
-    vars: {},
-    claudeBin: "claude"
+    vars: {}
   };
   for (let i = 0;i < args.length; i++) {
     const a = args[i];
@@ -9158,14 +9914,21 @@ function parseArgs(args) {
       if (opts.models.includes(v))
         throw new UsageError(`--model ${v} given twice`);
       opts.models.push(v);
+    } else if (flag === "--harness") {
+      const v = value();
+      if (!HARNESSES.has(v))
+        throw new UsageError(`Unknown harness ${JSON.stringify(v)} ${validHarnesses()}`);
+      if (opts.harnesses.includes(v))
+        throw new UsageError(`--harness ${v} given twice`);
+      opts.harnesses.push(v);
     } else if (flag === "--parallel") {
       const v = value();
       const n = Number(v);
       if (!/^\d+$/.test(v) || n < 1)
         throw new UsageError(`--parallel must be a positive integer, got ${v}`);
       opts.parallel = n;
-    } else if (flag === "--claude-bin")
-      opts.claudeBin = value();
+    } else if (flag === "--harness-bin")
+      opts.harnessBin = value();
     else if (a.startsWith("-"))
       throw new UsageError(`Unknown flag: ${a}`);
     else
@@ -9178,7 +9941,7 @@ function newRunId() {
   return `${ts}-${randomBytes2(2).toString("hex")}`;
 }
 function writeRunJson(r) {
-  writeFileSync4(join7(r.dir, "run.json"), `${JSON.stringify(r, null, 2)}
+  writeFileSync4(join10(r.dir, "run.json"), `${JSON.stringify(r, null, 2)}
 `);
 }
 async function uploadPreflight(opts, ui, project, runGroup, git) {
@@ -9219,18 +9982,18 @@ async function uploadPreflight(opts, ui, project, runGroup, git) {
     };
   }
 }
-function runFill(runId, n, testName, out, dir) {
+function runFill(runId, n, testName, out, agentDir) {
   return {
     "run.id": runId,
     "run.n": String(n),
     "test.name": testName,
     FIREGOAT_OUT: out,
-    FIREGOAT_WORKDIR: join7(dir, "work"),
-    FIREGOAT_HOME: join7(dir, "home")
+    FIREGOAT_WORKDIR: join10(agentDir, "work"),
+    FIREGOAT_HOME: join10(agentDir, "home")
   };
 }
-function renderClaudeArgs(test, vars, fill) {
-  return test.claudeArgs.map((a, i) => renderValue(a, vars, fill, `claude_args[${i}]`));
+function renderHarnessArgs(test, vars, fill) {
+  return test.harnessArgs.map((a) => renderValue(a.arg, vars, fill, a.path));
 }
 function renderSteps(what, commands, vars, fill) {
   return commands.map((c, i) => renderStep(c, vars, fill, `${what}[${i}]`));
@@ -9245,24 +10008,29 @@ function renderRunSteps(test, top, vars, fill) {
 function renderBuild(config, vars, runId) {
   return renderSteps("build", config.build, vars, { "run.id": runId });
 }
-function planRuns(tests, models, runs) {
-  const matrix = models.length > 0 ? models : [null];
-  return tests.flatMap((t) => matrix.flatMap((model, m) => Array.from({ length: runs }, (_, i) => ({
+function planRuns(tests, harnesses, models, runs) {
+  const modelList = models.length > 0 ? models : [null];
+  const matrix = harnesses.flatMap((harness) => modelList.map((model) => ({ harness, model })));
+  return tests.flatMap((t) => matrix.flatMap((cell, c) => Array.from({ length: runs }, (_, i) => ({
     test: t.name,
-    n: m * runs + i + 1,
-    model
+    n: c * runs + i + 1,
+    ...cell
   }))));
 }
+function showsHarness(harnesses) {
+  return harnesses.length > 1 || harnesses[0] !== DEFAULT_HARNESS;
+}
 var DRY_RUN_ID = "<runId>";
 function dryRun(opts) {
-  const out = join7(opts.runsRoot, DRY_RUN_ID, "out");
+  const agentDir = join10(opts.agentDirs, "<random>");
+  const out = join10(agentDir, "out");
   const runs = opts.plan.map((p) => {
     const test = opts.tests.find((t) => t.name === p.test);
-    const fill = runFill(DRY_RUN_ID, p.n, test.name, out, join7(opts.runsRoot, DRY_RUN_ID, test.name, String(p.n)));
+    const fill = runFill(DRY_RUN_ID, p.n, test.name, out, agentDir);
     renderEnv(test.env, opts.vars, fill);
     return {
       ...p,
-      claudeArgs: renderClaudeArgs(test, opts.vars, fill),
+      harnessArgs: renderHarnessArgs(test, opts.vars, fill),
       ...renderRunSteps(test, opts.config, opts.vars, fill)
     };
   });
@@ -9274,22 +10042,27 @@ function dryRun(opts) {
   const count = (n, what) => `${n} ${what}${n === 1 ? "" : "s"}`;
   opts.say(`Dry run: ${count(runs.length, "run")}, build ${count(opts.config.build.length, "step")}. Nothing was started.`);
   for (const r of runs)
-    opts.say(`${r.test}#${r.n}${r.model === null ? "" : ` [${r.model}]`} install ${r.install.length}, checks ${r.checks.length}, cleanup ${r.cleanup.length}${r.claudeArgs.length > 0 ? `, claude_args ${JSON.stringify(r.claudeArgs)}` : ""}`);
+    opts.say(`${r.test}#${r.n}${runTag(opts.showHarness ? r.harness : null, r.model)} install ${r.install.length}, checks ${r.checks.length}, cleanup ${r.cleanup.length}${r.harnessArgs.length > 0 ? `, harness_args ${JSON.stringify(r.harnessArgs)}` : ""}`);
 }
 async function runOne(opts) {
   const { test, n } = opts;
   const started = Date.now();
-  const dir = join7(opts.runDir, test.name, String(n));
-  const workdir = join7(dir, "work");
-  mkdirSync4(workdir, { recursive: true });
-  const { home, envFile } = makeRunHome(dir);
-  const fill = runFill(opts.runId, n, test.name, opts.out, dir);
+  const dir = join10(opts.runDir, test.name, String(n));
+  mkdirSync5(dir, { recursive: true });
+  const agentDir = makeAgentDir(opts.agentDirs);
+  const workdir = join10(agentDir, "work");
+  mkdirSync5(workdir);
+  const { home, envFile } = makeRunHome(agentDir);
+  linkDir(workdir, join10(dir, "work"));
+  linkDir(home, join10(dir, "home"));
+  const fill = runFill(opts.runId, n, test.name, opts.out, agentDir);
   const r = {
     test: test.name,
     n,
+    harness: opts.harness.name,
     model: opts.model,
-    claudeModel: null,
-    claudeArgs: renderClaudeArgs(test, opts.vars, fill),
+    reportedModel: null,
+    harnessArgs: renderHarnessArgs(test, opts.vars, fill),
     status: "error",
     reason: "",
     prompt: render(test.prompt, opts.vars),
@@ -9299,11 +10072,11 @@ async function runOne(opts) {
     workdir,
     home,
     session: null,
-    transcript: join7(dir, "transcript.txt"),
-    stderr: join7(dir, "stderr.txt"),
-    checksLog: join7(dir, "checks.txt"),
+    transcript: join10(dir, "transcript.txt"),
+    stderr: join10(dir, "stderr.txt"),
+    checksLog: join10(dir, "checks.txt"),
     cleanup: [],
-    cleanupLog: join7(dir, "cleanup.txt"),
+    cleanupLog: join10(dir, "cleanup.txt"),
     num_turns: null,
     cost: null,
     duration_ms: 0,
@@ -9316,7 +10089,9 @@ async function runOne(opts) {
     ...env,
     ...homeEnv(home),
     FIREGOAT_WORKDIR: workdir,
-    FIREGOAT_HOME: home
+    FIREGOAT_HOME: home,
+    FIREGOAT_RUN_DIR: dir,
+    FIREGOAT_REPO: opts.repo
   };
   const steps = renderRunSteps(test, opts, opts.vars, fill);
   const [status, reason] = await attempt(opts, r, steps, env, stepEnv, envFile);
@@ -9333,6 +10108,9 @@ async function runOne(opts) {
   writeRunJson(r);
   return r;
 }
+function withoutOldPwd(env) {
+  return Object.fromEntries(Object.entries(env).filter(([k]) => k !== "OLDPWD"));
+}
 async function attempt(opts, r, steps, env, stepEnv, homeEnvFile) {
   const { test, n } = opts;
   const { dir, workdir } = r;
@@ -9342,7 +10120,7 @@ async function attempt(opts, r, steps, env, stepEnv, homeEnvFile) {
       commands: steps.install,
       cwd: workdir,
       env: stepEnv,
-      logPath: join7(dir, "install.txt")
+      logPath: join10(dir, "install.txt")
     });
     if (error)
       return ["error", `not graded: ${error}`];
@@ -9354,9 +10132,13 @@ async function attempt(opts, r, steps, env, stepEnv, homeEnvFile) {
     prompt: r.prompt,
     cwd: workdir,
     env: {
-      ...hostEnv(),
+      ...withoutOldPwd(hostEnv()),
       ...env,
-      ...harness.env({ configDir: opts.configDir, homeEnvFile }),
+      ...harness.env({
+        configDir: opts.configDir,
+        run: { home: r.home, homeEnvFile }
+      }),
+      PWD: workdir,
       PATH: opts.stepEnv.PATH,
       FIREGOAT_OUT: opts.out,
       FIREGOAT_WORKDIR: workdir,
@@ -9365,18 +10147,22 @@ async function attempt(opts, r, steps, env, stepEnv, homeEnvFile) {
     stderr,
     timeoutMs: test.timeout * 1000,
     model: opts.model ?? undefined,
-    extraArgs: r.claudeArgs,
+    extraArgs: r.harnessArgs,
     onProgress: opts.onProgress
   });
-  r.claudeModel = outcome.reportedModel;
   if (outcome.version !== null)
     evalStats({ claudeVersion: claudeVersion(outcome.version) });
   await new Promise((res) => stderr.end(res));
   if (outcome.sessionId)
     r.session = harness.findSession(opts.configDir, outcome.sessionId);
+  r.reportedModel = outcome.reportedModel ?? (r.session && harness.sessionModel ? harness.sessionModel(r.session) : null);
+  const about = [
+    opts.shownHarness,
+    opts.model === null ? null : `model ${opts.model}`
+  ].filter((x) => x !== null);
   writeTranscript({
     path: r.transcript,
-    title: `${test.name}#${n}${opts.model ? ` (model ${opts.model})` : ""}`,
+    title: `${test.name}#${n}${about.length > 0 ? ` (${about.join(", ")})` : ""}`,
     prompt: r.prompt,
     steps: outcome.steps,
     final: outcome.final,
@@ -9419,6 +10205,9 @@ async function attempt(opts, r, steps, env, stepEnv, homeEnvFile) {
     test.checks.length === 0 ? "no script checks" : `${r.checks.length}/${r.checks.length} checks passed`
   ];
 }
+function notUploaded(h) {
+  return `firegoat.ai doesn't read ${h.name} sessions yet: its runs stay local`;
+}
 var MAX_LISTED_URLS = 5;
 function uploadedSummary(origin, results, runDir) {
   const urls = results.flatMap((r) => r.upload?.status === "uploaded" ? [r.upload.url] : []);
@@ -9426,7 +10215,7 @@ function uploadedSummary(origin, results, runDir) {
     return;
   const head = `Uploaded ${urls.length} session${urls.length === 1 ? "" : "s"} to ${new URL(origin).host} (private)`;
   if (urls.length > MAX_LISTED_URLS) {
-    console.log(`${head}: each run's upload.url in ${join7(runDir, "summary.json")}`);
+    console.log(`${head}: each run's upload.url in ${join10(runDir, "summary.json")}`);
     return;
   }
   console.log(`${head}:`);
@@ -9481,9 +10270,22 @@ async function evalCommand(args, root) {
   }
   const runs = opts.runs ?? config.runs;
   const models = opts.models.length > 0 ? opts.models : config.models;
+  const harnessNames = opts.harnesses.length > 0 ? opts.harnesses : config.harnesses;
+  if (opts.harnessBin !== undefined && harnessNames.length !== 1)
+    return fatal(EXIT.usage, `--harness-bin needs exactly one harness, this eval runs ${harnessNames.length} (${harnessNames.join(", ")}): pick one with --harness`, "usage");
+  const harnesses = harnessNames.map((h) => HARNESSES.get(h));
+  const showHarness = showsHarness(harnessNames);
+  try {
+    for (const h of harnesses)
+      checkHarnessArgs(tests, h);
+  } catch (e) {
+    if (e instanceof ConfigError)
+      return fatal(EXIT.usage, e.message, "config");
+    throw e;
+  }
   evalStats({
     testCount: tests.length,
-    runCount: tests.length * runs * Math.max(models.length, 1),
+    runCount: tests.length * runs * Math.max(models.length, 1) * harnesses.length,
     modelCount: models.length,
     parallel: opts.parallel
   });
@@ -9493,7 +10295,7 @@ async function evalCommand(args, root) {
       t.prompt,
       ...t.checks.map((c) => c.run),
       ...Object.values(t.env),
-      ...t.claudeArgs,
+      ...t.harnessArgs.map((a) => a.arg),
       ...t.install,
       ...t.cleanup
     ]).concat(config.build, config.install, config.cleanup).flatMap(varRefs).filter((k) => !(k in vars)))
@@ -9511,9 +10313,10 @@ async function evalCommand(args, root) {
       dryRun({
         config,
         tests,
-        plan: planRuns(tests, models, runs),
+        plan: planRuns(tests, harnessNames, models, runs),
+        showHarness,
         vars,
-        runsRoot: join7(repo?.mainCheckout ?? root, ".firegoat", "runs"),
+        agentDirs: projectAgentDir(repo?.mainCheckout ?? root),
         json: opts.json,
         say
       });
@@ -9524,28 +10327,42 @@ async function evalCommand(args, root) {
     }
     return EXIT.pass;
   }
-  const harness = claude;
-  evalStats({ harness: harness.name });
-  const binPath = resolveBin(opts.claudeBin);
-  if (!binPath)
-    return fatal(EXIT.notGraded, `Not graded: ${harness.name} binary ${JSON.stringify(opts.claudeBin)} not found. ${harness.installHint}`, "claude_missing");
+  evalStats({ harness: harnessNames.join(",") });
+  const bins = new Map;
+  for (const h of harnesses) {
+    const bin = opts.harnessBin ?? h.bin;
+    const path = resolveBin(bin);
+    if (!path)
+      return fatal(EXIT.notGraded, `Not graded: ${h.name} binary ${JSON.stringify(bin)} not found. ${h.installHint}`, "harness_missing");
+    bins.set(h, path);
+  }
   let repo;
   try {
     repo = gitRepo(root);
   } catch (e) {
     return fatal(EXIT.usage, e.message, "config");
   }
-  const firegoatDir = ensureFiregoatDir(repo?.mainCheckout ?? root);
-  const configDir = join7(firegoatDir, harness.configDirName);
-  mkdirSync4(configDir, { recursive: true });
-  ui.auth(harness.authLabel());
-  const loginProblem = harness.checkLogin({
-    bin: binPath,
-    env: { ...hostEnv(), ...harness.env({ configDir }) },
-    cwd: root
-  });
-  if (loginProblem)
-    return fatal(EXIT.notGraded, `Not graded: ${loginProblem.reason}`, loginProblem.loggedOut ? "claude_logged_out" : "claude_auth_status");
+  const project = repo?.mainCheckout ?? root;
+  const firegoatDir = ensureFiregoatDir(project);
+  const agentDirs = ensureProjectAgentDir(project);
+  const configDirs = new Map;
+  for (const h of harnesses) {
+    const configDir = join10(agentDirs, h.configDirName);
+    mkdirSync5(configDir, { recursive: true });
+    configDirs.set(h, configDir);
+  }
+  ui.auth(harnesses.length === 1 ? harnesses[0].authLabel() : harnesses.map((h) => `${h.name}: ${h.authLabel()}`).join("; "));
+  for (const h of harnesses) {
+    const configDir = configDirs.get(h);
+    const loginProblem = h.checkLogin({
+      bin: bins.get(h),
+      env: { ...hostEnv(), ...h.env({ configDir }) },
+      cwd: root,
+      configDir
+    });
+    if (loginProblem)
+      return fatal(EXIT.notGraded, `Not graded: ${loginProblem.reason}`, loginProblem.loggedOut ? "harness_logged_out" : "harness_auth_status");
+  }
   const evalStarted = Date.now();
   const runId = newRunId();
   let upload;
@@ -9554,23 +10371,32 @@ async function evalCommand(args, root) {
     if (pre && "exitCode" in pre)
       return fatal(pre.exitCode, pre.error, pre.code);
     upload = pre;
+    if (upload) {
+      for (const h of harnesses)
+        if (!h.uploads)
+          ui.note(notUploaded(h));
+    }
   } catch (e) {
     if (e instanceof ConfigError)
       return fatal(EXIT.usage, e.message, "config");
     throw e;
   }
-  const runsRoot = join7(firegoatDir, "runs");
-  const runDir = join7(runsRoot, runId);
-  const out = join7(runDir, "out");
-  mkdirSync4(out, { recursive: true });
+  const runsRoot = join10(firegoatDir, "runs");
+  const runDir = join10(runsRoot, runId);
+  mkdirSync5(runDir, { recursive: true });
+  const out = join10(makeAgentDir(agentDirs), "out");
+  mkdirSync5(out);
+  linkDir(out, join10(runDir, "out"));
   const stepEnv = {
     ...hostEnv(),
     ...vars,
     FIREGOAT_OUT: out,
     PATH: [out, process.env.PATH ?? ""].join(delimiter2)
   };
-  const rows = planRuns(tests, models, runs).map((p) => ({
+  const plan = planRuns(tests, harnessNames, models, runs);
+  const rows = plan.map((p) => ({
     ...p,
+    harness: showHarness ? p.harness : null,
     state: "queued",
     turns: 0
   }));
@@ -9585,7 +10411,7 @@ async function evalCommand(args, root) {
         ...stepEnv,
         ...renderEnv(Object.fromEntries(Object.entries(config.env).filter(([, v]) => isEvalLevel(v))), vars, { "run.id": runId, FIREGOAT_OUT: out })
       },
-      logPath: join7(runDir, "build.txt")
+      logPath: join10(runDir, "build.txt")
     });
     const secs = Math.round((Date.now() - buildStarted) / 1000);
     ui.build(error ? "fail" : "ok", secs);
@@ -9597,6 +10423,7 @@ async function evalCommand(args, root) {
   const runAt = async (i) => {
     const row = rows[i];
     const test = tests.find((t) => t.name === row.test);
+    const harness = HARNESSES.get(plan[i].harness);
     row.state = "running";
     ui.update();
     const r = await runOne({
@@ -9608,11 +10435,14 @@ async function evalCommand(args, root) {
       test,
       n: row.n,
       model: row.model,
+      shownHarness: row.harness,
       runId,
       runDir,
+      agentDirs,
+      repo: root,
       harness,
-      bin: binPath,
-      configDir,
+      bin: bins.get(harness),
+      configDir: configDirs.get(harness),
       install: config.install,
       cleanup: config.cleanup,
       out,
@@ -9630,7 +10460,10 @@ async function evalCommand(args, root) {
       command: undefined
     });
     ui.update();
-    if (upload) {
+    if (upload && !harness.uploads) {
+      r.upload = { status: "skipped", reason: notUploaded(harness) };
+      writeRunJson(r);
+    } else if (upload) {
       row.upload = "uploading";
       ui.update();
       r.upload = await uploadRun(upload, r);
@@ -9669,20 +10502,22 @@ async function evalCommand(args, root) {
     runsDir: runDir,
     dir: root,
     ...repo && { git: repo.info },
+    harnesses: harnessNames,
     upload: upload && { origin: upload.origin, email: upload.email },
     runs: results,
     totals,
     status,
     exitCode
   };
-  writeFileSync4(join7(runDir, "summary.json"), `${JSON.stringify(doc, null, 2)}
+  writeFileSync4(join10(runDir, "summary.json"), `${JSON.stringify(doc, null, 2)}
 `);
-  appendFileSync(join7(runsRoot, "index.jsonl"), `${JSON.stringify({
+  appendFileSync(join10(runsRoot, "index.jsonl"), `${JSON.stringify({
     runId,
     startedAt: new Date(evalStarted).toISOString(),
     dir: root,
     branch: repo?.info.branch ?? null,
     commit: repo?.info.commit ?? null,
+    harnesses: harnessNames,
     passed: totals.pass,
     failed: totals.fail,
     notGraded: totals.error,
@@ -9715,7 +10550,7 @@ async function evalCommand(args, root) {
 }
 
 // src/feedback.ts
-import { readFileSync as readFileSync4 } from "node:fs";
+import { readFileSync as readFileSync6 } from "node:fs";
 
 // node_modules/@firegoat/feedback/dist/index.js
 function memoryDeviceStore() {
@@ -9934,7 +10769,7 @@ async function feedbackCommand(args) {
     try {
       if (a === "--body") {
         const v = value();
-        body = v === "-" ? readFileSync4(0, "utf8") : v;
+        body = v === "-" ? readFileSync6(0, "utf8") : v;
       } else if (a === "--type") {
         const v = value();
         if (!TYPES.includes(v))
@@ -9987,9 +10822,9 @@ firegoat ${package_default.version}` : `firegoat ${package_default.version}`,
 }
 
 // src/init.ts
-import { spawn as spawn3, spawnSync as spawnSync3 } from "node:child_process";
-import { existsSync as existsSync4, writeFileSync as writeFileSync5 } from "node:fs";
-import { basename as basename4, join as join8 } from "node:path";
+import { spawn as spawn3, spawnSync as spawnSync4 } from "node:child_process";
+import { existsSync as existsSync6, writeFileSync as writeFileSync5 } from "node:fs";
+import { basename as basename4, join as join11 } from "node:path";
 
 // src/login.ts
 import { spawn as spawn2 } from "node:child_process";
@@ -10000,7 +10835,7 @@ var FORM = { "Content-Type": "application/x-www-form-urlencoded" };
 var UPLOADS_ON = "Eval uploads each session to firegoat.ai as a private trace; --no-upload keeps runs local.";
 var UPLOAD_NOTICE = `${UPLOADS_ON} A session holds prompts, tool outputs and file contents: possibly secrets from your env or repo. The agent runs as you and can read ~/.firegoat (and a FIREGOAT_API_KEY in firegoat's own process env).`;
 var TERMS = "By logging in, you agree to the firegoat.ai Terms of Service (https://firegoat.ai/terms) and Privacy Policy (https://firegoat.ai/privacy).";
-function str2(o, k, step) {
+function str3(o, k, step) {
   const v = o?.[k];
   if (typeof v !== "string" || v === "")
     throw new ApiError(`${step}: response has no ${k}`);
@@ -10031,9 +10866,9 @@ async function loginFlow(show, interactive) {
   });
   if (cfg.status !== 200)
     throw replyError(cfgStep, cfg);
-  const clientId = str2(cfg.json, "client_id", cfgStep);
-  const deviceUrl = assertSafeUrl(str2(cfg.json, "device_authorization_endpoint", cfgStep), "device_authorization_endpoint");
-  const tokenUrl = assertSafeUrl(str2(cfg.json, "token_endpoint", cfgStep), "token_endpoint");
+  const clientId = str3(cfg.json, "client_id", cfgStep);
+  const deviceUrl = assertSafeUrl(str3(cfg.json, "device_authorization_endpoint", cfgStep), "device_authorization_endpoint");
+  const tokenUrl = assertSafeUrl(str3(cfg.json, "token_endpoint", cfgStep), "token_endpoint");
   if (tokenUrl.origin !== origin)
     throw new ApiError(`token_endpoint must be on ${origin}, got ${printable(tokenUrl.origin, 100)}`);
   const devStep = "Device authorization";
@@ -10047,11 +10882,11 @@ async function loginFlow(show, interactive) {
   });
   if (dev.status !== 200)
     throw replyError(devStep, dev);
-  const deviceCode = str2(dev.json, "device_code", devStep);
-  const userCode = str2(dev.json, "user_code", devStep);
+  const deviceCode = str3(dev.json, "device_code", devStep);
+  const userCode = str3(dev.json, "user_code", devStep);
   if (!/^[A-Za-z0-9-]{1,64}$/.test(userCode))
     throw new ApiError(`${devStep}: user_code has unexpected characters`);
-  const pageRaw = typeof dev.json?.verification_uri_complete === "string" ? dev.json.verification_uri_complete : str2(dev.json, "verification_uri", devStep);
+  const pageRaw = typeof dev.json?.verification_uri_complete === "string" ? dev.json.verification_uri_complete : str3(dev.json, "verification_uri", devStep);
   const page = assertSafeUrl(pageRaw, "verification_uri");
   if (page.protocol !== "https:" || pageRaw.startsWith("-"))
     throw new ApiError("verification_uri must be https");
@@ -10288,18 +11123,23 @@ function skillsDir(scope) {
   if (scope === "local")
     return ".claude/skills/";
   const config = process.env.CLAUDE_CONFIG_DIR?.trim();
-  return config ? `${join8(config, "skills")}/` : "~/.claude/skills/";
+  return config ? `${join11(config, "skills")}/` : "~/.claude/skills/";
 }
 function template(product) {
   return `# firegoat.yaml: evals for how well coding agents use ${product}.
-# Each test runs your claude in an empty folder (.firegoat/runs/...) with \`prompt\`
-# as the first message, then runs \`checks\` there. Reference: npx firegoat init --help
+# Each test runs your agent (claude by default) in an empty folder
+# (~/.firegoat/runs/...) with \`prompt\` as the first message, then runs \`checks\`
+# there. Reference: npx firegoat init --help
 
 # How many times to run each test.
 runs: 1
 
-# Optional: claude's --model. A list runs every test \`runs\` times per model.
-# Unset: claude's default. eval --model NAME overrides it.
+# Optional: the agent CLI each test runs on: claude-code (default), codex-cli or grok-cli.
+# A list runs every test \`runs\` times per harness. eval --harness NAME overrides it.
+# harness: [claude-code, codex-cli]
+
+# Optional: the harness's --model. A list runs every test \`runs\` times per model.
+# Unset: the harness's default. eval --model NAME overrides it.
 # model: [claude-haiku-5-5, claude-sonnet-5-5]
 
 # Optional: test your local build instead of the published package.
@@ -10380,7 +11220,7 @@ var CLIPBOARD = [
 ];
 function copyToClipboard(text) {
   return CLIPBOARD.some(([cmd, args]) => {
-    const r = spawnSync3(cmd, args, { input: text, env: hostEnv() });
+    const r = spawnSync4(cmd, args, { input: text, env: hostEnv() });
     return !r.error && r.status === 0;
   });
 }
@@ -10545,8 +11385,8 @@ ${INIT_HELP}`);
     console.log(rule);
   }
   ensureFiregoatDir(cwd);
-  const configPath = join8(cwd, CONFIG_FILE);
-  const keep = existsSync4(configPath) && !force;
+  const configPath = join11(cwd, CONFIG_FILE);
+  const keep = existsSync6(configPath) && !force;
   if (!keep)
     writeFileSync5(configPath, template(basename4(cwd)));
   if (ui) {
@@ -10590,7 +11430,7 @@ ${INIT_HELP}`);
       block.draw([ui.done(3, skill, `Installed → ${where}`)]);
     else
       console.log(`Installed skill firegoat-onboarding in ${where}`);
-    const ignored = scope === "local" && spawnSync3("git", ["check-ignore", "-q", ".claude/skills"], {
+    const ignored = scope === "local" && spawnSync4("git", ["check-ignore", "-q", ".claude/skills"], {
       cwd,
       env: hostEnv()
     }).status === 0;