firegoat 0.19.0 → 0.21.0

package/README.md

@@ -12,7 +12,7 @@ npx firegoat@latest init     # Creates firegoat.yaml and .firegoat/, installs th
 npx firegoat@latest eval     # Runs your claude on each test, in .firegoat/runs/<runId>/<test>/<n>/work
 ```
 
-That's it: two commands. Use `npx firegoat@latest`: plain `npx firegoat` can keep running an old cached version. `eval` prints its version first (`firegoat 0.18.0`), and `summary.json` records it as `firegoatVersion`. `claude` must be installed and logged in (run `claude` once).
+That's it: two commands. Use `npx firegoat@latest`: plain `npx firegoat` can keep running an old cached version. `eval` prints its version first (`firegoat X.Y.Z`), and `summary.json` records it as `firegoatVersion`. `claude` must be installed and logged in (run `claude` once).
 
 `init` installs the firegoat-onboarding skill globally, into `~/.claude/skills/` (in a terminal it asks: global, this repo, or none). `--local-skills` puts it in this repo's `.claude/skills/` instead (commit it so teammates and CI get it); `--no-skills` skips it.
 
@@ -156,7 +156,28 @@ tests:
     claude_args: [--chrome]                    # Claude in Chrome, this test only
 ```
 
-`claude_args:` (top level, and per test after the top-level ones) is passed to claude after firegoat's own flags and `--model`. Each entry is one argument, as-is, with no shell splitting: `[--plugin-dir, ./plugins/acme]`, not `["--plugin-dir ./plugins/acme"]`. Use it for `--chrome`, `--plugin-dir`, `--mcp-config`, `--agents`, `--append-system-prompt`. The flags firegoat sets itself are config errors, also as `--flag=value`: `-p`, `--print`, `--output-format`, `--verbose`, `--setting-sources`, `--strict-mcp-config`, `--disable-slash-commands`, `--dangerously-skip-permissions`, `--model` (use `model:`), and `--session-id`, `--resume`, `-r`, `--continue`, `-c` (every run is a fresh session). Each run's `run.json` records `claudeArgs`.
+`claude_args:` (top level, and per test after the top-level ones) is passed to claude after firegoat's own flags and `--model`. Each entry is one argument, as-is, with no shell splitting: `[--plugin-dir, ./plugins/acme]`, not `["--plugin-dir ./plugins/acme"]`. Use it for `--chrome`, `--plugin-dir`, `--mcp-config`, `--agents`, `--append-system-prompt`. The flags firegoat sets itself are config errors, also as `--flag=value`: `-p`, `--print`, `--output-format`, `--verbose`, `--setting-sources`, `--strict-mcp-config`, `--disable-slash-commands`, `--dangerously-skip-permissions`, `--model` (use `model:`), and `--session-id`, `--resume`, `-r`, `--continue`, `-c` (every run is a fresh session). Entries expand like `env:` values: `$FIREGOAT_WORKDIR`, `$FIREGOAT_HOME`, `$FIREGOAT_OUT` (also `${...}`), `{{vars.KEY}}`, `{{run.id}}`, `{{run.n}}`, `{{test.name}}`; no other `$` expansion. Each run's `run.json` records the expanded `claudeArgs`.
+
+### Testing an MCP server
+
+Commit the server's config as `fixtures/mcp.json`:
+
+```json
+{"mcpServers": {"acme": {"command": "./node_modules/.bin/acme-mcp",
+  "env": {"HOME": "${FIREGOAT_HOME}"}}}}
+```
+
+Copy it in `build:` and pass it with `claude_args:`:
+
+```yaml
+build:
+  - npm pack --pack-destination "$FIREGOAT_OUT"
+  - cp -R fixtures/. "$FIREGOAT_OUT"
+install: npm init -y >/dev/null && npm install "$FIREGOAT_OUT"/*.tgz
+claude_args: [--mcp-config, $FIREGOAT_OUT/mcp.json]
+```
+
+claude gets `FIREGOAT_HOME`, `FIREGOAT_WORKDIR` and `FIREGOAT_OUT` in its own env, set per run, and expands `${VAR}` in an MCP config's `command`, `args`, `env`, `url` and `headers` from its env (an unset var stays the literal text). Relative paths in the config are from the run's folder, claude's cwd: `./node_modules/.bin/acme-mcp` is the build under test, installed by `install:`. MCP servers that claude starts inherit claude's own env, which keeps your real `HOME` (claude needs it for its login): set `"env": {"HOME": "${FIREGOAT_HOME}"}` on the server, as above, or it writes to your real home. Per-run state the server keeps elsewhere goes the same way, e.g. `"ACME_DB": "${FIREGOAT_WORKDIR}/acme.db"`. `--strict-mcp-config` is always on, so only the configs you pass load. To check that the agent used a tool, grep the transcript: `grep -Eq '^\[[0-9]+\] mcp__acme__search' "$FIREGOAT_TRANSCRIPT"`.
 
 With `--chrome`, claude points the Claude in Chrome extension's native host (`~/Library/Application Support/<browser>/NativeMessagingHosts/com.anthropic.claude_code_browser_extension.json`) at `.firegoat/.claude`. If you delete `.firegoat/`, run a normal `claude --chrome` once to point it back.
 
@@ -198,9 +219,9 @@ Limits:
 ## What eval does
 
 - claude runs on its own login (or `ANTHROPIC_API_KEY`, if set) with `CLAUDE_CONFIG_DIR=.firegoat/.claude` and `--setting-sources "" --strict-mcp-config --disable-slash-commands`: none of your settings, CLAUDE.md, plugins, skills or MCP servers (add flags with `claude_args:`, see above). claude saves its session there, in its own format.
-- No sandbox: the agent's folder is inside `.firegoat/runs/`, and it can reach the rest of your machine. `.firegoat/` holds local run output and ignores itself in git. Keep test fixtures (scripts, sample data) outside it, e.g. in `fixtures/` next to `firegoat.yaml`, and commit them.
+- No sandbox: the agent's folder is inside `.firegoat/runs/`, and it can reach the rest of your machine. `.firegoat/` holds local run output and ignores itself in git. Keep test fixtures (scripts, sample data) in `fixtures/` next to `firegoat.yaml`, committed.
 - In a linked git worktree (`git worktree add`), `.firegoat/runs/` and `.firegoat/.claude/` are in the main checkout, so runs (and their session files) survive `git worktree remove`; `firegoat.yaml`, fixtures and `build:` stay in the worktree. Each `summary.json` records `dir` (where eval ran) and `git: {branch, commit, dirty}`, and `.firegoat/runs/index.jsonl` gets one line per eval (`runId, startedAt, dir, branch, commit, passed, failed, notGraded, cost, uploaded, uploadFailed`) to tell runs from different branches apart.
-- The agent gets your environment plus `env:`, never `vars:`. Vars that npm/npx set for firegoat itself (`npm_config_*`, `npm_lifecycle_*`, `npm_package_*`, `npm_execpath`, `npm_node_execpath`, `npm_command`, `INIT_CWD`) are removed from every env firegoat starts (agent, build, install, checks): `npx -y firegoat` sets `npm_config_yes=true`, which would make the agent's `npm init` / `npx create-*` skip their prompts. Set one under `env:` to keep it. Every `FIREGOAT_*` var firegoat was started with (`FIREGOAT_API_KEY` above all) is removed too, and so are the vars a parent Claude Code session sets (when firegoat runs from claude, or its agent runs firegoat), so the claude under test doesn't act as a nested session: `CLAUDECODE`, `CLAUDE_CODE_*`, `CLAUDE_PID`, `CLAUDE_ENV_FILE`, `CLAUDE_SESSION_ID`, `CLAUDE_PROJECT_DIR`, `CLAUDE_EFFORT`, `CLAUDE_AGENT_SDK_*`, `CLAUDE_PLUGIN_ROOT`, `CLAUDE_PLUGIN_DATA`, `CLAUDE_SKILL_DIR`. Login and provider vars stay: `ANTHROPIC_*`, `CLAUDE_CODE_OAUTH_TOKEN`, `CLAUDE_CODE_OAUTH_REFRESH_TOKEN`, `CLAUDE_CODE_USE_*` (Bedrock, Vertex, ...), `CLAUDE_CODE_SKIP_*_AUTH`, `CLAUDE_CODE_CLIENT_CERT`, `CLAUDE_CODE_CLIENT_KEY`, `CLAUDE_CODE_CLIENT_KEY_PASSPHRASE`.
+- The agent gets your environment plus `env:`, never `vars:`. Vars that npm/npx set for firegoat itself (`npm_config_*`, `npm_lifecycle_*`, `npm_package_*`, `npm_execpath`, `npm_node_execpath`, `npm_command`, `INIT_CWD`) are removed from every env firegoat starts (agent, build, install, checks): `npx -y firegoat` sets `npm_config_yes=true`, which would make the agent's `npm init` / `npx create-*` skip their prompts. Set one under `env:` to keep it. Every `FIREGOAT_*` var firegoat was started with (`FIREGOAT_API_KEY` above all) is removed too (claude then gets the run's own `FIREGOAT_OUT`, `FIREGOAT_WORKDIR` and `FIREGOAT_HOME`, for MCP configs; the agent's commands see them too), and so are the vars a parent Claude Code session sets (when firegoat runs from claude, or its agent runs firegoat), so the claude under test doesn't act as a nested session: `CLAUDECODE`, `CLAUDE_CODE_*`, `CLAUDE_PID`, `CLAUDE_ENV_FILE`, `CLAUDE_SESSION_ID`, `CLAUDE_PROJECT_DIR`, `CLAUDE_EFFORT`, `CLAUDE_AGENT_SDK_*`, `CLAUDE_PLUGIN_ROOT`, `CLAUDE_PLUGIN_DATA`, `CLAUDE_SKILL_DIR`. Login and provider vars stay: `ANTHROPIC_*`, `CLAUDE_CODE_OAUTH_TOKEN`, `CLAUDE_CODE_OAUTH_REFRESH_TOKEN`, `CLAUDE_CODE_USE_*` (Bedrock, Vertex, ...), `CLAUDE_CODE_SKIP_*_AUTH`, `CLAUDE_CODE_CLIENT_CERT`, `CLAUDE_CODE_CLIENT_KEY`, `CLAUDE_CODE_CLIENT_KEY_PASSPHRASE`.
 - Runs (tests × models × `runs`) go 4 at a time, each in its own folder with its own session and HOME; `--parallel N` changes that, `--parallel 1` runs them one after another. `summary.json` lists them in `firegoat.yaml` order.
 - Script checks run in the agent's folder afterwards and set the exit code. `end_goal` is not graded by firegoat: it's for you, or your coding agent, to judge from `run.json`, the session and the folder.
 - Output per run, in `.firegoat/runs/<runId>/<test>/<n>/`: `transcript.txt` (each tool call with its command and short output, the final answer, files made; checks get it as `$FIREGOAT_TRANSCRIPT`; greppable line by line: every line of a shell call starts with `[n] $ `, every line of the agent's text and final answer with `agent: `, output is indented, so `grep -E '^\[[0-9]+\] \$ .*npm install'` matches only commands, also inside heredocs and multi-line scripts), `run.json` (status, prompt, end_goal, check results, paths), `work/`, `home/` and `home.env` (the run's HOME), logs. Read the transcript to judge `end_goal`; claude's full session file is ~50x larger. `summary.json` covers the whole eval; `--json` prints it.

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.19.0",
+  version: "0.21.0",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -7211,6 +7211,7 @@ function parseClaudeArgs(raw, path) {
       for (const f of flags)
         if (arg === f || arg.startsWith(`${f}=`))
           fail(p, `${f} is not allowed: ${why}`);
+    checkPlaceholders(arg, p);
     return arg;
   });
 }
@@ -7266,15 +7267,18 @@ function parseEnv(raw, path) {
       fail(p, "is set by firegoat and cannot be overridden (reserved: PATH, CLAUDE_CONFIG_DIR, CLAUDE_SECURESTORAGE_CONFIG_DIR, FIREGOAT_*)");
     if (typeof v !== "string")
       fail(p, "must be a string (quote numbers and booleans)");
-    for (const m of v.matchAll(ANY_PLACEHOLDER)) {
-      const name = m[1];
-      if (!/^vars\.[A-Za-z0-9_]+$/.test(name) && !RUN_PLACEHOLDERS.includes(name))
-        fail(p, `unknown placeholder ${m[0]} (allowed: {{vars.KEY}}, ${RUN_PLACEHOLDERS.map((n) => `{{${n}}}`).join(", ")})`);
-    }
+    checkPlaceholders(v, p);
     out[k] = v;
   }
   return out;
 }
+function checkPlaceholders(v, p) {
+  for (const m of v.matchAll(ANY_PLACEHOLDER)) {
+    const name = m[1];
+    if (!/^vars\.[A-Za-z0-9_]+$/.test(name) && !RUN_PLACEHOLDERS.includes(name))
+      fail(p, `unknown placeholder ${m[0]} (allowed: {{vars.KEY}}, ${RUN_PLACEHOLDERS.map((n) => `{{${n}}}`).join(", ")})`);
+  }
+}
 function isEvalLevel(value) {
   const dirsOk = [...value.matchAll(DIR_REF)].every((m) => (m[1] ?? m[2]) === "FIREGOAT_OUT");
   return dirsOk && [...value.matchAll(ANY_PLACEHOLDER)].every((m) => {
@@ -7282,16 +7286,19 @@ function isEvalLevel(value) {
     return name.startsWith("vars.") || EVAL_PLACEHOLDERS.includes(name);
   });
 }
+function renderValue(value, vars, run, where) {
+  return value.replace(ENV_REF, (whole, name, braced, bare) => {
+    const key = name ?? braced ?? bare;
+    const filled = key.startsWith("vars.") ? vars[key.slice("vars.".length)] : run[key];
+    if (filled === undefined)
+      throw new ConfigError(`${where}: no value for ${whole}`);
+    return filled;
+  });
+}
 function renderEnv(env, vars, run) {
   const out = {};
   for (const [k, v] of Object.entries(env))
-    out[k] = v.replace(ENV_REF, (whole, name, braced, bare) => {
-      const key = name ?? braced ?? bare;
-      const filled = key.startsWith("vars.") ? vars[key.slice("vars.".length)] : run[key];
-      if (filled === undefined)
-        throw new ConfigError(`env.${k}: no value for ${whole}`);
-      return filled;
-    });
+    out[k] = renderValue(v, vars, run, `env.${k}`);
   return out;
 }
 var PLACEHOLDER = /\{\{\s*vars\.([A-Za-z0-9_]+)\s*\}\}/g;
@@ -7441,8 +7448,15 @@ async function runClaude(opts) {
       reason: `claude did not start a session (exit ${exited.code ?? "by signal"})${said ? `: ${said}` : ", see stderr.txt"}`
     };
   }
-  const sessionId = String(init.session_id);
   const claudeModel = typeof init.model === "string" ? init.model : null;
+  if (typeof init.session_id !== "string" || init.session_id === "")
+    return {
+      events,
+      claudeModel,
+      ok: false,
+      reason: "claude's init line has no session_id"
+    };
+  const sessionId = init.session_id;
   const result = events.findLast((e) => e?.type === "result");
   if (exited.timedOut)
     return {
@@ -7639,7 +7653,7 @@ async function send(opts) {
 function replyError(step, r) {
   const j = r.json ?? {};
   const error = typeof j.error === "string" ? j.error : "no error message";
-  const more = j.error_description ?? j.details ?? j.message;
+  const more = j.error_description ?? j.details;
   const details = more === undefined ? "" : ` ${typeof more === "string" ? more : JSON.stringify(more)}`;
   return new ApiError(printable(`${step} ${r.status}: ${error}${details}`), r.status);
 }
@@ -7809,7 +7823,12 @@ var GITIGNORE = `*
 function ensureFiregoatDir(root) {
   const dir = join2(root, ".firegoat");
   mkdirSync2(dir, { recursive: true });
-  writeFileSync(join2(dir, ".gitignore"), GITIGNORE);
+  try {
+    writeFileSync(join2(dir, ".gitignore"), GITIGNORE, { flag: "wx" });
+  } catch (e) {
+    if (e.code !== "EEXIST")
+      throw e;
+  }
   return dir;
 }
 
@@ -7911,8 +7930,8 @@ firegoat-onboarding skill globally, into ~/.claude/skills/ (npx skills add
 firegoat-ai/skills -g; in a terminal it asks: global, this repo, or none), offers
 firegoat login (in a terminal only; otherwise prints how), then prints a prompt to
 paste into your coding agent. .firegoat/ holds local run output and ignores itself
-in git. Keep test fixtures (helper scripts, sample data) outside it, e.g. in
-fixtures/ next to firegoat.yaml, and commit them.
+in git. Keep test fixtures (helper scripts, sample data) in fixtures/ next to
+firegoat.yaml, committed.
 
   --force          Replace an existing firegoat.yaml
   --local-skills   Install the skill into this repo's .claude/skills/ instead
@@ -7957,7 +7976,10 @@ firegoat.yaml reference:
                               as-is (no shell splitting): [--chrome] (Claude in
                               Chrome, for tests that need a browser),
                               [--plugin-dir, ./plugins/x], --mcp-config, --agents,
-                              --append-system-prompt. Flags firegoat sets are
+                              --append-system-prompt. Entries expand like env:
+                              values ($FIREGOAT_WORKDIR, $FIREGOAT_HOME,
+                              $FIREGOAT_OUT, {{vars.KEY}}, {{run.*}}); run.json
+                              claudeArgs has the result. Flags firegoat sets are
                               errors, also as --flag=value: -p, --print,
                               --output-format, --verbose, --setting-sources,
                               --strict-mcp-config, --disable-slash-commands,
@@ -7998,7 +8020,23 @@ firegoat.yaml reference:
   (npm_config_*, npm_lifecycle_*, npm_package_*, npm_execpath, npm_node_execpath,
   npm_command, INIT_CWD; e.g. npm_config_yes=true from npx -y) are removed from
   every env firegoat starts, unless env: sets them. So is every FIREGOAT_* var
-  (FIREGOAT_API_KEY above all) firegoat itself was started with.
+  (FIREGOAT_API_KEY above all) firegoat itself was started with. claude itself
+  gets $FIREGOAT_OUT, $FIREGOAT_WORKDIR and $FIREGOAT_HOME (for MCP configs, below),
+  so the agent's commands see them too.
+
+Testing an MCP server: commit its config as fixtures/mcp.json, copy it in build:
+and pass it with claude_args. Only the configs you pass load: --strict-mcp-config
+is always on. An MCP server inherits claude's env, which keeps your real HOME
+(claude's login is there), so set the server's HOME to the run's home, or it
+writes to yours. claude gets $FIREGOAT_HOME, $FIREGOAT_WORKDIR and $FIREGOAT_OUT
+in its env and expands \${VAR} in the config (an unset one stays literal);
+relative paths in it are from the run's folder, claude's cwd:
+  fixtures/mcp.json:
+    {"mcpServers": {"acme": {"command": "./node_modules/.bin/acme-mcp",
+      "env": {"HOME": "\${FIREGOAT_HOME}"}}}}
+  firegoat.yaml:
+    build: cp -R fixtures/. "$FIREGOAT_OUT"
+    claude_args: [--mcp-config, $FIREGOAT_OUT/mcp.json]
 
 Example:
   build: npm pack --pack-destination "$FIREGOAT_OUT"
@@ -8114,8 +8152,7 @@ fixtures and build: stay in the current folder.
   --json             The summary as JSON on stdout (or {error, exitCode} if eval
                      stops early); progress goes to stderr
   --no-upload        Keep runs local even when logged in to firegoat.ai
-  --claude-bin PATH  Path to claude (env FIREGOAT_CLAUDE_BIN; default: claude on
-                     PATH)
+  --claude-bin PATH  Path to claude (default: claude on PATH)
 
 Upload: after firegoat login (or with FIREGOAT_API_KEY), each run's claude session,
 checks.txt and stderr.txt go to firegoat.ai as a private trace, and its URL is
@@ -8132,8 +8169,8 @@ Output: .firegoat/runs/<runId>/
                           git {branch, commit, dirty} (no git outside a repo)
                           and upload {origin, email} (null when not uploading)
   ../index.jsonl          One line per eval: runId, startedAt, dir, branch,
-                          commit, passed, failed, notGraded, cost, uploaded,
-                          uploadFailed
+                          commit, passed, failed, notGraded, cost (null when
+                          any run's cost is unknown), uploaded, uploadFailed
   <test>/<n>/run.json     One run: status and reason (from the checks), model
                           (asked for, or null) and claudeModel (what claude's
                           init line reported), claudeArgs (claude_args
@@ -8284,6 +8321,13 @@ function describeCall(step, name, input) {
       return `[${step}] ${name} ${input[key]}`;
   return `[${step}] ${name} ${truncate(JSON.stringify(input), 1, 300)}`;
 }
+function describeToolUse(step, b) {
+  if (typeof b.name !== "string" || b.name === "")
+    return `[${step}] (tool call without a name)`;
+  if (typeof b.input !== "object" || b.input === null)
+    return `[${step}] ${b.name} (no input)`;
+  return describeCall(step, b.name, b.input);
+}
 function resultText(content) {
   if (typeof content === "string")
     return content;
@@ -8319,7 +8363,7 @@ function writeTranscript(opts) {
     if (e?.type === "assistant")
       for (const b of content) {
         if (b.type === "tool_use")
-          out.push(describeCall(++step, String(b.name), b.input ?? {}));
+          out.push(describeToolUse(++step, b));
         else if (b.type === "text" && typeof b.text === "string") {
           lastText = out.length;
           out.push(prose(truncate(b.text, 30, TEXT_CHARS)));
@@ -8396,23 +8440,38 @@ function ramp(r) {
   const end = r.state === "pass" ? "▲" : r.state === "fail" ? "╳" : r.state === "error" ? "╌" : r.state === "running" ? "●" : "";
   return (blocks + end).padEnd(8);
 }
-function usage(r) {
+function usageParts(r) {
   const parts = [];
-  if (r.cost !== null && r.cost !== undefined)
+  if (typeof r.cost === "number")
     parts.push(`$${r.cost.toFixed(2)}`);
-  parts.push(`${r.turns} turns`);
+  if (typeof r.reportedTurns === "number")
+    parts.push(`${r.reportedTurns} turns`);
   if (r.seconds !== undefined)
     parts.push(`${r.seconds}s`);
-  return parts.join("  ");
+  return parts;
+}
+function usage(r) {
+  return usageParts(r).join("  ");
 }
 function checksPassed(r) {
   const all = r.checks ?? [];
   return `${all.filter((c) => c.code === 0).length}/${all.length} checks`;
 }
-function total(rows) {
-  const cost = rows.reduce((s, r) => s + (r.cost ?? 0), 0);
-  const turns = rows.reduce((s, r) => s + r.turns, 0);
-  return { cost, turns };
+function totalCost(rows) {
+  const done = rows.filter((r) => r.state !== "queued" && r.state !== "running");
+  const unknown = done.filter((r) => typeof r.cost !== "number").length;
+  if (unknown > 0)
+    return `cost unknown for ${unknown} run${unknown === 1 ? "" : "s"}`;
+  return `$${done.reduce((s, r) => s + r.cost, 0).toFixed(2)}`;
+}
+function totalTurns(rows) {
+  let sum = 0;
+  for (const r of rows) {
+    if (typeof r.reportedTurns !== "number")
+      return null;
+    sum += r.reportedTurns;
+  }
+  return sum;
 }
 function duration(seconds) {
   return seconds >= 60 ? `${Math.floor(seconds / 60)}m ${seconds % 60}s` : `${seconds}s`;
@@ -8454,9 +8513,9 @@ function plainUI(write) {
           continue;
         printed.add(r);
         const label = r.state === "error" ? "ERROR" : r.state.toUpperCase();
-        const u = `$${(r.cost ?? 0).toFixed(2)}, ${r.turns} turns, ${r.seconds ?? 0}s`;
+        const u = usageParts(r).join(", ");
         const model = r.model === null ? "" : ` [${r.model}]`;
-        write(`${label} ${r.test}#${r.n}${model} ${r.reason} (${u})`);
+        write(`${label} ${r.test}#${r.n}${model} ${r.reason}${u ? ` (${u})` : ""}`);
       }
       for (const r of rows) {
         if (uploadPrinted.has(r) || !r.upload || r.upload === "uploading")
@@ -8467,10 +8526,9 @@ function plainUI(write) {
     },
     done: ({ runsDir, rows: all }) => {
       const c = counts(all);
-      const t = total(all);
       const tr = tier(all);
       write("");
-      write(`${c.pass} passed, ${c.fail} failed, ${c.error} not graded (${all.length} runs, $${t.cost.toFixed(2)})${tr ? `, tier ${tr}` : ""}`);
+      write(`${c.pass} passed, ${c.fail} failed, ${c.error} not graded (${all.length} runs, ${totalCost(all)})${tr ? `, tier ${tr}` : ""}`);
       write(`Runs: ${runsDir} (end_goal is for you to judge: read <test>/<n>/transcript.txt)`);
     }
   };
@@ -8563,12 +8621,11 @@ function ttyUI(out, opts) {
         out.push(p(C.amber, `  │        ${cut(r.upload.status === "failed" ? `↑ Upload failed: ${r.upload.error}` : `↑ Not uploaded: ${r.upload.reason}`, w - 12)}`));
     });
     out.push(rule);
-    const t = total(rows);
     if (!finished) {
       const track = Math.max(10, w - 34);
       const at = rows.length ? Math.round(doneCount / rows.length * (track - 1)) : 0;
       const progress = `${"·".repeat(at)}▶${"·".repeat(Math.max(0, track - at - 1))}`;
-      out.push(`  ${p(C.ember, progress)}  ${p(C.gray, `${doneCount}/${rows.length} · $${t.cost.toFixed(2)} so far`)}`);
+      out.push(`  ${p(C.ember, progress)}  ${p(C.gray, `${doneCount}/${rows.length} · ${totalCost(rows)} so far`)}`);
       return out;
     }
     const tr = tier(rows);
@@ -8577,7 +8634,8 @@ function ttyUI(out, opts) {
     const left = goated ? `${rows.length}/${rows.length} runs passed their checks` : tally;
     const right = goated ? `GOATED · tier ${tr}` : tr ? `Tier ${tr}` : "";
     out.push(`  ${goated ? p(C.green, left) : left}${" ".repeat(Math.max(2, w - left.length - right.length - 3))}${goated ? p(C.ember, right) : right}`);
-    out.push(p(C.gray, `  Total $${t.cost.toFixed(2)} · ${t.turns} turns · ${duration(finished.seconds)}`));
+    const turns = totalTurns(rows);
+    out.push(p(C.gray, `  Total ${totalCost(rows)}${turns === null ? "" : ` · ${turns} turns`} · ${duration(finished.seconds)}`));
     out.push(p(C.gray, `  Runs  ${shortPath(finished.runsDir)}/`));
     out.push(`  Checks are a score, not the verdict: judge each end_goal from ${p(C.cyan, "<test>/<n>/transcript.txt")}`);
     return out;
@@ -8644,6 +8702,12 @@ function label(v, what) {
     throw new Error(`${what} is ${v.length} characters, the upload takes at most ${MAX_LABEL}`);
   return v;
 }
+function projectName(v) {
+  const name = label(v, "Project name");
+  if (name === null)
+    throw new Error("Project name is empty");
+  return name;
+}
 function runModel(r) {
   return {
     requested: label(r.model, "Model"),
@@ -8709,9 +8773,11 @@ async function uploadRun(t, r) {
       };
     bytes[name] = b;
   }
+  let project;
   let model;
   let git;
   try {
+    project = projectName(t.project);
     model = runModel(r);
     git = t.git ? runGit(t.git) : undefined;
   } catch (e) {
@@ -8720,7 +8786,7 @@ async function uploadRun(t, r) {
   const body = {
     sessionId,
     runGroup: t.runGroup,
-    project: t.project,
+    project,
     timestamp: r.startedAt,
     run: {
       test: r.test,
@@ -8814,7 +8880,7 @@ function parseArgs(args) {
     json: false,
     noUpload: false,
     vars: {},
-    claudeBin: process.env.FIREGOAT_CLAUDE_BIN || "claude"
+    claudeBin: "claude"
   };
   for (let i = 0;i < args.length; i++) {
     const a = args[i];
@@ -8930,12 +8996,20 @@ async function runOne(opts) {
   const workdir = join6(dir, "work");
   mkdirSync4(workdir, { recursive: true });
   const { home, envFile } = makeRunHome(dir);
+  const fill = {
+    "run.id": opts.runId,
+    "run.n": String(n),
+    "test.name": test.name,
+    FIREGOAT_OUT: opts.out,
+    FIREGOAT_WORKDIR: workdir,
+    FIREGOAT_HOME: home
+  };
   const r = {
     test: test.name,
     n,
     model: opts.model,
     claudeModel: null,
-    claudeArgs: test.claudeArgs,
+    claudeArgs: test.claudeArgs.map((a, i) => renderValue(a, opts.vars, fill, `claude_args[${i}]`)),
     status: "error",
     reason: "",
     prompt: render(test.prompt, opts.vars),
@@ -8956,14 +9030,7 @@ async function runOne(opts) {
     startedAt: new Date(started).toISOString(),
     upload: null
   };
-  const env = renderEnv(test.env, opts.vars, {
-    "run.id": opts.runId,
-    "run.n": String(n),
-    "test.name": test.name,
-    FIREGOAT_OUT: opts.out,
-    FIREGOAT_WORKDIR: workdir,
-    FIREGOAT_HOME: home
-  });
+  const env = renderEnv(test.env, opts.vars, fill);
   const stepEnv = {
     ...opts.stepEnv,
     ...env,
@@ -9008,7 +9075,10 @@ async function attempt(opts, r, env, stepEnv, homeEnvFile) {
     cwd: workdir,
     env: {
       ...claudeEnv(opts.configDir, env, homeEnvFile),
-      PATH: opts.stepEnv.PATH
+      PATH: opts.stepEnv.PATH,
+      FIREGOAT_OUT: opts.out,
+      FIREGOAT_WORKDIR: workdir,
+      FIREGOAT_HOME: r.home
     },
     stderr,
     timeoutMs: test.timeout * 1000,
@@ -9064,6 +9134,20 @@ async function attempt(opts, r, env, stepEnv, homeEnvFile) {
     test.checks.length === 0 ? "no script checks" : `${r.checks.length}/${r.checks.length} checks passed`
   ];
 }
+var MAX_LISTED_URLS = 5;
+function uploadedSummary(origin, results, runDir) {
+  const urls = results.flatMap((r) => r.upload?.status === "uploaded" ? [r.upload.url] : []);
+  if (urls.length === 0)
+    return;
+  const head = `Uploaded ${urls.length} session${urls.length === 1 ? "" : "s"} to ${new URL(origin).host} (private)`;
+  if (urls.length > MAX_LISTED_URLS) {
+    console.log(`${head}: each run's upload.url in ${join6(runDir, "summary.json")}`);
+    return;
+  }
+  console.log(`${head}:`);
+  for (const u of urls)
+    console.log(`  ${u}`);
+}
 async function evalCommand(args, root) {
   if (args.includes("--help") || args.includes("-h")) {
     console.log(EVAL_HELP);
@@ -9116,14 +9200,15 @@ async function evalCommand(args, root) {
     ...new Set(tests.flatMap((t) => [
       t.prompt,
       ...t.checks.map((c) => c.run),
-      ...Object.values(t.env)
+      ...Object.values(t.env),
+      ...t.claudeArgs
     ]).flatMap(varRefs).filter((k) => !(k in vars)))
   ];
   if (missing.length > 0)
     return fatal(EXIT.usage, `No value for ${missing.map((k) => `{{vars.${k}}}`).join(", ")}: pass --var ${missing[0]}=... or set it under vars: in firegoat.yaml`);
   const claudePath = resolveBin(opts.claudeBin);
   if (!claudePath)
-    return fatal(EXIT.notGraded, `Not graded: claude binary ${JSON.stringify(opts.claudeBin)} not found. Install Claude Code (https://claude.com/claude-code) or pass --claude-bin PATH / FIREGOAT_CLAUDE_BIN.`);
+    return fatal(EXIT.notGraded, `Not graded: claude binary ${JSON.stringify(opts.claudeBin)} not found. Install Claude Code (https://claude.com/claude-code) or pass --claude-bin PATH.`);
   let repo;
   try {
     repo = gitRepo(root);
@@ -9145,7 +9230,7 @@ async function evalCommand(args, root) {
   const runId = newRunId();
   let upload;
   try {
-    const pre = await uploadPreflight(opts, ui, basename3(repo?.mainCheckout ?? repo?.toplevel ?? root).slice(0, 256), runId, repo?.info ?? null);
+    const pre = await uploadPreflight(opts, ui, basename3(repo?.mainCheckout ?? repo?.toplevel ?? root), runId, repo?.info ?? null);
     if (pre && "exitCode" in pre)
       return fatal(pre.exitCode, pre.error);
     upload = pre;
@@ -9228,7 +9313,7 @@ async function evalCommand(args, root) {
     results[i] = r;
     Object.assign(row, {
       state: r.status,
-      turns: r.num_turns ?? row.turns,
+      reportedTurns: r.num_turns,
       reason: r.reason,
       checks: r.checks,
       cost: r.cost,
@@ -9276,7 +9361,7 @@ async function evalCommand(args, root) {
     passed: totals.pass,
     failed: totals.fail,
     notGraded: totals.error,
-    cost: results.reduce((sum, r) => sum + (r.cost ?? 0), 0),
+    cost: results.every((r) => r.cost !== null) ? results.reduce((sum, r) => sum + r.cost, 0) : null,
     uploaded: results.filter((r) => r.upload?.status === "uploaded").length,
     uploadFailed: uploadsFailed
   })}
@@ -9287,6 +9372,8 @@ async function evalCommand(args, root) {
     rows,
     seconds: Math.round((Date.now() - evalStarted) / 1000)
   });
+  if (upload && !opts.json)
+    uploadedSummary(upload.origin, results, runDir);
   for (const r of results)
     for (const c of r.cleanup)
       if (c.code !== 0)

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.19.0",
+	"version": "0.21.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/firegoat-ai/firegoat-cli.git"