firegoat 0.20.0 → 0.21.0

package/README.md

@@ -12,7 +12,7 @@ npx firegoat@latest init     # Creates firegoat.yaml and .firegoat/, installs th
 npx firegoat@latest eval     # Runs your claude on each test, in .firegoat/runs/<runId>/<test>/<n>/work
 ```
 
-That's it: two commands. Use `npx firegoat@latest`: plain `npx firegoat` can keep running an old cached version. `eval` prints its version first (`firegoat 0.18.0`), and `summary.json` records it as `firegoatVersion`. `claude` must be installed and logged in (run `claude` once).
+That's it: two commands. Use `npx firegoat@latest`: plain `npx firegoat` can keep running an old cached version. `eval` prints its version first (`firegoat X.Y.Z`), and `summary.json` records it as `firegoatVersion`. `claude` must be installed and logged in (run `claude` once).
 
 `init` installs the firegoat-onboarding skill globally, into `~/.claude/skills/` (in a terminal it asks: global, this repo, or none). `--local-skills` puts it in this repo's `.claude/skills/` instead (commit it so teammates and CI get it); `--no-skills` skips it.
 
@@ -160,16 +160,24 @@ tests:
 
 ### Testing an MCP server
 
+Commit the server's config as `fixtures/mcp.json`:
+
+```json
+{"mcpServers": {"acme": {"command": "./node_modules/.bin/acme-mcp",
+  "env": {"HOME": "${FIREGOAT_HOME}"}}}}
+```
+
+Copy it in `build:` and pass it with `claude_args:`:
+
 ```yaml
-install: |
-  cat > .mcp.json <<EOF
-  {"mcpServers": {"acme": {"command": "npx", "args": ["-y", "acme-mcp"],
-    "env": {"HOME": "$FIREGOAT_HOME"}}}}
-  EOF
-claude_args: [--mcp-config, .mcp.json]
+build:
+  - npm pack --pack-destination "$FIREGOAT_OUT"
+  - cp -R fixtures/. "$FIREGOAT_OUT"
+install: npm init -y >/dev/null && npm install "$FIREGOAT_OUT"/*.tgz
+claude_args: [--mcp-config, $FIREGOAT_OUT/mcp.json]
 ```
 
-Write the MCP config per run in `install:` (or copy a committed one to `$FIREGOAT_OUT` in `build:` and pass `"$FIREGOAT_OUT/mcp.json"`, but then it can't hold the run's home), and pass it with `claude_args: [--mcp-config, .mcp.json]`: relative paths are from the run's folder, claude's cwd. `--strict-mcp-config` is always on, so only the configs you pass load. MCP servers that claude starts inherit claude's own env, which keeps your real `HOME` (claude needs it for its login): set `"env": {"HOME": "<the run's $FIREGOAT_HOME>"}` on the server, as above, or it writes to your real home.
+claude gets `FIREGOAT_HOME`, `FIREGOAT_WORKDIR` and `FIREGOAT_OUT` in its own env, set per run, and expands `${VAR}` in an MCP config's `command`, `args`, `env`, `url` and `headers` from its env (an unset var stays the literal text). Relative paths in the config are from the run's folder, claude's cwd: `./node_modules/.bin/acme-mcp` is the build under test, installed by `install:`. MCP servers that claude starts inherit claude's own env, which keeps your real `HOME` (claude needs it for its login): set `"env": {"HOME": "${FIREGOAT_HOME}"}` on the server, as above, or it writes to your real home. Per-run state the server keeps elsewhere goes the same way, e.g. `"ACME_DB": "${FIREGOAT_WORKDIR}/acme.db"`. `--strict-mcp-config` is always on, so only the configs you pass load. To check that the agent used a tool, grep the transcript: `grep -Eq '^\[[0-9]+\] mcp__acme__search' "$FIREGOAT_TRANSCRIPT"`.
 
 With `--chrome`, claude points the Claude in Chrome extension's native host (`~/Library/Application Support/<browser>/NativeMessagingHosts/com.anthropic.claude_code_browser_extension.json`) at `.firegoat/.claude`. If you delete `.firegoat/`, run a normal `claude --chrome` once to point it back.
 
@@ -211,9 +219,9 @@ Limits:
 ## What eval does
 
 - claude runs on its own login (or `ANTHROPIC_API_KEY`, if set) with `CLAUDE_CONFIG_DIR=.firegoat/.claude` and `--setting-sources "" --strict-mcp-config --disable-slash-commands`: none of your settings, CLAUDE.md, plugins, skills or MCP servers (add flags with `claude_args:`, see above). claude saves its session there, in its own format.
-- No sandbox: the agent's folder is inside `.firegoat/runs/`, and it can reach the rest of your machine. `.firegoat/` holds local run output and ignores itself in git. Keep test fixtures (scripts, sample data) outside it, e.g. in `fixtures/` next to `firegoat.yaml`, and commit them.
+- No sandbox: the agent's folder is inside `.firegoat/runs/`, and it can reach the rest of your machine. `.firegoat/` holds local run output and ignores itself in git. Keep test fixtures (scripts, sample data) in `fixtures/` next to `firegoat.yaml`, committed.
 - In a linked git worktree (`git worktree add`), `.firegoat/runs/` and `.firegoat/.claude/` are in the main checkout, so runs (and their session files) survive `git worktree remove`; `firegoat.yaml`, fixtures and `build:` stay in the worktree. Each `summary.json` records `dir` (where eval ran) and `git: {branch, commit, dirty}`, and `.firegoat/runs/index.jsonl` gets one line per eval (`runId, startedAt, dir, branch, commit, passed, failed, notGraded, cost, uploaded, uploadFailed`) to tell runs from different branches apart.
-- The agent gets your environment plus `env:`, never `vars:`. Vars that npm/npx set for firegoat itself (`npm_config_*`, `npm_lifecycle_*`, `npm_package_*`, `npm_execpath`, `npm_node_execpath`, `npm_command`, `INIT_CWD`) are removed from every env firegoat starts (agent, build, install, checks): `npx -y firegoat` sets `npm_config_yes=true`, which would make the agent's `npm init` / `npx create-*` skip their prompts. Set one under `env:` to keep it. Every `FIREGOAT_*` var firegoat was started with (`FIREGOAT_API_KEY` above all) is removed too, and so are the vars a parent Claude Code session sets (when firegoat runs from claude, or its agent runs firegoat), so the claude under test doesn't act as a nested session: `CLAUDECODE`, `CLAUDE_CODE_*`, `CLAUDE_PID`, `CLAUDE_ENV_FILE`, `CLAUDE_SESSION_ID`, `CLAUDE_PROJECT_DIR`, `CLAUDE_EFFORT`, `CLAUDE_AGENT_SDK_*`, `CLAUDE_PLUGIN_ROOT`, `CLAUDE_PLUGIN_DATA`, `CLAUDE_SKILL_DIR`. Login and provider vars stay: `ANTHROPIC_*`, `CLAUDE_CODE_OAUTH_TOKEN`, `CLAUDE_CODE_OAUTH_REFRESH_TOKEN`, `CLAUDE_CODE_USE_*` (Bedrock, Vertex, ...), `CLAUDE_CODE_SKIP_*_AUTH`, `CLAUDE_CODE_CLIENT_CERT`, `CLAUDE_CODE_CLIENT_KEY`, `CLAUDE_CODE_CLIENT_KEY_PASSPHRASE`.
+- The agent gets your environment plus `env:`, never `vars:`. Vars that npm/npx set for firegoat itself (`npm_config_*`, `npm_lifecycle_*`, `npm_package_*`, `npm_execpath`, `npm_node_execpath`, `npm_command`, `INIT_CWD`) are removed from every env firegoat starts (agent, build, install, checks): `npx -y firegoat` sets `npm_config_yes=true`, which would make the agent's `npm init` / `npx create-*` skip their prompts. Set one under `env:` to keep it. Every `FIREGOAT_*` var firegoat was started with (`FIREGOAT_API_KEY` above all) is removed too (claude then gets the run's own `FIREGOAT_OUT`, `FIREGOAT_WORKDIR` and `FIREGOAT_HOME`, for MCP configs; the agent's commands see them too), and so are the vars a parent Claude Code session sets (when firegoat runs from claude, or its agent runs firegoat), so the claude under test doesn't act as a nested session: `CLAUDECODE`, `CLAUDE_CODE_*`, `CLAUDE_PID`, `CLAUDE_ENV_FILE`, `CLAUDE_SESSION_ID`, `CLAUDE_PROJECT_DIR`, `CLAUDE_EFFORT`, `CLAUDE_AGENT_SDK_*`, `CLAUDE_PLUGIN_ROOT`, `CLAUDE_PLUGIN_DATA`, `CLAUDE_SKILL_DIR`. Login and provider vars stay: `ANTHROPIC_*`, `CLAUDE_CODE_OAUTH_TOKEN`, `CLAUDE_CODE_OAUTH_REFRESH_TOKEN`, `CLAUDE_CODE_USE_*` (Bedrock, Vertex, ...), `CLAUDE_CODE_SKIP_*_AUTH`, `CLAUDE_CODE_CLIENT_CERT`, `CLAUDE_CODE_CLIENT_KEY`, `CLAUDE_CODE_CLIENT_KEY_PASSPHRASE`.
 - Runs (tests × models × `runs`) go 4 at a time, each in its own folder with its own session and HOME; `--parallel N` changes that, `--parallel 1` runs them one after another. `summary.json` lists them in `firegoat.yaml` order.
 - Script checks run in the agent's folder afterwards and set the exit code. `end_goal` is not graded by firegoat: it's for you, or your coding agent, to judge from `run.json`, the session and the folder.
 - Output per run, in `.firegoat/runs/<runId>/<test>/<n>/`: `transcript.txt` (each tool call with its command and short output, the final answer, files made; checks get it as `$FIREGOAT_TRANSCRIPT`; greppable line by line: every line of a shell call starts with `[n] $ `, every line of the agent's text and final answer with `agent: `, output is indented, so `grep -E '^\[[0-9]+\] \$ .*npm install'` matches only commands, also inside heredocs and multi-line scripts), `run.json` (status, prompt, end_goal, check results, paths), `work/`, `home/` and `home.env` (the run's HOME), logs. Read the transcript to judge `end_goal`; claude's full session file is ~50x larger. `summary.json` covers the whole eval; `--json` prints it.

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.20.0",
+  version: "0.21.0",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -7448,8 +7448,15 @@ async function runClaude(opts) {
       reason: `claude did not start a session (exit ${exited.code ?? "by signal"})${said ? `: ${said}` : ", see stderr.txt"}`
     };
   }
-  const sessionId = String(init.session_id);
   const claudeModel = typeof init.model === "string" ? init.model : null;
+  if (typeof init.session_id !== "string" || init.session_id === "")
+    return {
+      events,
+      claudeModel,
+      ok: false,
+      reason: "claude's init line has no session_id"
+    };
+  const sessionId = init.session_id;
   const result = events.findLast((e) => e?.type === "result");
   if (exited.timedOut)
     return {
@@ -7646,7 +7653,7 @@ async function send(opts) {
 function replyError(step, r) {
   const j = r.json ?? {};
   const error = typeof j.error === "string" ? j.error : "no error message";
-  const more = j.error_description ?? j.details ?? j.message;
+  const more = j.error_description ?? j.details;
   const details = more === undefined ? "" : ` ${typeof more === "string" ? more : JSON.stringify(more)}`;
   return new ApiError(printable(`${step} ${r.status}: ${error}${details}`), r.status);
 }
@@ -7816,7 +7823,12 @@ var GITIGNORE = `*
 function ensureFiregoatDir(root) {
   const dir = join2(root, ".firegoat");
   mkdirSync2(dir, { recursive: true });
-  writeFileSync(join2(dir, ".gitignore"), GITIGNORE);
+  try {
+    writeFileSync(join2(dir, ".gitignore"), GITIGNORE, { flag: "wx" });
+  } catch (e) {
+    if (e.code !== "EEXIST")
+      throw e;
+  }
   return dir;
 }
 
@@ -7918,8 +7930,8 @@ firegoat-onboarding skill globally, into ~/.claude/skills/ (npx skills add
 firegoat-ai/skills -g; in a terminal it asks: global, this repo, or none), offers
 firegoat login (in a terminal only; otherwise prints how), then prints a prompt to
 paste into your coding agent. .firegoat/ holds local run output and ignores itself
-in git. Keep test fixtures (helper scripts, sample data) outside it, e.g. in
-fixtures/ next to firegoat.yaml, and commit them.
+in git. Keep test fixtures (helper scripts, sample data) in fixtures/ next to
+firegoat.yaml, committed.
 
   --force          Replace an existing firegoat.yaml
   --local-skills   Install the skill into this repo's .claude/skills/ instead
@@ -8008,19 +8020,23 @@ firegoat.yaml reference:
   (npm_config_*, npm_lifecycle_*, npm_package_*, npm_execpath, npm_node_execpath,
   npm_command, INIT_CWD; e.g. npm_config_yes=true from npx -y) are removed from
   every env firegoat starts, unless env: sets them. So is every FIREGOAT_* var
-  (FIREGOAT_API_KEY above all) firegoat itself was started with.
-
-Testing an MCP server: write its config per run in install: and pass it with
-claude_args (relative paths are from the run's folder, claude's cwd). Only the
-configs you pass load: --strict-mcp-config is always on. An MCP server inherits
-claude's env, which keeps your real HOME (claude's login is there), so set the
-server's HOME to the run's home, or it writes to yours:
-  install: |
-    cat > .mcp.json <<EOF
-    {"mcpServers": {"acme": {"command": "npx", "args": ["-y", "acme-mcp"],
-      "env": {"HOME": "$FIREGOAT_HOME"}}}}
-    EOF
-  claude_args: [--mcp-config, .mcp.json]
+  (FIREGOAT_API_KEY above all) firegoat itself was started with. claude itself
+  gets $FIREGOAT_OUT, $FIREGOAT_WORKDIR and $FIREGOAT_HOME (for MCP configs, below),
+  so the agent's commands see them too.
+
+Testing an MCP server: commit its config as fixtures/mcp.json, copy it in build:
+and pass it with claude_args. Only the configs you pass load: --strict-mcp-config
+is always on. An MCP server inherits claude's env, which keeps your real HOME
+(claude's login is there), so set the server's HOME to the run's home, or it
+writes to yours. claude gets $FIREGOAT_HOME, $FIREGOAT_WORKDIR and $FIREGOAT_OUT
+in its env and expands \${VAR} in the config (an unset one stays literal);
+relative paths in it are from the run's folder, claude's cwd:
+  fixtures/mcp.json:
+    {"mcpServers": {"acme": {"command": "./node_modules/.bin/acme-mcp",
+      "env": {"HOME": "\${FIREGOAT_HOME}"}}}}
+  firegoat.yaml:
+    build: cp -R fixtures/. "$FIREGOAT_OUT"
+    claude_args: [--mcp-config, $FIREGOAT_OUT/mcp.json]
 
 Example:
   build: npm pack --pack-destination "$FIREGOAT_OUT"
@@ -8136,8 +8152,7 @@ fixtures and build: stay in the current folder.
   --json             The summary as JSON on stdout (or {error, exitCode} if eval
                      stops early); progress goes to stderr
   --no-upload        Keep runs local even when logged in to firegoat.ai
-  --claude-bin PATH  Path to claude (env FIREGOAT_CLAUDE_BIN; default: claude on
-                     PATH)
+  --claude-bin PATH  Path to claude (default: claude on PATH)
 
 Upload: after firegoat login (or with FIREGOAT_API_KEY), each run's claude session,
 checks.txt and stderr.txt go to firegoat.ai as a private trace, and its URL is
@@ -8154,8 +8169,8 @@ Output: .firegoat/runs/<runId>/
                           git {branch, commit, dirty} (no git outside a repo)
                           and upload {origin, email} (null when not uploading)
   ../index.jsonl          One line per eval: runId, startedAt, dir, branch,
-                          commit, passed, failed, notGraded, cost, uploaded,
-                          uploadFailed
+                          commit, passed, failed, notGraded, cost (null when
+                          any run's cost is unknown), uploaded, uploadFailed
   <test>/<n>/run.json     One run: status and reason (from the checks), model
                           (asked for, or null) and claudeModel (what claude's
                           init line reported), claudeArgs (claude_args
@@ -8306,6 +8321,13 @@ function describeCall(step, name, input) {
       return `[${step}] ${name} ${input[key]}`;
   return `[${step}] ${name} ${truncate(JSON.stringify(input), 1, 300)}`;
 }
+function describeToolUse(step, b) {
+  if (typeof b.name !== "string" || b.name === "")
+    return `[${step}] (tool call without a name)`;
+  if (typeof b.input !== "object" || b.input === null)
+    return `[${step}] ${b.name} (no input)`;
+  return describeCall(step, b.name, b.input);
+}
 function resultText(content) {
   if (typeof content === "string")
     return content;
@@ -8341,7 +8363,7 @@ function writeTranscript(opts) {
     if (e?.type === "assistant")
       for (const b of content) {
         if (b.type === "tool_use")
-          out.push(describeCall(++step, String(b.name), b.input ?? {}));
+          out.push(describeToolUse(++step, b));
         else if (b.type === "text" && typeof b.text === "string") {
           lastText = out.length;
           out.push(prose(truncate(b.text, 30, TEXT_CHARS)));
@@ -8418,23 +8440,38 @@ function ramp(r) {
   const end = r.state === "pass" ? "▲" : r.state === "fail" ? "╳" : r.state === "error" ? "╌" : r.state === "running" ? "●" : "";
   return (blocks + end).padEnd(8);
 }
-function usage(r) {
+function usageParts(r) {
   const parts = [];
-  if (r.cost !== null && r.cost !== undefined)
+  if (typeof r.cost === "number")
     parts.push(`$${r.cost.toFixed(2)}`);
-  parts.push(`${r.turns} turns`);
+  if (typeof r.reportedTurns === "number")
+    parts.push(`${r.reportedTurns} turns`);
   if (r.seconds !== undefined)
     parts.push(`${r.seconds}s`);
-  return parts.join("  ");
+  return parts;
+}
+function usage(r) {
+  return usageParts(r).join("  ");
 }
 function checksPassed(r) {
   const all = r.checks ?? [];
   return `${all.filter((c) => c.code === 0).length}/${all.length} checks`;
 }
-function total(rows) {
-  const cost = rows.reduce((s, r) => s + (r.cost ?? 0), 0);
-  const turns = rows.reduce((s, r) => s + r.turns, 0);
-  return { cost, turns };
+function totalCost(rows) {
+  const done = rows.filter((r) => r.state !== "queued" && r.state !== "running");
+  const unknown = done.filter((r) => typeof r.cost !== "number").length;
+  if (unknown > 0)
+    return `cost unknown for ${unknown} run${unknown === 1 ? "" : "s"}`;
+  return `$${done.reduce((s, r) => s + r.cost, 0).toFixed(2)}`;
+}
+function totalTurns(rows) {
+  let sum = 0;
+  for (const r of rows) {
+    if (typeof r.reportedTurns !== "number")
+      return null;
+    sum += r.reportedTurns;
+  }
+  return sum;
 }
 function duration(seconds) {
   return seconds >= 60 ? `${Math.floor(seconds / 60)}m ${seconds % 60}s` : `${seconds}s`;
@@ -8476,9 +8513,9 @@ function plainUI(write) {
           continue;
         printed.add(r);
         const label = r.state === "error" ? "ERROR" : r.state.toUpperCase();
-        const u = `$${(r.cost ?? 0).toFixed(2)}, ${r.turns} turns, ${r.seconds ?? 0}s`;
+        const u = usageParts(r).join(", ");
         const model = r.model === null ? "" : ` [${r.model}]`;
-        write(`${label} ${r.test}#${r.n}${model} ${r.reason} (${u})`);
+        write(`${label} ${r.test}#${r.n}${model} ${r.reason}${u ? ` (${u})` : ""}`);
       }
       for (const r of rows) {
         if (uploadPrinted.has(r) || !r.upload || r.upload === "uploading")
@@ -8489,10 +8526,9 @@ function plainUI(write) {
     },
     done: ({ runsDir, rows: all }) => {
       const c = counts(all);
-      const t = total(all);
       const tr = tier(all);
       write("");
-      write(`${c.pass} passed, ${c.fail} failed, ${c.error} not graded (${all.length} runs, $${t.cost.toFixed(2)})${tr ? `, tier ${tr}` : ""}`);
+      write(`${c.pass} passed, ${c.fail} failed, ${c.error} not graded (${all.length} runs, ${totalCost(all)})${tr ? `, tier ${tr}` : ""}`);
       write(`Runs: ${runsDir} (end_goal is for you to judge: read <test>/<n>/transcript.txt)`);
     }
   };
@@ -8585,12 +8621,11 @@ function ttyUI(out, opts) {
         out.push(p(C.amber, `  │        ${cut(r.upload.status === "failed" ? `↑ Upload failed: ${r.upload.error}` : `↑ Not uploaded: ${r.upload.reason}`, w - 12)}`));
     });
     out.push(rule);
-    const t = total(rows);
     if (!finished) {
       const track = Math.max(10, w - 34);
       const at = rows.length ? Math.round(doneCount / rows.length * (track - 1)) : 0;
       const progress = `${"·".repeat(at)}▶${"·".repeat(Math.max(0, track - at - 1))}`;
-      out.push(`  ${p(C.ember, progress)}  ${p(C.gray, `${doneCount}/${rows.length} · $${t.cost.toFixed(2)} so far`)}`);
+      out.push(`  ${p(C.ember, progress)}  ${p(C.gray, `${doneCount}/${rows.length} · ${totalCost(rows)} so far`)}`);
       return out;
     }
     const tr = tier(rows);
@@ -8599,7 +8634,8 @@ function ttyUI(out, opts) {
     const left = goated ? `${rows.length}/${rows.length} runs passed their checks` : tally;
     const right = goated ? `GOATED · tier ${tr}` : tr ? `Tier ${tr}` : "";
     out.push(`  ${goated ? p(C.green, left) : left}${" ".repeat(Math.max(2, w - left.length - right.length - 3))}${goated ? p(C.ember, right) : right}`);
-    out.push(p(C.gray, `  Total $${t.cost.toFixed(2)} · ${t.turns} turns · ${duration(finished.seconds)}`));
+    const turns = totalTurns(rows);
+    out.push(p(C.gray, `  Total ${totalCost(rows)}${turns === null ? "" : ` · ${turns} turns`} · ${duration(finished.seconds)}`));
     out.push(p(C.gray, `  Runs  ${shortPath(finished.runsDir)}/`));
     out.push(`  Checks are a score, not the verdict: judge each end_goal from ${p(C.cyan, "<test>/<n>/transcript.txt")}`);
     return out;
@@ -8666,6 +8702,12 @@ function label(v, what) {
     throw new Error(`${what} is ${v.length} characters, the upload takes at most ${MAX_LABEL}`);
   return v;
 }
+function projectName(v) {
+  const name = label(v, "Project name");
+  if (name === null)
+    throw new Error("Project name is empty");
+  return name;
+}
 function runModel(r) {
   return {
     requested: label(r.model, "Model"),
@@ -8731,9 +8773,11 @@ async function uploadRun(t, r) {
       };
     bytes[name] = b;
   }
+  let project;
   let model;
   let git;
   try {
+    project = projectName(t.project);
     model = runModel(r);
     git = t.git ? runGit(t.git) : undefined;
   } catch (e) {
@@ -8742,7 +8786,7 @@ async function uploadRun(t, r) {
   const body = {
     sessionId,
     runGroup: t.runGroup,
-    project: t.project,
+    project,
     timestamp: r.startedAt,
     run: {
       test: r.test,
@@ -8836,7 +8880,7 @@ function parseArgs(args) {
     json: false,
     noUpload: false,
     vars: {},
-    claudeBin: process.env.FIREGOAT_CLAUDE_BIN || "claude"
+    claudeBin: "claude"
   };
   for (let i = 0;i < args.length; i++) {
     const a = args[i];
@@ -9031,7 +9075,10 @@ async function attempt(opts, r, env, stepEnv, homeEnvFile) {
     cwd: workdir,
     env: {
       ...claudeEnv(opts.configDir, env, homeEnvFile),
-      PATH: opts.stepEnv.PATH
+      PATH: opts.stepEnv.PATH,
+      FIREGOAT_OUT: opts.out,
+      FIREGOAT_WORKDIR: workdir,
+      FIREGOAT_HOME: r.home
     },
     stderr,
     timeoutMs: test.timeout * 1000,
@@ -9161,7 +9208,7 @@ async function evalCommand(args, root) {
     return fatal(EXIT.usage, `No value for ${missing.map((k) => `{{vars.${k}}}`).join(", ")}: pass --var ${missing[0]}=... or set it under vars: in firegoat.yaml`);
   const claudePath = resolveBin(opts.claudeBin);
   if (!claudePath)
-    return fatal(EXIT.notGraded, `Not graded: claude binary ${JSON.stringify(opts.claudeBin)} not found. Install Claude Code (https://claude.com/claude-code) or pass --claude-bin PATH / FIREGOAT_CLAUDE_BIN.`);
+    return fatal(EXIT.notGraded, `Not graded: claude binary ${JSON.stringify(opts.claudeBin)} not found. Install Claude Code (https://claude.com/claude-code) or pass --claude-bin PATH.`);
   let repo;
   try {
     repo = gitRepo(root);
@@ -9183,7 +9230,7 @@ async function evalCommand(args, root) {
   const runId = newRunId();
   let upload;
   try {
-    const pre = await uploadPreflight(opts, ui, basename3(repo?.mainCheckout ?? repo?.toplevel ?? root).slice(0, 256), runId, repo?.info ?? null);
+    const pre = await uploadPreflight(opts, ui, basename3(repo?.mainCheckout ?? repo?.toplevel ?? root), runId, repo?.info ?? null);
     if (pre && "exitCode" in pre)
       return fatal(pre.exitCode, pre.error);
     upload = pre;
@@ -9266,7 +9313,7 @@ async function evalCommand(args, root) {
     results[i] = r;
     Object.assign(row, {
       state: r.status,
-      turns: r.num_turns ?? row.turns,
+      reportedTurns: r.num_turns,
       reason: r.reason,
       checks: r.checks,
       cost: r.cost,
@@ -9314,7 +9361,7 @@ async function evalCommand(args, root) {
     passed: totals.pass,
     failed: totals.fail,
     notGraded: totals.error,
-    cost: results.reduce((sum, r) => sum + (r.cost ?? 0), 0),
+    cost: results.every((r) => r.cost !== null) ? results.reduce((sum, r) => sum + r.cost, 0) : null,
     uploaded: results.filter((r) => r.upload?.status === "uploaded").length,
     uploadFailed: uploadsFailed
   })}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.20.0",
+	"version": "0.21.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/firegoat-ai/firegoat-cli.git"