firegoat 0.21.0 → 0.22.0

package/README.md

@@ -12,7 +12,7 @@ npx firegoat@latest init     # Creates firegoat.yaml and .firegoat/, installs th
 npx firegoat@latest eval     # Runs your claude on each test, in .firegoat/runs/<runId>/<test>/<n>/work
 ```
 
-That's it: two commands. Use `npx firegoat@latest`: plain `npx firegoat` can keep running an old cached version. `eval` prints its version first (`firegoat X.Y.Z`), and `summary.json` records it as `firegoatVersion`. `claude` must be installed and logged in (run `claude` once).
+That's it: two commands. `npx firegoat@latest eval --dry-run` checks `firegoat.yaml` and lists the runs it would start (`<test>#<n> [model]`, with step counts) without building, running claude or uploading; config errors exit 3 as in a real eval. Use `npx firegoat@latest`: plain `npx firegoat` can keep running an old cached version. `eval` prints its version first (`firegoat X.Y.Z`), and `summary.json` records it as `firegoatVersion`. `claude` must be installed and logged in (run `claude` once).
 
 `init` installs the firegoat-onboarding skill globally, into `~/.claude/skills/` (in a terminal it asks: global, this repo, or none). `--local-skills` puts it in this repo's `.claude/skills/` instead (commit it so teammates and CI get it); `--no-skills` skips it.
 
@@ -224,7 +224,17 @@ Limits:
 - The agent gets your environment plus `env:`, never `vars:`. Vars that npm/npx set for firegoat itself (`npm_config_*`, `npm_lifecycle_*`, `npm_package_*`, `npm_execpath`, `npm_node_execpath`, `npm_command`, `INIT_CWD`) are removed from every env firegoat starts (agent, build, install, checks): `npx -y firegoat` sets `npm_config_yes=true`, which would make the agent's `npm init` / `npx create-*` skip their prompts. Set one under `env:` to keep it. Every `FIREGOAT_*` var firegoat was started with (`FIREGOAT_API_KEY` above all) is removed too (claude then gets the run's own `FIREGOAT_OUT`, `FIREGOAT_WORKDIR` and `FIREGOAT_HOME`, for MCP configs; the agent's commands see them too), and so are the vars a parent Claude Code session sets (when firegoat runs from claude, or its agent runs firegoat), so the claude under test doesn't act as a nested session: `CLAUDECODE`, `CLAUDE_CODE_*`, `CLAUDE_PID`, `CLAUDE_ENV_FILE`, `CLAUDE_SESSION_ID`, `CLAUDE_PROJECT_DIR`, `CLAUDE_EFFORT`, `CLAUDE_AGENT_SDK_*`, `CLAUDE_PLUGIN_ROOT`, `CLAUDE_PLUGIN_DATA`, `CLAUDE_SKILL_DIR`. Login and provider vars stay: `ANTHROPIC_*`, `CLAUDE_CODE_OAUTH_TOKEN`, `CLAUDE_CODE_OAUTH_REFRESH_TOKEN`, `CLAUDE_CODE_USE_*` (Bedrock, Vertex, ...), `CLAUDE_CODE_SKIP_*_AUTH`, `CLAUDE_CODE_CLIENT_CERT`, `CLAUDE_CODE_CLIENT_KEY`, `CLAUDE_CODE_CLIENT_KEY_PASSPHRASE`.
 - Runs (tests × models × `runs`) go 4 at a time, each in its own folder with its own session and HOME; `--parallel N` changes that, `--parallel 1` runs them one after another. `summary.json` lists them in `firegoat.yaml` order.
 - Script checks run in the agent's folder afterwards and set the exit code. `end_goal` is not graded by firegoat: it's for you, or your coding agent, to judge from `run.json`, the session and the folder.
-- Output per run, in `.firegoat/runs/<runId>/<test>/<n>/`: `transcript.txt` (each tool call with its command and short output, the final answer, files made; checks get it as `$FIREGOAT_TRANSCRIPT`; greppable line by line: every line of a shell call starts with `[n] $ `, every line of the agent's text and final answer with `agent: `, output is indented, so `grep -E '^\[[0-9]+\] \$ .*npm install'` matches only commands, also inside heredocs and multi-line scripts), `run.json` (status, prompt, end_goal, check results, paths), `work/`, `home/` and `home.env` (the run's HOME), logs. Read the transcript to judge `end_goal`; claude's full session file is ~50x larger. `summary.json` covers the whole eval; `--json` prints it.
+- Output per run, in `.firegoat/runs/<runId>/<test>/<n>/`: `transcript.txt` (each tool call with its command and short output, the final answer, files made; checks get it as `$FIREGOAT_TRANSCRIPT`; format below), `run.json` (status, prompt, end_goal, check results, paths), `work/`, `home/` and `home.env` (the run's HOME), logs. Read the transcript to judge `end_goal`; claude's full session file is ~50x larger. `summary.json` covers the whole eval; `--json` prints it.
+
+### The transcript
+
+`transcript.txt` is greppable line by line; `n` counts the agent's tool calls:
+
+- `[n] $ <command>`: a shell call, on every line of the command (heredocs and multi-line scripts too), never cut. So `grep -E '^\[[0-9]+\] \$ .*npm install'` matches only commands.
+- `[n] <Tool> <target>`: a tool (MCP ones too) whose input has a `file_path`, `path`, `url`, `pattern` or `query`: just that value, e.g. `[2] Write src/app.ts`. The written body is not in the transcript: check the file in the workdir.
+- `[n] <Tool> <json>`: any other tool, MCP tools included, with its whole input as one line of JSON, cut at 4000 chars: `[3] mcp__acme__push_html {"canvas":"demo","html":"<p>Hi</p>"}`. So `grep -Eq '^\[[0-9]+\] mcp__acme__push_html .*<h1>' "$FIREGOAT_TRANSCRIPT"` checks what the agent pushed.
+- `    → ok` or `    → error`, then the tool's output indented 4 spaces, cut at 15 lines or 1500 chars (`… (N more chars)`).
+- `agent: ` on every line of the agent's text (cut at 30 lines or 1500 chars) and of its final answer (uncut), then the files in the workdir (without `node_modules`, `.git`, `.venv`, `__pycache__`).
 
 `firegoat.yaml` keys: `runs`, `model`, `build`, `install`, `cleanup`, `vars`, `env`, `claude_args`, and `tests` (`name`, `prompt`, `end_goal`, `checks`, `timeout`, `env`, `install`, `cleanup`, `claude_args`). Unknown keys are errors. Full reference: `npx firegoat init --help`.
 

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.21.0",
+  version: "0.22.0",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -8010,11 +8010,9 @@ firegoat.yaml reference:
   vars, plus $FIREGOAT_OUT and $FIREGOAT_WORKDIR; install, checks and cleanup also
   get $FIREGOAT_HOME, the run's own home, as HOME (build keeps your real HOME).
   Checks also get $FIREGOAT_TRANSCRIPT, a short plain-text
-  transcript, greppable line by line: "[1] $ <command>" on every line of a shell
-  call (one line per line of the command, same n), "[2] Write <path>" for other
-  tools, each output indented, and "agent: " on every line of the agent's text and
-  final answer. So '^\\[[0-9]+\\] \\$ .*foo' matches only commands. Checks also
-  get $FIREGOAT_SESSION (claude's full session file, JSON lines).
+  transcript, greppable line by line (format: eval --help, Transcript). So
+  '^\\[[0-9]+\\] \\$ .*foo' matches only commands. Checks also get
+  $FIREGOAT_SESSION (claude's full session file, JSON lines).
 
   The agent never gets vars, only env:. The vars npm/npx set for firegoat itself
   (npm_config_*, npm_lifecycle_*, npm_package_*, npm_execpath, npm_node_execpath,
@@ -8098,7 +8096,8 @@ page.
 Exit codes: 0 logged out (or was not logged in), 3 the file or ~/.firegoat is a
 symlink, not a regular file/directory, or someone else's (left alone).`;
 var EVAL_HELP = `Usage: firegoat eval [test..] [--runs N] [--model NAME].. [--parallel N]
-                     [--var KEY=value].. [--json] [--no-upload] [--claude-bin PATH]
+                     [--var KEY=value].. [--json] [--no-upload] [--dry-run]
+                     [--claude-bin PATH]
 
 Runs each test in firegoat.yaml (in the current folder): your claude gets the prompt
 in a fresh, empty folder, .firegoat/runs/<runId>/<test>/<n>/work, with build:'s
@@ -8152,6 +8151,14 @@ fixtures and build: stay in the current folder.
   --json             The summary as JSON on stdout (or {error, exitCode} if eval
                      stops early); progress goes to stderr
   --no-upload        Keep runs local even when logged in to firegoat.ai
+  --dry-run          Check firegoat.yaml as eval would (test names, --var,
+                     --model, missing vars), print each planned run
+                     ("<test>#<n> [model]" with its install, checks and cleanup
+                     counts and rendered claude_args, paths under
+                     runs/<runId>/) and exit 0. Builds nothing, starts no
+                     claude, uploads nothing, writes no files. Config errors
+                     exit 3. With --json: {dryRun, build, runs: [{test, n,
+                     model, claudeArgs, install, checks, cleanup}]}
   --claude-bin PATH  Path to claude (default: claude on PATH)
 
 Upload: after firegoat login (or with FIREGOAT_API_KEY), each run's claude session,
@@ -8181,14 +8188,25 @@ Output: .firegoat/runs/<runId>/
                           checksLog, cleanupLog; startedAt; upload ({status: "uploaded",
                           url} | {status: "failed", error} | {status:
                           "skipped", reason}, null when not uploading)
-  <test>/<n>/transcript.txt  What the agent did: each tool call with its command
-                          ("[n] $ " on every line of a shell call), short output
-                          (indented), its text and final answer ("agent: " on
-                          every line), files it made
+  <test>/<n>/transcript.txt  What the agent did, below
   <test>/<n>/home/, home.env  The run's HOME, and the file claude sources
                           before the agent's Bash commands
   build.txt, <test>/<n>/install.txt, checks.txt, cleanup.txt   Logs
 
+Transcript ($FIREGOAT_TRANSCRIPT), greppable line by line; n counts tool calls:
+  [n] $ <command>         A shell call, on every line of the command, uncut
+  [n] <Tool> <target>     A tool (MCP too) with a file_path, path, url, pattern
+                          or query input: that value, e.g. "[2] Write src/a.ts"
+  [n] <Tool> <json>       Any other tool, MCP ones included: its input as one
+                          line of JSON, e.g. [3] mcp__<server>__<tool> {"q":"x"};
+                          cut at 4000 chars
+      → ok | → error      Then the tool's output, indented 4 spaces, cut at 15
+                          lines or 1500 chars ("… (N more chars)")
+  agent: <text>           Every line of the agent's text (cut at 30 lines or
+                          1500 chars) and of its final answer (uncut)
+  Then the files in the workdir (not node_modules, .git, .venv, __pycache__).
+  A written file's body is not in the transcript: check the file in the workdir.
+
 Grading: firegoat runs only the script checks; they set the exit code. end_goal is
 yours to judge: read transcript.txt first (a few KB); the session file has everything
 (often 50x larger).
@@ -8199,6 +8217,7 @@ Examples:
   npx firegoat eval --model claude-haiku-5-5 --model claude-sonnet-5-5
                                             Each test on both models
   npx firegoat eval --var SHA=$(git rev-parse HEAD)
+  npx firegoat eval --dry-run               Validate firegoat.yaml, list the runs
 
 A run can take minutes (default timeout 600s per test).
 
@@ -8295,6 +8314,7 @@ import { join as join4, relative } from "node:path";
 var OUTPUT_LINES = 15;
 var OUTPUT_CHARS = 1500;
 var TEXT_CHARS = 1500;
+var INPUT_CHARS = 4000;
 var SKIP_DIRS = new Set(["node_modules", ".git", ".venv", "__pycache__"]);
 function truncate(text, maxLines, maxChars) {
   let out = text.length > maxChars ? text.slice(0, maxChars) : text;
@@ -8319,7 +8339,7 @@ function describeCall(step, name, input) {
   for (const key of ["file_path", "path", "url", "pattern", "query"])
     if (typeof input[key] === "string")
       return `[${step}] ${name} ${input[key]}`;
-  return `[${step}] ${name} ${truncate(JSON.stringify(input), 1, 300)}`;
+  return `[${step}] ${name} ${truncate(JSON.stringify(input), 1, INPUT_CHARS)}`;
 }
 function describeToolUse(step, b) {
   if (typeof b.name !== "string" || b.name === "")
@@ -8879,6 +8899,7 @@ function parseArgs(args) {
     parallel: 4,
     json: false,
     noUpload: false,
+    dryRun: false,
     vars: {},
     claudeBin: "claude"
   };
@@ -8899,6 +8920,8 @@ function parseArgs(args) {
       opts.json = true;
     else if (flag === "--no-upload")
       opts.noUpload = true;
+    else if (flag === "--dry-run")
+      opts.dryRun = true;
     else if (flag === "--var") {
       const kv = value();
       const at = kv.indexOf("=");
@@ -8989,6 +9012,51 @@ async function uploadPreflight(opts, ui, project, runGroup, git) {
     };
   }
 }
+function runFill(runId, n, testName, out, dir) {
+  return {
+    "run.id": runId,
+    "run.n": String(n),
+    "test.name": testName,
+    FIREGOAT_OUT: out,
+    FIREGOAT_WORKDIR: join6(dir, "work"),
+    FIREGOAT_HOME: join6(dir, "home")
+  };
+}
+function renderClaudeArgs(test, vars, fill) {
+  return test.claudeArgs.map((a, i) => renderValue(a, vars, fill, `claude_args[${i}]`));
+}
+function planRuns(tests, models, runs) {
+  const matrix = models.length > 0 ? models : [null];
+  return tests.flatMap((t) => matrix.flatMap((model, m) => Array.from({ length: runs }, (_, i) => ({
+    test: t.name,
+    n: m * runs + i + 1,
+    model
+  }))));
+}
+var DRY_RUN_ID = "<runId>";
+function dryRun(opts) {
+  const out = join6(opts.runsRoot, DRY_RUN_ID, "out");
+  const runs = opts.plan.map((p) => {
+    const test = opts.tests.find((t) => t.name === p.test);
+    const fill = runFill(DRY_RUN_ID, p.n, test.name, out, join6(opts.runsRoot, DRY_RUN_ID, test.name, String(p.n)));
+    renderEnv(test.env, opts.vars, fill);
+    return {
+      ...p,
+      claudeArgs: renderClaudeArgs(test, opts.vars, fill),
+      install: [...opts.config.install, ...test.install],
+      checks: test.checks.map((c) => render(c.run, opts.vars)),
+      cleanup: [...test.cleanup, ...opts.config.cleanup]
+    };
+  });
+  if (opts.json) {
+    console.log(JSON.stringify({ dryRun: true, build: opts.config.build, runs }, null, 2));
+    return;
+  }
+  const count = (n, what) => `${n} ${what}${n === 1 ? "" : "s"}`;
+  opts.say(`Dry run: ${count(runs.length, "run")}, build ${count(opts.config.build.length, "step")}. Nothing was started.`);
+  for (const r of runs)
+    opts.say(`${r.test}#${r.n}${r.model === null ? "" : ` [${r.model}]`} install ${r.install.length}, checks ${r.checks.length}, cleanup ${r.cleanup.length}${r.claudeArgs.length > 0 ? `, claude_args ${JSON.stringify(r.claudeArgs)}` : ""}`);
+}
 async function runOne(opts) {
   const { test, n } = opts;
   const started = Date.now();
@@ -8996,20 +9064,13 @@ async function runOne(opts) {
   const workdir = join6(dir, "work");
   mkdirSync4(workdir, { recursive: true });
   const { home, envFile } = makeRunHome(dir);
-  const fill = {
-    "run.id": opts.runId,
-    "run.n": String(n),
-    "test.name": test.name,
-    FIREGOAT_OUT: opts.out,
-    FIREGOAT_WORKDIR: workdir,
-    FIREGOAT_HOME: home
-  };
+  const fill = runFill(opts.runId, n, test.name, opts.out, dir);
   const r = {
     test: test.name,
     n,
     model: opts.model,
     claudeModel: null,
-    claudeArgs: test.claudeArgs.map((a, i) => renderValue(a, opts.vars, fill, `claude_args[${i}]`)),
+    claudeArgs: renderClaudeArgs(test, opts.vars, fill),
     status: "error",
     reason: "",
     prompt: render(test.prompt, opts.vars),
@@ -9206,6 +9267,30 @@ async function evalCommand(args, root) {
   ];
   if (missing.length > 0)
     return fatal(EXIT.usage, `No value for ${missing.map((k) => `{{vars.${k}}}`).join(", ")}: pass --var ${missing[0]}=... or set it under vars: in firegoat.yaml`);
+  if (opts.dryRun) {
+    let repo;
+    try {
+      repo = gitRepo(root);
+    } catch (e) {
+      return fatal(EXIT.usage, e.message);
+    }
+    try {
+      dryRun({
+        config,
+        tests,
+        plan: planRuns(tests, models, runs),
+        vars,
+        runsRoot: join6(repo?.mainCheckout ?? root, ".firegoat", "runs"),
+        json: opts.json,
+        say
+      });
+    } catch (e) {
+      if (e instanceof ConfigError)
+        return fatal(EXIT.usage, e.message);
+      throw e;
+    }
+    return EXIT.pass;
+  }
   const claudePath = resolveBin(opts.claudeBin);
   if (!claudePath)
     return fatal(EXIT.notGraded, `Not graded: claude binary ${JSON.stringify(opts.claudeBin)} not found. Install Claude Code (https://claude.com/claude-code) or pass --claude-bin PATH.`);
@@ -9249,14 +9334,11 @@ async function evalCommand(args, root) {
     FIREGOAT_OUT: out,
     PATH: [out, process.env.PATH ?? ""].join(delimiter2)
   };
-  const matrix = models.length > 0 ? models : [null];
-  const rows = tests.flatMap((t) => matrix.flatMap((model, m) => Array.from({ length: runs }, (_, i) => ({
-    test: t.name,
-    n: m * runs + i + 1,
-    model,
+  const rows = planRuns(tests, models, runs).map((p) => ({
+    ...p,
     state: "queued",
     turns: 0
-  }))));
+  }));
   if (config.build.length > 0) {
     const buildStarted = Date.now();
     ui.build("running");
@@ -9664,7 +9746,8 @@ var TIMEOUT_MS2 = 30000;
 var MAX_EXPIRES_S = 900;
 var MAX_INTERVAL_S = 60;
 var FORM = { "Content-Type": "application/x-www-form-urlencoded" };
-var UPLOAD_NOTICE = "Each eval uploads every agent session (prompts, tool outputs, file contents, possibly secrets from your env or repo) to firegoat.ai as private traces; --no-upload keeps runs local. The agent runs as you and can read ~/.firegoat (and a FIREGOAT_API_KEY in firegoat's own process env).";
+var UPLOADS_ON = "Eval uploads each session to firegoat.ai as a private trace; --no-upload keeps runs local.";
+var UPLOAD_NOTICE = `${UPLOADS_ON} A session holds prompts, tool outputs and file contents: possibly secrets from your env or repo. The agent runs as you and can read ~/.firegoat (and a FIREGOAT_API_KEY in firegoat's own process env).`;
 var TERMS = "By logging in, you agree to the firegoat.ai Terms of Service (https://firegoat.ai/terms) and Privacy Policy (https://firegoat.ai/privacy).";
 function str2(o, k, step) {
   const v = o?.[k];
@@ -9967,13 +10050,16 @@ async function askScope(question) {
 var LOGIN_HINT = "Runs stay local: npx firegoat login uploads them to firegoat.ai";
 async function loginStep(interactive, done, paint) {
   const label = "firegoat.ai login";
+  const uploadsOn = () => console.log(`      ${paint("90", UPLOADS_ON)}`);
   const key = resolveKey(baseUrl());
   if (key) {
     const who = key.source === "env" ? "FIREGOAT_API_KEY is set" : `Logged in as ${key.email}`;
-    if (interactive)
+    if (interactive) {
       done(label, who);
-    else
-      console.log(who);
+      uploadsOn();
+    } else
+      console.log(`${who}
+${UPLOADS_ON}`);
     return EXIT.pass;
   }
   if (!interactive) {
@@ -9989,6 +10075,7 @@ async function loginStep(interactive, done, paint) {
   try {
     const email = await loginFlow((l) => console.log(`  ${l}`), true);
     done(label, `Logged in as ${email}`);
+    uploadsOn();
     return EXIT.pass;
   } catch (e) {
     if (!(e instanceof ApiError))

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.21.0",
+	"version": "0.22.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/firegoat-ai/firegoat-cli.git"