firegoat 0.19.0 → 0.20.0

package/README.md

@@ -156,7 +156,20 @@ tests:
     claude_args: [--chrome]                    # Claude in Chrome, this test only
 ```
 
-`claude_args:` (top level, and per test after the top-level ones) is passed to claude after firegoat's own flags and `--model`. Each entry is one argument, as-is, with no shell splitting: `[--plugin-dir, ./plugins/acme]`, not `["--plugin-dir ./plugins/acme"]`. Use it for `--chrome`, `--plugin-dir`, `--mcp-config`, `--agents`, `--append-system-prompt`. The flags firegoat sets itself are config errors, also as `--flag=value`: `-p`, `--print`, `--output-format`, `--verbose`, `--setting-sources`, `--strict-mcp-config`, `--disable-slash-commands`, `--dangerously-skip-permissions`, `--model` (use `model:`), and `--session-id`, `--resume`, `-r`, `--continue`, `-c` (every run is a fresh session). Each run's `run.json` records `claudeArgs`.
+`claude_args:` (top level, and per test after the top-level ones) is passed to claude after firegoat's own flags and `--model`. Each entry is one argument, as-is, with no shell splitting: `[--plugin-dir, ./plugins/acme]`, not `["--plugin-dir ./plugins/acme"]`. Use it for `--chrome`, `--plugin-dir`, `--mcp-config`, `--agents`, `--append-system-prompt`. The flags firegoat sets itself are config errors, also as `--flag=value`: `-p`, `--print`, `--output-format`, `--verbose`, `--setting-sources`, `--strict-mcp-config`, `--disable-slash-commands`, `--dangerously-skip-permissions`, `--model` (use `model:`), and `--session-id`, `--resume`, `-r`, `--continue`, `-c` (every run is a fresh session). Entries expand like `env:` values: `$FIREGOAT_WORKDIR`, `$FIREGOAT_HOME`, `$FIREGOAT_OUT` (also `${...}`), `{{vars.KEY}}`, `{{run.id}}`, `{{run.n}}`, `{{test.name}}`; no other `$` expansion. Each run's `run.json` records the expanded `claudeArgs`.
+
+### Testing an MCP server
+
+```yaml
+install: |
+  cat > .mcp.json <<EOF
+  {"mcpServers": {"acme": {"command": "npx", "args": ["-y", "acme-mcp"],
+    "env": {"HOME": "$FIREGOAT_HOME"}}}}
+  EOF
+claude_args: [--mcp-config, .mcp.json]
+```
+
+Write the MCP config per run in `install:` (or copy a committed one to `$FIREGOAT_OUT` in `build:` and pass `"$FIREGOAT_OUT/mcp.json"`, but then it can't hold the run's home), and pass it with `claude_args: [--mcp-config, .mcp.json]`: relative paths are from the run's folder, claude's cwd. `--strict-mcp-config` is always on, so only the configs you pass load. MCP servers that claude starts inherit claude's own env, which keeps your real `HOME` (claude needs it for its login): set `"env": {"HOME": "<the run's $FIREGOAT_HOME>"}` on the server, as above, or it writes to your real home.
 
 With `--chrome`, claude points the Claude in Chrome extension's native host (`~/Library/Application Support/<browser>/NativeMessagingHosts/com.anthropic.claude_code_browser_extension.json`) at `.firegoat/.claude`. If you delete `.firegoat/`, run a normal `claude --chrome` once to point it back.
 

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.19.0",
+  version: "0.20.0",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -7211,6 +7211,7 @@ function parseClaudeArgs(raw, path) {
       for (const f of flags)
         if (arg === f || arg.startsWith(`${f}=`))
           fail(p, `${f} is not allowed: ${why}`);
+    checkPlaceholders(arg, p);
     return arg;
   });
 }
@@ -7266,15 +7267,18 @@ function parseEnv(raw, path) {
       fail(p, "is set by firegoat and cannot be overridden (reserved: PATH, CLAUDE_CONFIG_DIR, CLAUDE_SECURESTORAGE_CONFIG_DIR, FIREGOAT_*)");
     if (typeof v !== "string")
       fail(p, "must be a string (quote numbers and booleans)");
-    for (const m of v.matchAll(ANY_PLACEHOLDER)) {
-      const name = m[1];
-      if (!/^vars\.[A-Za-z0-9_]+$/.test(name) && !RUN_PLACEHOLDERS.includes(name))
-        fail(p, `unknown placeholder ${m[0]} (allowed: {{vars.KEY}}, ${RUN_PLACEHOLDERS.map((n) => `{{${n}}}`).join(", ")})`);
-    }
+    checkPlaceholders(v, p);
     out[k] = v;
   }
   return out;
 }
+function checkPlaceholders(v, p) {
+  for (const m of v.matchAll(ANY_PLACEHOLDER)) {
+    const name = m[1];
+    if (!/^vars\.[A-Za-z0-9_]+$/.test(name) && !RUN_PLACEHOLDERS.includes(name))
+      fail(p, `unknown placeholder ${m[0]} (allowed: {{vars.KEY}}, ${RUN_PLACEHOLDERS.map((n) => `{{${n}}}`).join(", ")})`);
+  }
+}
 function isEvalLevel(value) {
   const dirsOk = [...value.matchAll(DIR_REF)].every((m) => (m[1] ?? m[2]) === "FIREGOAT_OUT");
   return dirsOk && [...value.matchAll(ANY_PLACEHOLDER)].every((m) => {
@@ -7282,16 +7286,19 @@ function isEvalLevel(value) {
     return name.startsWith("vars.") || EVAL_PLACEHOLDERS.includes(name);
   });
 }
+function renderValue(value, vars, run, where) {
+  return value.replace(ENV_REF, (whole, name, braced, bare) => {
+    const key = name ?? braced ?? bare;
+    const filled = key.startsWith("vars.") ? vars[key.slice("vars.".length)] : run[key];
+    if (filled === undefined)
+      throw new ConfigError(`${where}: no value for ${whole}`);
+    return filled;
+  });
+}
 function renderEnv(env, vars, run) {
   const out = {};
   for (const [k, v] of Object.entries(env))
-    out[k] = v.replace(ENV_REF, (whole, name, braced, bare) => {
-      const key = name ?? braced ?? bare;
-      const filled = key.startsWith("vars.") ? vars[key.slice("vars.".length)] : run[key];
-      if (filled === undefined)
-        throw new ConfigError(`env.${k}: no value for ${whole}`);
-      return filled;
-    });
+    out[k] = renderValue(v, vars, run, `env.${k}`);
   return out;
 }
 var PLACEHOLDER = /\{\{\s*vars\.([A-Za-z0-9_]+)\s*\}\}/g;
@@ -7957,7 +7964,10 @@ firegoat.yaml reference:
                               as-is (no shell splitting): [--chrome] (Claude in
                               Chrome, for tests that need a browser),
                               [--plugin-dir, ./plugins/x], --mcp-config, --agents,
-                              --append-system-prompt. Flags firegoat sets are
+                              --append-system-prompt. Entries expand like env:
+                              values ($FIREGOAT_WORKDIR, $FIREGOAT_HOME,
+                              $FIREGOAT_OUT, {{vars.KEY}}, {{run.*}}); run.json
+                              claudeArgs has the result. Flags firegoat sets are
                               errors, also as --flag=value: -p, --print,
                               --output-format, --verbose, --setting-sources,
                               --strict-mcp-config, --disable-slash-commands,
@@ -8000,6 +8010,18 @@ firegoat.yaml reference:
   every env firegoat starts, unless env: sets them. So is every FIREGOAT_* var
   (FIREGOAT_API_KEY above all) firegoat itself was started with.
 
+Testing an MCP server: write its config per run in install: and pass it with
+claude_args (relative paths are from the run's folder, claude's cwd). Only the
+configs you pass load: --strict-mcp-config is always on. An MCP server inherits
+claude's env, which keeps your real HOME (claude's login is there), so set the
+server's HOME to the run's home, or it writes to yours:
+  install: |
+    cat > .mcp.json <<EOF
+    {"mcpServers": {"acme": {"command": "npx", "args": ["-y", "acme-mcp"],
+      "env": {"HOME": "$FIREGOAT_HOME"}}}}
+    EOF
+  claude_args: [--mcp-config, .mcp.json]
+
 Example:
   build: npm pack --pack-destination "$FIREGOAT_OUT"
   install: npm init -y >/dev/null && npm install "$FIREGOAT_OUT"/*.tgz
@@ -8930,12 +8952,20 @@ async function runOne(opts) {
   const workdir = join6(dir, "work");
   mkdirSync4(workdir, { recursive: true });
   const { home, envFile } = makeRunHome(dir);
+  const fill = {
+    "run.id": opts.runId,
+    "run.n": String(n),
+    "test.name": test.name,
+    FIREGOAT_OUT: opts.out,
+    FIREGOAT_WORKDIR: workdir,
+    FIREGOAT_HOME: home
+  };
   const r = {
     test: test.name,
     n,
     model: opts.model,
     claudeModel: null,
-    claudeArgs: test.claudeArgs,
+    claudeArgs: test.claudeArgs.map((a, i) => renderValue(a, opts.vars, fill, `claude_args[${i}]`)),
     status: "error",
     reason: "",
     prompt: render(test.prompt, opts.vars),
@@ -8956,14 +8986,7 @@ async function runOne(opts) {
     startedAt: new Date(started).toISOString(),
     upload: null
   };
-  const env = renderEnv(test.env, opts.vars, {
-    "run.id": opts.runId,
-    "run.n": String(n),
-    "test.name": test.name,
-    FIREGOAT_OUT: opts.out,
-    FIREGOAT_WORKDIR: workdir,
-    FIREGOAT_HOME: home
-  });
+  const env = renderEnv(test.env, opts.vars, fill);
   const stepEnv = {
     ...opts.stepEnv,
     ...env,
@@ -9064,6 +9087,20 @@ async function attempt(opts, r, env, stepEnv, homeEnvFile) {
     test.checks.length === 0 ? "no script checks" : `${r.checks.length}/${r.checks.length} checks passed`
   ];
 }
+var MAX_LISTED_URLS = 5;
+function uploadedSummary(origin, results, runDir) {
+  const urls = results.flatMap((r) => r.upload?.status === "uploaded" ? [r.upload.url] : []);
+  if (urls.length === 0)
+    return;
+  const head = `Uploaded ${urls.length} session${urls.length === 1 ? "" : "s"} to ${new URL(origin).host} (private)`;
+  if (urls.length > MAX_LISTED_URLS) {
+    console.log(`${head}: each run's upload.url in ${join6(runDir, "summary.json")}`);
+    return;
+  }
+  console.log(`${head}:`);
+  for (const u of urls)
+    console.log(`  ${u}`);
+}
 async function evalCommand(args, root) {
   if (args.includes("--help") || args.includes("-h")) {
     console.log(EVAL_HELP);
@@ -9116,7 +9153,8 @@ async function evalCommand(args, root) {
     ...new Set(tests.flatMap((t) => [
       t.prompt,
       ...t.checks.map((c) => c.run),
-      ...Object.values(t.env)
+      ...Object.values(t.env),
+      ...t.claudeArgs
     ]).flatMap(varRefs).filter((k) => !(k in vars)))
   ];
   if (missing.length > 0)
@@ -9287,6 +9325,8 @@ async function evalCommand(args, root) {
     rows,
     seconds: Math.round((Date.now() - evalStarted) / 1000)
   });
+  if (upload && !opts.json)
+    uploadedSummary(upload.origin, results, runDir);
   for (const r of results)
     for (const c of r.cleanup)
       if (c.code !== 0)

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.19.0",
+	"version": "0.20.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/firegoat-ai/firegoat-cli.git"