firegoat 0.13.0 → 0.14.0

package/README.md

@@ -88,6 +88,7 @@ npx firegoat eval --var SHA=$(git rev-parse HEAD)
 env:
   ACME_HOME: "/tmp/acme-{{run.id}}-{{run.n}}"   # sessions/config never collide across runs
   ACME_TELEMETRY: "off"
+  UV_FIND_LINKS: "$FIREGOAT_OUT"                # resolve the wheel setup built
 tests:
   - name: login
     prompt: Log in to acme with the CLI and print the current user.
@@ -98,12 +99,13 @@ tests:
       - run: ls "$ACME_HOME"/sessions | grep -q .
 ```
 
-`env:` goes to the agent, and to `workdir_setup` and checks of the same run, so checks see the same values. Placeholders: `{{vars.KEY}}`, `{{run.id}}` (the eval's run id), `{{run.n}}` (1..runs), `{{run.workdir}}`, `{{test.name}}`. `setup` runs once, before any run: it gets only the top-level entries that use nothing but `{{vars.*}}` and `{{run.id}}`. Values are strings (quote numbers). `PATH`, `CLAUDE_CONFIG_DIR` and `FIREGOAT_*` are firegoat's and can't be set.
+`env:` goes to the agent, and to `workdir_setup` and checks of the same run, so checks see the same values. Placeholders: `{{vars.KEY}}`, `{{run.id}}` (the eval's run id), `{{run.n}}` (1..runs), `{{test.name}}`, and `$FIREGOAT_OUT` / `$FIREGOAT_WORKDIR` (also `${...}`), the same dirs `setup`, `workdir_setup` and checks see. There is no other `$` expansion: `$HOME` stays the literal text `$HOME`. `setup` runs once, before any run: it gets only the top-level entries that use nothing but `{{vars.*}}`, `{{run.id}}` and `$FIREGOAT_OUT`. Values are strings (quote numbers). `PATH`, `CLAUDE_CONFIG_DIR` and `FIREGOAT_*` are firegoat's and can't be set.
 
 ## What eval does
 
 - claude runs on its own login (or `ANTHROPIC_API_KEY`, if set) with `CLAUDE_CONFIG_DIR=.firegoat/.claude` and `--setting-sources "" --strict-mcp-config --disable-slash-commands`: none of your settings, CLAUDE.md, plugins, skills or MCP servers. claude saves its session there, in its own format.
 - No sandbox: the agent's folder is inside `.firegoat/runs/`, and it can reach the rest of your machine. In `.firegoat/`, git ignores `runs/` and `.claude/`; keep test fixtures (scripts, sample data) in `.firegoat/fixtures/` and commit them with `firegoat.yaml`.
+- In a linked git worktree (`git worktree add`), `.firegoat/runs/` and `.firegoat/.claude/` are in the main checkout, so runs (and their session files) survive `git worktree remove`; `firegoat.yaml`, fixtures and `setup:` stay in the worktree. Each `summary.json` records `dir` (where eval ran) and `git: {branch, commit, dirty}`, and `.firegoat/runs/index.jsonl` gets one line per eval (`runId, startedAt, dir, branch, commit, passed, failed, notGraded, cost`) to tell runs from different branches apart.
 - The agent gets your environment plus `env:`, never `vars:`. Vars that npm/npx set for firegoat itself (`npm_config_*`, `npm_lifecycle_*`, `npm_package_*`, `npm_execpath`, `npm_node_execpath`, `npm_command`, `INIT_CWD`) are removed from every env firegoat starts (agent, setup, workdir_setup, checks): `npx -y firegoat` sets `npm_config_yes=true`, which would make the agent's `npm init` / `npx create-*` skip their prompts. Set one under `env:` to keep it.
 - Runs (tests × `runs`) go 4 at a time, each in its own folder with its own session; `--parallel N` changes that, `--parallel 1` runs them one after another. `summary.json` lists them in `firegoat.yaml` order.
 - Script checks run in the agent's folder afterwards and set the exit code. `end_goal` is not graded by firegoat: it's for you, or your coding agent, to judge from `run.json`, the session and the folder.

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.13.0",
+  version: "0.14.0",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -7160,9 +7160,11 @@ var RESERVED_ENV = new Set([
   "CLAUDE_CONFIG_DIR",
   "CLAUDE_SECURESTORAGE_CONFIG_DIR"
 ]);
-var RUN_PLACEHOLDERS = ["run.id", "run.n", "run.workdir", "test.name"];
+var RUN_PLACEHOLDERS = ["run.id", "run.n", "test.name"];
 var EVAL_PLACEHOLDERS = ["run.id"];
 var ANY_PLACEHOLDER = /\{\{\s*([^{}]*?)\s*\}\}/g;
+var DIR_REF = /\$(?:\{(FIREGOAT_OUT|FIREGOAT_WORKDIR)\}|(FIREGOAT_OUT|FIREGOAT_WORKDIR)(?![A-Za-z0-9_]))/g;
+var ENV_REF = new RegExp(`${ANY_PLACEHOLDER.source}|${DIR_REF.source}`, "g");
 function parseEnv(raw, path) {
   if (raw === undefined)
     return {};
@@ -7187,7 +7189,8 @@ function parseEnv(raw, path) {
   return out;
 }
 function isEvalLevel(value) {
-  return [...value.matchAll(ANY_PLACEHOLDER)].every((m) => {
+  const dirsOk = [...value.matchAll(DIR_REF)].every((m) => (m[1] ?? m[2]) === "FIREGOAT_OUT");
+  return dirsOk && [...value.matchAll(ANY_PLACEHOLDER)].every((m) => {
     const name = m[1];
     return name.startsWith("vars.") || EVAL_PLACEHOLDERS.includes(name);
   });
@@ -7195,8 +7198,9 @@ function isEvalLevel(value) {
 function renderEnv(env, vars, run) {
   const out = {};
   for (const [k, v] of Object.entries(env))
-    out[k] = v.replace(ANY_PLACEHOLDER, (whole, name) => {
-      const filled = name.startsWith("vars.") ? vars[name.slice("vars.".length)] : run[name];
+    out[k] = v.replace(ENV_REF, (whole, name, braced, bare) => {
+      const key = name ?? braced ?? bare;
+      const filled = key.startsWith("vars.") ? vars[key.slice("vars.".length)] : run[key];
       if (filled === undefined)
         throw new ConfigError(`env.${k}: no value for ${whole}`);
       return filled;
@@ -7225,13 +7229,14 @@ function loadConfig(dir) {
 // src/eval.ts
 import { randomBytes } from "node:crypto";
 import {
+  appendFileSync,
   createWriteStream as createWriteStream2,
   existsSync as existsSync2,
   mkdirSync as mkdirSync2,
   readdirSync as readdirSync2,
   writeFileSync as writeFileSync3
 } from "node:fs";
-import { basename, delimiter as delimiter2, join as join4 } from "node:path";
+import { basename as basename2, delimiter as delimiter2, join as join4 } from "node:path";
 
 // src/claude.ts
 import { spawn, spawnSync } from "node:child_process";
@@ -7425,6 +7430,51 @@ function ensureFiregoatDir(root) {
   return dir;
 }
 
+// src/git.ts
+import { spawnSync as spawnSync2 } from "node:child_process";
+import { basename, dirname } from "node:path";
+function git(dir, args) {
+  const r = spawnSync2("git", args, { cwd: dir, encoding: "utf8" });
+  if (r.error)
+    throw r.error;
+  return { code: r.status ?? -1, stdout: r.stdout.trim() };
+}
+function gitRepo(dir) {
+  const r = spawnSync2("git", ["rev-parse", "--path-format=absolute", "--git-dir", "--git-common-dir"], { cwd: dir, encoding: "utf8" });
+  if (r.error?.code === "ENOENT")
+    return null;
+  if (r.error)
+    throw r.error;
+  if (r.status !== 0) {
+    if (/not a git repository/.test(r.stderr))
+      return null;
+    throw new Error(`git rev-parse in ${dir} failed: ${r.stderr.trim()}`);
+  }
+  const [gitDir, commonDir] = r.stdout.trim().split(`
+`);
+  if (!gitDir || !commonDir)
+    throw new Error(`git rev-parse in ${dir} printed ${r.stdout}`);
+  let mainCheckout = null;
+  if (gitDir !== commonDir) {
+    if (basename(commonDir) !== ".git")
+      throw new Error(`${dir} is a linked git worktree, but its common git dir ${commonDir} is not a checkout's .git (a bare repo?), so there is no main checkout to keep .firegoat/runs in`);
+    mainCheckout = dirname(commonDir);
+  }
+  const branch = git(dir, ["symbolic-ref", "-q", "--short", "HEAD"]);
+  const commit = git(dir, ["rev-parse", "-q", "--verify", "HEAD"]);
+  const status = git(dir, ["status", "--porcelain"]);
+  if (status.code !== 0)
+    throw new Error(`git status in ${dir} failed`);
+  return {
+    info: {
+      branch: branch.code === 0 ? branch.stdout : null,
+      commit: commit.code === 0 ? commit.stdout : null,
+      dirty: status.stdout !== ""
+    },
+    mainCheckout
+  };
+}
+
 // src/help.ts
 var EXIT_CODES = `Exit codes:
   0  every run's script checks passed
@@ -7472,8 +7522,10 @@ firegoat.yaml reference:
   env: {NAME: value}          env vars for the agent, and for workdir_setup and
                               checks of the same run; values are strings and can
                               use {{vars.KEY}}, {{run.id}}, {{run.n}},
-                              {{run.workdir}}, {{test.name}}. setup gets only the
-                              entries that use nothing but {{vars.*}}/{{run.id}}.
+                              {{test.name}}, $FIREGOAT_OUT, $FIREGOAT_WORKDIR
+                              (no other $ expansion: $HOME stays literal).
+                              setup gets only the entries that use nothing but
+                              {{vars.*}}, {{run.id}} and $FIREGOAT_OUT.
                               PATH, CLAUDE_CONFIG_DIR and FIREGOAT_* are reserved.
   tests:
     - name: quickstart        letters, digits, '.', '_', '-'; unique
@@ -7526,6 +7578,10 @@ with a fresh config dir, .firegoat/.claude: none of your settings, CLAUDE.md,
 plugins, skills or MCP servers. claude saves its session there. There is no
 sandbox: the agent can reach the rest of your machine, including the repo.
 
+In a linked git worktree (git worktree add), .firegoat/runs/ and .firegoat/.claude
+live in the main checkout instead, so runs outlive the worktree; firegoat.yaml,
+fixtures and setup: stay in the current folder.
+
   test..             only these tests (by name)
   --runs N           runs per test (overrides runs: in firegoat.yaml)
   --parallel N       runs at once (default 4; 1 runs them one after another)
@@ -7535,7 +7591,10 @@ sandbox: the agent can reach the rest of your machine, including the repo.
   --claude-bin PATH  claude binary (env FIREGOAT_CLAUDE_BIN; default: claude on PATH)
 
 Output: .firegoat/runs/<runId>/
-  summary.json            every run, plus totals
+  summary.json            every run, plus totals, dir (where eval ran) and
+                          git {branch, commit, dirty} (no git outside a repo)
+  ../index.jsonl          one line per eval: runId, startedAt, dir, branch,
+                          commit, passed, failed, notGraded, cost
   <test>/<n>/run.json     one run: status and reason (from the checks), prompt,
                           end_goal, checks [{run, code}], and paths: workdir,
                           transcript, session (claude's session file), stderr,
@@ -7692,10 +7751,14 @@ ${indent(truncate(text, OUTPUT_LINES, OUTPUT_CHARS))}` : `    → ${status}`);
 }
 
 // src/ui.ts
-import { relative as relative2 } from "node:path";
+import { isAbsolute, relative as relative2 } from "node:path";
 var RAMP = "▁▂▃▄▅▆▇";
 var ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, "g");
 var SPIN = ["◐", "◓", "◑", "◒"];
+function shortPath(path) {
+  const rel = relative2(process.cwd(), path);
+  return rel.startsWith("..") || isAbsolute(rel) ? path : rel;
+}
 function painter(color) {
   return (code, text) => color ? `\x1B[${code}m${text}\x1B[0m` : text;
 }
@@ -7891,7 +7954,7 @@ function ttyUI(out, opts) {
     const right = goated ? `GOATED · tier ${tr}` : tr ? `tier ${tr}` : "";
     out.push(`  ${goated ? p(C.green, left) : left}${" ".repeat(Math.max(2, w - left.length - right.length - 3))}${goated ? p(C.ember, right) : right}`);
     out.push(p(C.gray, `  total $${t.cost.toFixed(2)} · ${t.turns} turns · ${duration(finished.seconds)}`));
-    out.push(p(C.gray, `  runs  ${relative2(process.cwd(), finished.runsDir)}/`));
+    out.push(p(C.gray, `  runs  ${shortPath(finished.runsDir)}/`));
     out.push(`  checks are a score, not the verdict: judge each end_goal from ${p(C.cyan, "<test>/<n>/transcript.txt")}`);
     return out;
   };
@@ -8061,8 +8124,9 @@ async function runOne(opts) {
   const env = renderEnv(test.env, opts.vars, {
     "run.id": opts.runId,
     "run.n": String(n),
-    "run.workdir": workdir,
-    "test.name": test.name
+    "test.name": test.name,
+    FIREGOAT_OUT: opts.out,
+    FIREGOAT_WORKDIR: workdir
   });
   const stepEnv = { ...opts.stepEnv, ...env, FIREGOAT_WORKDIR: workdir };
   if (opts.workdirSetup.length > 0) {
@@ -8146,7 +8210,7 @@ async function evalCommand(args, root) {
   }
   const say = opts.json ? console.error : console.log;
   const stream = opts.json ? process.stderr : process.stdout;
-  const ui = stream.isTTY ? ttyUI(stream, { project: basename(root), color: !process.env.NO_COLOR }) : plainUI(say);
+  const ui = stream.isTTY ? ttyUI(stream, { project: basename2(root), color: !process.env.NO_COLOR }) : plainUI(say);
   let config;
   try {
     config = loadConfig(root);
@@ -8177,7 +8241,13 @@ async function evalCommand(args, root) {
   const claudePath = resolveBin(opts.claudeBin);
   if (!claudePath)
     return fatal(EXIT.notGraded, `not graded: claude binary ${JSON.stringify(opts.claudeBin)} not found. Install Claude Code (https://claude.com/claude-code) or pass --claude-bin PATH / FIREGOAT_CLAUDE_BIN.`);
-  const firegoatDir = ensureFiregoatDir(root);
+  let repo;
+  try {
+    repo = gitRepo(root);
+  } catch (e) {
+    return fatal(EXIT.usage, e.message);
+  }
+  const firegoatDir = ensureFiregoatDir(repo?.mainCheckout ?? root);
   const configDir = join4(firegoatDir, ".claude");
   mkdirSync2(configDir, { recursive: true });
   ui.auth(process.env.ANTHROPIC_API_KEY ? "ANTHROPIC_API_KEY (claude bills the API key; unset it to use your claude login)" : "your claude login");
@@ -8190,7 +8260,8 @@ async function evalCommand(args, root) {
     return fatal(EXIT.notGraded, `not graded: ${loginProblem}`);
   const evalStarted = Date.now();
   const runId = newRunId();
-  const runDir = join4(firegoatDir, "runs", runId);
+  const runsRoot = join4(firegoatDir, "runs");
+  const runDir = join4(runsRoot, runId);
   const out = join4(runDir, "out");
   mkdirSync2(out, { recursive: true });
   const stepEnv = {
@@ -8214,7 +8285,7 @@ async function evalCommand(args, root) {
       cwd: root,
       env: {
         ...stepEnv,
-        ...renderEnv(Object.fromEntries(Object.entries(config.env).filter(([, v]) => isEvalLevel(v))), vars, { "run.id": runId })
+        ...renderEnv(Object.fromEntries(Object.entries(config.env).filter(([, v]) => isEvalLevel(v))), vars, { "run.id": runId, FIREGOAT_OUT: out })
       },
       logPath: join4(runDir, "setup.txt")
     });
@@ -8252,6 +8323,7 @@ async function evalCommand(args, root) {
       claudeBin: claudePath,
       configDir,
       workdirSetup: config.workdirSetup,
+      out,
       stepEnv,
       vars
     });
@@ -8276,8 +8348,27 @@ async function evalCommand(args, root) {
   const totals = { pass: 0, fail: 0, error: 0 };
   for (const r of results)
     totals[r.status]++;
-  const doc = { runId, runsDir: runDir, totals, runs: results };
+  const doc = {
+    runId,
+    runsDir: runDir,
+    dir: root,
+    ...repo && { git: repo.info },
+    totals,
+    runs: results
+  };
   writeFileSync3(join4(runDir, "summary.json"), `${JSON.stringify(doc, null, 2)}
+`);
+  appendFileSync(join4(runsRoot, "index.jsonl"), `${JSON.stringify({
+    runId,
+    startedAt: new Date(evalStarted).toISOString(),
+    dir: root,
+    branch: repo?.info.branch ?? null,
+    commit: repo?.info.commit ?? null,
+    passed: totals.pass,
+    failed: totals.fail,
+    notGraded: totals.error,
+    cost: results.reduce((sum, r) => sum + (r.cost ?? 0), 0)
+  })}
 `);
   ui.done({
     runId,
@@ -8466,9 +8557,9 @@ firegoat ${package_default.version}` : `firegoat ${package_default.version}`,
 }
 
 // src/init.ts
-import { spawnSync as spawnSync2 } from "node:child_process";
+import { spawnSync as spawnSync3 } from "node:child_process";
 import { existsSync as existsSync3, writeFileSync as writeFileSync4 } from "node:fs";
-import { basename as basename2, join as join5 } from "node:path";
+import { basename as basename3, join as join5 } from "node:path";
 import { createInterface } from "node:readline/promises";
 var SKILLS_ARGV = [
   "skills",
@@ -8514,9 +8605,11 @@ runs: 1
 
 # Optional: env vars for the agent (and for workdir_setup and checks of the same
 # run). The agent gets none of vars:, only these. Placeholders: {{vars.KEY}},
-# {{run.id}}, {{run.n}}, {{run.workdir}}, {{test.name}}. A test can add its own env:.
+# {{run.id}}, {{run.n}}, {{test.name}}, and $FIREGOAT_OUT / $FIREGOAT_WORKDIR (no
+# other $ expansion: $HOME stays literal). A test can add its own env:.
 # env:
 #   MYTOOL_HOME: "/tmp/mytool-{{run.id}}-{{run.n}}"
+#   UV_FIND_LINKS: "$FIREGOAT_OUT"
 
 tests:
   - name: quickstart
@@ -8545,7 +8638,7 @@ var CLIPBOARD = [
 ];
 function copyToClipboard(text) {
   return CLIPBOARD.some(([cmd, args]) => {
-    const r = spawnSync2(cmd, args, { input: text });
+    const r = spawnSync3(cmd, args, { input: text });
     return !r.error && r.status === 0;
   });
 }
@@ -8592,7 +8685,7 @@ ${INIT_HELP}`);
   const configPath = join5(cwd, CONFIG_FILE);
   const keep = existsSync3(configPath) && !force;
   if (!keep)
-    writeFileSync4(configPath, template(basename2(cwd)));
+    writeFileSync4(configPath, template(basename3(cwd)));
   if (interactive) {
     step(1, CONFIG_FILE, keep ? "kept (--force replaces it)" : "created");
     console.log(paint("90", "  │"));
@@ -8609,7 +8702,7 @@ ${INIT_HELP}`);
       console.log(`installing skills: npx ${SKILLS_ARGV.join(" ")}`);
     const env = { ...process.env };
     delete env.npm_config_package;
-    const r = spawnSync2("npx", ["-y", ...SKILLS_ARGV], {
+    const r = spawnSync3("npx", ["-y", ...SKILLS_ARGV], {
       cwd,
       env,
       stdio: "pipe",
@@ -8624,7 +8717,7 @@ ${INIT_HELP}`);
       step(3, "firegoat-onboarding skill", "installed → .claude/skills/");
     else
       console.log("installed skills firegoat-onboarding, suggesting-eval-prompts in .claude/skills/");
-    const ignored = spawnSync2("git", ["check-ignore", "-q", ".claude/skills"], {
+    const ignored = spawnSync3("git", ["check-ignore", "-q", ".claude/skills"], {
       cwd
     });
     if (ignored.status === 0)

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.13.0",
+	"version": "0.14.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/firegoat-ai/firegoat-cli.git"