firegoat 0.12.0 → 0.14.0

package/README.md

@@ -88,6 +88,7 @@ npx firegoat eval --var SHA=$(git rev-parse HEAD)
 env:
   ACME_HOME: "/tmp/acme-{{run.id}}-{{run.n}}"   # sessions/config never collide across runs
   ACME_TELEMETRY: "off"
+  UV_FIND_LINKS: "$FIREGOAT_OUT"                # resolve the wheel setup built
 tests:
   - name: login
     prompt: Log in to acme with the CLI and print the current user.
@@ -98,13 +99,15 @@ tests:
       - run: ls "$ACME_HOME"/sessions | grep -q .
 ```
 
-`env:` goes to the agent, and to `workdir_setup` and checks of the same run, so checks see the same values. Placeholders: `{{vars.KEY}}`, `{{run.id}}` (the eval's run id), `{{run.n}}` (1..runs), `{{run.workdir}}`, `{{test.name}}`. `setup` runs once, before any run: it gets only the top-level entries that use nothing but `{{vars.*}}` and `{{run.id}}`. Values are strings (quote numbers). `PATH`, `CLAUDE_CONFIG_DIR` and `FIREGOAT_*` are firegoat's and can't be set.
+`env:` goes to the agent, and to `workdir_setup` and checks of the same run, so checks see the same values. Placeholders: `{{vars.KEY}}`, `{{run.id}}` (the eval's run id), `{{run.n}}` (1..runs), `{{test.name}}`, and `$FIREGOAT_OUT` / `$FIREGOAT_WORKDIR` (also `${...}`), the same dirs `setup`, `workdir_setup` and checks see. There is no other `$` expansion: `$HOME` stays the literal text `$HOME`. `setup` runs once, before any run: it gets only the top-level entries that use nothing but `{{vars.*}}`, `{{run.id}}` and `$FIREGOAT_OUT`. Values are strings (quote numbers). `PATH`, `CLAUDE_CONFIG_DIR` and `FIREGOAT_*` are firegoat's and can't be set.
 
 ## What eval does
 
 - claude runs on its own login (or `ANTHROPIC_API_KEY`, if set) with `CLAUDE_CONFIG_DIR=.firegoat/.claude` and `--setting-sources "" --strict-mcp-config --disable-slash-commands`: none of your settings, CLAUDE.md, plugins, skills or MCP servers. claude saves its session there, in its own format.
 - No sandbox: the agent's folder is inside `.firegoat/runs/`, and it can reach the rest of your machine. In `.firegoat/`, git ignores `runs/` and `.claude/`; keep test fixtures (scripts, sample data) in `.firegoat/fixtures/` and commit them with `firegoat.yaml`.
+- In a linked git worktree (`git worktree add`), `.firegoat/runs/` and `.firegoat/.claude/` are in the main checkout, so runs (and their session files) survive `git worktree remove`; `firegoat.yaml`, fixtures and `setup:` stay in the worktree. Each `summary.json` records `dir` (where eval ran) and `git: {branch, commit, dirty}`, and `.firegoat/runs/index.jsonl` gets one line per eval (`runId, startedAt, dir, branch, commit, passed, failed, notGraded, cost`) to tell runs from different branches apart.
 - The agent gets your environment plus `env:`, never `vars:`. Vars that npm/npx set for firegoat itself (`npm_config_*`, `npm_lifecycle_*`, `npm_package_*`, `npm_execpath`, `npm_node_execpath`, `npm_command`, `INIT_CWD`) are removed from every env firegoat starts (agent, setup, workdir_setup, checks): `npx -y firegoat` sets `npm_config_yes=true`, which would make the agent's `npm init` / `npx create-*` skip their prompts. Set one under `env:` to keep it.
+- Runs (tests × `runs`) go 4 at a time, each in its own folder with its own session; `--parallel N` changes that, `--parallel 1` runs them one after another. `summary.json` lists them in `firegoat.yaml` order.
 - Script checks run in the agent's folder afterwards and set the exit code. `end_goal` is not graded by firegoat: it's for you, or your coding agent, to judge from `run.json`, the session and the folder.
 - Output per run, in `.firegoat/runs/<runId>/<test>/<n>/`: `transcript.txt` (each tool call with its command and short output, the final answer, files made; checks get it as `$FIREGOAT_TRANSCRIPT`), `run.json` (status, prompt, end_goal, check results, paths), `work/`, logs. Read the transcript to judge `end_goal`; claude's full session file is ~50x larger. `summary.json` covers the whole eval; `--json` prints it.
 
@@ -119,6 +122,16 @@ Want a plain `npm i acme-sdk` to get your build, with no hint in the prompt? See
 | 2 | at least one run was not graded (claude missing or logged out, crash, timeout, setup failed); wins over 1 |
 | 3 | usage or config error |
 
+## Feedback
+
+Something in firegoat got in your way (or your agent's)? Tell us from the terminal:
+
+```sh
+npx firegoat feedback "eval hid claude's spend-limit error" --type bug --body "ran npx firegoat eval, got 'not graded' x3"
+```
+
+It goes to the firegoat team with the firegoat version, your OS and node version, and which coding agent sent it. Don't include secrets: a report that looks like it has one is refused.
+
 ## Develop
 
 ```bash

package/dist/cli.js

@@ -6947,13 +6947,14 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.12.0",
+  version: "0.14.0",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
   },
   devDependencies: {
     "@biomejs/biome": "^2.5.15",
+    "@firegoat/feedback": "0.0.0",
     "@types/bun": "latest",
     typescript: "^7",
     yaml: "^2.9.1"
@@ -7159,9 +7160,11 @@ var RESERVED_ENV = new Set([
   "CLAUDE_CONFIG_DIR",
   "CLAUDE_SECURESTORAGE_CONFIG_DIR"
 ]);
-var RUN_PLACEHOLDERS = ["run.id", "run.n", "run.workdir", "test.name"];
+var RUN_PLACEHOLDERS = ["run.id", "run.n", "test.name"];
 var EVAL_PLACEHOLDERS = ["run.id"];
 var ANY_PLACEHOLDER = /\{\{\s*([^{}]*?)\s*\}\}/g;
+var DIR_REF = /\$(?:\{(FIREGOAT_OUT|FIREGOAT_WORKDIR)\}|(FIREGOAT_OUT|FIREGOAT_WORKDIR)(?![A-Za-z0-9_]))/g;
+var ENV_REF = new RegExp(`${ANY_PLACEHOLDER.source}|${DIR_REF.source}`, "g");
 function parseEnv(raw, path) {
   if (raw === undefined)
     return {};
@@ -7186,7 +7189,8 @@ function parseEnv(raw, path) {
   return out;
 }
 function isEvalLevel(value) {
-  return [...value.matchAll(ANY_PLACEHOLDER)].every((m) => {
+  const dirsOk = [...value.matchAll(DIR_REF)].every((m) => (m[1] ?? m[2]) === "FIREGOAT_OUT");
+  return dirsOk && [...value.matchAll(ANY_PLACEHOLDER)].every((m) => {
     const name = m[1];
     return name.startsWith("vars.") || EVAL_PLACEHOLDERS.includes(name);
   });
@@ -7194,8 +7198,9 @@ function isEvalLevel(value) {
 function renderEnv(env, vars, run) {
   const out = {};
   for (const [k, v] of Object.entries(env))
-    out[k] = v.replace(ANY_PLACEHOLDER, (whole, name) => {
-      const filled = name.startsWith("vars.") ? vars[name.slice("vars.".length)] : run[name];
+    out[k] = v.replace(ENV_REF, (whole, name, braced, bare) => {
+      const key = name ?? braced ?? bare;
+      const filled = key.startsWith("vars.") ? vars[key.slice("vars.".length)] : run[key];
       if (filled === undefined)
         throw new ConfigError(`env.${k}: no value for ${whole}`);
       return filled;
@@ -7224,13 +7229,14 @@ function loadConfig(dir) {
 // src/eval.ts
 import { randomBytes } from "node:crypto";
 import {
+  appendFileSync,
   createWriteStream as createWriteStream2,
   existsSync as existsSync2,
   mkdirSync as mkdirSync2,
   readdirSync as readdirSync2,
   writeFileSync as writeFileSync3
 } from "node:fs";
-import { basename, delimiter as delimiter2, join as join4 } from "node:path";
+import { basename as basename2, delimiter as delimiter2, join as join4 } from "node:path";
 
 // src/claude.ts
 import { spawn, spawnSync } from "node:child_process";
@@ -7424,6 +7430,51 @@ function ensureFiregoatDir(root) {
   return dir;
 }
 
+// src/git.ts
+import { spawnSync as spawnSync2 } from "node:child_process";
+import { basename, dirname } from "node:path";
+function git(dir, args) {
+  const r = spawnSync2("git", args, { cwd: dir, encoding: "utf8" });
+  if (r.error)
+    throw r.error;
+  return { code: r.status ?? -1, stdout: r.stdout.trim() };
+}
+function gitRepo(dir) {
+  const r = spawnSync2("git", ["rev-parse", "--path-format=absolute", "--git-dir", "--git-common-dir"], { cwd: dir, encoding: "utf8" });
+  if (r.error?.code === "ENOENT")
+    return null;
+  if (r.error)
+    throw r.error;
+  if (r.status !== 0) {
+    if (/not a git repository/.test(r.stderr))
+      return null;
+    throw new Error(`git rev-parse in ${dir} failed: ${r.stderr.trim()}`);
+  }
+  const [gitDir, commonDir] = r.stdout.trim().split(`
+`);
+  if (!gitDir || !commonDir)
+    throw new Error(`git rev-parse in ${dir} printed ${r.stdout}`);
+  let mainCheckout = null;
+  if (gitDir !== commonDir) {
+    if (basename(commonDir) !== ".git")
+      throw new Error(`${dir} is a linked git worktree, but its common git dir ${commonDir} is not a checkout's .git (a bare repo?), so there is no main checkout to keep .firegoat/runs in`);
+    mainCheckout = dirname(commonDir);
+  }
+  const branch = git(dir, ["symbolic-ref", "-q", "--short", "HEAD"]);
+  const commit = git(dir, ["rev-parse", "-q", "--verify", "HEAD"]);
+  const status = git(dir, ["status", "--porcelain"]);
+  if (status.code !== 0)
+    throw new Error(`git status in ${dir} failed`);
+  return {
+    info: {
+      branch: branch.code === 0 ? branch.stdout : null,
+      commit: commit.code === 0 ? commit.stdout : null,
+      dirty: status.stdout !== ""
+    },
+    mainCheckout
+  };
+}
+
 // src/help.ts
 var EXIT_CODES = `Exit codes:
   0  every run's script checks passed
@@ -7438,6 +7489,7 @@ script checks.
 
   npx firegoat init     create firegoat.yaml and .firegoat/, install the skills
   npx firegoat eval     run the tests (claude must be logged in: run \`claude\` once)
+  npx firegoat feedback "<what got in the way>"   tell the firegoat team
 
 Typical flow for a coding agent driving firegoat:
   1. read npx firegoat init --help, then edit firegoat.yaml
@@ -7470,8 +7522,10 @@ firegoat.yaml reference:
   env: {NAME: value}          env vars for the agent, and for workdir_setup and
                               checks of the same run; values are strings and can
                               use {{vars.KEY}}, {{run.id}}, {{run.n}},
-                              {{run.workdir}}, {{test.name}}. setup gets only the
-                              entries that use nothing but {{vars.*}}/{{run.id}}.
+                              {{test.name}}, $FIREGOAT_OUT, $FIREGOAT_WORKDIR
+                              (no other $ expansion: $HOME stays literal).
+                              setup gets only the entries that use nothing but
+                              {{vars.*}}, {{run.id}} and $FIREGOAT_OUT.
                               PATH, CLAUDE_CONFIG_DIR and FIREGOAT_* are reserved.
   tests:
     - name: quickstart        letters, digits, '.', '_', '-'; unique
@@ -7512,8 +7566,8 @@ Example:
             ! grep -Eq '$ .*npm (install|i|add) .*acme' "$FIREGOAT_TRANSCRIPT"
 
 Exit codes: 0 done, 2 the skills install failed, 3 usage error.`;
-var EVAL_HELP = `Usage: firegoat eval [test..] [--runs N] [--var KEY=value].. [--json]
-                     [--claude-bin PATH]
+var EVAL_HELP = `Usage: firegoat eval [test..] [--runs N] [--parallel N] [--var KEY=value]..
+                     [--json] [--claude-bin PATH]
 
 Runs each test in firegoat.yaml (in the current folder): your claude gets the prompt
 in a fresh, empty folder, .firegoat/runs/<runId>/<test>/<n>/work, with setup:'s
@@ -7524,15 +7578,23 @@ with a fresh config dir, .firegoat/.claude: none of your settings, CLAUDE.md,
 plugins, skills or MCP servers. claude saves its session there. There is no
 sandbox: the agent can reach the rest of your machine, including the repo.
 
+In a linked git worktree (git worktree add), .firegoat/runs/ and .firegoat/.claude
+live in the main checkout instead, so runs outlive the worktree; firegoat.yaml,
+fixtures and setup: stay in the current folder.
+
   test..             only these tests (by name)
   --runs N           runs per test (overrides runs: in firegoat.yaml)
+  --parallel N       runs at once (default 4; 1 runs them one after another)
   --var KEY=value    fills {{vars.KEY}}; env for setup, workdir_setup and checks
   --json             the summary as JSON on stdout (or {error, exitCode} if eval
                      stops early); progress goes to stderr
   --claude-bin PATH  claude binary (env FIREGOAT_CLAUDE_BIN; default: claude on PATH)
 
 Output: .firegoat/runs/<runId>/
-  summary.json            every run, plus totals
+  summary.json            every run, plus totals, dir (where eval ran) and
+                          git {branch, commit, dirty} (no git outside a repo)
+  ../index.jsonl          one line per eval: runId, startedAt, dir, branch,
+                          commit, passed, failed, notGraded, cost
   <test>/<n>/run.json     one run: status and reason (from the checks), prompt,
                           end_goal, checks [{run, code}], and paths: workdir,
                           transcript, session (claude's session file), stderr,
@@ -7553,6 +7615,25 @@ Examples:
 A run can take minutes (default timeout 600s per test).
 
 ${EXIT_CODES}`;
+var FEEDBACK_HELP = `Usage: firegoat feedback "<title>" [--body TEXT|-] [--type bug|feature_request|feedback]
+
+Sends feedback about firegoat to the firegoat team (firegoat.ai). Coding agents: if
+firegoat confused you, failed you or cost you tokens, say so here: one concrete
+title, then what you ran and what happened in --body (- reads stdin).
+
+  "<title>"     8-200 characters, e.g. "eval hid claude's spend-limit error"
+  --body TEXT   details: commands, output, what you expected
+  --type        bug, feature_request or feedback (default)
+
+Sent along: the firegoat version, your OS and node version, and which agent you are
+(claude-code/codex/cursor, from the environment). Don't include secrets: a report that
+looks like it contains one is refused.
+
+Example:
+  npx firegoat feedback "checks can't grep commands with quotes" --type bug \\
+    --body "grep '"command":"...' stops at the first escaped quote"
+
+Exit codes: 0 sent, 2 the service failed (or rate limit), 3 usage error.`;
 
 // src/setup.ts
 import { createWriteStream } from "node:fs";
@@ -7670,10 +7751,14 @@ ${indent(truncate(text, OUTPUT_LINES, OUTPUT_CHARS))}` : `    → ${status}`);
 }
 
 // src/ui.ts
-import { relative as relative2 } from "node:path";
+import { isAbsolute, relative as relative2 } from "node:path";
 var RAMP = "▁▂▃▄▅▆▇";
 var ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, "g");
 var SPIN = ["◐", "◓", "◑", "◒"];
+function shortPath(path) {
+  const rel = relative2(process.cwd(), path);
+  return rel.startsWith("..") || isAbsolute(rel) ? path : rel;
+}
 function painter(color) {
   return (code, text) => color ? `\x1B[${code}m${text}\x1B[0m` : text;
 }
@@ -7686,6 +7771,17 @@ var C = {
   bold: "1",
   cyan: "36"
 };
+var BUTTERFLY = ["  \\  /", " (_\\/_)", " (_/\\_)"];
+var LOGO_COLS = 10;
+function logoHeader(paint, title, right, width) {
+  return BUTTERFLY.map((wing, i) => {
+    const art = paint(C.ember, wing);
+    if (i !== 1)
+      return art;
+    const used = LOGO_COLS + title.length + right.replace(ANSI, "").length;
+    return `${art}${" ".repeat(LOGO_COLS - wing.length)}${paint(C.bold, title)}${" ".repeat(Math.max(2, width - used - 1))}${right}`;
+  });
+}
 function tier(rows) {
   const graded = rows.filter((r) => r.state === "pass" || r.state === "fail");
   if (graded.length === 0)
@@ -7720,9 +7816,10 @@ function checksPassed(r) {
 function total(rows) {
   const cost = rows.reduce((s, r) => s + (r.cost ?? 0), 0);
   const turns = rows.reduce((s, r) => s + r.turns, 0);
-  const seconds = rows.reduce((s, r) => s + (r.seconds ?? 0), 0);
-  const time = seconds >= 60 ? `${Math.floor(seconds / 60)}m ${seconds % 60}s` : `${seconds}s`;
-  return { cost, turns, time };
+  return { cost, turns };
+}
+function duration(seconds) {
+  return seconds >= 60 ? `${Math.floor(seconds / 60)}m ${seconds % 60}s` : `${seconds}s`;
 }
 function counts(rows) {
   return {
@@ -7809,9 +7906,8 @@ function ttyUI(out, opts) {
     const c = counts(rows);
     const doneCount = c.pass + c.fail + c.error;
     const badge = finished ? `DONE · RUN ${finished.runId.slice(-4)}` : `RUNNING · ${doneCount}/${rows.length}`;
-    const title = ` \uD83E\uDD8B firegoat eval · ${opts.project}`;
     const out = [
-      `${p(C.ember, " \uD83E\uDD8B")}${title.slice(3)}${" ".repeat(Math.max(1, w - title.length - badge.length - 1))}${p(finished ? C.green : C.ember, badge)}`,
+      ...logoHeader(p, `firegoat eval · ${opts.project}`, p(finished ? C.green : C.ember, badge), w),
       rule
     ];
     if (authText)
@@ -7847,22 +7943,23 @@ function ttyUI(out, opts) {
     if (!finished) {
       const track = Math.max(10, w - 34);
       const at = rows.length ? Math.round(doneCount / rows.length * (track - 1)) : 0;
-      const goat = `${"·".repeat(at)}\uD83D\uDC10${"·".repeat(Math.max(0, track - at - 1))}`;
-      out.push(`  ${p(C.ember, goat)}  ${p(C.gray, `${doneCount}/${rows.length} · $${t.cost.toFixed(2)} so far`)}`);
+      const progress = `${"·".repeat(at)}▶${"·".repeat(Math.max(0, track - at - 1))}`;
+      out.push(`  ${p(C.ember, progress)}  ${p(C.gray, `${doneCount}/${rows.length} · $${t.cost.toFixed(2)} so far`)}`);
       return out;
     }
     const tr = tier(rows);
     const goated = c.pass === rows.length && rows.length > 0;
     const tally = `${c.pass} PASS · ${c.fail} FAIL · ${c.error} NOT GRADED`;
-    const right = goated ? p(C.ember, `\uD83D\uDC10 GOATED · tier ${tr}`) : tr ? `tier ${tr}` : "";
-    out.push(`  ${goated ? p(C.green, `${rows.length}/${rows.length} runs passed their checks`) : tally}${" ".repeat(Math.max(2, w - tally.length - 22))}${right}`);
-    out.push(p(C.gray, `  total $${t.cost.toFixed(2)} · ${t.turns} turns · ${t.time}`));
-    out.push(p(C.gray, `  runs  ${relative2(process.cwd(), finished.runsDir)}/`));
+    const left = goated ? `${rows.length}/${rows.length} runs passed their checks` : tally;
+    const right = goated ? `GOATED · tier ${tr}` : tr ? `tier ${tr}` : "";
+    out.push(`  ${goated ? p(C.green, left) : left}${" ".repeat(Math.max(2, w - left.length - right.length - 3))}${goated ? p(C.ember, right) : right}`);
+    out.push(p(C.gray, `  total $${t.cost.toFixed(2)} · ${t.turns} turns · ${duration(finished.seconds)}`));
+    out.push(p(C.gray, `  runs  ${shortPath(finished.runsDir)}/`));
     out.push(`  checks are a score, not the verdict: judge each end_goal from ${p(C.cyan, "<test>/<n>/transcript.txt")}`);
     return out;
   };
   const rowsOf = (line) => {
-    const visible = line.replace(ANSI, "").replace(/🐐|🦋/g, "gg");
+    const visible = line.replace(ANSI, "");
     return Math.max(1, Math.ceil(visible.length / (out.columns || 80)));
   };
   const render = () => {
@@ -7892,9 +7989,9 @@ function ttyUI(out, opts) {
       render();
     },
     update: render,
-    done: ({ runId, runsDir }) => {
+    done: ({ runId, runsDir, seconds }) => {
       clearInterval(timer);
-      finished = { runId, runsDir };
+      finished = { runId, runsDir, seconds };
       render();
     }
   };
@@ -7929,6 +8026,7 @@ class UsageError extends Error {
 function parseArgs(args) {
   const opts = {
     names: [],
+    parallel: 4,
     json: false,
     vars: {},
     claudeBin: process.env.FIREGOAT_CLAUDE_BIN || "claude"
@@ -7961,6 +8059,12 @@ function parseArgs(args) {
       if (!/^\d+$/.test(v) || n < 1)
         throw new UsageError(`--runs must be a positive integer, got ${v}`);
       opts.runs = n;
+    } else if (flag === "--parallel") {
+      const v = value();
+      const n = Number(v);
+      if (!/^\d+$/.test(v) || n < 1)
+        throw new UsageError(`--parallel must be a positive integer, got ${v}`);
+      opts.parallel = n;
     } else if (flag === "--claude-bin")
       opts.claudeBin = value();
     else if (a.startsWith("-"))
@@ -8020,8 +8124,9 @@ async function runOne(opts) {
   const env = renderEnv(test.env, opts.vars, {
     "run.id": opts.runId,
     "run.n": String(n),
-    "run.workdir": workdir,
-    "test.name": test.name
+    "test.name": test.name,
+    FIREGOAT_OUT: opts.out,
+    FIREGOAT_WORKDIR: workdir
   });
   const stepEnv = { ...opts.stepEnv, ...env, FIREGOAT_WORKDIR: workdir };
   if (opts.workdirSetup.length > 0) {
@@ -8105,7 +8210,7 @@ async function evalCommand(args, root) {
   }
   const say = opts.json ? console.error : console.log;
   const stream = opts.json ? process.stderr : process.stdout;
-  const ui = stream.isTTY ? ttyUI(stream, { project: basename(root), color: !process.env.NO_COLOR }) : plainUI(say);
+  const ui = stream.isTTY ? ttyUI(stream, { project: basename2(root), color: !process.env.NO_COLOR }) : plainUI(say);
   let config;
   try {
     config = loadConfig(root);
@@ -8136,7 +8241,13 @@ async function evalCommand(args, root) {
   const claudePath = resolveBin(opts.claudeBin);
   if (!claudePath)
     return fatal(EXIT.notGraded, `not graded: claude binary ${JSON.stringify(opts.claudeBin)} not found. Install Claude Code (https://claude.com/claude-code) or pass --claude-bin PATH / FIREGOAT_CLAUDE_BIN.`);
-  const firegoatDir = ensureFiregoatDir(root);
+  let repo;
+  try {
+    repo = gitRepo(root);
+  } catch (e) {
+    return fatal(EXIT.usage, e.message);
+  }
+  const firegoatDir = ensureFiregoatDir(repo?.mainCheckout ?? root);
   const configDir = join4(firegoatDir, ".claude");
   mkdirSync2(configDir, { recursive: true });
   ui.auth(process.env.ANTHROPIC_API_KEY ? "ANTHROPIC_API_KEY (claude bills the API key; unset it to use your claude login)" : "your claude login");
@@ -8147,8 +8258,10 @@ async function evalCommand(args, root) {
   });
   if (loginProblem)
     return fatal(EXIT.notGraded, `not graded: ${loginProblem}`);
+  const evalStarted = Date.now();
   const runId = newRunId();
-  const runDir = join4(firegoatDir, "runs", runId);
+  const runsRoot = join4(firegoatDir, "runs");
+  const runDir = join4(runsRoot, runId);
   const out = join4(runDir, "out");
   mkdirSync2(out, { recursive: true });
   const stepEnv = {
@@ -8172,7 +8285,7 @@ async function evalCommand(args, root) {
       cwd: root,
       env: {
         ...stepEnv,
-        ...renderEnv(Object.fromEntries(Object.entries(config.env).filter(([, v]) => isEvalLevel(v))), vars, { "run.id": runId })
+        ...renderEnv(Object.fromEntries(Object.entries(config.env).filter(([, v]) => isEvalLevel(v))), vars, { "run.id": runId, FIREGOAT_OUT: out })
       },
       logPath: join4(runDir, "setup.txt")
     });
@@ -8182,65 +8295,271 @@ async function evalCommand(args, root) {
       return fatal(EXIT.notGraded, `not graded: ${error}`);
   }
   ui.runs(rows);
-  const results = [];
-  for (const test of tests)
-    for (let n = 1;n <= runs; n++) {
-      const row = rows.find((x) => x.test === test.name && x.n === n);
-      row.state = "running";
-      ui.update();
-      const r = await runOne({
-        onEvent: (e) => {
-          if (e.type !== "assistant")
-            return;
-          row.turns++;
-          const content = e.message?.content;
-          if (!Array.isArray(content))
-            return;
-          for (const b of content) {
-            const cmd = b.input?.command;
-            if (b.type === "tool_use" && typeof cmd === "string")
-              row.command = cmd.split(`
+  const results = new Array(rows.length);
+  const runAt = async (i) => {
+    const row = rows[i];
+    const test = tests.find((t) => t.name === row.test);
+    row.state = "running";
+    ui.update();
+    const r = await runOne({
+      onEvent: (e) => {
+        if (e.type !== "assistant")
+          return;
+        row.turns++;
+        const content = e.message?.content;
+        if (!Array.isArray(content))
+          return;
+        for (const b of content) {
+          const cmd = b.input?.command;
+          if (b.type === "tool_use" && typeof cmd === "string")
+            row.command = cmd.split(`
 `)[0];
-          }
-        },
-        test,
-        n,
-        runId,
-        runDir,
-        claudeBin: claudePath,
-        configDir,
-        workdirSetup: config.workdirSetup,
-        stepEnv,
-        vars
-      });
-      results.push(r);
-      Object.assign(row, {
-        state: r.status,
-        turns: r.num_turns ?? row.turns,
-        reason: r.reason,
-        checks: r.checks,
-        cost: r.cost,
-        seconds: Math.round(r.duration_ms / 1000),
-        command: undefined
-      });
-      ui.update();
-    }
+        }
+      },
+      test,
+      n: row.n,
+      runId,
+      runDir,
+      claudeBin: claudePath,
+      configDir,
+      workdirSetup: config.workdirSetup,
+      out,
+      stepEnv,
+      vars
+    });
+    results[i] = r;
+    Object.assign(row, {
+      state: r.status,
+      turns: r.num_turns ?? row.turns,
+      reason: r.reason,
+      checks: r.checks,
+      cost: r.cost,
+      seconds: Math.round(r.duration_ms / 1000),
+      command: undefined
+    });
+    ui.update();
+  };
+  let next = 0;
+  const worker = async () => {
+    while (next < rows.length)
+      await runAt(next++);
+  };
+  await Promise.all(Array.from({ length: Math.min(opts.parallel, rows.length) }, worker));
   const totals = { pass: 0, fail: 0, error: 0 };
   for (const r of results)
     totals[r.status]++;
-  const doc = { runId, runsDir: runDir, totals, runs: results };
+  const doc = {
+    runId,
+    runsDir: runDir,
+    dir: root,
+    ...repo && { git: repo.info },
+    totals,
+    runs: results
+  };
   writeFileSync3(join4(runDir, "summary.json"), `${JSON.stringify(doc, null, 2)}
 `);
-  ui.done({ runId, runsDir: runDir, rows });
+  appendFileSync(join4(runsRoot, "index.jsonl"), `${JSON.stringify({
+    runId,
+    startedAt: new Date(evalStarted).toISOString(),
+    dir: root,
+    branch: repo?.info.branch ?? null,
+    commit: repo?.info.commit ?? null,
+    passed: totals.pass,
+    failed: totals.fail,
+    notGraded: totals.error,
+    cost: results.reduce((sum, r) => sum + (r.cost ?? 0), 0)
+  })}
+`);
+  ui.done({
+    runId,
+    runsDir: runDir,
+    rows,
+    seconds: Math.round((Date.now() - evalStarted) / 1000)
+  });
   if (opts.json)
     console.log(JSON.stringify(doc, null, 2));
   return totals.error > 0 ? EXIT.notGraded : totals.fail > 0 ? EXIT.fail : EXIT.pass;
 }
 
+// src/feedback.ts
+import { readFileSync as readFileSync2 } from "node:fs";
+
+// node_modules/@firegoat/feedback/dist/index.js
+var VERSION = "0.0.0";
+var DEFAULT_BASE_URL = "https://firegoat.ai/evals";
+var TIMEOUT_MS = 15000;
+function envBaseUrl() {
+  return globalThis.process?.env?.FEEDBACK_API_URL;
+}
+
+class FeedbackError extends Error {
+  status;
+  code;
+  retryAfterSeconds;
+  detected;
+  constructor(status, code, message, retryAfterSeconds, detected) {
+    super(message);
+    this.status = status;
+    this.code = code;
+    this.retryAfterSeconds = retryAfterSeconds;
+    this.detected = detected;
+    this.name = "FeedbackError";
+  }
+}
+var TITLE_MIN = 8;
+var TITLE_MAX = 200;
+
+class FeedbackClient {
+  domain;
+  baseUrl;
+  token;
+  fetchImpl;
+  constructor(options) {
+    if (!options.domain?.trim()) {
+      throw new Error("FeedbackClient: `domain` is required (your product's domain, e.g. example.com)");
+    }
+    this.domain = options.domain.trim();
+    this.baseUrl = (options.baseUrl ?? envBaseUrl() ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+    this.token = options.token;
+    this.fetchImpl = options.fetch ?? fetch;
+  }
+  async registerDevice() {
+    const data = await this.post("/api/feedback/device", undefined);
+    if (typeof data.token !== "string" || !data.token) {
+      throw new FeedbackError(201, "invalid_response", "server response missing token");
+    }
+    this.token = data.token;
+    return data.token;
+  }
+  async send(input) {
+    const title = input.title.trim();
+    if (title.length < TITLE_MIN || title.length > TITLE_MAX) {
+      throw new FeedbackError(400, "invalid_request", `title must be ${TITLE_MIN}-${TITLE_MAX} characters, got ${title.length}`);
+    }
+    const meta = {
+      ...input.meta,
+      cliVersion: `@firegoat/feedback@${VERSION}`,
+      feedbackType: input.type
+    };
+    const data = await this.post("/api/feedback", {
+      domain: this.domain,
+      title,
+      body: input.body ?? "",
+      meta,
+      maskSecrets: input.maskSecrets ?? false
+    });
+    if (typeof data.id !== "string" || !data.id) {
+      throw new FeedbackError(201, "invalid_response", "server response missing id");
+    }
+    return { id: data.id, similarCount: data.similarCount ?? 0 };
+  }
+  async post(path, body) {
+    const headers = {
+      "content-type": "application/json"
+    };
+    if (this.token)
+      headers["x-feedback-token"] = this.token;
+    const res = await this.fetchImpl(`${this.baseUrl}${path}`, {
+      method: "POST",
+      headers,
+      body: body === undefined ? undefined : JSON.stringify(body),
+      signal: AbortSignal.timeout(TIMEOUT_MS)
+    });
+    const text = await res.text();
+    let data;
+    try {
+      data = JSON.parse(text);
+    } catch {
+      throw new FeedbackError(res.status, "invalid_response", `non-JSON response (${res.status}): ${text.slice(0, 200)}`);
+    }
+    if (!res.ok) {
+      throw new FeedbackError(res.status, String(data.error ?? "unknown_error"), String(data.message ?? `request failed with status ${res.status}`), typeof data.retryAfterSeconds === "number" ? data.retryAfterSeconds : undefined, Array.isArray(data.detected) ? data.detected : undefined);
+    }
+    return data;
+  }
+}
+
+// src/feedback.ts
+var TYPES = ["bug", "feature_request", "feedback"];
+function agentName() {
+  if (process.env.CLAUDECODE)
+    return "claude-code";
+  if (process.env.CODEX_HOME || process.env.CODEX_SANDBOX)
+    return "codex";
+  if (process.env.CURSOR_TRACE_ID)
+    return "cursor";
+  return;
+}
+async function feedbackCommand(args) {
+  if (args.length === 0 || args.includes("--help") || args.includes("-h")) {
+    console.log(FEEDBACK_HELP);
+    return args.length === 0 ? EXIT.usage : EXIT.pass;
+  }
+  const titleParts = [];
+  let body;
+  let type = "feedback";
+  for (let i = 0;i < args.length; i++) {
+    const a = args[i];
+    const value = () => {
+      const v = args[++i];
+      if (v === undefined)
+        throw new Error(`${a} needs a value`);
+      return v;
+    };
+    try {
+      if (a === "--body") {
+        const v = value();
+        body = v === "-" ? readFileSync2(0, "utf8") : v;
+      } else if (a === "--type") {
+        const v = value();
+        if (!TYPES.includes(v))
+          throw new Error(`--type must be one of ${TYPES.join(", ")}, got ${v}`);
+        type = v;
+      } else if (a.startsWith("-"))
+        throw new Error(`unknown flag: ${a}`);
+      else
+        titleParts.push(a);
+    } catch (e) {
+      console.error(`firegoat feedback: ${e.message}`);
+      return EXIT.usage;
+    }
+  }
+  const title = titleParts.join(" ").trim();
+  if (title.length < 8 || title.length > 200) {
+    console.error(`firegoat feedback: the title must be 8-200 characters (got ${title.length})`);
+    return EXIT.usage;
+  }
+  const client = new FeedbackClient({ domain: "firegoat.ai" });
+  try {
+    const agent = agentName();
+    const { id, similarCount } = await client.send({
+      title,
+      body: body?.trim() ? `${body.trim()}
+
+firegoat ${package_default.version}` : `firegoat ${package_default.version}`,
+      type,
+      meta: {
+        ...agent ? { agent } : {},
+        os: process.platform,
+        nodeVersion: process.version
+      }
+    });
+    console.log(`sent ${type} ${id} to the firegoat team${similarCount > 0 ? ` (${similarCount} similar report${similarCount === 1 ? "" : "s"} already)` : ""}. thanks.`);
+    return EXIT.pass;
+  } catch (e) {
+    if (e instanceof FeedbackError) {
+      const hint = e.code === "possible_secret" ? " Remove the secret from the title/body and send again." : e.code === "rate_limited" && e.retryAfterSeconds ? ` Try again in ${e.retryAfterSeconds}s.` : "";
+      console.error(`firegoat feedback: ${e.code}: ${e.message}.${hint}`);
+      return e.code === "invalid_request" || e.code === "possible_secret" ? EXIT.usage : EXIT.notGraded;
+    }
+    throw e;
+  }
+}
+
 // src/init.ts
-import { spawnSync as spawnSync2 } from "node:child_process";
+import { spawnSync as spawnSync3 } from "node:child_process";
 import { existsSync as existsSync3, writeFileSync as writeFileSync4 } from "node:fs";
-import { basename as basename2, join as join5 } from "node:path";
+import { basename as basename3, join as join5 } from "node:path";
 import { createInterface } from "node:readline/promises";
 var SKILLS_ARGV = [
   "skills",
@@ -8286,9 +8605,11 @@ runs: 1
 
 # Optional: env vars for the agent (and for workdir_setup and checks of the same
 # run). The agent gets none of vars:, only these. Placeholders: {{vars.KEY}},
-# {{run.id}}, {{run.n}}, {{run.workdir}}, {{test.name}}. A test can add its own env:.
+# {{run.id}}, {{run.n}}, {{test.name}}, and $FIREGOAT_OUT / $FIREGOAT_WORKDIR (no
+# other $ expansion: $HOME stays literal). A test can add its own env:.
 # env:
 #   MYTOOL_HOME: "/tmp/mytool-{{run.id}}-{{run.n}}"
+#   UV_FIND_LINKS: "$FIREGOAT_OUT"
 
 tests:
   - name: quickstart
@@ -8317,7 +8638,7 @@ var CLIPBOARD = [
 ];
 function copyToClipboard(text) {
   return CLIPBOARD.some(([cmd, args]) => {
-    const r = spawnSync2(cmd, args, { input: text });
+    const r = spawnSync3(cmd, args, { input: text });
     return !r.error && r.status === 0;
   });
 }
@@ -8356,16 +8677,15 @@ ${INIT_HELP}`);
   const rule = paint("90", ` ${"─".repeat(width - 2)}`);
   const step = (n, label, status) => console.log(`  ${paint("32", "●")}  ${paint("90", String(n).padStart(2, "0"))}  ${label.padEnd(28)}${paint("90", status)}`);
   if (interactive) {
-    const title = " \uD83E\uDD8B firegoat init";
-    const tag = "Get agent-ready.";
-    console.log(`${paint("38;5;208", " \uD83E\uDD8B")} firegoat init${" ".repeat(Math.max(2, width - title.length - tag.length - 2))}${paint("90", tag)}`);
+    for (const line of logoHeader(paint, "firegoat init", paint("90", "Get agent-ready."), width))
+      console.log(line);
     console.log(rule);
   }
   ensureFiregoatDir(cwd);
   const configPath = join5(cwd, CONFIG_FILE);
   const keep = existsSync3(configPath) && !force;
   if (!keep)
-    writeFileSync4(configPath, template(basename2(cwd)));
+    writeFileSync4(configPath, template(basename3(cwd)));
   if (interactive) {
     step(1, CONFIG_FILE, keep ? "kept (--force replaces it)" : "created");
     console.log(paint("90", "  │"));
@@ -8382,7 +8702,7 @@ ${INIT_HELP}`);
       console.log(`installing skills: npx ${SKILLS_ARGV.join(" ")}`);
     const env = { ...process.env };
     delete env.npm_config_package;
-    const r = spawnSync2("npx", ["-y", ...SKILLS_ARGV], {
+    const r = spawnSync3("npx", ["-y", ...SKILLS_ARGV], {
       cwd,
       env,
       stdio: "pipe",
@@ -8397,7 +8717,7 @@ ${INIT_HELP}`);
       step(3, "firegoat-onboarding skill", "installed → .claude/skills/");
     else
       console.log("installed skills firegoat-onboarding, suggesting-eval-prompts in .claude/skills/");
-    const ignored = spawnSync2("git", ["check-ignore", "-q", ".claude/skills"], {
+    const ignored = spawnSync3("git", ["check-ignore", "-q", ".claude/skills"], {
       cwd
     });
     if (ignored.status === 0)
@@ -8436,6 +8756,8 @@ async function main(argv) {
       return initCommand(rest, process.cwd());
     case "eval":
       return evalCommand(rest, process.cwd());
+    case "feedback":
+      return feedbackCommand(rest);
     default:
       console.error(`unknown command: ${cmd}
 

package/package.json

@@ -1,12 +1,13 @@
 {
 	"name": "firegoat",
-	"version": "0.12.0",
+	"version": "0.14.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/firegoat-ai/firegoat-cli.git"
 	},
 	"devDependencies": {
 		"@biomejs/biome": "^2.5.15",
+		"@firegoat/feedback": "0.0.0",
 		"@types/bun": "latest",
 		"typescript": "^7",
 		"yaml": "^2.9.1"