firegoat 0.10.0 → 0.12.0

package/README.md

@@ -2,7 +2,7 @@
 
 Local AX evals: how well coding agents use your SDK, CLI, API and docs. firegoat gives your own `claude` one prompt in a fresh folder, saves the session, and runs your checks.
 
-Status: early (v0.10). See `plan.md`.
+Status: early (v0.11). See `plan.md`.
 
 ## Quickstart
 
@@ -82,14 +82,33 @@ tests:
 npx firegoat eval --var SHA=$(git rev-parse HEAD)
 ```
 
+### Env vars for the agent: a home dir per run, a feature flag
+
+```yaml
+env:
+  ACME_HOME: "/tmp/acme-{{run.id}}-{{run.n}}"   # sessions/config never collide across runs
+  ACME_TELEMETRY: "off"
+tests:
+  - name: login
+    prompt: Log in to acme with the CLI and print the current user.
+    end_goal: The agent completes acme's login flow and prints the user.
+    env:
+      ACME_BETA_LOGIN: "1"                        # this test only, on top of env:
+    checks:
+      - run: ls "$ACME_HOME"/sessions | grep -q .
+```
+
+`env:` goes to the agent, and to `workdir_setup` and checks of the same run, so checks see the same values. Placeholders: `{{vars.KEY}}`, `{{run.id}}` (the eval's run id), `{{run.n}}` (1..runs), `{{run.workdir}}`, `{{test.name}}`. `setup` runs once, before any run: it gets only the top-level entries that use nothing but `{{vars.*}}` and `{{run.id}}`. Values are strings (quote numbers). `PATH`, `CLAUDE_CONFIG_DIR` and `FIREGOAT_*` are firegoat's and can't be set.
+
 ## What eval does
 
 - claude runs on its own login (or `ANTHROPIC_API_KEY`, if set) with `CLAUDE_CONFIG_DIR=.firegoat/.claude` and `--setting-sources "" --strict-mcp-config --disable-slash-commands`: none of your settings, CLAUDE.md, plugins, skills or MCP servers. claude saves its session there, in its own format.
 - No sandbox: the agent's folder is inside `.firegoat/runs/`, and it can reach the rest of your machine. In `.firegoat/`, git ignores `runs/` and `.claude/`; keep test fixtures (scripts, sample data) in `.firegoat/fixtures/` and commit them with `firegoat.yaml`.
+- The agent gets your environment plus `env:`, never `vars:`. Vars that npm/npx set for firegoat itself (`npm_config_*`, `npm_lifecycle_*`, `npm_package_*`, `npm_execpath`, `npm_node_execpath`, `npm_command`, `INIT_CWD`) are removed from every env firegoat starts (agent, setup, workdir_setup, checks): `npx -y firegoat` sets `npm_config_yes=true`, which would make the agent's `npm init` / `npx create-*` skip their prompts. Set one under `env:` to keep it.
 - Script checks run in the agent's folder afterwards and set the exit code. `end_goal` is not graded by firegoat: it's for you, or your coding agent, to judge from `run.json`, the session and the folder.
 - Output per run, in `.firegoat/runs/<runId>/<test>/<n>/`: `transcript.txt` (each tool call with its command and short output, the final answer, files made; checks get it as `$FIREGOAT_TRANSCRIPT`), `run.json` (status, prompt, end_goal, check results, paths), `work/`, logs. Read the transcript to judge `end_goal`; claude's full session file is ~50x larger. `summary.json` covers the whole eval; `--json` prints it.
 
-`firegoat.yaml` keys: `runs`, `setup`, `workdir_setup`, `vars`, and `tests` (`name`, `prompt`, `end_goal`, `checks`, `timeout`). Unknown keys are errors. Full reference: `npx firegoat init --help`.
+`firegoat.yaml` keys: `runs`, `setup`, `workdir_setup`, `vars`, `env`, and `tests` (`name`, `prompt`, `end_goal`, `checks`, `timeout`, `env`). Unknown keys are errors. Full reference: `npx firegoat init --help`.
 
 Want a plain `npm i acme-sdk` to get your build, with no hint in the prompt? See [`examples/local-npm-registry`](examples/local-npm-registry/README.md).
 

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.10.0",
+  version: "0.12.0",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -7072,11 +7072,19 @@ function parseConfig(text) {
   }
   if (!isRecord(raw))
     fail("(root)", "must be a mapping");
-  onlyKeys(raw, "(root)", ["runs", "setup", "workdir_setup", "vars", "tests"]);
+  onlyKeys(raw, "(root)", [
+    "runs",
+    "setup",
+    "workdir_setup",
+    "vars",
+    "env",
+    "tests"
+  ]);
   const runs = posInt(raw.runs, "runs", 1);
   const setup = commands(raw.setup, "setup");
   const workdirSetup = commands(raw.workdir_setup, "workdir_setup");
   const vars = parseVars(raw.vars);
+  const env = parseEnv(raw.env, "env");
   if (!Array.isArray(raw.tests) || raw.tests.length === 0)
     fail("tests", "must be a non-empty list");
   const names = new Set;
@@ -7084,7 +7092,7 @@ function parseConfig(text) {
     const p = `tests[${i}]`;
     if (!isRecord(t))
       fail(p, "must be a mapping");
-    onlyKeys(t, p, ["name", "prompt", "end_goal", "checks", "timeout"]);
+    onlyKeys(t, p, ["name", "prompt", "end_goal", "checks", "timeout", "env"]);
     const name = str(t.name, `${p}.name`);
     if (!NAME.test(name))
       fail(`${p}.name`, "may only contain letters, digits, '.', '_' and '-'");
@@ -7107,7 +7115,8 @@ function parseConfig(text) {
       prompt: str(t.prompt, `${p}.prompt`),
       end_goal: str(t.end_goal, `${p}.end_goal`),
       checks,
-      timeout: posInt(t.timeout, `${p}.timeout`, DEFAULT_TIMEOUT)
+      timeout: posInt(t.timeout, `${p}.timeout`, DEFAULT_TIMEOUT),
+      env: { ...env, ...parseEnv(t.env, `${p}.env`) }
     };
   });
   return {
@@ -7115,6 +7124,7 @@ function parseConfig(text) {
     setup,
     workdirSetup,
     vars,
+    env,
     tests
   };
 }
@@ -7144,6 +7154,54 @@ function parseVars(raw) {
   }
   return out;
 }
+var RESERVED_ENV = new Set([
+  "PATH",
+  "CLAUDE_CONFIG_DIR",
+  "CLAUDE_SECURESTORAGE_CONFIG_DIR"
+]);
+var RUN_PLACEHOLDERS = ["run.id", "run.n", "run.workdir", "test.name"];
+var EVAL_PLACEHOLDERS = ["run.id"];
+var ANY_PLACEHOLDER = /\{\{\s*([^{}]*?)\s*\}\}/g;
+function parseEnv(raw, path) {
+  if (raw === undefined)
+    return {};
+  if (!isRecord(raw))
+    fail(path, "must be a mapping of NAME to value");
+  const out = {};
+  for (const [k, v] of Object.entries(raw)) {
+    const p = `${path}.${k}`;
+    if (!VAR_KEY.test(k))
+      fail(p, "name must be letters, digits and _, not starting with a digit");
+    if (RESERVED_ENV.has(k) || k.startsWith("FIREGOAT_"))
+      fail(p, "is set by firegoat and cannot be overridden (reserved: PATH, CLAUDE_CONFIG_DIR, CLAUDE_SECURESTORAGE_CONFIG_DIR, FIREGOAT_*)");
+    if (typeof v !== "string")
+      fail(p, "must be a string (quote numbers and booleans)");
+    for (const m of v.matchAll(ANY_PLACEHOLDER)) {
+      const name = m[1];
+      if (!/^vars\.[A-Za-z0-9_]+$/.test(name) && !RUN_PLACEHOLDERS.includes(name))
+        fail(p, `unknown placeholder ${m[0]} (allowed: {{vars.KEY}}, ${RUN_PLACEHOLDERS.map((n) => `{{${n}}}`).join(", ")})`);
+    }
+    out[k] = v;
+  }
+  return out;
+}
+function isEvalLevel(value) {
+  return [...value.matchAll(ANY_PLACEHOLDER)].every((m) => {
+    const name = m[1];
+    return name.startsWith("vars.") || EVAL_PLACEHOLDERS.includes(name);
+  });
+}
+function renderEnv(env, vars, run) {
+  const out = {};
+  for (const [k, v] of Object.entries(env))
+    out[k] = v.replace(ANY_PLACEHOLDER, (whole, name) => {
+      const filled = name.startsWith("vars.") ? vars[name.slice("vars.".length)] : run[name];
+      if (filled === undefined)
+        throw new ConfigError(`env.${k}: no value for ${whole}`);
+      return filled;
+    });
+  return out;
+}
 var PLACEHOLDER = /\{\{\s*vars\.([A-Za-z0-9_]+)\s*\}\}/g;
 function varRefs(text) {
   return [...text.matchAll(PLACEHOLDER)].map((m) => m[1]);
@@ -7172,7 +7230,7 @@ import {
   readdirSync as readdirSync2,
   writeFileSync as writeFileSync3
 } from "node:fs";
-import { delimiter as delimiter2, join as join4 } from "node:path";
+import { basename, delimiter as delimiter2, join as join4 } from "node:path";
 
 // src/claude.ts
 import { spawn, spawnSync } from "node:child_process";
@@ -7186,9 +7244,14 @@ var CLAUDE_FLAGS = [
   "",
   "--disable-slash-commands"
 ];
-function claudeEnv(configDir) {
+var NPM_INJECTED = /^(npm_config_|npm_lifecycle_|npm_package_)|^(npm_execpath|npm_node_execpath|npm_command|INIT_CWD)$/;
+function hostEnv() {
+  return Object.fromEntries(Object.entries(process.env).filter(([k]) => !NPM_INJECTED.test(k)));
+}
+function claudeEnv(configDir, env = {}) {
   return {
-    ...process.env,
+    ...hostEnv(),
+    ...env,
     CLAUDE_CONFIG_DIR: configDir,
     CLAUDE_SECURESTORAGE_CONFIG_DIR: ""
   };
@@ -7226,6 +7289,7 @@ function run(opts) {
 }
 async function runClaude(opts) {
   let stdout = "";
+  let pending = "";
   const exited = await run({
     cmd: opts.bin,
     args: ["-p", opts.prompt, ...CLAUDE_FLAGS],
@@ -7233,7 +7297,19 @@ async function runClaude(opts) {
     env: opts.env,
     timeoutMs: opts.timeoutMs,
     onStdout: (c) => {
-      stdout += c.toString();
+      const text = c.toString();
+      stdout += text;
+      if (!opts.onEvent)
+        return;
+      pending += text;
+      const lines = pending.split(`
+`);
+      pending = lines.pop() ?? "";
+      for (const line of lines) {
+        try {
+          opts.onEvent(JSON.parse(line));
+        } catch {}
+      }
     },
     onStderr: (c) => opts.stderr.write(c)
   });
@@ -7391,6 +7467,12 @@ firegoat.yaml reference:
   workdir_setup: <sh> | [...] in each fresh folder before the agent starts
   vars: {KEY: value}          defaults for {{vars.KEY}} in prompts and checks;
                               --var KEY=value overrides
+  env: {NAME: value}          env vars for the agent, and for workdir_setup and
+                              checks of the same run; values are strings and can
+                              use {{vars.KEY}}, {{run.id}}, {{run.n}},
+                              {{run.workdir}}, {{test.name}}. setup gets only the
+                              entries that use nothing but {{vars.*}}/{{run.id}}.
+                              PATH, CLAUDE_CONFIG_DIR and FIREGOAT_* are reserved.
   tests:
     - name: quickstart        letters, digits, '.', '_', '-'; unique
       prompt: ...             what a user would ask the agent; one concrete outcome
@@ -7399,6 +7481,7 @@ firegoat.yaml reference:
       checks:                 optional shell commands, run in the agent's folder
         - run: ...            afterwards; exit 0 = pass; they set eval's exit code
       timeout: 600            optional, seconds
+      env: {NAME: value}      optional, on top of the top-level env:
 
   Unknown keys are errors. Keep helper files out of firegoat.yaml: put them in
   .firegoat/fixtures/ and copy them in setup or workdir_setup. setup, workdir_setup and checks get the vars as env vars,
@@ -7407,6 +7490,11 @@ firegoat.yaml reference:
   each output), and
   $FIREGOAT_SESSION (claude's full session file, JSON lines).
 
+  The agent never gets vars, only env:. The vars npm/npx set for firegoat itself
+  (npm_config_*, npm_lifecycle_*, npm_package_*, npm_execpath, npm_node_execpath,
+  npm_command, INIT_CWD; e.g. npm_config_yes=true from npx -y) are removed from
+  every env firegoat starts, unless env: sets them.
+
 Example:
   setup: npm pack --pack-destination "$FIREGOAT_OUT"
   workdir_setup: npm init -y >/dev/null && npm install "$FIREGOAT_OUT"/*.tgz
@@ -7581,6 +7669,237 @@ ${indent(truncate(text, OUTPUT_LINES, OUTPUT_CHARS))}` : `    → ${status}`);
 `);
 }
 
+// src/ui.ts
+import { relative as relative2 } from "node:path";
+var RAMP = "▁▂▃▄▅▆▇";
+var ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, "g");
+var SPIN = ["◐", "◓", "◑", "◒"];
+function painter(color) {
+  return (code, text) => color ? `\x1B[${code}m${text}\x1B[0m` : text;
+}
+var C = {
+  ember: "38;5;208",
+  green: "32",
+  red: "31",
+  amber: "33",
+  gray: "90",
+  bold: "1",
+  cyan: "36"
+};
+function tier(rows) {
+  const graded = rows.filter((r) => r.state === "pass" || r.state === "fail");
+  if (graded.length === 0)
+    return null;
+  const share = graded.filter((r) => r.state === "pass").length / graded.length;
+  return share === 1 ? "S" : share >= 0.75 ? "A" : share >= 0.5 ? "B" : "C";
+}
+function notGradedWhy(reason) {
+  const said = /claude result is an error \(subtype=[^)]*\): (.*)$/s.exec(reason);
+  if (said)
+    return `claude said: "${said[1]}"`;
+  return reason.replace(/^not graded: /, "");
+}
+function ramp(r) {
+  const blocks = Array.from({ length: Math.min(r.turns, 7) }, (_, i) => RAMP[i]).join("");
+  const end = r.state === "pass" ? "▲" : r.state === "fail" ? "╳" : r.state === "error" ? "╌" : r.state === "running" ? "●" : "";
+  return (blocks + end).padEnd(8);
+}
+function usage(r) {
+  const parts = [];
+  if (r.cost !== null && r.cost !== undefined)
+    parts.push(`$${r.cost.toFixed(2)}`);
+  parts.push(`${r.turns} turns`);
+  if (r.seconds !== undefined)
+    parts.push(`${r.seconds}s`);
+  return parts.join("  ");
+}
+function checksPassed(r) {
+  const all = r.checks ?? [];
+  return `${all.filter((c) => c.code === 0).length}/${all.length} checks`;
+}
+function total(rows) {
+  const cost = rows.reduce((s, r) => s + (r.cost ?? 0), 0);
+  const turns = rows.reduce((s, r) => s + r.turns, 0);
+  const seconds = rows.reduce((s, r) => s + (r.seconds ?? 0), 0);
+  const time = seconds >= 60 ? `${Math.floor(seconds / 60)}m ${seconds % 60}s` : `${seconds}s`;
+  return { cost, turns, time };
+}
+function counts(rows) {
+  return {
+    pass: rows.filter((r) => r.state === "pass").length,
+    fail: rows.filter((r) => r.state === "fail").length,
+    error: rows.filter((r) => r.state === "error").length
+  };
+}
+function plainUI(write) {
+  let rows = [];
+  const printed = new Set;
+  return {
+    auth: (text) => write(`auth: ${text}`),
+    setup: (state, seconds) => {
+      if (state === "running")
+        write("setup...");
+      else if (state === "fail")
+        write(`setup failed (${seconds}s)`);
+    },
+    runs: (r) => {
+      rows = r;
+    },
+    update: () => {
+      for (const r of rows) {
+        if (printed.has(r) || r.state === "queued" || r.state === "running")
+          continue;
+        printed.add(r);
+        const label = r.state === "error" ? "ERROR" : r.state.toUpperCase();
+        const u = `$${(r.cost ?? 0).toFixed(2)}, ${r.turns} turns, ${r.seconds ?? 0}s`;
+        write(`${label} ${r.test}#${r.n} ${r.reason} (${u})`);
+      }
+    },
+    done: ({ runsDir, rows: all }) => {
+      const c = counts(all);
+      const t = total(all);
+      const tr = tier(all);
+      write("");
+      write(`${c.pass} passed, ${c.fail} failed, ${c.error} not graded (${all.length} runs, $${t.cost.toFixed(2)})${tr ? `, tier ${tr}` : ""}`);
+      write(`runs: ${runsDir} (end_goal is for you to judge: read <test>/<n>/transcript.txt)`);
+    }
+  };
+}
+function ttyUI(out, opts) {
+  const p = painter(opts.color);
+  let rows = [];
+  let setup = null;
+  let authText = "";
+  let frame = 0;
+  let drawn = 0;
+  let finished = null;
+  const width = () => Math.min(out.columns || 80, 100);
+  const cut = (text, max) => text.length > max ? `${text.slice(0, max - 1)}…` : text;
+  const statusWord = (r) => {
+    switch (r.state) {
+      case "pass":
+        return p(C.green, "PASS      ");
+      case "fail":
+        return p(C.red, "FAIL      ");
+      case "error":
+        return p(C.amber, "NOT GRADED");
+      case "running":
+        return p(C.ember, "running   ");
+      default:
+        return p(C.gray, "queued    ");
+    }
+  };
+  const glyph = (r) => {
+    switch (r.state) {
+      case "pass":
+        return p(C.green, "●");
+      case "fail":
+        return p(C.red, "✗");
+      case "error":
+        return p(C.amber, "?");
+      case "running":
+        return p(C.ember, SPIN[frame % SPIN.length]);
+      default:
+        return p(C.gray, "○");
+    }
+  };
+  const lines = () => {
+    const w = width();
+    const rule = p(C.gray, ` ${"─".repeat(w - 2)}`);
+    const c = counts(rows);
+    const doneCount = c.pass + c.fail + c.error;
+    const badge = finished ? `DONE · RUN ${finished.runId.slice(-4)}` : `RUNNING · ${doneCount}/${rows.length}`;
+    const title = ` \uD83E\uDD8B firegoat eval · ${opts.project}`;
+    const out = [
+      `${p(C.ember, " \uD83E\uDD8B")}${title.slice(3)}${" ".repeat(Math.max(1, w - title.length - badge.length - 1))}${p(finished ? C.green : C.ember, badge)}`,
+      rule
+    ];
+    if (authText)
+      out.push(p(C.gray, `  auth  ${authText}`));
+    if (setup) {
+      const mark = setup.state === "ok" ? p(C.green, "✓") : setup.state === "fail" ? p(C.red, "✗") : p(C.ember, SPIN[frame % SPIN.length]);
+      out.push(`  ${setup.state === "running" ? p(C.ember, "◐") : p(C.green, "●")}  setup${" ".repeat(Math.max(1, w - 20))}${mark}${setup.seconds !== undefined ? ` ${setup.seconds}s` : ""}`);
+    }
+    rows.forEach((r, i) => {
+      out.push(p(C.gray, "  │"));
+      const num = String(i + 1).padStart(2, "0");
+      const name = cut(`${r.test}${r.n > 1 ? `#${r.n}` : ""}`, 16).padEnd(16);
+      let detail = "";
+      if (r.state === "running")
+        detail = p(C.gray, `turn ${r.turns}`);
+      else if (r.state === "pass" || r.state === "fail")
+        detail = `${checksPassed(r).padEnd(11)} ${p(C.gray, usage(r))}`;
+      else if (r.state === "error")
+        detail = p(C.gray, usage(r));
+      out.push(`  ${glyph(r)}  ${p(C.gray, num)}  ${p(C.bold, name)} ${p(C.ember, ramp(r))} ${statusWord(r)}  ${detail}`);
+      if (r.state === "running" && r.command)
+        out.push(p(C.gray, `  ┊        └ $ ${cut(r.command, w - 16)}`));
+      if (r.state === "fail")
+        for (const ck of (r.checks ?? []).filter((x) => x.code !== 0)) {
+          out.push(`  ${p(C.gray, "│")}        ${p(C.red, "✗")} ${cut(ck.run, w - 14)}`);
+          out.push(p(C.gray, `  │          ${ck.error ?? `exit ${ck.code}`}`));
+        }
+      if (r.state === "error")
+        out.push(p(C.amber, `           ${cut(notGradedWhy(r.reason ?? ""), w - 12)}`));
+    });
+    out.push(rule);
+    const t = total(rows);
+    if (!finished) {
+      const track = Math.max(10, w - 34);
+      const at = rows.length ? Math.round(doneCount / rows.length * (track - 1)) : 0;
+      const goat = `${"·".repeat(at)}\uD83D\uDC10${"·".repeat(Math.max(0, track - at - 1))}`;
+      out.push(`  ${p(C.ember, goat)}  ${p(C.gray, `${doneCount}/${rows.length} · $${t.cost.toFixed(2)} so far`)}`);
+      return out;
+    }
+    const tr = tier(rows);
+    const goated = c.pass === rows.length && rows.length > 0;
+    const tally = `${c.pass} PASS · ${c.fail} FAIL · ${c.error} NOT GRADED`;
+    const right = goated ? p(C.ember, `\uD83D\uDC10 GOATED · tier ${tr}`) : tr ? `tier ${tr}` : "";
+    out.push(`  ${goated ? p(C.green, `${rows.length}/${rows.length} runs passed their checks`) : tally}${" ".repeat(Math.max(2, w - tally.length - 22))}${right}`);
+    out.push(p(C.gray, `  total $${t.cost.toFixed(2)} · ${t.turns} turns · ${t.time}`));
+    out.push(p(C.gray, `  runs  ${relative2(process.cwd(), finished.runsDir)}/`));
+    out.push(`  checks are a score, not the verdict: judge each end_goal from ${p(C.cyan, "<test>/<n>/transcript.txt")}`);
+    return out;
+  };
+  const rowsOf = (line) => {
+    const visible = line.replace(ANSI, "").replace(/🐐|🦋/g, "gg");
+    return Math.max(1, Math.ceil(visible.length / (out.columns || 80)));
+  };
+  const render = () => {
+    const text = lines();
+    let s = drawn > 0 ? `\x1B[${drawn}A\x1B[J` : "";
+    s += `${text.join(`
+`)}
+`;
+    out.write(s);
+    drawn = text.reduce((n, l) => n + rowsOf(l), 0);
+  };
+  const timer = setInterval(() => {
+    frame++;
+    render();
+  }, 120);
+  timer.unref();
+  return {
+    auth: (text) => {
+      authText = text;
+    },
+    setup: (state, seconds) => {
+      setup = { state, seconds };
+      render();
+    },
+    runs: (r) => {
+      rows = r;
+      render();
+    },
+    update: render,
+    done: ({ runId, runsDir }) => {
+      clearInterval(timer);
+      finished = { runId, runsDir };
+      render();
+    }
+  };
+}
+
 // src/which.ts
 import { accessSync, constants, statSync as statSync2 } from "node:fs";
 import { delimiter, join as join3, resolve } from "node:path";
@@ -7662,15 +7981,6 @@ function findSession(configDir, sessionId) {
   }
   return null;
 }
-function usage(r) {
-  const parts = [];
-  if (r.cost !== null)
-    parts.push(`$${r.cost.toFixed(2)}`);
-  if (r.num_turns !== null)
-    parts.push(`${r.num_turns} turns`);
-  parts.push(`${Math.round(r.duration_ms / 1000)}s`);
-  return parts.join(", ");
-}
 function newRunId() {
   const ts = new Date().toISOString().replace(/\.\d+Z$/, "Z").replace(/[-:]/g, "");
   return `${ts}-${randomBytes(2).toString("hex")}`;
@@ -7707,7 +8017,13 @@ async function runOne(opts) {
 `);
     return r;
   };
-  const stepEnv = { ...opts.stepEnv, FIREGOAT_WORKDIR: workdir };
+  const env = renderEnv(test.env, opts.vars, {
+    "run.id": opts.runId,
+    "run.n": String(n),
+    "run.workdir": workdir,
+    "test.name": test.name
+  });
+  const stepEnv = { ...opts.stepEnv, ...env, FIREGOAT_WORKDIR: workdir };
   if (opts.workdirSetup.length > 0) {
     const error = await runSteps({
       what: "workdir_setup",
@@ -7724,9 +8040,10 @@ async function runOne(opts) {
     bin: opts.claudeBin,
     prompt: r.prompt,
     cwd: workdir,
-    env: { ...claudeEnv(opts.configDir), PATH: opts.stepEnv.PATH },
+    env: { ...claudeEnv(opts.configDir, env), PATH: opts.stepEnv.PATH },
     stderr,
-    timeoutMs: test.timeout * 1000
+    timeoutMs: test.timeout * 1000,
+    onEvent: opts.onEvent
   });
   await new Promise((res) => stderr.end(res));
   if (outcome.sessionId)
@@ -7787,6 +8104,8 @@ async function evalCommand(args, root) {
     throw e;
   }
   const say = opts.json ? console.error : console.log;
+  const stream = opts.json ? process.stderr : process.stdout;
+  const ui = stream.isTTY ? ttyUI(stream, { project: basename(root), color: !process.env.NO_COLOR }) : plainUI(say);
   let config;
   try {
     config = loadConfig(root);
@@ -7806,7 +8125,11 @@ async function evalCommand(args, root) {
   const runs = opts.runs ?? config.runs;
   const vars = { ...config.vars, ...opts.vars };
   const missing = [
-    ...new Set(tests.flatMap((t) => [t.prompt, ...t.checks.map((c) => c.run)]).flatMap(varRefs).filter((k) => !(k in vars)))
+    ...new Set(tests.flatMap((t) => [
+      t.prompt,
+      ...t.checks.map((c) => c.run),
+      ...Object.values(t.env)
+    ]).flatMap(varRefs).filter((k) => !(k in vars)))
   ];
   if (missing.length > 0)
     return fatal(EXIT.usage, `no value for ${missing.map((k) => `{{vars.${k}}}`).join(", ")}: pass --var ${missing[0]}=... or set it under vars: in firegoat.yaml`);
@@ -7816,7 +8139,7 @@ async function evalCommand(args, root) {
   const firegoatDir = ensureFiregoatDir(root);
   const configDir = join4(firegoatDir, ".claude");
   mkdirSync2(configDir, { recursive: true });
-  say(process.env.ANTHROPIC_API_KEY ? "auth: ANTHROPIC_API_KEY (claude bills the API key; unset it to use your claude login)" : "auth: your claude login");
+  ui.auth(process.env.ANTHROPIC_API_KEY ? "ANTHROPIC_API_KEY (claude bills the API key; unset it to use your claude login)" : "your claude login");
   const loginProblem = checkLogin({
     bin: claudePath,
     env: claudeEnv(configDir),
@@ -7829,29 +8152,60 @@ async function evalCommand(args, root) {
   const out = join4(runDir, "out");
   mkdirSync2(out, { recursive: true });
   const stepEnv = {
-    ...process.env,
+    ...hostEnv(),
     ...vars,
     FIREGOAT_OUT: out,
     PATH: [out, process.env.PATH ?? ""].join(delimiter2)
   };
+  const rows = tests.flatMap((t) => Array.from({ length: runs }, (_, i) => ({
+    test: t.name,
+    n: i + 1,
+    state: "queued",
+    turns: 0
+  })));
   if (config.setup.length > 0) {
-    say("setup...");
+    const setupStarted = Date.now();
+    ui.setup("running");
     const error = await runSteps({
       what: "setup",
       commands: config.setup,
       cwd: root,
-      env: stepEnv,
+      env: {
+        ...stepEnv,
+        ...renderEnv(Object.fromEntries(Object.entries(config.env).filter(([, v]) => isEvalLevel(v))), vars, { "run.id": runId })
+      },
       logPath: join4(runDir, "setup.txt")
     });
+    const secs = Math.round((Date.now() - setupStarted) / 1000);
+    ui.setup(error ? "fail" : "ok", secs);
     if (error)
       return fatal(EXIT.notGraded, `not graded: ${error}`);
   }
+  ui.runs(rows);
   const results = [];
   for (const test of tests)
     for (let n = 1;n <= runs; n++) {
+      const row = rows.find((x) => x.test === test.name && x.n === n);
+      row.state = "running";
+      ui.update();
       const r = await runOne({
+        onEvent: (e) => {
+          if (e.type !== "assistant")
+            return;
+          row.turns++;
+          const content = e.message?.content;
+          if (!Array.isArray(content))
+            return;
+          for (const b of content) {
+            const cmd = b.input?.command;
+            if (b.type === "tool_use" && typeof cmd === "string")
+              row.command = cmd.split(`
+`)[0];
+          }
+        },
         test,
         n,
+        runId,
         runDir,
         claudeBin: claudePath,
         configDir,
@@ -7860,7 +8214,16 @@ async function evalCommand(args, root) {
         vars
       });
       results.push(r);
-      say(`${r.status.toUpperCase()} ${r.test}#${r.n} ${r.reason} (${usage(r)})`);
+      Object.assign(row, {
+        state: r.status,
+        turns: r.num_turns ?? row.turns,
+        reason: r.reason,
+        checks: r.checks,
+        cost: r.cost,
+        seconds: Math.round(r.duration_ms / 1000),
+        command: undefined
+      });
+      ui.update();
     }
   const totals = { pass: 0, fail: 0, error: 0 };
   for (const r of results)
@@ -7868,10 +8231,7 @@ async function evalCommand(args, root) {
   const doc = { runId, runsDir: runDir, totals, runs: results };
   writeFileSync3(join4(runDir, "summary.json"), `${JSON.stringify(doc, null, 2)}
 `);
-  const cost = results.reduce((sum, r) => sum + (r.cost ?? 0), 0);
-  say(`
-${totals.pass} passed, ${totals.fail} failed, ${totals.error} not graded (${results.length} runs, $${cost.toFixed(2)})`);
-  say(`runs: ${runDir} (end_goal is for you to judge: see run.json per run)`);
+  ui.done({ runId, runsDir: runDir, rows });
   if (opts.json)
     console.log(JSON.stringify(doc, null, 2));
   return totals.error > 0 ? EXIT.notGraded : totals.fail > 0 ? EXIT.fail : EXIT.pass;
@@ -7880,7 +8240,7 @@ ${totals.pass} passed, ${totals.fail} failed, ${totals.error} not graded (${resu
 // src/init.ts
 import { spawnSync as spawnSync2 } from "node:child_process";
 import { existsSync as existsSync3, writeFileSync as writeFileSync4 } from "node:fs";
-import { basename, join as join5 } from "node:path";
+import { basename as basename2, join as join5 } from "node:path";
 import { createInterface } from "node:readline/promises";
 var SKILLS_ARGV = [
   "skills",
@@ -7924,6 +8284,12 @@ runs: 1
 # setup runs in this folder, so: setup: cp -R .firegoat/fixtures/. "$FIREGOAT_OUT"
 # then e.g. workdir_setup: cp "$FIREGOAT_OUT/wizard.mjs" .
 
+# Optional: env vars for the agent (and for workdir_setup and checks of the same
+# run). The agent gets none of vars:, only these. Placeholders: {{vars.KEY}},
+# {{run.id}}, {{run.n}}, {{run.workdir}}, {{test.name}}. A test can add its own env:.
+# env:
+#   MYTOOL_HOME: "/tmp/mytool-{{run.id}}-{{run.n}}"
+
 tests:
   - name: quickstart
     # What a user would ask their agent. Edit this.
@@ -7983,48 +8349,73 @@ ${INIT_HELP}`);
       return EXIT.usage;
     }
   }
+  const interactive = Boolean(process.stdin.isTTY && process.stdout.isTTY);
+  const color = interactive && !process.env.NO_COLOR;
+  const paint = (code, text) => color ? `\x1B[${code}m${text}\x1B[0m` : text;
+  const width = Math.min(process.stdout.columns || 80, 80);
+  const rule = paint("90", ` ${"─".repeat(width - 2)}`);
+  const step = (n, label, status) => console.log(`  ${paint("32", "●")}  ${paint("90", String(n).padStart(2, "0"))}  ${label.padEnd(28)}${paint("90", status)}`);
+  if (interactive) {
+    const title = " \uD83E\uDD8B firegoat init";
+    const tag = "Get agent-ready.";
+    console.log(`${paint("38;5;208", " \uD83E\uDD8B")} firegoat init${" ".repeat(Math.max(2, width - title.length - tag.length - 2))}${paint("90", tag)}`);
+    console.log(rule);
+  }
   ensureFiregoatDir(cwd);
   const configPath = join5(cwd, CONFIG_FILE);
-  if (existsSync3(configPath) && !force)
-    console.log(`kept ${CONFIG_FILE} (--force replaces it)`);
-  else {
-    writeFileSync4(configPath, template(basename(cwd)));
-    console.log(`wrote ${CONFIG_FILE}: edit the quickstart test to fit your users`);
-  }
-  const interactive = process.stdin.isTTY && process.stdout.isTTY;
-  if (skills && interactive)
-    skills = await askYes("Install the firegoat-onboarding skill for your coding agent?");
+  const keep = existsSync3(configPath) && !force;
+  if (!keep)
+    writeFileSync4(configPath, template(basename2(cwd)));
+  if (interactive) {
+    step(1, CONFIG_FILE, keep ? "kept (--force replaces it)" : "created");
+    console.log(paint("90", "  │"));
+    step(2, ".firegoat/", "ready");
+  } else
+    console.log(keep ? `kept ${CONFIG_FILE} (--force replaces it)` : `wrote ${CONFIG_FILE}: edit the quickstart test to fit your users`);
+  if (skills && interactive) {
+    console.log(paint("90", "  │"));
+    skills = await askYes(`  ${paint("38;5;208", "◉")}  ${paint("90", "03")}  Install the firegoat-onboarding skill?`);
+    process.stdout.write("\x1B[1A\x1B[2K");
+  }
   if (skills) {
-    console.log(`installing skills: npx ${SKILLS_ARGV.join(" ")}`);
+    if (!interactive)
+      console.log(`installing skills: npx ${SKILLS_ARGV.join(" ")}`);
     const env = { ...process.env };
     delete env.npm_config_package;
     const r = spawnSync2("npx", ["-y", ...SKILLS_ARGV], {
       cwd,
       env,
-      stdio: interactive ? "inherit" : "pipe",
+      stdio: "pipe",
       encoding: "utf8"
     });
     if (r.error || r.status !== 0) {
-      if (!interactive)
-        process.stderr.write(`${r.stdout ?? ""}${r.stderr ?? ""}`);
+      process.stderr.write(`${r.stdout ?? ""}${r.stderr ?? ""}`);
       console.error(`firegoat init: installing the skills failed (${r.error?.message ?? `exit ${r.status}`}). Run it yourself: npx ${SKILLS_ARGV.join(" ")}`);
       return EXIT.notGraded;
     }
-    console.log("installed skills firegoat-onboarding, suggesting-eval-prompts in .claude/skills/");
+    if (interactive)
+      step(3, "firegoat-onboarding skill", "installed → .claude/skills/");
+    else
+      console.log("installed skills firegoat-onboarding, suggesting-eval-prompts in .claude/skills/");
     const ignored = spawnSync2("git", ["check-ignore", "-q", ".claude/skills"], {
       cwd
     });
     if (ignored.status === 0)
       console.error("warning: git ignores .claude/skills, so teammates and CI won't get the skills. Commit them with: git add -f .claude/skills");
-  }
-  console.log(interactive ? `
-Paste this into your coding agent:
-
-${AGENT_PROMPT}
-` : `
+  } else if (interactive)
+    step(3, "firegoat-onboarding skill", "skipped");
+  if (!interactive) {
+    console.log(`
 Prompt for your coding agent: ${AGENT_PROMPT}`);
-  if (interactive && copyToClipboard(AGENT_PROMPT))
-    console.log("\x1B[32mPrompt copied!\x1B[0m");
+    return EXIT.pass;
+  }
+  console.log(rule);
+  console.log(`  next: hand this to your coding agent
+`);
+  console.log(AGENT_PROMPT);
+  console.log("");
+  const copied = copyToClipboard(AGENT_PROMPT);
+  console.log(`  ${copied ? `${paint("32", "✓ Prompt copied!")}   ` : ""}then: ${paint("36", "npx firegoat eval")}`);
   return EXIT.pass;
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.10.0",
+	"version": "0.12.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/firegoat-ai/firegoat-cli.git"