firegoat 0.28.0 → 0.30.0

package/README.md

@@ -22,6 +22,8 @@ In a terminal it asks where to install the firegoat-onboarding skill (global by
 npx firegoat@latest eval
 ```
 
+Run by an agent, eval ends with a results block to paste as-is (also in `.firegoat/runs/<runId>/results.txt`): ask it to show you the results table.
+
 ## Example
 
 ```yaml

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.28.0",
+  version: "0.30.0",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -7265,8 +7265,146 @@ import { homedir } from "node:os";
 import { join as join3 } from "node:path";
 
 // src/home.ts
+import { execFile } from "node:child_process";
 import { mkdirSync, writeFileSync } from "node:fs";
 import { join as join2 } from "node:path";
+import { promisify } from "node:util";
+
+// src/process.ts
+import { execFileSync, spawn } from "node:child_process";
+var NPM_INJECTED = /^(npm_config_|npm_lifecycle_|npm_package_)|^(npm_execpath|npm_node_execpath|npm_command|INIT_CWD)$/;
+var CLAUDE_SESSION = /^(CLAUDECODE|CLAUDE_CODE_.+|CLAUDE_PID|CLAUDE_ENV_FILE|CLAUDE_SESSION_ID|CLAUDE_PROJECT_DIR|CLAUDE_EFFORT|CLAUDE_AGENT_SDK_.+|CLAUDE_PLUGIN_ROOT|CLAUDE_PLUGIN_DATA|CLAUDE_SKILL_DIR)$/;
+var CLAUDE_AUTH = /^CLAUDE_CODE_(OAUTH_TOKEN|OAUTH_REFRESH_TOKEN|USE_[A-Z_]+|SKIP_[A-Z]+_AUTH|CLIENT_CERT|CLIENT_KEY|CLIENT_KEY_PASSPHRASE)$/;
+function hostEnv() {
+  return Object.fromEntries(Object.entries(process.env).filter(([k]) => !NPM_INJECTED.test(k) && !k.startsWith("FIREGOAT_") && (!CLAUDE_SESSION.test(k) || CLAUDE_AUTH.test(k))));
+}
+var GRACE_MS = 5000;
+var liveGroups = new Set;
+var signalsHooked = false;
+function hookSignals() {
+  if (signalsHooked)
+    return;
+  signalsHooked = true;
+  const onSignal = (signal) => {
+    for (const g of liveGroups)
+      try {
+        process.kill(-g, "SIGKILL");
+      } catch {}
+    process.kill(process.pid, signal);
+  };
+  for (const s of ["SIGINT", "SIGTERM", "SIGHUP"])
+    process.once(s, onSignal);
+}
+function forgetGroup(pgid) {
+  liveGroups.delete(pgid);
+}
+function processTree(pid) {
+  const children = new Map;
+  const ps = execFileSync("ps", ["-A", "-o", "pid=,ppid="], {
+    encoding: "utf8"
+  });
+  for (const line of ps.split(`
+`)) {
+    const [child, parent] = line.trim().split(/\s+/).map(Number);
+    if (!child || parent === undefined)
+      continue;
+    children.set(parent, [...children.get(parent) ?? [], child]);
+  }
+  const tree = [pid];
+  for (let i = 0;i < tree.length; i++)
+    tree.push(...children.get(tree[i]) ?? []);
+  return tree;
+}
+function killTree(pid, group) {
+  if (group)
+    try {
+      process.kill(-pid, "SIGKILL");
+    } catch {}
+  let tree = [pid];
+  let problem;
+  try {
+    tree = processTree(pid);
+  } catch (e) {
+    problem = e.message;
+  }
+  for (const p of tree)
+    try {
+      process.kill(p, "SIGKILL");
+    } catch {}
+  return problem;
+}
+function run(opts) {
+  return new Promise((resolve) => {
+    const detached = opts.detached === true;
+    if (detached)
+      hookSignals();
+    const child = spawn(opts.cmd, opts.args, {
+      cwd: opts.cwd,
+      env: opts.env,
+      stdio: ["ignore", "pipe", "pipe"],
+      detached
+    });
+    if (child.pid !== undefined) {
+      if (detached)
+        liveGroups.add(child.pid);
+      opts.onSpawn?.(child.pid);
+    }
+    let timedOut = false;
+    let settled = false;
+    const timer = setTimeout(() => {
+      timedOut = true;
+      if (child.pid === undefined)
+        return;
+      const problem = killTree(child.pid, detached);
+      if (problem)
+        opts.onStderr(Buffer.from(`firegoat: timed out; could not list ${opts.cmd}'s child processes (${problem}), killed only ${opts.cmd}
+`));
+    }, opts.timeoutMs);
+    const done = (r) => {
+      if (settled)
+        return;
+      settled = true;
+      clearTimeout(timer);
+      resolve(r);
+    };
+    child.stdout.on("data", opts.onStdout);
+    child.stderr.on("data", opts.onStderr);
+    opts.finalSeen?.then(() => setTimeout(() => done({ code: null, timedOut: false, ended: true }), GRACE_MS).unref());
+    child.on("error", (error) => done({ code: null, timedOut, error }));
+    child.on("close", (code) => done({ code, timedOut }));
+    child.on("exit", (code) => setTimeout(() => {
+      child.stdout.destroy();
+      child.stderr.destroy();
+      done({ code, timedOut });
+    }, 2000).unref());
+  });
+}
+async function runShell(opts) {
+  opts.log.write(`$ ${opts.run}
+`);
+  const r = await run({
+    cmd: "/bin/sh",
+    args: ["-c", opts.run],
+    cwd: opts.cwd,
+    env: opts.env,
+    timeoutMs: opts.timeoutMs,
+    onStdout: (c) => opts.log.write(c),
+    onStderr: (c) => opts.log.write(c)
+  });
+  let error;
+  if (r.error)
+    error = `could not start /bin/sh: ${r.error.message}`;
+  else if (r.timedOut)
+    error = `timed out after ${opts.timeoutMs / 1000}s`;
+  else if (r.code === null)
+    error = "killed by a signal";
+  opts.log.write(`[exit ${r.code ?? "none"}${error ? `: ${error}` : ""}]
+
+`);
+  return error ? { run: opts.run, code: r.code, error } : { run: opts.run, code: r.code };
+}
+
+// src/home.ts
 function homeEnv(home) {
   return {
     HOME: home,
@@ -7287,6 +7425,23 @@ function makeRunHome(dir) {
 `).join(""));
   return { home, envFile };
 }
+async function makeRunKeychain(home) {
+  const keychain = join2(home, "Library", "Keychains", "login.keychain-db");
+  const env = { ...hostEnv(), HOME: home };
+  mkdirSync(join2(home, "Library", "Keychains"), { recursive: true });
+  for (const args of [
+    ["create-keychain", "-p", "", keychain],
+    ["set-keychain-settings", keychain]
+  ]) {
+    try {
+      await promisify(execFile)("security", args, { env, timeout: 30000 });
+    } catch (e) {
+      const err = e;
+      return `could not create the run's login keychain (security ${args[0]} ${keychain}): ${err.stderr?.trim() || err.message}`;
+    }
+  }
+  return null;
+}
 
 // src/harnesses/codex.ts
 var FLAGS2 = [
@@ -7879,6 +8034,7 @@ function validHarnesses() {
 // src/config.ts
 var CONFIG_FILE = "firegoat.yaml";
 var DEFAULT_TIMEOUT = 600;
+var DEFAULT_PARALLEL = 4;
 
 class ConfigError extends Error {
 }
@@ -7916,6 +8072,7 @@ function parseConfig(text) {
     fail("(root)", "must be a mapping");
   onlyKeys(raw, "(root)", [
     "runs",
+    "parallel",
     "harness",
     "model",
     "build",
@@ -7927,6 +8084,7 @@ function parseConfig(text) {
     "tests"
   ]);
   const runs = posInt(raw.runs, "runs", 1);
+  const parallel = posInt(raw.parallel, "parallel", DEFAULT_PARALLEL);
   const harnesses = raw.harness === undefined ? [DEFAULT_HARNESS] : nameList(raw.harness, "harness", "a harness name");
   harnesses.forEach((h, i) => {
     if (!HARNESSES.has(h))
@@ -7990,6 +8148,7 @@ function parseConfig(text) {
   });
   return {
     runs,
+    parallel,
     harnesses,
     models,
     build,
@@ -8173,6 +8332,7 @@ import { randomBytes as randomBytes2 } from "node:crypto";
 import {
   appendFileSync,
   createWriteStream as createWriteStream2,
+  existsSync as existsSync6,
   mkdirSync as mkdirSync5,
   writeFileSync as writeFileSync5
 } from "node:fs";
@@ -8530,142 +8690,6 @@ function ensureFiregoatDir(root) {
 // src/git.ts
 import { spawnSync as spawnSync3 } from "node:child_process";
 import { basename, dirname } from "node:path";
-
-// src/process.ts
-import { execFileSync, spawn } from "node:child_process";
-var NPM_INJECTED = /^(npm_config_|npm_lifecycle_|npm_package_)|^(npm_execpath|npm_node_execpath|npm_command|INIT_CWD)$/;
-var CLAUDE_SESSION = /^(CLAUDECODE|CLAUDE_CODE_.+|CLAUDE_PID|CLAUDE_ENV_FILE|CLAUDE_SESSION_ID|CLAUDE_PROJECT_DIR|CLAUDE_EFFORT|CLAUDE_AGENT_SDK_.+|CLAUDE_PLUGIN_ROOT|CLAUDE_PLUGIN_DATA|CLAUDE_SKILL_DIR)$/;
-var CLAUDE_AUTH = /^CLAUDE_CODE_(OAUTH_TOKEN|OAUTH_REFRESH_TOKEN|USE_[A-Z_]+|SKIP_[A-Z]+_AUTH|CLIENT_CERT|CLIENT_KEY|CLIENT_KEY_PASSPHRASE)$/;
-function hostEnv() {
-  return Object.fromEntries(Object.entries(process.env).filter(([k]) => !NPM_INJECTED.test(k) && !k.startsWith("FIREGOAT_") && (!CLAUDE_SESSION.test(k) || CLAUDE_AUTH.test(k))));
-}
-var GRACE_MS = 5000;
-var liveGroups = new Set;
-var signalsHooked = false;
-function hookSignals() {
-  if (signalsHooked)
-    return;
-  signalsHooked = true;
-  const onSignal = (signal) => {
-    for (const g of liveGroups)
-      try {
-        process.kill(-g, "SIGKILL");
-      } catch {}
-    process.kill(process.pid, signal);
-  };
-  for (const s of ["SIGINT", "SIGTERM", "SIGHUP"])
-    process.once(s, onSignal);
-}
-function forgetGroup(pgid) {
-  liveGroups.delete(pgid);
-}
-function processTree(pid) {
-  const children = new Map;
-  const ps = execFileSync("ps", ["-A", "-o", "pid=,ppid="], {
-    encoding: "utf8"
-  });
-  for (const line of ps.split(`
-`)) {
-    const [child, parent] = line.trim().split(/\s+/).map(Number);
-    if (!child || parent === undefined)
-      continue;
-    children.set(parent, [...children.get(parent) ?? [], child]);
-  }
-  const tree = [pid];
-  for (let i = 0;i < tree.length; i++)
-    tree.push(...children.get(tree[i]) ?? []);
-  return tree;
-}
-function killTree(pid, group) {
-  if (group)
-    try {
-      process.kill(-pid, "SIGKILL");
-    } catch {}
-  let tree = [pid];
-  let problem;
-  try {
-    tree = processTree(pid);
-  } catch (e) {
-    problem = e.message;
-  }
-  for (const p of tree)
-    try {
-      process.kill(p, "SIGKILL");
-    } catch {}
-  return problem;
-}
-function run(opts) {
-  return new Promise((resolve) => {
-    const detached = opts.detached === true;
-    if (detached)
-      hookSignals();
-    const child = spawn(opts.cmd, opts.args, {
-      cwd: opts.cwd,
-      env: opts.env,
-      stdio: ["ignore", "pipe", "pipe"],
-      detached
-    });
-    if (child.pid !== undefined) {
-      if (detached)
-        liveGroups.add(child.pid);
-      opts.onSpawn?.(child.pid);
-    }
-    let timedOut = false;
-    let settled = false;
-    const timer = setTimeout(() => {
-      timedOut = true;
-      if (child.pid === undefined)
-        return;
-      const problem = killTree(child.pid, detached);
-      if (problem)
-        opts.onStderr(Buffer.from(`firegoat: timed out; could not list ${opts.cmd}'s child processes (${problem}), killed only ${opts.cmd}
-`));
-    }, opts.timeoutMs);
-    const done = (r) => {
-      if (settled)
-        return;
-      settled = true;
-      clearTimeout(timer);
-      resolve(r);
-    };
-    child.stdout.on("data", opts.onStdout);
-    child.stderr.on("data", opts.onStderr);
-    opts.finalSeen?.then(() => setTimeout(() => done({ code: null, timedOut: false, ended: true }), GRACE_MS).unref());
-    child.on("error", (error) => done({ code: null, timedOut, error }));
-    child.on("close", (code) => done({ code, timedOut }));
-    child.on("exit", (code) => setTimeout(() => {
-      child.stdout.destroy();
-      child.stderr.destroy();
-      done({ code, timedOut });
-    }, 2000).unref());
-  });
-}
-async function runShell(opts) {
-  opts.log.write(`$ ${opts.run}
-`);
-  const r = await run({
-    cmd: "/bin/sh",
-    args: ["-c", opts.run],
-    cwd: opts.cwd,
-    env: opts.env,
-    timeoutMs: opts.timeoutMs,
-    onStdout: (c) => opts.log.write(c),
-    onStderr: (c) => opts.log.write(c)
-  });
-  let error;
-  if (r.error)
-    error = `could not start /bin/sh: ${r.error.message}`;
-  else if (r.timedOut)
-    error = `timed out after ${opts.timeoutMs / 1000}s`;
-  else if (r.code === null)
-    error = "killed by a signal";
-  opts.log.write(`[exit ${r.code ?? "none"}${error ? `: ${error}` : ""}]
-
-`);
-  return error ? { run: opts.run, code: r.code, error } : { run: opts.run, code: r.code };
-}
-
-// src/git.ts
 function git(dir, args) {
   const r = spawnSync3("git", args, {
     cwd: dir,
@@ -8834,6 +8858,9 @@ firegoat.yaml, committed.
 
 firegoat.yaml reference:
   runs: 1                     Runs per test
+  parallel: 4                 Runs at once (default 4; 1 runs them one after
+                              another, e.g. when tests share one cloud
+                              account); eval --parallel overrides
   harness: <name> | [<name>, ...]
                               The agent CLI: claude-code (default), codex-cli
                               or grok-cli.
@@ -8946,7 +8973,11 @@ firegoat.yaml reference:
   Checks also get $FIREGOAT_TRANSCRIPT, a short plain-text
   transcript, greppable line by line (format: eval --help, Transcript). So
   '^\\[[0-9]+\\] \\$ .*foo' matches only commands. Checks also get
-  $FIREGOAT_SESSION (the harness's full session file, JSON lines).
+  $FIREGOAT_SESSION (the harness's full session file, JSON lines). cleanup gets
+  $FIREGOAT_TRANSCRIPT once the agent has run (also after a claude error or a
+  timeout), and $FIREGOAT_SESSION when the harness saved a session file; neither
+  is set when install failed before the agent started, so a cleanup that reads
+  them should fail on an unset one: grep ... "\${FIREGOAT_TRANSCRIPT:?}".
 
   The agent never gets vars, only env:. The vars npm/npx set for firegoat itself
   (npm_config_*, npm_lifecycle_*, npm_package_*, npm_execpath, npm_node_execpath,
@@ -9102,10 +9133,13 @@ Every run gets its own HOME, ~/.firegoat/runs/<hash>/<random>/home next to work
 $FIREGOAT_HOME: tools that keep state in ~ start fresh and parallel runs don't
 collide. claude itself keeps your real HOME (its login is there) and sources
 home.env next to it (CLAUDE_ENV_FILE) before each Bash command the agent runs; install,
-checks and cleanup get the run's HOME directly; build keeps yours. Limits: claude's
-own Read/Write tools expand ~ to your real home; getpwuid lookups (os.userInfo,
-python pwd) still see it; keychain logins (gh, docker, git credential helpers) look
-logged out inside a run, so log tools in from install: or with env: tokens; the
+checks and cleanup get the run's HOME directly; build keeps yours. On macOS the run's
+home has its own empty, password-less login keychain (Library/Keychains), so a
+browser the agent starts doesn't ask you for one; your keychain is never touched.
+Limits: claude's own Read/Write tools expand ~ to your real home; getpwuid lookups
+(os.userInfo, python pwd) still see it; keychain logins (gh, docker, git credential
+helpers) look logged out inside a run, so log tools in from install: or with env:
+tokens; the
 fresh home has no .gitconfig, so set user.name/user.email in install: if the agent
 commits; claude's shell snapshot still sources your real ~/.zshrc.
 
@@ -9129,14 +9163,16 @@ firegoat.yaml, fixtures and build: stay in the current folder.
   --model NAME       The harness's --model, replacing model: in firegoat.yaml;
                      repeat it (--model a --model b) to run each test runs times
                      per model
-  --parallel N       Runs at once (default 4; 1 runs them one after another).
+  --parallel N       Runs at once (overrides parallel: in firegoat.yaml;
+                     default 4; 1 runs them one after another).
                      Each run has its own HOME; state outside it (a fixed port,
                      /tmp paths) can still collide: use 1 then
   --var KEY=value    Fills {{vars.KEY}} (prompts, checks, build, install,
                      cleanup); env for build, install, checks and cleanup
   --json             The summary as JSON on stdout (or {error, exitCode} if eval
                      stops early); progress goes to stderr. The verdict comes
-                     last, after runs: totals {pass, fail, error}, status
+                     last, after runs: totals {pass, fail, error}, tier
+                     (S/A/B/C, as printed; null when no run was graded), status
                      ("pass", "fail" or "error" = not graded) and exitCode
   --no-upload        Keep runs local even when logged in to firegoat.ai
   --dry-run          Check firegoat.yaml as eval would (test names, --var,
@@ -9167,6 +9203,14 @@ says "N uploads failed" on stderr. A file over 20 MiB fails that run's upload.
 Runs without a session file are not uploaded. FIREGOAT_BASE_URL points elsewhere
 (default https://firegoat.ai/evals). See firegoat login --help.
 
+Results block: without a terminal (an agent, a pipe, CI) and without --json, eval
+ends with the terminal's final timeline as one plain block (no color, at most 100
+columns): the logo header, a row per run (status, a ramp of one block per agent turn
+ending ▲ pass, ╳ fail or ╌ not graded, checks passed, cost, turns, time), its failed
+checks and trace URL, the totals and the tier ("GOATED · tier S" when all pass). It
+sits between two lines of ═, after a "Results (paste the block ...)" line; the last
+line says where it is saved, results.txt. Show it to your user as-is in a code block.
+
 Output: .firegoat/runs/<runId>/
   out                     A symlink to $FIREGOAT_OUT,
                           ~/.firegoat/runs/<hash>/<random>/out
@@ -9174,7 +9218,8 @@ Output: .firegoat/runs/<runId>/
                           ran), git {branch, commit, dirty} (no git outside a
                           repo), harnesses, upload {origin, email} (null when
                           not uploading), runs (every run.json), then the
-                          verdict: totals, status and exitCode
+                          verdict: totals, tier, status and exitCode
+  results.txt             The results block, also with --json or in a terminal
   ../index.jsonl          One line per eval: runId, startedAt, dir, branch,
                           commit, harnesses, passed, failed, notGraded, cost (null when
                           any run's cost is unknown), uploaded, uploadFailed
@@ -9191,7 +9236,8 @@ Output: .firegoat/runs/<runId>/
                           claudeSession (the session.jsonl uploaded: claude's
                           own file, else <test>/<n>/session.jsonl made from the
                           stream) and sessionId (its id, the trace's), stderr,
-                          checksLog, cleanupLog; startedAt; upload ({status: "uploaded",
+                          checksLog, cleanupLog; turns and cost (null when the
+                          harness reports none), durationMs, startedAt; upload ({status: "uploaded",
                           url} | {status: "failed", error} | {status:
                           "skipped", reason}, null when not uploading)
   <test>/<n>/transcript.txt  What the agent did, below
@@ -10276,17 +10322,9 @@ function plainUI(write) {
     }
   };
 }
-function ttyUI(out, opts) {
-  const p = painter(opts.color);
-  let rows = [];
-  let build = null;
-  let authText = "";
-  const notes = [];
-  let frame = 0;
-  let drawn = 0;
-  let finished = null;
-  const width = () => Math.min(out.columns || 80, 100);
-  const cut = (text, max) => text.length > max ? `${text.slice(0, max - 1)}…` : text;
+var cut = (text, max) => text.length > max ? `${text.slice(0, max - 1)}…` : text;
+function timeline(t, p, w) {
+  const { rows, build, finished, frame } = t;
   const statusWord = (r) => {
     switch (r.state) {
       case "pass":
@@ -10315,77 +10353,109 @@ function ttyUI(out, opts) {
         return p(C.gray, "○");
     }
   };
-  const lines = () => {
-    const w = width();
-    const rule = p(C.gray, ` ${"─".repeat(w - 2)}`);
-    const c = counts(rows);
-    const doneCount = c.pass + c.fail + c.error;
-    const badge = finished ? `DONE · RUN ${finished.runId.slice(-4)}` : `RUNNING · ${doneCount}/${rows.length}`;
-    const out = [
-      ...logoHeader(p, `firegoat ${opts.version} eval · ${opts.project}`, p(finished ? C.green : C.ember, badge), w),
-      rule
-    ];
-    if (authText)
-      out.push(p(C.gray, `  Auth  ${authText}`));
-    for (const n of notes)
-      out.push(p(C.gray, `  ${cut(n, w - 2)}`));
-    if (build) {
-      const mark = build.state === "ok" ? p(C.green, "✓") : build.state === "fail" ? p(C.red, "✗") : p(C.ember, SPIN[frame % SPIN.length]);
-      out.push(`  ${build.state === "running" ? p(C.ember, "◐") : p(C.green, "●")}  Build${" ".repeat(Math.max(1, w - 20))}${mark}${build.seconds !== undefined ? ` ${build.seconds}s` : ""}`);
-    }
-    const column = (width) => Math.min(24, Math.max(0, ...rows.map(width)));
-    const harnessCols = column((r) => r.harness?.length ?? 0);
-    const modelCols = column((r) => r.model?.length ?? 0);
-    rows.forEach((r, i) => {
-      out.push(p(C.gray, "  │"));
-      const num = String(i + 1).padStart(2, "0");
-      const name = cut(`${r.test}${r.n > 1 ? `#${r.n}` : ""}`, 16).padEnd(16);
-      const harness = harnessCols > 0 ? ` ${p(C.cyan, cut(r.harness ?? "", harnessCols).padEnd(harnessCols))}` : "";
-      const model = modelCols > 0 ? ` ${p(C.cyan, cut(r.model ?? "", modelCols).padEnd(modelCols))}` : "";
-      let detail = "";
-      if (r.state === "running")
-        detail = p(C.gray, `Turn ${r.turns}`);
-      else if (r.state === "pass" || r.state === "fail")
-        detail = `${checksPassed(r).padEnd(11)} ${p(C.gray, usage(r))}`;
-      else if (r.state === "error")
-        detail = p(C.gray, usage(r));
-      out.push(`  ${glyph(r)}  ${p(C.gray, num)}  ${p(C.bold, name)}${harness}${model} ${p(C.ember, ramp(r))} ${statusWord(r)}  ${detail}`);
-      if (r.state === "running" && r.command)
-        out.push(p(C.gray, `  ┊        └ $ ${cut(r.command, w - 16)}`));
-      if (r.state === "fail")
-        for (const ck of (r.checks ?? []).filter((x) => x.code !== 0)) {
-          out.push(`  ${p(C.gray, "│")}        ${p(C.red, "✗")} ${cut(ck.run, w - 14)}`);
-          out.push(p(C.gray, `  │          ${ck.error ?? `exit ${ck.code}`}`));
-        }
-      if (r.state === "error")
-        out.push(p(C.amber, `           ${cut(notGradedWhy(r.reason ?? ""), w - 12)}`));
-      if (r.upload === "uploading")
-        out.push(p(C.gray, "  │        ↑ Uploading…"));
-      else if (r.upload?.status === "uploaded")
-        out.push(`  ${p(C.gray, "│")}        ${p(C.cyan, `↑ ${r.upload.url}`)}`);
-      else if (r.upload)
-        out.push(p(C.amber, `  │        ${cut(r.upload.status === "failed" ? `↑ Upload failed: ${r.upload.error}` : `↑ Not uploaded: ${r.upload.reason}`, w - 12)}`));
-    });
-    out.push(rule);
-    if (!finished) {
-      const track = Math.max(10, w - 34);
-      const at = rows.length ? Math.round(doneCount / rows.length * (track - 1)) : 0;
-      const progress = `${"·".repeat(at)}▶${"·".repeat(Math.max(0, track - at - 1))}`;
-      out.push(`  ${p(C.ember, progress)}  ${p(C.gray, `${doneCount}/${rows.length} · ${totalCost(rows)} so far`)}`);
-      return out;
-    }
-    const tr = tier(rows);
-    const goated = c.pass === rows.length && rows.length > 0;
-    const tally = `${c.pass} PASS · ${c.fail} FAIL · ${c.error} NOT GRADED`;
-    const left = goated ? `${rows.length}/${rows.length} runs passed their checks` : tally;
-    const right = goated ? `GOATED · tier ${tr}` : tr ? `Tier ${tr}` : "";
-    out.push(`  ${goated ? p(C.green, left) : left}${" ".repeat(Math.max(2, w - left.length - right.length - 3))}${goated ? p(C.ember, right) : right}`);
-    const turns = totalTurns(rows);
-    out.push(p(C.gray, `  Total ${totalCost(rows)}${turns === null ? "" : ` · ${turns} turns`} · ${duration(finished.seconds)}`));
-    out.push(p(C.gray, `  Runs  ${shortPath(finished.runsDir)}/`));
-    out.push(`  Checks are a score, not the verdict: judge each end_goal from ${p(C.cyan, "<test>/<n>/transcript.txt")}`);
+  const rule = p(C.gray, ` ${"─".repeat(w - 2)}`);
+  const c = counts(rows);
+  const doneCount = c.pass + c.fail + c.error;
+  const badge = finished ? `DONE · RUN ${finished.runId.slice(-4)}` : `RUNNING · ${doneCount}/${rows.length}`;
+  const out = [
+    ...logoHeader(p, `firegoat ${t.version} eval · ${t.project}`, p(finished ? C.green : C.ember, badge), w),
+    rule
+  ];
+  if (t.authText)
+    out.push(p(C.gray, `  Auth  ${t.authText}`));
+  for (const n of t.notes)
+    out.push(p(C.gray, `  ${cut(n, w - 2)}`));
+  if (build) {
+    const mark = build.state === "ok" ? p(C.green, "✓") : build.state === "fail" ? p(C.red, "✗") : p(C.ember, SPIN[frame % SPIN.length]);
+    out.push(`  ${build.state === "running" ? p(C.ember, "◐") : p(C.green, "●")}  Build${" ".repeat(Math.max(1, w - 20))}${mark}${build.seconds !== undefined ? ` ${build.seconds}s` : ""}`);
+  }
+  const column = (width) => Math.min(24, Math.max(0, ...rows.map(width)));
+  const harnessCols = column((r) => r.harness?.length ?? 0);
+  const modelCols = column((r) => r.model?.length ?? 0);
+  const detailCols = w - 1 - (47 + (harnessCols > 0 ? harnessCols + 1 : 0) + (modelCols > 0 ? modelCols + 1 : 0));
+  rows.forEach((r, i) => {
+    out.push(p(C.gray, "  │"));
+    const num = String(i + 1).padStart(2, "0");
+    const name = cut(`${r.test}${r.n > 1 ? `#${r.n}` : ""}`, 16).padEnd(16);
+    const harness = harnessCols > 0 ? ` ${p(C.cyan, cut(r.harness ?? "", harnessCols).padEnd(harnessCols))}` : "";
+    const model = modelCols > 0 ? ` ${p(C.cyan, cut(r.model ?? "", modelCols).padEnd(modelCols))}` : "";
+    let detail = "";
+    if (r.state === "running")
+      detail = p(C.gray, `Turn ${r.turns}`);
+    else if (r.state === "pass" || r.state === "fail")
+      detail = `${checksPassed(r).padEnd(11)} ${p(C.gray, cut(usage(r), Math.max(1, detailCols - 12)))}`;
+    else if (r.state === "error")
+      detail = p(C.gray, cut(usage(r), Math.max(1, detailCols)));
+    out.push(`  ${glyph(r)}  ${p(C.gray, num)}  ${p(C.bold, name)}${harness}${model} ${p(C.ember, ramp(r))} ${statusWord(r)}  ${detail}`);
+    if (r.state === "running" && r.command)
+      out.push(p(C.gray, `  ┊        └ $ ${cut(r.command, w - 16)}`));
+    if (r.state === "fail")
+      for (const ck of (r.checks ?? []).filter((x) => x.code !== 0)) {
+        out.push(`  ${p(C.gray, "│")}        ${p(C.red, "✗")} ${cut(ck.run, w - 14)}`);
+        out.push(p(C.gray, `  │          ${cut(ck.error ?? `exit ${ck.code}`, w - 14)}`));
+      }
+    if (r.state === "error")
+      out.push(p(C.amber, `           ${cut(notGradedWhy(r.reason ?? ""), w - 12)}`));
+    if (r.upload === "uploading")
+      out.push(p(C.gray, "  │        ↑ Uploading…"));
+    else if (r.upload?.status === "uploaded")
+      out.push(`  ${p(C.gray, "│")}        ${p(C.cyan, `↑ ${r.upload.url}`)}`);
+    else if (r.upload)
+      out.push(p(C.amber, `  │        ${cut(r.upload.status === "failed" ? `↑ Upload failed: ${r.upload.error}` : `↑ Not uploaded: ${r.upload.reason}`, w - 12)}`));
+  });
+  out.push(rule);
+  if (!finished) {
+    const track = Math.max(10, w - 34);
+    const at = rows.length ? Math.round(doneCount / rows.length * (track - 1)) : 0;
+    const progress = `${"·".repeat(at)}▶${"·".repeat(Math.max(0, track - at - 1))}`;
+    out.push(`  ${p(C.ember, progress)}  ${p(C.gray, `${doneCount}/${rows.length} · ${totalCost(rows)} so far`)}`);
     return out;
-  };
+  }
+  const tr = tier(rows);
+  const goated = c.pass === rows.length && rows.length > 0;
+  const tally = `${c.pass} PASS · ${c.fail} FAIL · ${c.error} NOT GRADED`;
+  const left = goated ? `${rows.length}/${rows.length} runs passed their checks` : tally;
+  const right = goated ? `GOATED · tier ${tr}` : tr ? `Tier ${tr}` : "";
+  out.push(`  ${goated ? p(C.green, left) : left}${" ".repeat(Math.max(2, w - left.length - right.length - 3))}${goated ? p(C.ember, right) : right}`);
+  const turns = totalTurns(rows);
+  out.push(p(C.gray, `  Total ${totalCost(rows)}${turns === null ? "" : ` · ${turns} turns`} · ${duration(finished.seconds)}`));
+  out.push(p(C.gray, `  Runs  ${shortPath(finished.runsDir)}/`));
+  out.push(`  Checks are a score, not the verdict: judge each end_goal from ${p(C.cyan, "<test>/<n>/transcript.txt")}`);
+  return out;
+}
+var RESULTS_WIDTH = 100;
+function resultsBlock(t) {
+  return timeline({
+    project: t.project,
+    version: t.version,
+    rows: t.rows,
+    build: null,
+    authText: "",
+    notes: [],
+    finished: { runId: t.runId, runsDir: t.runsDir, seconds: t.seconds },
+    frame: 0
+  }, painter(false), RESULTS_WIDTH);
+}
+function ttyUI(out, opts) {
+  const p = painter(opts.color);
+  let rows = [];
+  let build = null;
+  let authText = "";
+  const notes = [];
+  let frame = 0;
+  let drawn = 0;
+  let finished = null;
+  const width = () => Math.min(out.columns || 80, 100);
+  const lines = () => timeline({
+    project: opts.project,
+    version: opts.version,
+    rows,
+    build,
+    authText,
+    notes,
+    finished,
+    frame
+  }, p, width());
   const rowsOf = (line) => {
     const visible = line.replace(ANSI, "");
     return Math.max(1, Math.ceil(visible.length / (out.columns || 80)));
@@ -10543,9 +10613,9 @@ async function uploadRun(t, r) {
         run: clip(c.run, MAX_TEXT, "…"),
         code: c.code
       })),
-      num_turns: r.num_turns,
+      num_turns: r.turns,
       cost: r.cost,
-      duration_ms: r.duration_ms,
+      duration_ms: r.durationMs,
       harness: r.harness,
       model,
       ...git && { git }
@@ -10624,7 +10694,6 @@ function parseArgs(args) {
     names: [],
     models: [],
     harnesses: [],
-    parallel: 4,
     json: false,
     noUpload: false,
     dryRun: false,
@@ -10835,9 +10904,9 @@ async function runOne(opts) {
     cleanup: [],
     cleanupLog: join10(dir, "cleanup.txt"),
     leftover: [],
-    num_turns: null,
+    turns: null,
     cost: null,
-    duration_ms: 0,
+    durationMs: 0,
     startedAt: new Date(started).toISOString(),
     upload: null
   };
@@ -10858,13 +10927,17 @@ async function runOne(opts) {
     r.cleanup = await runAllSteps({
       commands: steps.cleanup,
       cwd: workdir,
-      env: stepEnv,
+      env: {
+        ...stepEnv,
+        ...existsSync6(r.transcript) && { FIREGOAT_TRANSCRIPT: r.transcript },
+        ...r.session !== null && { FIREGOAT_SESSION: r.session }
+      },
       logPath: r.cleanupLog
     });
   r.leftover = await stopLeftovers({ group: agent.group, workdir });
   r.status = status;
   r.reason = reason;
-  r.duration_ms = Date.now() - started;
+  r.durationMs = Date.now() - started;
   writeRunJson(r);
   return r;
 }
@@ -10874,6 +10947,11 @@ function withoutOldPwd(env) {
 async function attempt(opts, r, steps, env, stepEnv, homeEnvFile, agent) {
   const { test, n } = opts;
   const { dir, workdir } = r;
+  if (process.platform === "darwin") {
+    const error = await makeRunKeychain(r.home);
+    if (error)
+      return ["error", `not graded: ${error}`];
+  }
   if (steps.install.length > 0) {
     const error = await runSteps({
       what: "install",
@@ -10947,7 +11025,7 @@ async function attempt(opts, r, steps, env, stepEnv, homeEnvFile, agent) {
     final: outcome.final,
     workdir
   });
-  r.num_turns = outcome.turns;
+  r.turns = outcome.turns;
   r.cost = outcome.cost;
   if (!outcome.ok)
     return ["error", `not graded: ${outcome.reason}`];
@@ -10998,6 +11076,16 @@ function uploadedSummary(origin, results, runDir) {
   for (const u of urls)
     console.log(`  ${u}`);
 }
+var RESULTS_EDGE = "═".repeat(RESULTS_WIDTH);
+function printResults(block, file) {
+  console.log("");
+  console.log("Results (paste the block between the ═ lines as-is, in a code block):");
+  console.log(RESULTS_EDGE);
+  for (const line of block)
+    console.log(line);
+  console.log(RESULTS_EDGE);
+  console.log(`Results block saved to ${file}`);
+}
 async function evalCommand(args, root) {
   if (args.includes("--help") || args.includes("-h")) {
     console.log(EVAL_HELP);
@@ -11045,6 +11133,7 @@ async function evalCommand(args, root) {
     tests = tests.filter((t) => opts.names.includes(t.name));
   }
   const runs = opts.runs ?? config.runs;
+  const parallel = opts.parallel ?? config.parallel;
   const models = opts.models.length > 0 ? opts.models : config.models;
   const harnessNames = opts.harnesses.length > 0 ? opts.harnesses : config.harnesses;
   if (opts.harnessBin !== undefined && harnessNames.length !== 1)
@@ -11055,7 +11144,7 @@ async function evalCommand(args, root) {
     testCount: tests.length,
     runCount: tests.length * runs * Math.max(models.length, 1) * harnesses.length,
     modelCount: models.length,
-    parallel: opts.parallel
+    parallel
   });
   const vars = { ...config.vars, ...opts.vars };
   const missing = [
@@ -11215,11 +11304,11 @@ async function evalCommand(args, root) {
     results[i] = r;
     Object.assign(row, {
       state: r.status,
-      reportedTurns: r.num_turns,
+      reportedTurns: r.turns,
       reason: r.reason,
       checks: r.checks,
       cost: r.cost,
-      seconds: Math.round(r.duration_ms / 1000),
+      seconds: Math.round(r.durationMs / 1000),
       command: undefined
     });
     ui.update();
@@ -11237,7 +11326,7 @@ async function evalCommand(args, root) {
     while (next < rows.length)
       await runAt(next++);
   };
-  await Promise.all(Array.from({ length: Math.min(opts.parallel, rows.length) }, worker));
+  await Promise.all(Array.from({ length: Math.min(parallel, rows.length) }, worker));
   const totals = { pass: 0, fail: 0, error: 0 };
   for (const r of results)
     totals[r.status]++;
@@ -11266,6 +11355,7 @@ async function evalCommand(args, root) {
     upload: upload && { origin: upload.origin, email: upload.email },
     runs: results,
     totals,
+    tier: tier(rows),
     status,
     exitCode
   };
@@ -11286,12 +11376,8 @@ async function evalCommand(args, root) {
     uploadFailed: uploadsFailed
   })}
 `);
-  ui.done({
-    runId,
-    runsDir: runDir,
-    rows,
-    seconds: Math.round((Date.now() - evalStarted) / 1000)
-  });
+  const seconds = Math.round((Date.now() - evalStarted) / 1000);
+  ui.done({ runId, runsDir: runDir, rows, seconds });
   if (upload && !opts.json)
     uploadedSummary(upload.origin, results, runDir);
   for (const r of results)
@@ -11303,6 +11389,20 @@ async function evalCommand(args, root) {
       console.error(`firegoat eval: Warning: ${r.test}#${r.n} left ${r.leftover.length} process${r.leftover.length === 1 ? "" : "es"} running, stopped after cleanup: ${r.leftover.map((l) => `${l.pid} ${l.command}`).join("; ")} (run.json leftover)`);
   if (uploadsFailed > 0)
     console.error(`firegoat eval: ${uploadsFailed} upload${uploadsFailed === 1 ? "" : "s"} failed (each run's upload.error in summary.json)`);
+  const block = resultsBlock({
+    project: basename2(root),
+    version: package_default.version,
+    rows,
+    runId,
+    runsDir: runDir,
+    seconds
+  });
+  const resultsFile = join10(runDir, "results.txt");
+  writeFileSync5(resultsFile, `${block.join(`
+`)}
+`);
+  if (!opts.json && !stream.isTTY)
+    printResults(block, resultsFile);
   if (opts.json)
     console.log(JSON.stringify(doc, null, 2));
   if (status === "error")
@@ -11586,7 +11686,7 @@ firegoat ${package_default.version}` : `firegoat ${package_default.version}`,
 
 // src/init.ts
 import { spawn as spawn4, spawnSync as spawnSync4 } from "node:child_process";
-import { existsSync as existsSync6, writeFileSync as writeFileSync6 } from "node:fs";
+import { existsSync as existsSync7, writeFileSync as writeFileSync6 } from "node:fs";
 import { basename as basename3, join as join11 } from "node:path";
 
 // src/login.ts
@@ -11900,6 +12000,10 @@ function template(product) {
 # How many times to run each test.
 runs: 1
 
+# Optional: how many runs at once (default 4). 1 runs them one after another,
+# e.g. when tests share one cloud account. eval --parallel N overrides it.
+# parallel: 1
+
 # Optional: the agent CLI each test runs on: claude-code (default), codex-cli or grok-cli.
 # A list runs every test \`runs\` times per harness. eval --harness NAME overrides it.
 # harness: [claude-code, codex-cli]
@@ -11998,7 +12102,7 @@ function copyToClipboard(text) {
   });
 }
 var LOGIN_HINT = "Runs stay local: npx firegoat login uploads them to firegoat.ai";
-function cut(s, n) {
+function cut2(s, n) {
   return s.length <= n ? s : `${s.slice(0, Math.max(0, n - 1))}~`;
 }
 
@@ -12013,8 +12117,8 @@ class Timeline {
   line(mark, n, label, status = "") {
     const head = `  ${mark}  ${this.paint("90", String(n).padStart(2, "0"))}  `;
     if (!status)
-      return head + cut(label, this.width - 9);
-    return `${head}${label.padEnd(28)}${this.paint("90", cut(status, this.width - 37))}`;
+      return head + cut2(label, this.width - 9);
+    return `${head}${label.padEnd(28)}${this.paint("90", cut2(status, this.width - 37))}`;
   }
   done(n, label, status) {
     return this.line(this.paint("32", "●"), n, label, status);
@@ -12037,7 +12141,7 @@ class Timeline {
     return select(block, options.length, (current) => [
       this.line(this.paint("38;5;208", "◉"), n, question),
       ...options.map(([label, note], i) => {
-        const text = cut(`${label.padEnd(cols)}${note}`, room);
+        const text = cut2(`${label.padEnd(cols)}${note}`, room);
         return i === current ? `${Timeline.INDENT}${this.paint("38;5;208", `● ${text}`)}` : this.paint("90", `${Timeline.INDENT}○ ${text}`);
       }),
       this.paint("90", `${Timeline.INDENT}Up/down to move, enter to pick`)
@@ -12162,7 +12266,7 @@ ${INIT_HELP}`);
   }
   ensureFiregoatDir(cwd);
   const configPath = join11(cwd, CONFIG_FILE);
-  const keep = existsSync6(configPath) && !force;
+  const keep = existsSync7(configPath) && !force;
   if (!keep)
     writeFileSync6(configPath, template(basename3(cwd)));
   if (ui) {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.28.0",
+	"version": "0.30.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/firegoat-ai/firegoat-cli.git"