firegoat 0.25.0 → 0.27.0

package/README.md

@@ -37,6 +37,8 @@ tests:
       - run: grep -qE 'evt_[A-Za-z0-9]+' "$FIREGOAT_TRANSCRIPT"
 ```
 
+Each run starts in a fresh, empty folder outside your repo, `~/.firegoat/runs/<hash>/<random>/work`, so its path doesn't name your project and the agent doesn't wander into your source. Logs and results stay in the repo, in `.firegoat/runs/`.
+
 ## Uploads
 
 Log in to your firegoat.ai account and each trace is uploaded to our system automatically. Secrets are masked in everything you see, but the uploaded files are stored as sent. You can opt out of uploading with `eval --no-upload`. You can also delete your traces on the dashboard, and the files and the record are gone from our site.

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.25.0",
+  version: "0.27.0",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -6976,7 +6976,7 @@ var package_default = {
   },
   scripts: {
     build: "bun build src/cli.ts --target node --outfile dist/cli.js",
-    test: "bun test",
+    test: "bun test --timeout 30000",
     typecheck: "bun check",
     lint: "biome check .",
     format: "biome check --write .",
@@ -7157,7 +7157,7 @@ function read(stdout, exited, timeoutMs) {
       reportedModel,
       version,
       ok: false,
-      reason: `claude result is an error (subtype=${result.subtype})${result.result ? `: ${result.result.slice(0, 300)}` : ""}`,
+      reason: `claude reported an error${result.subtype === undefined || result.subtype === "success" ? "" : ` (${result.subtype})`}${result.result ? `: ${result.result.slice(0, 300)}` : ""}`,
       sessionId
     };
   return { ...trace, reportedModel, version, ok: true, sessionId };
@@ -7231,6 +7231,7 @@ var claude = {
   }),
   authLabel: () => process.env.ANTHROPIC_API_KEY ? "ANTHROPIC_API_KEY (claude bills the API key; unset it to use your claude login)" : "your claude login",
   checkLogin,
+  isFinal: (line) => parse(line)?.type === "result",
   progress,
   read,
   sessionsDir: (configDir) => join(configDir, "projects"),
@@ -7549,6 +7550,10 @@ var codex = {
   }),
   authLabel: () => `your codex login (${join3(userCodexHome(), "auth.json")})${process.env.CODEX_API_KEY ? "; CODEX_API_KEY is set and passes through" : ""}`,
   checkLogin: checkLogin2,
+  isFinal: (line) => {
+    const type = parse2(line)?.type;
+    return type === "turn.completed" || type === "turn.failed";
+  },
   progress: progress2,
   read: read2,
   sessionsDir: (configDir) => join3(configDir, "sessions"),
@@ -7653,7 +7658,7 @@ function steps3(events) {
 function resultError(result) {
   const errors = Array.isArray(result.errors) ? result.errors.filter((x) => typeof x === "string").join("; ") : "";
   const said = errors || result.result || "";
-  return `grok result is an error (subtype=${result.subtype})${said ? `: ${said.slice(0, 300)}` : ""}`;
+  return `grok reported an error${result.subtype === undefined || result.subtype === "success" ? "" : ` (${result.subtype})`}${said ? `: ${said.slice(0, 300)}` : ""}`;
 }
 function read3(stdout, exited, timeoutMs) {
   const events = stdout.split(`
@@ -7836,6 +7841,7 @@ var grok = {
   }),
   authLabel: () => `your grok login (${USER_AUTH})`,
   checkLogin: checkLogin3,
+  isFinal: (line) => parse3(line)?.type === "result",
   progress: progress3,
   read: read3,
   sessionsDir: (configDir) => join4(configDir, "sessions"),
@@ -8150,10 +8156,54 @@ import { randomBytes as randomBytes2 } from "node:crypto";
 import {
   appendFileSync,
   createWriteStream as createWriteStream2,
-  mkdirSync as mkdirSync4,
+  mkdirSync as mkdirSync5,
   writeFileSync as writeFileSync4
 } from "node:fs";
-import { basename as basename3, delimiter as delimiter2, join as join9 } from "node:path";
+import { basename as basename3, delimiter as delimiter2, join as join10 } from "node:path";
+
+// src/agent-dirs.ts
+import { createHash, randomInt } from "node:crypto";
+import { mkdirSync as mkdirSync2, realpathSync, symlinkSync as symlinkSync3 } from "node:fs";
+import { homedir as homedir3 } from "node:os";
+import { join as join5 } from "node:path";
+var ID_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789";
+var ID_LENGTH = 8;
+var MAX_TRIES = 10;
+function projectAgentDir(projectRoot) {
+  const hash = createHash("sha256").update(realpathSync(projectRoot)).digest("hex").slice(0, 12);
+  return join5(firegoatHome(), "runs", hash);
+}
+function firegoatHome() {
+  return join5(realpathSync(homedir3()), ".firegoat");
+}
+function randomId() {
+  let id = "";
+  for (let i = 0;i < ID_LENGTH; i++)
+    id += ID_CHARS[randomInt(ID_CHARS.length)];
+  return id;
+}
+function ensureProjectAgentDir(projectRoot) {
+  mkdirSync2(firegoatHome(), { recursive: true, mode: 448 });
+  const dir = projectAgentDir(projectRoot);
+  mkdirSync2(dir, { recursive: true });
+  return dir;
+}
+function makeAgentDir(projectDir) {
+  for (let i = 0;i < MAX_TRIES; i++) {
+    const dir = join5(projectDir, randomId());
+    try {
+      mkdirSync2(dir);
+      return dir;
+    } catch (e) {
+      if (e.code !== "EEXIST")
+        throw e;
+    }
+  }
+  throw new Error(`Could not create a new run folder in ${projectDir}: ${MAX_TRIES} random names were taken`);
+}
+function linkDir(target, link) {
+  symlinkSync3(target, link, "dir");
+}
 
 // src/credentials.ts
 import { randomBytes } from "node:crypto";
@@ -8162,15 +8212,15 @@ import {
   closeSync as closeSync2,
   fsyncSync,
   lstatSync as lstatSync3,
-  mkdirSync as mkdirSync2,
+  mkdirSync as mkdirSync3,
   openSync as openSync2,
   readFileSync as readFileSync4,
   renameSync,
   unlinkSync as unlinkSync3,
   writeSync
 } from "node:fs";
-import { homedir as homedir3 } from "node:os";
-import { join as join5 } from "node:path";
+import { homedir as homedir4 } from "node:os";
+import { join as join6 } from "node:path";
 
 // src/firegoat-api.ts
 var DEFAULT_BASE_URL = "https://firegoat.ai/evals";
@@ -8322,10 +8372,10 @@ async function whoami(base, key) {
 // src/credentials.ts
 var KEY = /^fire_[\x21-\x7e]+$/;
 function credentialsDir() {
-  return join5(homedir3(), ".firegoat");
+  return join6(homedir4(), ".firegoat");
 }
 function credentialsPath() {
-  return join5(credentialsDir(), "credentials.json");
+  return join6(credentialsDir(), "credentials.json");
 }
 function uid() {
   if (!process.getuid)
@@ -8388,7 +8438,7 @@ function saveCredentials(c) {
   if (st)
     assertOwned(dir, st, "dir");
   else
-    mkdirSync2(dir, { mode: 448 });
+    mkdirSync3(dir, { mode: 448 });
   chmodSync(dir, 448);
   const tmp = `${credentialsPath()}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`;
   const fd = openSync2(tmp, "wx", 384);
@@ -8444,15 +8494,15 @@ function resolveKey(base) {
 var EXIT = { pass: 0, fail: 1, notGraded: 2, usage: 3 };
 
 // src/firegoat-dir.ts
-import { mkdirSync as mkdirSync3, writeFileSync as writeFileSync2 } from "node:fs";
-import { join as join6 } from "node:path";
+import { mkdirSync as mkdirSync4, writeFileSync as writeFileSync2 } from "node:fs";
+import { join as join7 } from "node:path";
 var GITIGNORE = `*
 `;
 function ensureFiregoatDir(root) {
-  const dir = join6(root, ".firegoat");
-  mkdirSync3(dir, { recursive: true });
+  const dir = join7(root, ".firegoat");
+  mkdirSync4(dir, { recursive: true });
   try {
-    writeFileSync2(join6(dir, ".gitignore"), GITIGNORE, { flag: "wx" });
+    writeFileSync2(join7(dir, ".gitignore"), GITIGNORE, { flag: "wx" });
   } catch (e) {
     if (e.code !== "EEXIST")
       throw e;
@@ -8472,6 +8522,26 @@ var CLAUDE_AUTH = /^CLAUDE_CODE_(OAUTH_TOKEN|OAUTH_REFRESH_TOKEN|USE_[A-Z_]+|SKI
 function hostEnv() {
   return Object.fromEntries(Object.entries(process.env).filter(([k]) => !NPM_INJECTED.test(k) && !k.startsWith("FIREGOAT_") && (!CLAUDE_SESSION.test(k) || CLAUDE_AUTH.test(k))));
 }
+var GRACE_MS = 5000;
+var liveGroups = new Set;
+var signalsHooked = false;
+function hookSignals() {
+  if (signalsHooked)
+    return;
+  signalsHooked = true;
+  const onSignal = (signal) => {
+    for (const g of liveGroups)
+      try {
+        process.kill(-g, "SIGKILL");
+      } catch {}
+    process.kill(process.pid, signal);
+  };
+  for (const s of ["SIGINT", "SIGTERM", "SIGHUP"])
+    process.once(s, onSignal);
+}
+function forgetGroup(pgid) {
+  liveGroups.delete(pgid);
+}
 function processTree(pid) {
   const children = new Map;
   const ps = execFileSync("ps", ["-A", "-o", "pid=,ppid="], {
@@ -8489,7 +8559,11 @@ function processTree(pid) {
     tree.push(...children.get(tree[i]) ?? []);
   return tree;
 }
-function killTree(pid) {
+function killTree(pid, group) {
+  if (group)
+    try {
+      process.kill(-pid, "SIGKILL");
+    } catch {}
   let tree = [pid];
   let problem;
   try {
@@ -8505,18 +8579,27 @@ function killTree(pid) {
 }
 function run(opts) {
   return new Promise((resolve) => {
+    const detached = opts.detached === true;
+    if (detached)
+      hookSignals();
     const child = spawn(opts.cmd, opts.args, {
       cwd: opts.cwd,
       env: opts.env,
-      stdio: ["ignore", "pipe", "pipe"]
+      stdio: ["ignore", "pipe", "pipe"],
+      detached
     });
+    if (child.pid !== undefined) {
+      if (detached)
+        liveGroups.add(child.pid);
+      opts.onSpawn?.(child.pid);
+    }
     let timedOut = false;
     let settled = false;
     const timer = setTimeout(() => {
       timedOut = true;
       if (child.pid === undefined)
         return;
-      const problem = killTree(child.pid);
+      const problem = killTree(child.pid, detached);
       if (problem)
         opts.onStderr(Buffer.from(`firegoat: timed out; could not list ${opts.cmd}'s child processes (${problem}), killed only ${opts.cmd}
 `));
@@ -8530,6 +8613,7 @@ function run(opts) {
     };
     child.stdout.on("data", opts.onStdout);
     child.stderr.on("data", opts.onStderr);
+    opts.finalSeen?.then(() => setTimeout(() => done({ code: null, timedOut: false, ended: true }), GRACE_MS).unref());
     child.on("error", (error) => done({ code: null, timedOut, error }));
     child.on("close", (code) => done({ code, timedOut }));
     child.on("exit", (code) => setTimeout(() => {
@@ -8622,41 +8706,54 @@ function gitRepo(dir) {
 async function runAgent(harness, opts) {
   let stdout = "";
   let pending = "";
+  let group;
+  let markFinal = () => {};
+  const finalSeen = new Promise((r) => {
+    markFinal = r;
+  });
   const exited = await run({
     cmd: opts.bin,
     args: harness.args(opts),
     cwd: opts.cwd,
     env: opts.env,
     timeoutMs: opts.timeoutMs,
+    detached: true,
+    finalSeen,
+    onSpawn: (pid) => {
+      group = pid;
+    },
     onStdout: (c) => {
       const text = c.toString();
       stdout += text;
-      if (!opts.onProgress)
-        return;
       pending += text;
       const lines = pending.split(`
 `);
       pending = lines.pop() ?? "";
       for (const line of lines) {
-        const p = harness.progress(line);
+        if (harness.isFinal(line))
+          markFinal();
+        const p = opts.onProgress && harness.progress(line);
         if (p)
-          opts.onProgress(p);
+          opts.onProgress?.(p);
       }
     },
     onStderr: (c) => opts.stderr.write(c)
   });
   if (exited.error)
     return {
-      ok: false,
-      reason: `could not start ${opts.bin}: ${exited.error.message}`,
-      steps: [],
-      final: "",
-      reportedModel: null,
-      version: null,
-      turns: null,
-      cost: null
+      group,
+      outcome: {
+        ok: false,
+        reason: `could not start ${opts.bin}: ${exited.error.message}`,
+        steps: [],
+        final: "",
+        reportedModel: null,
+        version: null,
+        turns: null,
+        cost: null
+      }
     };
-  return harness.read(stdout, exited, opts.timeoutMs);
+  return { group, outcome: harness.read(stdout, exited, opts.timeoutMs) };
 }
 
 // src/help.ts
@@ -8734,8 +8831,10 @@ firegoat.yaml reference:
   install: <sh> | [<sh>, ...] In each fresh folder, before the agent starts; a
                               failure: that run is not graded
   cleanup: <sh> | [<sh>, ...] In each run's folder once the run is over: after the
-                              checks, and also after an error or timeout. Stops
-                              what the agent left running (servers, daemons).
+                              checks, and also after an error or timeout. For
+                              what lives outside the run (e.g. seeded data);
+                              processes the agent left running are stopped by
+                              firegoat after cleanup (see eval --help).
                               Every step runs; a failure is a warning (run.json
                               cleanup, <test>/<n>/cleanup.txt), never the verdict
   vars: {KEY: value}          Defaults for {{vars.KEY}} in prompts, checks and
@@ -8812,7 +8911,11 @@ firegoat.yaml reference:
   for any value.
   The build, install and cleanup commands and the checks get the vars as env
   vars, plus $FIREGOAT_OUT and $FIREGOAT_WORKDIR; install, checks and cleanup also
-  get $FIREGOAT_HOME, the run's own home, as HOME (build keeps your real HOME).
+  get $FIREGOAT_HOME, the run's own home, as HOME (build keeps your real HOME),
+  $FIREGOAT_RUN_DIR (the run's .firegoat/runs/<runId>/<test>/<n>/, where its logs
+  go) and $FIREGOAT_REPO (the folder holding firegoat.yaml). The agent gets
+  neither: keep values it must not find in $FIREGOAT_RUN_DIR, e.g. install:
+  echo "$EXPECTED" > "$FIREGOAT_RUN_DIR/expected", and a check reads it back.
   Checks also get $FIREGOAT_TRANSCRIPT, a short plain-text
   transcript, greppable line by line (format: eval --help, Transcript). So
   '^\\[[0-9]+\\] \\$ .*foo' matches only commands. Checks also get
@@ -8843,7 +8946,6 @@ relative paths in it are from the run's folder, claude's cwd:
 Example:
   build: npm pack --pack-destination "$FIREGOAT_OUT"
   install: npm init -y >/dev/null && npm install "$FIREGOAT_OUT"/*.tgz
-  cleanup: pkill -f "$FIREGOAT_WORKDIR" || true
   tests:
     - name: first-call
       prompt: >-
@@ -8911,8 +9013,13 @@ var EVAL_HELP = `Usage: firegoat eval [test..] [--runs N] [--model NAME].. [--pa
 
 Runs each test in firegoat.yaml (in the current folder): your agent harness
 (claude-code by default, codex-cli or grok-cli; harness: in firegoat.yaml) gets the prompt
-in a fresh, empty folder, .firegoat/runs/<runId>/<test>/<n>/work, with build:'s
-outputs on PATH. Then the test's checks run in that folder.
+in a fresh, empty folder outside the repo, ~/.firegoat/runs/<hash>/<random>/work,
+with build:'s outputs on PATH. Then the test's checks run in that folder. <hash>
+is a hash of the repo's path (a linked worktree uses its main checkout's) and
+<random> is new for each run, so the agent's paths name neither the project nor
+the test, and git there sees no repo. The run's logs stay in the repo, in
+.firegoat/runs/<runId>/<test>/<n>/, whose work and home are symlinks to the
+run's folders.
 
 Step order: build: once, in this folder; then per run, in its folder: install:
 (top-level, then the test's), the agent, checks, cleanup: (the test's, then
@@ -8920,39 +9027,51 @@ top-level). cleanup: runs whenever the folder exists, also after a failed instal
 a claude error or a timeout; a failing cleanup step prints a warning and never
 changes the run's status or the exit code.
 
+The agent's run ends with its final result (claude's and grok's result line,
+codex's turn.completed or turn.failed): a harness that hasn't exited 5s later
+(a server it started in the background keeps it alive) is left running, so the
+checks can still reach that server. After cleanup, firegoat stops whatever is
+still running from the run (SIGTERM, then SIGKILL after 2s): the harness's
+process group, every process below it, and every process whose working
+directory is inside the run's folder (lsof on macOS, /proc on Linux). They are
+listed in run.json leftover, with one warning per run; never the verdict. Not
+found: a daemon that left the group and changed its working directory out of
+the run's folder.
+
 claude runs on its own login (or ANTHROPIC_API_KEY if set, as claude always does)
-with a fresh config dir, .firegoat/.claude: none of your settings, CLAUDE.md,
-plugins, skills or MCP servers (harness_args: in firegoat.yaml adds claude flags,
-e.g. --chrome or --plugin-dir). claude saves its session there. There is no
-sandbox: the agent can reach the rest of your machine, including the repo.
+with a fresh config dir, ~/.firegoat/runs/<hash>/.claude: none of your settings,
+CLAUDE.md, plugins, skills or MCP servers (harness_args: in firegoat.yaml adds
+claude flags, e.g. --chrome or --plugin-dir). claude saves its session there. There
+is no sandbox: the agent can still reach the rest of your machine, including the
+repo, if it goes looking.
 
 codex-cli runs codex exec --json --dangerously-bypass-approvals-and-sandbox with a
-fresh CODEX_HOME, .firegoat/.codex, holding a link to your codex login
+fresh CODEX_HOME, ~/.firegoat/runs/<hash>/.codex, holding a link to your codex login
 (~/.codex/auth.json, or $CODEX_HOME/auth.json; run codex login first), and with
 -c project_doc_max_bytes=0: no AGENTS.md, config.toml, skills or plugins of
 yours. codex's own HOME is the run's home (below), also for its Read/Write. It
-saves its session (rollout) under .firegoat/.codex/sessions/. firegoat.ai doesn't
+saves its session (rollout) under that CODEX_HOME's sessions/. firegoat.ai doesn't
 read codex sessions yet: those runs are not uploaded. codex reports no turns or
 cost; its model comes from the session file.
 
 grok-cli runs ~/.grok/bin/grok (where grok's installer puts it; never a grok on
 PATH, which may be a wrapper such as cmux's: a #! script is refused) as grok -p
 --output-format streaming-messages-json --always-approve --no-auto-update, with a
-fresh GROK_HOME, .firegoat/.grok, holding a link to your grok login
+fresh GROK_HOME, ~/.firegoat/runs/<hash>/.grok, holding a link to your grok login
 ($GROK_HOME/auth.json or ~/.grok/auth.json; run grok login first; XAI_API_KEY alone
 does not log headless grok in). grok's own HOME is the run's home (below): with
 your real one it loads Claude Code's skills, plugins, MCP servers and hooks. eval
 refuses a login that expires within 5 minutes (run grok -p hi once to refresh it):
 a refresh inside a run would replace the link with a file, which the next eval
 asks you to move back over ~/.grok/auth.json. The session is a directory,
-.firegoat/.grok/sessions/<cwd>/<id>/ ($FIREGOAT_SESSION). firegoat.ai doesn't read
+$GROK_HOME/sessions/<cwd>/<id>/ ($FIREGOAT_SESSION). firegoat.ai doesn't read
 grok sessions yet: those runs are not uploaded.
 
-Every run gets its own HOME, .firegoat/runs/<runId>/<test>/<n>/home (with
-XDG_CONFIG_HOME, XDG_CACHE_HOME, XDG_DATA_HOME, XDG_STATE_HOME inside it), as
+Every run gets its own HOME, ~/.firegoat/runs/<hash>/<random>/home next to work
+(with XDG_CONFIG_HOME, XDG_CACHE_HOME, XDG_DATA_HOME, XDG_STATE_HOME inside it), as
 $FIREGOAT_HOME: tools that keep state in ~ start fresh and parallel runs don't
 collide. claude itself keeps your real HOME (its login is there) and sources
-<n>/home.env (CLAUDE_ENV_FILE) before each Bash command the agent runs; install,
+home.env next to it (CLAUDE_ENV_FILE) before each Bash command the agent runs; install,
 checks and cleanup get the run's HOME directly; build keeps yours. Limits: claude's
 own Read/Write tools expand ~ to your real home; getpwuid lookups (os.userInfo,
 python pwd) still see it; keychain logins (gh, docker, git credential helpers) look
@@ -8968,9 +9087,9 @@ nested session. Login and provider vars stay: ANTHROPIC_*, CLAUDE_CODE_OAUTH_TOK
 CLAUDE_CODE_OAUTH_REFRESH_TOKEN, CLAUDE_CODE_USE_*, CLAUDE_CODE_SKIP_*_AUTH,
 CLAUDE_CODE_CLIENT_CERT, CLAUDE_CODE_CLIENT_KEY, CLAUDE_CODE_CLIENT_KEY_PASSPHRASE.
 
-In a linked git worktree (git worktree add), .firegoat/runs/ and the harness
-config dirs (.firegoat/.claude, .firegoat/.codex, .firegoat/.grok) live in the main checkout instead, so runs outlive the worktree; firegoat.yaml,
-fixtures and build: stay in the current folder.
+In a linked git worktree (git worktree add), .firegoat/runs/ lives in the main
+checkout instead, and <hash> is the main checkout's, so runs outlive the worktree;
+firegoat.yaml, fixtures and build: stay in the current folder.
 
   test..             Only these tests (by name)
   --runs N           Runs per test (overrides runs: in firegoat.yaml)
@@ -8994,8 +9113,9 @@ fixtures and build: stay in the current folder.
                      --model, --harness, missing vars), print each planned
                      run ("<test>#<n> [harness model]" with its install, checks
                      and cleanup counts and rendered harness_args, paths under
-                     runs/<runId>/) and exit 0. Builds nothing, starts no
-                     agent, uploads nothing, writes no files. Config errors
+                     ~/.firegoat/runs/<hash>/<random>/) and exit 0. Builds
+                     nothing, starts no agent, uploads nothing, writes no
+                     files. Config errors
                      exit 3. With --json: {dryRun, build, runs: [{test, n,
                      harness, model, harnessArgs, install, checks,
                      cleanup}]}, steps and checks with their placeholders
@@ -9017,6 +9137,8 @@ Runs without a session file are not uploaded. FIREGOAT_BASE_URL points elsewhere
 (default https://firegoat.ai/evals). See firegoat login --help.
 
 Output: .firegoat/runs/<runId>/
+  out                     A symlink to $FIREGOAT_OUT,
+                          ~/.firegoat/runs/<hash>/<random>/out
   summary.json            firegoatVersion, runId, runsDir, dir (where eval
                           ran), git {branch, commit, dirty} (no git outside a
                           repo), harnesses, upload {origin, email} (null when
@@ -9030,14 +9152,18 @@ Output: .firegoat/runs/<runId>/
                           the harness reported running, or null), harnessArgs
                           (harness_args passed), prompt, end_goal,
                           checks [{run, code}], cleanup [{run, code}],
-                          and paths: workdir, home (the run's HOME),
+                          leftover [{pid, command}] (processes stopped after
+                          cleanup), and paths: dir (this folder), workdir, home
+                          (the run's HOME; both real paths, under ~/.firegoat),
                           transcript, session (the harness's session file), stderr,
                           checksLog, cleanupLog; startedAt; upload ({status: "uploaded",
                           url} | {status: "failed", error} | {status:
                           "skipped", reason}, null when not uploading)
   <test>/<n>/transcript.txt  What the agent did, below
-  <test>/<n>/home/, home.env  The run's HOME, and the file claude sources
-                          before the agent's Bash commands
+  <test>/<n>/work, home   Symlinks to the agent's folder and the run's HOME
+                          (home.env, which claude sources before the agent's
+                          Bash commands, is next to them)
+
   build.txt, <test>/<n>/install.txt, checks.txt, cleanup.txt   Logs
 
 Transcript ($FIREGOAT_TRANSCRIPT), greppable line by line; n counts tool calls:
@@ -9093,6 +9219,121 @@ Example:
 
 Exit codes: 0 sent, 2 the service failed (or rate limit), 3 usage error.`;
 
+// src/leftover.ts
+import { spawn as spawn2 } from "node:child_process";
+import { readdirSync as readdirSync4, readlinkSync as readlinkSync2, realpathSync as realpathSync2 } from "node:fs";
+var TERM_WAIT_MS = 2000;
+function output(cmd, args) {
+  return new Promise((resolve, reject) => {
+    const child = spawn2(cmd, args, { stdio: ["ignore", "pipe", "pipe"] });
+    let out = "";
+    let err = "";
+    child.stdout.on("data", (c) => {
+      out += c;
+    });
+    child.stderr.on("data", (c) => {
+      err += c;
+    });
+    child.on("error", (e) => reject(new Error(`could not run ${cmd}: ${e.message}`)));
+    child.on("close", (code) => code === 0 || out !== "" ? resolve(out) : reject(new Error(`${cmd} exited ${code}: ${err.trim().slice(0, 300)}`)));
+  });
+}
+async function processes() {
+  const ps = await output("ps", [
+    "-A",
+    "-o",
+    "pid=,ppid=,pgid=,stat=,command="
+  ]);
+  const procs = [];
+  for (const line of ps.split(`
+`)) {
+    const m = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s*(.*)$/.exec(line);
+    if (!m || m[4]?.startsWith("Z"))
+      continue;
+    procs.push({
+      pid: Number(m[1]),
+      ppid: Number(m[2]),
+      pgid: Number(m[3]),
+      command: m[5] ?? ""
+    });
+  }
+  return procs;
+}
+async function cwdInside(dir) {
+  const inside = (p) => p === dir || p.startsWith(`${dir}/`);
+  if (process.platform === "linux") {
+    const pids = [];
+    for (const name of readdirSync4("/proc")) {
+      if (!/^\d+$/.test(name))
+        continue;
+      try {
+        if (inside(readlinkSync2(`/proc/${name}/cwd`)))
+          pids.push(Number(name));
+      } catch {}
+    }
+    return pids;
+  }
+  if (process.platform === "darwin") {
+    const pids = [];
+    let pid = 0;
+    for (const line of (await output("lsof", ["-d", "cwd", "-Fpn", "-w"])).split(`
+`)) {
+      if (line.startsWith("p"))
+        pid = Number(line.slice(1));
+      else if (line.startsWith("n") && inside(line.slice(1)))
+        pids.push(pid);
+    }
+    return pids;
+  }
+  throw new Error(`finding processes by cwd is not supported on ${process.platform}`);
+}
+function alive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function signal(pids, group, sig) {
+  for (const p of group === undefined ? pids : [...pids, -group])
+    try {
+      process.kill(p, sig);
+    } catch {}
+}
+async function stopLeftovers(opts) {
+  const workdir = realpathSync2(opts.workdir);
+  const [procs, inWorkdir] = await Promise.all([
+    processes(),
+    cwdInside(workdir)
+  ]);
+  const found = new Set(inWorkdir);
+  if (opts.group !== undefined) {
+    for (const p of procs)
+      if (p.pgid === opts.group)
+        found.add(p.pid);
+  }
+  for (let grew = true;grew; ) {
+    grew = false;
+    for (const p of procs)
+      if (found.has(p.ppid) && !found.has(p.pid)) {
+        found.add(p.pid);
+        grew = true;
+      }
+  }
+  found.delete(process.pid);
+  const leftover = procs.filter((p) => found.has(p.pid)).map((p) => ({ pid: p.pid, command: printable(p.command, 200) }));
+  const pids = leftover.map((l) => l.pid);
+  signal(pids, opts.group, "SIGTERM");
+  const deadline = Date.now() + TERM_WAIT_MS;
+  while (pids.some(alive) && Date.now() < deadline)
+    await new Promise((r) => setTimeout(r, 100));
+  signal(pids.filter(alive), opts.group, "SIGKILL");
+  if (opts.group !== undefined)
+    forgetGroup(opts.group);
+  return leftover;
+}
+
 // src/steps.ts
 import { createWriteStream } from "node:fs";
 var STEP_TIMEOUT_MS = 600000;
@@ -9227,8 +9468,8 @@ async function sendTelemetry(event, exitCode, startedAt) {
 }
 
 // src/transcript.ts
-import { readdirSync as readdirSync4, statSync as statSync2, writeFileSync as writeFileSync3 } from "node:fs";
-import { join as join7, relative } from "node:path";
+import { readdirSync as readdirSync5, statSync as statSync2, writeFileSync as writeFileSync3 } from "node:fs";
+import { join as join8, relative } from "node:path";
 var OUTPUT_LINES = 15;
 var OUTPUT_CHARS = 1500;
 var TEXT_CHARS = 1500;
@@ -9277,10 +9518,10 @@ function describeToolCall(step, c) {
 function workdirFiles(dir) {
   const out = [];
   const walk = (d) => {
-    for (const name of readdirSync4(d)) {
+    for (const name of readdirSync5(d)) {
       if (SKIP_DIRS.has(name))
         continue;
-      const p = join7(d, name);
+      const p = join8(d, name);
       if (statSync2(p).isDirectory())
         walk(p);
       else
@@ -9360,7 +9601,7 @@ function runTag(harness, model) {
   return parts.length > 0 ? ` [${parts.join(" ")}]` : "";
 }
 function notGradedWhy(reason) {
-  const said = /claude result is an error \(subtype=[^)]*\): (.*)$/s.exec(reason);
+  const said = /claude reported an error(?: \([^)]*\))?: (.*)$/s.exec(reason);
   if (said)
     return `claude said: "${said[1]}"`;
   return reason.replace(/^not graded: /, "");
@@ -9780,7 +10021,7 @@ async function uploadRun(t, r) {
 
 // src/which.ts
 import { accessSync, constants, statSync as statSync3 } from "node:fs";
-import { delimiter, join as join8, resolve } from "node:path";
+import { delimiter, join as join9, resolve } from "node:path";
 function resolveBin(bin) {
   const executable = (p) => {
     try {
@@ -9793,8 +10034,8 @@ function resolveBin(bin) {
   if (bin.includes("/"))
     return executable(bin) ? resolve(bin) : undefined;
   for (const dir of (process.env.PATH ?? "").split(delimiter)) {
-    if (dir && executable(join8(dir, bin)))
-      return join8(dir, bin);
+    if (dir && executable(join9(dir, bin)))
+      return join9(dir, bin);
   }
   return;
 }
@@ -9881,7 +10122,7 @@ function newRunId() {
   return `${ts}-${randomBytes2(2).toString("hex")}`;
 }
 function writeRunJson(r) {
-  writeFileSync4(join9(r.dir, "run.json"), `${JSON.stringify(r, null, 2)}
+  writeFileSync4(join10(r.dir, "run.json"), `${JSON.stringify(r, null, 2)}
 `);
 }
 async function uploadPreflight(opts, ui, project, runGroup, git) {
@@ -9922,14 +10163,14 @@ async function uploadPreflight(opts, ui, project, runGroup, git) {
     };
   }
 }
-function runFill(runId, n, testName, out, dir) {
+function runFill(runId, n, testName, out, agentDir) {
   return {
     "run.id": runId,
     "run.n": String(n),
     "test.name": testName,
     FIREGOAT_OUT: out,
-    FIREGOAT_WORKDIR: join9(dir, "work"),
-    FIREGOAT_HOME: join9(dir, "home")
+    FIREGOAT_WORKDIR: join10(agentDir, "work"),
+    FIREGOAT_HOME: join10(agentDir, "home")
   };
 }
 function renderHarnessArgs(test, vars, fill) {
@@ -9962,10 +10203,11 @@ function showsHarness(harnesses) {
 }
 var DRY_RUN_ID = "<runId>";
 function dryRun(opts) {
-  const out = join9(opts.runsRoot, DRY_RUN_ID, "out");
+  const agentDir = join10(opts.agentDirs, "<random>");
+  const out = join10(agentDir, "out");
   const runs = opts.plan.map((p) => {
     const test = opts.tests.find((t) => t.name === p.test);
-    const fill = runFill(DRY_RUN_ID, p.n, test.name, out, join9(opts.runsRoot, DRY_RUN_ID, test.name, String(p.n)));
+    const fill = runFill(DRY_RUN_ID, p.n, test.name, out, agentDir);
     renderEnv(test.env, opts.vars, fill);
     return {
       ...p,
@@ -9986,11 +10228,15 @@ function dryRun(opts) {
 async function runOne(opts) {
   const { test, n } = opts;
   const started = Date.now();
-  const dir = join9(opts.runDir, test.name, String(n));
-  const workdir = join9(dir, "work");
-  mkdirSync4(workdir, { recursive: true });
-  const { home, envFile } = makeRunHome(dir);
-  const fill = runFill(opts.runId, n, test.name, opts.out, dir);
+  const dir = join10(opts.runDir, test.name, String(n));
+  mkdirSync5(dir, { recursive: true });
+  const agentDir = makeAgentDir(opts.agentDirs);
+  const workdir = join10(agentDir, "work");
+  mkdirSync5(workdir);
+  const { home, envFile } = makeRunHome(agentDir);
+  linkDir(workdir, join10(dir, "work"));
+  linkDir(home, join10(dir, "home"));
+  const fill = runFill(opts.runId, n, test.name, opts.out, agentDir);
   const r = {
     test: test.name,
     n,
@@ -10007,11 +10253,12 @@ async function runOne(opts) {
     workdir,
     home,
     session: null,
-    transcript: join9(dir, "transcript.txt"),
-    stderr: join9(dir, "stderr.txt"),
-    checksLog: join9(dir, "checks.txt"),
+    transcript: join10(dir, "transcript.txt"),
+    stderr: join10(dir, "stderr.txt"),
+    checksLog: join10(dir, "checks.txt"),
     cleanup: [],
-    cleanupLog: join9(dir, "cleanup.txt"),
+    cleanupLog: join10(dir, "cleanup.txt"),
+    leftover: [],
     num_turns: null,
     cost: null,
     duration_ms: 0,
@@ -10024,10 +10271,13 @@ async function runOne(opts) {
     ...env,
     ...homeEnv(home),
     FIREGOAT_WORKDIR: workdir,
-    FIREGOAT_HOME: home
+    FIREGOAT_HOME: home,
+    FIREGOAT_RUN_DIR: dir,
+    FIREGOAT_REPO: opts.repo
   };
   const steps = renderRunSteps(test, opts, opts.vars, fill);
-  const [status, reason] = await attempt(opts, r, steps, env, stepEnv, envFile);
+  const agent = {};
+  const [status, reason] = await attempt(opts, r, steps, env, stepEnv, envFile, agent);
   if (steps.cleanup.length > 0)
     r.cleanup = await runAllSteps({
       commands: steps.cleanup,
@@ -10035,13 +10285,17 @@ async function runOne(opts) {
       env: stepEnv,
       logPath: r.cleanupLog
     });
+  r.leftover = await stopLeftovers({ group: agent.group, workdir });
   r.status = status;
   r.reason = reason;
   r.duration_ms = Date.now() - started;
   writeRunJson(r);
   return r;
 }
-async function attempt(opts, r, steps, env, stepEnv, homeEnvFile) {
+function withoutOldPwd(env) {
+  return Object.fromEntries(Object.entries(env).filter(([k]) => k !== "OLDPWD"));
+}
+async function attempt(opts, r, steps, env, stepEnv, homeEnvFile, agent) {
   const { test, n } = opts;
   const { dir, workdir } = r;
   if (steps.install.length > 0) {
@@ -10050,24 +10304,25 @@ async function attempt(opts, r, steps, env, stepEnv, homeEnvFile) {
       commands: steps.install,
       cwd: workdir,
       env: stepEnv,
-      logPath: join9(dir, "install.txt")
+      logPath: join10(dir, "install.txt")
     });
     if (error)
       return ["error", `not graded: ${error}`];
   }
   const stderr = createWriteStream2(r.stderr);
   const { harness } = opts;
-  const outcome = await runAgent(harness, {
+  const { outcome, group } = await runAgent(harness, {
     bin: opts.bin,
     prompt: r.prompt,
     cwd: workdir,
     env: {
-      ...hostEnv(),
+      ...withoutOldPwd(hostEnv()),
       ...env,
       ...harness.env({
         configDir: opts.configDir,
         run: { home: r.home, homeEnvFile }
       }),
+      PWD: workdir,
       PATH: opts.stepEnv.PATH,
       FIREGOAT_OUT: opts.out,
       FIREGOAT_WORKDIR: workdir,
@@ -10079,6 +10334,7 @@ async function attempt(opts, r, steps, env, stepEnv, homeEnvFile) {
     extraArgs: r.harnessArgs,
     onProgress: opts.onProgress
   });
+  agent.group = group;
   if (outcome.version !== null)
     evalStats({ claudeVersion: claudeVersion(outcome.version) });
   await new Promise((res) => stderr.end(res));
@@ -10144,7 +10400,7 @@ function uploadedSummary(origin, results, runDir) {
     return;
   const head = `Uploaded ${urls.length} session${urls.length === 1 ? "" : "s"} to ${new URL(origin).host} (private)`;
   if (urls.length > MAX_LISTED_URLS) {
-    console.log(`${head}: each run's upload.url in ${join9(runDir, "summary.json")}`);
+    console.log(`${head}: each run's upload.url in ${join10(runDir, "summary.json")}`);
     return;
   }
   console.log(`${head}:`);
@@ -10245,7 +10501,7 @@ async function evalCommand(args, root) {
         plan: planRuns(tests, harnessNames, models, runs),
         showHarness,
         vars,
-        runsRoot: join9(repo?.mainCheckout ?? root, ".firegoat", "runs"),
+        agentDirs: projectAgentDir(repo?.mainCheckout ?? root),
         json: opts.json,
         say
       });
@@ -10271,11 +10527,13 @@ async function evalCommand(args, root) {
   } catch (e) {
     return fatal(EXIT.usage, e.message, "config");
   }
-  const firegoatDir = ensureFiregoatDir(repo?.mainCheckout ?? root);
+  const project = repo?.mainCheckout ?? root;
+  const firegoatDir = ensureFiregoatDir(project);
+  const agentDirs = ensureProjectAgentDir(project);
   const configDirs = new Map;
   for (const h of harnesses) {
-    const configDir = join9(firegoatDir, h.configDirName);
-    mkdirSync4(configDir, { recursive: true });
+    const configDir = join10(agentDirs, h.configDirName);
+    mkdirSync5(configDir, { recursive: true });
     configDirs.set(h, configDir);
   }
   ui.auth(harnesses.length === 1 ? harnesses[0].authLabel() : harnesses.map((h) => `${h.name}: ${h.authLabel()}`).join("; "));
@@ -10308,10 +10566,12 @@ async function evalCommand(args, root) {
       return fatal(EXIT.usage, e.message, "config");
     throw e;
   }
-  const runsRoot = join9(firegoatDir, "runs");
-  const runDir = join9(runsRoot, runId);
-  const out = join9(runDir, "out");
-  mkdirSync4(out, { recursive: true });
+  const runsRoot = join10(firegoatDir, "runs");
+  const runDir = join10(runsRoot, runId);
+  mkdirSync5(runDir, { recursive: true });
+  const out = join10(makeAgentDir(agentDirs), "out");
+  mkdirSync5(out);
+  linkDir(out, join10(runDir, "out"));
   const stepEnv = {
     ...hostEnv(),
     ...vars,
@@ -10336,7 +10596,7 @@ async function evalCommand(args, root) {
         ...stepEnv,
         ...renderEnv(Object.fromEntries(Object.entries(config.env).filter(([, v]) => isEvalLevel(v))), vars, { "run.id": runId, FIREGOAT_OUT: out })
       },
-      logPath: join9(runDir, "build.txt")
+      logPath: join10(runDir, "build.txt")
     });
     const secs = Math.round((Date.now() - buildStarted) / 1000);
     ui.build(error ? "fail" : "ok", secs);
@@ -10363,6 +10623,8 @@ async function evalCommand(args, root) {
       shownHarness: row.harness,
       runId,
       runDir,
+      agentDirs,
+      repo: root,
       harness,
       bin: bins.get(harness),
       configDir: configDirs.get(harness),
@@ -10432,9 +10694,9 @@ async function evalCommand(args, root) {
     status,
     exitCode
   };
-  writeFileSync4(join9(runDir, "summary.json"), `${JSON.stringify(doc, null, 2)}
+  writeFileSync4(join10(runDir, "summary.json"), `${JSON.stringify(doc, null, 2)}
 `);
-  appendFileSync(join9(runsRoot, "index.jsonl"), `${JSON.stringify({
+  appendFileSync(join10(runsRoot, "index.jsonl"), `${JSON.stringify({
     runId,
     startedAt: new Date(evalStarted).toISOString(),
     dir: root,
@@ -10461,6 +10723,9 @@ async function evalCommand(args, root) {
     for (const c of r.cleanup)
       if (c.code !== 0)
         console.error(`firegoat eval: Warning: ${r.test}#${r.n} cleanup failed (${c.error ?? `exit ${c.code}`}): ${c.run} (log: ${r.cleanupLog})`);
+  for (const r of results)
+    if (r.leftover.length > 0)
+      console.error(`firegoat eval: Warning: ${r.test}#${r.n} left ${r.leftover.length} process${r.leftover.length === 1 ? "" : "es"} running, stopped after cleanup: ${r.leftover.map((l) => `${l.pid} ${l.command}`).join("; ")} (run.json leftover)`);
   if (uploadsFailed > 0)
     console.error(`firegoat eval: ${uploadsFailed} upload${uploadsFailed === 1 ? "" : "s"} failed (each run's upload.error in summary.json)`);
   if (opts.json)
@@ -10745,12 +11010,12 @@ firegoat ${package_default.version}` : `firegoat ${package_default.version}`,
 }
 
 // src/init.ts
-import { spawn as spawn3, spawnSync as spawnSync4 } from "node:child_process";
+import { spawn as spawn4, spawnSync as spawnSync4 } from "node:child_process";
 import { existsSync as existsSync6, writeFileSync as writeFileSync5 } from "node:fs";
-import { basename as basename4, join as join10 } from "node:path";
+import { basename as basename4, join as join11 } from "node:path";
 
 // src/login.ts
-import { spawn as spawn2 } from "node:child_process";
+import { spawn as spawn3 } from "node:child_process";
 var TIMEOUT_MS3 = 30000;
 var MAX_EXPIRES_S = 900;
 var MAX_INTERVAL_S = 60;
@@ -10767,7 +11032,7 @@ function str3(o, k, step) {
 function openBrowser(url) {
   const cmd = process.platform === "darwin" ? "open" : "xdg-open";
   try {
-    const child = spawn2(cmd, [url], {
+    const child = spawn3(cmd, [url], {
       env: hostEnv(),
       stdio: "ignore",
       detached: true
@@ -11046,12 +11311,12 @@ function skillsDir(scope) {
   if (scope === "local")
     return ".claude/skills/";
   const config = process.env.CLAUDE_CONFIG_DIR?.trim();
-  return config ? `${join10(config, "skills")}/` : "~/.claude/skills/";
+  return config ? `${join11(config, "skills")}/` : "~/.claude/skills/";
 }
 function template(product) {
   return `# firegoat.yaml: evals for how well coding agents use ${product}.
 # Each test runs your agent (claude by default) in an empty folder
-# (.firegoat/runs/...) with \`prompt\` as the first message, then runs \`checks\`
+# (~/.firegoat/runs/...) with \`prompt\` as the first message, then runs \`checks\`
 # there. Reference: npx firegoat init --help
 
 # How many times to run each test.
@@ -11091,10 +11356,12 @@ runs: 1
 # install: cp -R "$FIREGOAT_OUT/starter-app/." .
 
 # Optional: cleanup: runs in each run's folder once the run is over (after the
-# checks, also after an error or timeout), e.g. to stop a server the agent left
-# running. A failing cleanup step is a warning, never the verdict.
+# checks, also after an error or timeout), for what lives outside the run, e.g.
+# data install: seeded in a shared service. Servers the agent left running need
+# none: firegoat stops them after cleanup. A failing cleanup step is a warning,
+# never the verdict.
 # cleanup:
-#   - pkill -f "$FIREGOAT_WORKDIR" || true
+#   - acme db drop "eval-{{run.id}}-{{run.n}}" || true
 #
 # A test can add its own install: (after this one) and cleanup: (before this one).
 
@@ -11196,7 +11463,7 @@ class Timeline {
 }
 function runQuiet(cmd, args, opts) {
   return new Promise((resolve) => {
-    const child = spawn3(cmd, args, {
+    const child = spawn4(cmd, args, {
       ...opts,
       stdio: ["ignore", "pipe", "pipe"]
     });
@@ -11308,7 +11575,7 @@ ${INIT_HELP}`);
     console.log(rule);
   }
   ensureFiregoatDir(cwd);
-  const configPath = join10(cwd, CONFIG_FILE);
+  const configPath = join11(cwd, CONFIG_FILE);
   const keep = existsSync6(configPath) && !force;
   if (!keep)
     writeFileSync5(configPath, template(basename4(cwd)));

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.25.0",
+	"version": "0.27.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -29,7 +29,7 @@
 	},
 	"scripts": {
 		"build": "bun build src/cli.ts --target node --outfile dist/cli.js",
-		"test": "bun test",
+		"test": "bun test --timeout 30000",
 		"typecheck": "bun check",
 		"lint": "biome check .",
 		"format": "biome check --write .",