firegoat 0.25.0 → 0.26.0

package/README.md

@@ -37,6 +37,8 @@ tests:
       - run: grep -qE 'evt_[A-Za-z0-9]+' "$FIREGOAT_TRANSCRIPT"
 ```
 
+Each run starts in a fresh, empty folder outside your repo, `~/.firegoat/runs/<hash>/<random>/work`, so its path doesn't name your project and the agent doesn't wander into your source. Logs and results stay in the repo, in `.firegoat/runs/`.
+
 ## Uploads
 
 Log in to your firegoat.ai account and each trace is uploaded to our system automatically. Secrets are masked in everything you see, but the uploaded files are stored as sent. You can opt out of uploading with `eval --no-upload`. You can also delete your traces on the dashboard, and the files and the record are gone from our site.

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.25.0",
+  version: "0.26.0",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -6976,7 +6976,7 @@ var package_default = {
   },
   scripts: {
     build: "bun build src/cli.ts --target node --outfile dist/cli.js",
-    test: "bun test",
+    test: "bun test --timeout 30000",
     typecheck: "bun check",
     lint: "biome check .",
     format: "biome check --write .",
@@ -8150,10 +8150,54 @@ import { randomBytes as randomBytes2 } from "node:crypto";
 import {
   appendFileSync,
   createWriteStream as createWriteStream2,
-  mkdirSync as mkdirSync4,
+  mkdirSync as mkdirSync5,
   writeFileSync as writeFileSync4
 } from "node:fs";
-import { basename as basename3, delimiter as delimiter2, join as join9 } from "node:path";
+import { basename as basename3, delimiter as delimiter2, join as join10 } from "node:path";
+
+// src/agent-dirs.ts
+import { createHash, randomInt } from "node:crypto";
+import { mkdirSync as mkdirSync2, realpathSync, symlinkSync as symlinkSync3 } from "node:fs";
+import { homedir as homedir3 } from "node:os";
+import { join as join5 } from "node:path";
+var ID_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789";
+var ID_LENGTH = 8;
+var MAX_TRIES = 10;
+function projectAgentDir(projectRoot) {
+  const hash = createHash("sha256").update(realpathSync(projectRoot)).digest("hex").slice(0, 12);
+  return join5(firegoatHome(), "runs", hash);
+}
+function firegoatHome() {
+  return join5(realpathSync(homedir3()), ".firegoat");
+}
+function randomId() {
+  let id = "";
+  for (let i = 0;i < ID_LENGTH; i++)
+    id += ID_CHARS[randomInt(ID_CHARS.length)];
+  return id;
+}
+function ensureProjectAgentDir(projectRoot) {
+  mkdirSync2(firegoatHome(), { recursive: true, mode: 448 });
+  const dir = projectAgentDir(projectRoot);
+  mkdirSync2(dir, { recursive: true });
+  return dir;
+}
+function makeAgentDir(projectDir) {
+  for (let i = 0;i < MAX_TRIES; i++) {
+    const dir = join5(projectDir, randomId());
+    try {
+      mkdirSync2(dir);
+      return dir;
+    } catch (e) {
+      if (e.code !== "EEXIST")
+        throw e;
+    }
+  }
+  throw new Error(`Could not create a new run folder in ${projectDir}: ${MAX_TRIES} random names were taken`);
+}
+function linkDir(target, link) {
+  symlinkSync3(target, link, "dir");
+}
 
 // src/credentials.ts
 import { randomBytes } from "node:crypto";
@@ -8162,15 +8206,15 @@ import {
   closeSync as closeSync2,
   fsyncSync,
   lstatSync as lstatSync3,
-  mkdirSync as mkdirSync2,
+  mkdirSync as mkdirSync3,
   openSync as openSync2,
   readFileSync as readFileSync4,
   renameSync,
   unlinkSync as unlinkSync3,
   writeSync
 } from "node:fs";
-import { homedir as homedir3 } from "node:os";
-import { join as join5 } from "node:path";
+import { homedir as homedir4 } from "node:os";
+import { join as join6 } from "node:path";
 
 // src/firegoat-api.ts
 var DEFAULT_BASE_URL = "https://firegoat.ai/evals";
@@ -8322,10 +8366,10 @@ async function whoami(base, key) {
 // src/credentials.ts
 var KEY = /^fire_[\x21-\x7e]+$/;
 function credentialsDir() {
-  return join5(homedir3(), ".firegoat");
+  return join6(homedir4(), ".firegoat");
 }
 function credentialsPath() {
-  return join5(credentialsDir(), "credentials.json");
+  return join6(credentialsDir(), "credentials.json");
 }
 function uid() {
   if (!process.getuid)
@@ -8388,7 +8432,7 @@ function saveCredentials(c) {
   if (st)
     assertOwned(dir, st, "dir");
   else
-    mkdirSync2(dir, { mode: 448 });
+    mkdirSync3(dir, { mode: 448 });
   chmodSync(dir, 448);
   const tmp = `${credentialsPath()}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`;
   const fd = openSync2(tmp, "wx", 384);
@@ -8444,15 +8488,15 @@ function resolveKey(base) {
 var EXIT = { pass: 0, fail: 1, notGraded: 2, usage: 3 };
 
 // src/firegoat-dir.ts
-import { mkdirSync as mkdirSync3, writeFileSync as writeFileSync2 } from "node:fs";
-import { join as join6 } from "node:path";
+import { mkdirSync as mkdirSync4, writeFileSync as writeFileSync2 } from "node:fs";
+import { join as join7 } from "node:path";
 var GITIGNORE = `*
 `;
 function ensureFiregoatDir(root) {
-  const dir = join6(root, ".firegoat");
-  mkdirSync3(dir, { recursive: true });
+  const dir = join7(root, ".firegoat");
+  mkdirSync4(dir, { recursive: true });
   try {
-    writeFileSync2(join6(dir, ".gitignore"), GITIGNORE, { flag: "wx" });
+    writeFileSync2(join7(dir, ".gitignore"), GITIGNORE, { flag: "wx" });
   } catch (e) {
     if (e.code !== "EEXIST")
       throw e;
@@ -8812,7 +8856,11 @@ firegoat.yaml reference:
   for any value.
   The build, install and cleanup commands and the checks get the vars as env
   vars, plus $FIREGOAT_OUT and $FIREGOAT_WORKDIR; install, checks and cleanup also
-  get $FIREGOAT_HOME, the run's own home, as HOME (build keeps your real HOME).
+  get $FIREGOAT_HOME, the run's own home, as HOME (build keeps your real HOME),
+  $FIREGOAT_RUN_DIR (the run's .firegoat/runs/<runId>/<test>/<n>/, where its logs
+  go) and $FIREGOAT_REPO (the folder holding firegoat.yaml). The agent gets
+  neither: keep values it must not find in $FIREGOAT_RUN_DIR, e.g. install:
+  echo "$EXPECTED" > "$FIREGOAT_RUN_DIR/expected", and a check reads it back.
   Checks also get $FIREGOAT_TRANSCRIPT, a short plain-text
   transcript, greppable line by line (format: eval --help, Transcript). So
   '^\\[[0-9]+\\] \\$ .*foo' matches only commands. Checks also get
@@ -8911,8 +8959,13 @@ var EVAL_HELP = `Usage: firegoat eval [test..] [--runs N] [--model NAME].. [--pa
 
 Runs each test in firegoat.yaml (in the current folder): your agent harness
 (claude-code by default, codex-cli or grok-cli; harness: in firegoat.yaml) gets the prompt
-in a fresh, empty folder, .firegoat/runs/<runId>/<test>/<n>/work, with build:'s
-outputs on PATH. Then the test's checks run in that folder.
+in a fresh, empty folder outside the repo, ~/.firegoat/runs/<hash>/<random>/work,
+with build:'s outputs on PATH. Then the test's checks run in that folder. <hash>
+is a hash of the repo's path (a linked worktree uses its main checkout's) and
+<random> is new for each run, so the agent's paths name neither the project nor
+the test, and git there sees no repo. The run's logs stay in the repo, in
+.firegoat/runs/<runId>/<test>/<n>/, whose work and home are symlinks to the
+run's folders.
 
 Step order: build: once, in this folder; then per run, in its folder: install:
 (top-level, then the test's), the agent, checks, cleanup: (the test's, then
@@ -8921,38 +8974,39 @@ a claude error or a timeout; a failing cleanup step prints a warning and never
 changes the run's status or the exit code.
 
 claude runs on its own login (or ANTHROPIC_API_KEY if set, as claude always does)
-with a fresh config dir, .firegoat/.claude: none of your settings, CLAUDE.md,
-plugins, skills or MCP servers (harness_args: in firegoat.yaml adds claude flags,
-e.g. --chrome or --plugin-dir). claude saves its session there. There is no
-sandbox: the agent can reach the rest of your machine, including the repo.
+with a fresh config dir, ~/.firegoat/runs/<hash>/.claude: none of your settings,
+CLAUDE.md, plugins, skills or MCP servers (harness_args: in firegoat.yaml adds
+claude flags, e.g. --chrome or --plugin-dir). claude saves its session there. There
+is no sandbox: the agent can still reach the rest of your machine, including the
+repo, if it goes looking.
 
 codex-cli runs codex exec --json --dangerously-bypass-approvals-and-sandbox with a
-fresh CODEX_HOME, .firegoat/.codex, holding a link to your codex login
+fresh CODEX_HOME, ~/.firegoat/runs/<hash>/.codex, holding a link to your codex login
 (~/.codex/auth.json, or $CODEX_HOME/auth.json; run codex login first), and with
 -c project_doc_max_bytes=0: no AGENTS.md, config.toml, skills or plugins of
 yours. codex's own HOME is the run's home (below), also for its Read/Write. It
-saves its session (rollout) under .firegoat/.codex/sessions/. firegoat.ai doesn't
+saves its session (rollout) under that CODEX_HOME's sessions/. firegoat.ai doesn't
 read codex sessions yet: those runs are not uploaded. codex reports no turns or
 cost; its model comes from the session file.
 
 grok-cli runs ~/.grok/bin/grok (where grok's installer puts it; never a grok on
 PATH, which may be a wrapper such as cmux's: a #! script is refused) as grok -p
 --output-format streaming-messages-json --always-approve --no-auto-update, with a
-fresh GROK_HOME, .firegoat/.grok, holding a link to your grok login
+fresh GROK_HOME, ~/.firegoat/runs/<hash>/.grok, holding a link to your grok login
 ($GROK_HOME/auth.json or ~/.grok/auth.json; run grok login first; XAI_API_KEY alone
 does not log headless grok in). grok's own HOME is the run's home (below): with
 your real one it loads Claude Code's skills, plugins, MCP servers and hooks. eval
 refuses a login that expires within 5 minutes (run grok -p hi once to refresh it):
 a refresh inside a run would replace the link with a file, which the next eval
 asks you to move back over ~/.grok/auth.json. The session is a directory,
-.firegoat/.grok/sessions/<cwd>/<id>/ ($FIREGOAT_SESSION). firegoat.ai doesn't read
+$GROK_HOME/sessions/<cwd>/<id>/ ($FIREGOAT_SESSION). firegoat.ai doesn't read
 grok sessions yet: those runs are not uploaded.
 
-Every run gets its own HOME, .firegoat/runs/<runId>/<test>/<n>/home (with
-XDG_CONFIG_HOME, XDG_CACHE_HOME, XDG_DATA_HOME, XDG_STATE_HOME inside it), as
+Every run gets its own HOME, ~/.firegoat/runs/<hash>/<random>/home next to work
+(with XDG_CONFIG_HOME, XDG_CACHE_HOME, XDG_DATA_HOME, XDG_STATE_HOME inside it), as
 $FIREGOAT_HOME: tools that keep state in ~ start fresh and parallel runs don't
 collide. claude itself keeps your real HOME (its login is there) and sources
-<n>/home.env (CLAUDE_ENV_FILE) before each Bash command the agent runs; install,
+home.env next to it (CLAUDE_ENV_FILE) before each Bash command the agent runs; install,
 checks and cleanup get the run's HOME directly; build keeps yours. Limits: claude's
 own Read/Write tools expand ~ to your real home; getpwuid lookups (os.userInfo,
 python pwd) still see it; keychain logins (gh, docker, git credential helpers) look
@@ -8968,9 +9022,9 @@ nested session. Login and provider vars stay: ANTHROPIC_*, CLAUDE_CODE_OAUTH_TOK
 CLAUDE_CODE_OAUTH_REFRESH_TOKEN, CLAUDE_CODE_USE_*, CLAUDE_CODE_SKIP_*_AUTH,
 CLAUDE_CODE_CLIENT_CERT, CLAUDE_CODE_CLIENT_KEY, CLAUDE_CODE_CLIENT_KEY_PASSPHRASE.
 
-In a linked git worktree (git worktree add), .firegoat/runs/ and the harness
-config dirs (.firegoat/.claude, .firegoat/.codex, .firegoat/.grok) live in the main checkout instead, so runs outlive the worktree; firegoat.yaml,
-fixtures and build: stay in the current folder.
+In a linked git worktree (git worktree add), .firegoat/runs/ lives in the main
+checkout instead, and <hash> is the main checkout's, so runs outlive the worktree;
+firegoat.yaml, fixtures and build: stay in the current folder.
 
   test..             Only these tests (by name)
   --runs N           Runs per test (overrides runs: in firegoat.yaml)
@@ -8994,8 +9048,9 @@ fixtures and build: stay in the current folder.
                      --model, --harness, missing vars), print each planned
                      run ("<test>#<n> [harness model]" with its install, checks
                      and cleanup counts and rendered harness_args, paths under
-                     runs/<runId>/) and exit 0. Builds nothing, starts no
-                     agent, uploads nothing, writes no files. Config errors
+                     ~/.firegoat/runs/<hash>/<random>/) and exit 0. Builds
+                     nothing, starts no agent, uploads nothing, writes no
+                     files. Config errors
                      exit 3. With --json: {dryRun, build, runs: [{test, n,
                      harness, model, harnessArgs, install, checks,
                      cleanup}]}, steps and checks with their placeholders
@@ -9017,6 +9072,8 @@ Runs without a session file are not uploaded. FIREGOAT_BASE_URL points elsewhere
 (default https://firegoat.ai/evals). See firegoat login --help.
 
 Output: .firegoat/runs/<runId>/
+  out                     A symlink to $FIREGOAT_OUT,
+                          ~/.firegoat/runs/<hash>/<random>/out
   summary.json            firegoatVersion, runId, runsDir, dir (where eval
                           ran), git {branch, commit, dirty} (no git outside a
                           repo), harnesses, upload {origin, email} (null when
@@ -9030,14 +9087,17 @@ Output: .firegoat/runs/<runId>/
                           the harness reported running, or null), harnessArgs
                           (harness_args passed), prompt, end_goal,
                           checks [{run, code}], cleanup [{run, code}],
-                          and paths: workdir, home (the run's HOME),
+                          and paths: dir (this folder), workdir, home (the
+                          run's HOME; both real paths, under ~/.firegoat),
                           transcript, session (the harness's session file), stderr,
                           checksLog, cleanupLog; startedAt; upload ({status: "uploaded",
                           url} | {status: "failed", error} | {status:
                           "skipped", reason}, null when not uploading)
   <test>/<n>/transcript.txt  What the agent did, below
-  <test>/<n>/home/, home.env  The run's HOME, and the file claude sources
-                          before the agent's Bash commands
+  <test>/<n>/work, home   Symlinks to the agent's folder and the run's HOME
+                          (home.env, which claude sources before the agent's
+                          Bash commands, is next to them)
+
   build.txt, <test>/<n>/install.txt, checks.txt, cleanup.txt   Logs
 
 Transcript ($FIREGOAT_TRANSCRIPT), greppable line by line; n counts tool calls:
@@ -9228,7 +9288,7 @@ async function sendTelemetry(event, exitCode, startedAt) {
 
 // src/transcript.ts
 import { readdirSync as readdirSync4, statSync as statSync2, writeFileSync as writeFileSync3 } from "node:fs";
-import { join as join7, relative } from "node:path";
+import { join as join8, relative } from "node:path";
 var OUTPUT_LINES = 15;
 var OUTPUT_CHARS = 1500;
 var TEXT_CHARS = 1500;
@@ -9280,7 +9340,7 @@ function workdirFiles(dir) {
     for (const name of readdirSync4(d)) {
       if (SKIP_DIRS.has(name))
         continue;
-      const p = join7(d, name);
+      const p = join8(d, name);
       if (statSync2(p).isDirectory())
         walk(p);
       else
@@ -9780,7 +9840,7 @@ async function uploadRun(t, r) {
 
 // src/which.ts
 import { accessSync, constants, statSync as statSync3 } from "node:fs";
-import { delimiter, join as join8, resolve } from "node:path";
+import { delimiter, join as join9, resolve } from "node:path";
 function resolveBin(bin) {
   const executable = (p) => {
     try {
@@ -9793,8 +9853,8 @@ function resolveBin(bin) {
   if (bin.includes("/"))
     return executable(bin) ? resolve(bin) : undefined;
   for (const dir of (process.env.PATH ?? "").split(delimiter)) {
-    if (dir && executable(join8(dir, bin)))
-      return join8(dir, bin);
+    if (dir && executable(join9(dir, bin)))
+      return join9(dir, bin);
   }
   return;
 }
@@ -9881,7 +9941,7 @@ function newRunId() {
   return `${ts}-${randomBytes2(2).toString("hex")}`;
 }
 function writeRunJson(r) {
-  writeFileSync4(join9(r.dir, "run.json"), `${JSON.stringify(r, null, 2)}
+  writeFileSync4(join10(r.dir, "run.json"), `${JSON.stringify(r, null, 2)}
 `);
 }
 async function uploadPreflight(opts, ui, project, runGroup, git) {
@@ -9922,14 +9982,14 @@ async function uploadPreflight(opts, ui, project, runGroup, git) {
     };
   }
 }
-function runFill(runId, n, testName, out, dir) {
+function runFill(runId, n, testName, out, agentDir) {
   return {
     "run.id": runId,
     "run.n": String(n),
     "test.name": testName,
     FIREGOAT_OUT: out,
-    FIREGOAT_WORKDIR: join9(dir, "work"),
-    FIREGOAT_HOME: join9(dir, "home")
+    FIREGOAT_WORKDIR: join10(agentDir, "work"),
+    FIREGOAT_HOME: join10(agentDir, "home")
   };
 }
 function renderHarnessArgs(test, vars, fill) {
@@ -9962,10 +10022,11 @@ function showsHarness(harnesses) {
 }
 var DRY_RUN_ID = "<runId>";
 function dryRun(opts) {
-  const out = join9(opts.runsRoot, DRY_RUN_ID, "out");
+  const agentDir = join10(opts.agentDirs, "<random>");
+  const out = join10(agentDir, "out");
   const runs = opts.plan.map((p) => {
     const test = opts.tests.find((t) => t.name === p.test);
-    const fill = runFill(DRY_RUN_ID, p.n, test.name, out, join9(opts.runsRoot, DRY_RUN_ID, test.name, String(p.n)));
+    const fill = runFill(DRY_RUN_ID, p.n, test.name, out, agentDir);
     renderEnv(test.env, opts.vars, fill);
     return {
       ...p,
@@ -9986,11 +10047,15 @@ function dryRun(opts) {
 async function runOne(opts) {
   const { test, n } = opts;
   const started = Date.now();
-  const dir = join9(opts.runDir, test.name, String(n));
-  const workdir = join9(dir, "work");
-  mkdirSync4(workdir, { recursive: true });
-  const { home, envFile } = makeRunHome(dir);
-  const fill = runFill(opts.runId, n, test.name, opts.out, dir);
+  const dir = join10(opts.runDir, test.name, String(n));
+  mkdirSync5(dir, { recursive: true });
+  const agentDir = makeAgentDir(opts.agentDirs);
+  const workdir = join10(agentDir, "work");
+  mkdirSync5(workdir);
+  const { home, envFile } = makeRunHome(agentDir);
+  linkDir(workdir, join10(dir, "work"));
+  linkDir(home, join10(dir, "home"));
+  const fill = runFill(opts.runId, n, test.name, opts.out, agentDir);
   const r = {
     test: test.name,
     n,
@@ -10007,11 +10072,11 @@ async function runOne(opts) {
     workdir,
     home,
     session: null,
-    transcript: join9(dir, "transcript.txt"),
-    stderr: join9(dir, "stderr.txt"),
-    checksLog: join9(dir, "checks.txt"),
+    transcript: join10(dir, "transcript.txt"),
+    stderr: join10(dir, "stderr.txt"),
+    checksLog: join10(dir, "checks.txt"),
     cleanup: [],
-    cleanupLog: join9(dir, "cleanup.txt"),
+    cleanupLog: join10(dir, "cleanup.txt"),
     num_turns: null,
     cost: null,
     duration_ms: 0,
@@ -10024,7 +10089,9 @@ async function runOne(opts) {
     ...env,
     ...homeEnv(home),
     FIREGOAT_WORKDIR: workdir,
-    FIREGOAT_HOME: home
+    FIREGOAT_HOME: home,
+    FIREGOAT_RUN_DIR: dir,
+    FIREGOAT_REPO: opts.repo
   };
   const steps = renderRunSteps(test, opts, opts.vars, fill);
   const [status, reason] = await attempt(opts, r, steps, env, stepEnv, envFile);
@@ -10041,6 +10108,9 @@ async function runOne(opts) {
   writeRunJson(r);
   return r;
 }
+function withoutOldPwd(env) {
+  return Object.fromEntries(Object.entries(env).filter(([k]) => k !== "OLDPWD"));
+}
 async function attempt(opts, r, steps, env, stepEnv, homeEnvFile) {
   const { test, n } = opts;
   const { dir, workdir } = r;
@@ -10050,7 +10120,7 @@ async function attempt(opts, r, steps, env, stepEnv, homeEnvFile) {
       commands: steps.install,
       cwd: workdir,
       env: stepEnv,
-      logPath: join9(dir, "install.txt")
+      logPath: join10(dir, "install.txt")
     });
     if (error)
       return ["error", `not graded: ${error}`];
@@ -10062,12 +10132,13 @@ async function attempt(opts, r, steps, env, stepEnv, homeEnvFile) {
     prompt: r.prompt,
     cwd: workdir,
     env: {
-      ...hostEnv(),
+      ...withoutOldPwd(hostEnv()),
       ...env,
       ...harness.env({
         configDir: opts.configDir,
         run: { home: r.home, homeEnvFile }
       }),
+      PWD: workdir,
       PATH: opts.stepEnv.PATH,
       FIREGOAT_OUT: opts.out,
       FIREGOAT_WORKDIR: workdir,
@@ -10144,7 +10215,7 @@ function uploadedSummary(origin, results, runDir) {
     return;
   const head = `Uploaded ${urls.length} session${urls.length === 1 ? "" : "s"} to ${new URL(origin).host} (private)`;
   if (urls.length > MAX_LISTED_URLS) {
-    console.log(`${head}: each run's upload.url in ${join9(runDir, "summary.json")}`);
+    console.log(`${head}: each run's upload.url in ${join10(runDir, "summary.json")}`);
     return;
   }
   console.log(`${head}:`);
@@ -10245,7 +10316,7 @@ async function evalCommand(args, root) {
         plan: planRuns(tests, harnessNames, models, runs),
         showHarness,
         vars,
-        runsRoot: join9(repo?.mainCheckout ?? root, ".firegoat", "runs"),
+        agentDirs: projectAgentDir(repo?.mainCheckout ?? root),
         json: opts.json,
         say
       });
@@ -10271,11 +10342,13 @@ async function evalCommand(args, root) {
   } catch (e) {
     return fatal(EXIT.usage, e.message, "config");
   }
-  const firegoatDir = ensureFiregoatDir(repo?.mainCheckout ?? root);
+  const project = repo?.mainCheckout ?? root;
+  const firegoatDir = ensureFiregoatDir(project);
+  const agentDirs = ensureProjectAgentDir(project);
   const configDirs = new Map;
   for (const h of harnesses) {
-    const configDir = join9(firegoatDir, h.configDirName);
-    mkdirSync4(configDir, { recursive: true });
+    const configDir = join10(agentDirs, h.configDirName);
+    mkdirSync5(configDir, { recursive: true });
     configDirs.set(h, configDir);
   }
   ui.auth(harnesses.length === 1 ? harnesses[0].authLabel() : harnesses.map((h) => `${h.name}: ${h.authLabel()}`).join("; "));
@@ -10308,10 +10381,12 @@ async function evalCommand(args, root) {
       return fatal(EXIT.usage, e.message, "config");
     throw e;
   }
-  const runsRoot = join9(firegoatDir, "runs");
-  const runDir = join9(runsRoot, runId);
-  const out = join9(runDir, "out");
-  mkdirSync4(out, { recursive: true });
+  const runsRoot = join10(firegoatDir, "runs");
+  const runDir = join10(runsRoot, runId);
+  mkdirSync5(runDir, { recursive: true });
+  const out = join10(makeAgentDir(agentDirs), "out");
+  mkdirSync5(out);
+  linkDir(out, join10(runDir, "out"));
   const stepEnv = {
     ...hostEnv(),
     ...vars,
@@ -10336,7 +10411,7 @@ async function evalCommand(args, root) {
         ...stepEnv,
         ...renderEnv(Object.fromEntries(Object.entries(config.env).filter(([, v]) => isEvalLevel(v))), vars, { "run.id": runId, FIREGOAT_OUT: out })
       },
-      logPath: join9(runDir, "build.txt")
+      logPath: join10(runDir, "build.txt")
     });
     const secs = Math.round((Date.now() - buildStarted) / 1000);
     ui.build(error ? "fail" : "ok", secs);
@@ -10363,6 +10438,8 @@ async function evalCommand(args, root) {
       shownHarness: row.harness,
       runId,
       runDir,
+      agentDirs,
+      repo: root,
       harness,
       bin: bins.get(harness),
       configDir: configDirs.get(harness),
@@ -10432,9 +10509,9 @@ async function evalCommand(args, root) {
     status,
     exitCode
   };
-  writeFileSync4(join9(runDir, "summary.json"), `${JSON.stringify(doc, null, 2)}
+  writeFileSync4(join10(runDir, "summary.json"), `${JSON.stringify(doc, null, 2)}
 `);
-  appendFileSync(join9(runsRoot, "index.jsonl"), `${JSON.stringify({
+  appendFileSync(join10(runsRoot, "index.jsonl"), `${JSON.stringify({
     runId,
     startedAt: new Date(evalStarted).toISOString(),
     dir: root,
@@ -10747,7 +10824,7 @@ firegoat ${package_default.version}` : `firegoat ${package_default.version}`,
 // src/init.ts
 import { spawn as spawn3, spawnSync as spawnSync4 } from "node:child_process";
 import { existsSync as existsSync6, writeFileSync as writeFileSync5 } from "node:fs";
-import { basename as basename4, join as join10 } from "node:path";
+import { basename as basename4, join as join11 } from "node:path";
 
 // src/login.ts
 import { spawn as spawn2 } from "node:child_process";
@@ -11046,12 +11123,12 @@ function skillsDir(scope) {
   if (scope === "local")
     return ".claude/skills/";
   const config = process.env.CLAUDE_CONFIG_DIR?.trim();
-  return config ? `${join10(config, "skills")}/` : "~/.claude/skills/";
+  return config ? `${join11(config, "skills")}/` : "~/.claude/skills/";
 }
 function template(product) {
   return `# firegoat.yaml: evals for how well coding agents use ${product}.
 # Each test runs your agent (claude by default) in an empty folder
-# (.firegoat/runs/...) with \`prompt\` as the first message, then runs \`checks\`
+# (~/.firegoat/runs/...) with \`prompt\` as the first message, then runs \`checks\`
 # there. Reference: npx firegoat init --help
 
 # How many times to run each test.
@@ -11308,7 +11385,7 @@ ${INIT_HELP}`);
     console.log(rule);
   }
   ensureFiregoatDir(cwd);
-  const configPath = join10(cwd, CONFIG_FILE);
+  const configPath = join11(cwd, CONFIG_FILE);
   const keep = existsSync6(configPath) && !force;
   if (!keep)
     writeFileSync5(configPath, template(basename4(cwd)));

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.25.0",
+	"version": "0.26.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -29,7 +29,7 @@
 	},
 	"scripts": {
 		"build": "bun build src/cli.ts --target node --outfile dist/cli.js",
-		"test": "bun test",
+		"test": "bun test --timeout 30000",
 		"typecheck": "bun check",
 		"lint": "biome check .",
 		"format": "biome check --write .",