firegoat 0.1.0 → 0.3.0

package/README.md

@@ -2,15 +2,50 @@
 
 Local-first AX evals: how well coding agents use your SDK, CLI, API and docs.
 
-Status: early (v0.1). See `plan.md`.
+Status: early (v0.3). See `plan.md`.
 
 ## Use
 
 ```bash
 npx firegoat init            # writes firegoat.yaml with starter evals; --skills adds the onboarding skill
-npx firegoat eval            # runs each test with your local `claude` in a fresh empty folder
+npx firegoat login           # once per machine: signs claude in to ~/.firegoat/claude
+npx firegoat eval            # runs each test with your local `claude`, sandboxed, in a fresh folder
+npx firegoat open            # opens the latest run's folder (--print to only print it)
 ```
 
+### Local builds
+
+`build:` builds your packages once per `eval` (on the host, from source) and preinstalls them into every run's fresh folder. The agent gets the build, never the source.
+
+```yaml
+build:
+  sdk: ./packages/sdk            # package.json: npm pack, then npm install into node_modules
+  py: ./python                   # pyproject.toml: wheel (uv build, or python3 -m build) into .venv, on PATH
+  cli:                           # anything else: your command writes outputs to $FIREGOAT_OUT,
+    run: go build -o "$FIREGOAT_OUT/acme" ./cmd/acme   # which go on the agent's PATH
+tests:
+  - name: quickstart
+    prompt: "Deploy commit {{vars.SHA}} with the acme CLI. The SDK is already installed; don't install it from the registry."
+    goal: ...
+```
+
+- `{{build.NAME}}` in a prompt or check is the path of that build's artifact inside the run's folder.
+- `{{vars.KEY}}` comes from `--var KEY=value` or `vars:` in `firegoat.yaml`; vars are also in the env of `build:` commands. A missing var is a usage error. Other `{{...}}` text is left alone.
+- A failed build or install means not graded (exit 2); logs are in `.firegoat/runs/<runId>/build/build.txt` and `<test>/<n>/setup.txt`.
+
+### Isolation
+
+On macOS, `eval` runs `claude` inside a Seatbelt (`sandbox-exec`) profile:
+
+- it signs in with `~/.firegoat/claude` (from `firegoat login`), never your `~/.claude`;
+- it cannot read your home dir or the repo, except toolchains (node, bun, cargo, ...) it needs to run;
+- it can write only in its run dir (`work/`, its own `TMPDIR` and package caches) and system temp;
+- `ANTHROPIC_API_KEY` is removed, so runs bill your subscription, not the API.
+
+A probe checks the sandbox before every run; if it does not hold, the run is not graded. Network is open.
+
+Off macOS there is no sandbox yet: run inside a throwaway container and pass `--host`.
+
 `eval` saves each run's transcript to `.firegoat/runs/<runId>/<test>/<n>/session.jsonl` and a `summary.json`, then runs the test's `checks` in the agent's folder. Nothing is uploaded. `firegoat.yaml` is looked up in the current directory, then parents up to the git root.
 
 Exit codes:

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.1.0",
+  version: "0.3.0",
   repository: {
     type: "git",
     url: "git+https://github.com/team2027/firegoat-cli.git"
@@ -7081,6 +7081,8 @@ function parseConfig(text, warn = console.error) {
     }
   });
   const runs = posInt(raw.runs, "runs", 1);
+  const build = parseBuild(raw.build);
+  const vars = parseVars(raw.vars);
   if (!Array.isArray(raw.tests) || raw.tests.length === 0)
     fail("tests", "must be a non-empty list");
   const names = new Set;
@@ -7089,7 +7091,7 @@ function parseConfig(text, warn = console.error) {
     if (!isRecord(t))
       fail(p, "must be a mapping");
     const name = str(t.name, `${p}.name`);
-    if (!/^[A-Za-z0-9._-]+$/.test(name))
+    if (!NAME.test(name))
       fail(`${p}.name`, "may only contain letters, digits, '.', '_' and '-'");
     if (names.has(name))
       fail(`${p}.name`, `duplicate test name ${JSON.stringify(name)}`);
@@ -7116,7 +7118,71 @@ function parseConfig(text, warn = console.error) {
       timeout: posInt(t.timeout, `${p}.timeout`, DEFAULT_TIMEOUT)
     };
   });
-  return { agents, runs, tests };
+  const builds = new Set(build.map((b) => b.name));
+  tests.forEach((t, i) => {
+    const texts = [t.prompt, ...t.checks.map((c) => c.run)];
+    for (const ref of texts.flatMap(placeholders)) {
+      if (ref.kind === "build" && !builds.has(ref.key))
+        fail(`tests[${i}]`, `{{build.${ref.key}}} names no entry under build: (known: ${[...builds].join(", ") || "none"})`);
+    }
+  });
+  return { agents, runs, build, vars, tests };
+}
+var NAME = /^[A-Za-z0-9._-]+$/;
+function parseBuild(raw) {
+  if (raw === undefined)
+    return [];
+  if (!isRecord(raw))
+    fail("build", "must be a mapping of name to path or {path, run}");
+  return Object.entries(raw).map(([name, v]) => {
+    const p = `build.${name}`;
+    if (!NAME.test(name))
+      fail(p, "name may only contain letters, digits, '.', '_' and '-'");
+    if (typeof v === "string")
+      return { name, path: str(v, p) };
+    if (!isRecord(v))
+      fail(p, "must be a path or a mapping with path and/or run");
+    if (v.path === undefined && v.run === undefined)
+      fail(p, "needs path, run, or both");
+    const b = {
+      name,
+      path: v.path === undefined ? "." : str(v.path, `${p}.path`)
+    };
+    if (v.run !== undefined)
+      b.run = str(v.run, `${p}.run`);
+    return b;
+  });
+}
+var VAR_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/;
+function parseVars(raw) {
+  if (raw === undefined)
+    return {};
+  if (!isRecord(raw))
+    fail("vars", "must be a mapping of KEY to value");
+  const out = {};
+  for (const [k, v] of Object.entries(raw)) {
+    if (!VAR_KEY.test(k))
+      fail(`vars.${k}`, "key must be letters, digits and _, not starting with a digit");
+    if (typeof v !== "string" && typeof v !== "number" && typeof v !== "boolean")
+      fail(`vars.${k}`, "must be a string, number or boolean");
+    out[k] = String(v);
+  }
+  return out;
+}
+var PLACEHOLDER = /\{\{\s*(build|vars)\.([A-Za-z0-9._-]+)\s*\}\}/g;
+function placeholders(text) {
+  return [...text.matchAll(PLACEHOLDER)].map((m) => ({
+    kind: m[1],
+    key: m[2]
+  }));
+}
+function render(text, values) {
+  return text.replace(PLACEHOLDER, (_, kind, key) => {
+    const v = values[kind][key];
+    if (v === undefined)
+      throw new ConfigError(`no value for {{${kind}.${key}}}`);
+    return v;
+  });
 }
 function loadConfig(dir, warn) {
   const file = `${dir}/${CONFIG_FILE}`;
@@ -7139,23 +7205,177 @@ function findConfigDir(cwd) {
 // src/eval.ts
 import { randomBytes } from "node:crypto";
 import {
-  accessSync,
-  constants,
-  createWriteStream as createWriteStream2,
-  existsSync as existsSync2,
-  mkdirSync,
+  createWriteStream as createWriteStream3,
+  existsSync as existsSync4,
+  mkdirSync as mkdirSync3,
   mkdtempSync,
   readFileSync as readFileSync2,
-  statSync,
-  writeFileSync
+  realpathSync as realpathSync2,
+  writeFileSync as writeFileSync2
 } from "node:fs";
 import { tmpdir } from "node:os";
-import { delimiter, join as join2, resolve } from "node:path";
+import { join as join5, resolve as resolve3 } from "node:path";
+
+// src/build.ts
+import {
+  cpSync,
+  createWriteStream as createWriteStream2,
+  existsSync as existsSync3,
+  mkdirSync as mkdirSync2,
+  readdirSync,
+  statSync as statSync2,
+  writeFileSync
+} from "node:fs";
+import { delimiter as delimiter2, join as join4, resolve as resolve2 } from "node:path";
 
 // src/claude.ts
-import { spawn } from "node:child_process";
+import { spawn, spawnSync as spawnSync2 } from "node:child_process";
 import { createWriteStream } from "node:fs";
 import { createInterface } from "node:readline";
+
+// src/sandbox.ts
+import { spawnSync } from "node:child_process";
+import { existsSync as existsSync2, mkdirSync, realpathSync } from "node:fs";
+import { homedir } from "node:os";
+import { basename, dirname as dirname2, join as join2 } from "node:path";
+var SANDBOX_EXEC = "/usr/bin/sandbox-exec";
+function firegoatHome() {
+  return process.env.FIREGOAT_HOME || join2(homedir(), ".firegoat");
+}
+function claudeConfigDir() {
+  return join2(firegoatHome(), "claude");
+}
+var TOOLCHAINS = [
+  ".nvm",
+  ".fnm",
+  ".volta",
+  ".bun",
+  ".deno",
+  ".cargo",
+  ".rustup",
+  ".pyenv",
+  ".rbenv",
+  ".asdf",
+  ".sdkman",
+  ".local/share/mise",
+  ".local/share/fnm",
+  ".local/share/claude",
+  ".local/share/uv",
+  "go",
+  "Library/pnpm"
+];
+var TOOLCHAIN_SECRETS = [".cargo/credentials", ".cargo/credentials.toml"];
+function real(p) {
+  return realpathSync(p);
+}
+function under(p, dir) {
+  return p === dir || p.startsWith(`${dir}/`);
+}
+function readableToolDirs(home, claudeBin) {
+  const dirs = new Set;
+  const add = (p) => {
+    if (existsSync2(p))
+      dirs.add(real(p));
+  };
+  for (const t of TOOLCHAINS)
+    add(join2(home, t));
+  for (const p of (process.env.PATH ?? "").split(":")) {
+    if (!p.startsWith("/") || !existsSync2(p))
+      continue;
+    const r = real(p);
+    if (!under(r, home) || r === home)
+      continue;
+    const prefix = dirname2(r);
+    dirs.add(basename(r) === "bin" && prefix !== home && prefix !== join2(home, ".local") ? prefix : r);
+  }
+  add(dirname2(dirname2(process.execPath)));
+  add(dirname2(claudeBin));
+  add(dirname2(real(claudeBin)));
+  return [...dirs];
+}
+var q = (s) => JSON.stringify(s);
+function sandboxProfile(opts) {
+  const home = real(homedir());
+  const reads = readableToolDirs(home, opts.claudeBin).map((d) => `(subpath ${q(d)})`);
+  return `(version 1)
+(allow default)
+
+(deny file-read* (subpath ${q(home)}))
+(allow file-read-metadata)
+(allow file-read*
+  ${reads.join(`
+  `)}
+  (subpath ${q(join2(home, "Library/Keychains"))})
+  (regex ${q(`^${home}/Library/Preferences/com\\.apple\\.security`)})
+  (regex ${q(`^${home}/Library/Preferences/\\.GlobalPreferences`)}))
+(deny file-read* file-write*
+  ${TOOLCHAIN_SECRETS.map((s) => `(literal ${q(join2(home, s))})`).join(`
+  `)})
+
+(deny file-write* (subpath "/"))
+(allow file-write*
+  (subpath "/dev")
+  (subpath "/private/tmp")
+  (subpath "/private/var/folders"))
+(allow file-read* file-write*
+  (subpath ${q(real(opts.configDir))})
+  (subpath ${q(real(opts.runRoot))}))
+
+(deny file-read* file-write* (subpath ${q(real(opts.repo))}))
+`;
+}
+function agentEnv(runRoot) {
+  const env = { ...process.env };
+  delete env.ANTHROPIC_API_KEY;
+  const cache = join2(runRoot, "cache");
+  const tmp = join2(runRoot, "tmp");
+  mkdirSync(cache, { recursive: true });
+  mkdirSync(tmp, { recursive: true });
+  return {
+    ...env,
+    CLAUDE_CONFIG_DIR: claudeConfigDir(),
+    TMPDIR: tmp,
+    XDG_CACHE_HOME: cache,
+    npm_config_cache: join2(cache, "npm"),
+    npm_config_store_dir: join2(cache, "pnpm-store"),
+    YARN_CACHE_FOLDER: join2(cache, "yarn"),
+    BUN_INSTALL_CACHE_DIR: join2(cache, "bun"),
+    PIP_CACHE_DIR: join2(cache, "pip"),
+    UV_CACHE_DIR: join2(cache, "uv"),
+    GOMODCACHE: join2(cache, "gomod"),
+    GOCACHE: join2(cache, "gobuild"),
+    CARGO_HOME: join2(cache, "cargo")
+  };
+}
+function wrap(profile, cmd, args) {
+  if (profile === null)
+    return { cmd, args };
+  return { cmd: SANDBOX_EXEC, args: ["-p", profile, cmd, ...args] };
+}
+function probeSandbox(opts) {
+  const outside = join2(homedir(), `.firegoat-escape-probe-${process.pid}-${Date.now()}`);
+  const script = 'touch "$1" 2>/dev/null && echo home-write; cat "$2" >/dev/null 2>&1 && echo repo-read; touch "$3" 2>/dev/null && echo work-write; rm -f "$1" "$3"';
+  const { cmd, args } = wrap(opts.profile, "/bin/sh", [
+    "-c",
+    script,
+    "probe",
+    outside,
+    opts.repoFile,
+    join2(opts.workdir, ".firegoat-probe")
+  ]);
+  const r = spawnSync(cmd, args, { encoding: "utf8", cwd: opts.workdir });
+  if (r.error)
+    return `sandbox probe could not start: ${r.error.message}`;
+  const got = r.stdout.trim().split(`
+`).filter(Boolean);
+  if (got.includes("home-write") || got.includes("repo-read"))
+    return `sandbox does not hold (escaped: ${got.filter((g) => g !== "work-write").join(", ")}); refusing to run the agent`;
+  if (!got.includes("work-write"))
+    return `sandbox blocks writes in the workdir (stderr: ${r.stderr.trim() || "none"})`;
+  return;
+}
+
+// src/claude.ts
 var CLAUDE_FLAGS = [
   "--output-format",
   "stream-json",
@@ -7169,11 +7389,6 @@ var CLAUDE_FLAGS = [
 function claudeArgv(prompt) {
   return ["-p", prompt, ...CLAUDE_FLAGS];
 }
-function agentEnv() {
-  const env = { ...process.env };
-  delete env.ANTHROPIC_API_KEY;
-  return env;
-}
 var live = new Set;
 function killGroup(child, signal) {
   if (child.pid === undefined)
@@ -7303,10 +7518,9 @@ async function runClaude(opts) {
   let badStart;
   let result;
   const spawned = await spawnWatched({
-    cmd: opts.bin,
-    args: claudeArgv(opts.prompt),
+    ...wrap(opts.profile, opts.bin, claudeArgv(opts.prompt)),
     cwd: opts.cwd,
-    env: agentEnv(),
+    env: opts.env,
     timeoutMs: opts.timeoutMs,
     onStderr: (c) => stderr.write(c),
     onLine: (line) => {
@@ -7376,7 +7590,7 @@ async function runClaude(opts) {
     };
   return { ok: true, result };
 }
-async function runCheck(opts) {
+async function runShell(opts) {
   opts.log.write(`$ ${opts.run}
 `);
   const r = await spawnWatched({
@@ -7405,6 +7619,162 @@ async function runCheck(opts) {
     out.notStarted = true;
   return out;
 }
+function checkLogin(opts) {
+  const { cmd, args } = wrap(opts.profile, opts.bin, ["auth", "status"]);
+  const r = spawnSync2(cmd, args, {
+    cwd: opts.cwd,
+    env: opts.env,
+    encoding: "utf8",
+    timeout: 30000
+  });
+  if (r.error)
+    return `could not run claude auth status: ${r.error.message}`;
+  let status;
+  try {
+    status = JSON.parse(r.stdout);
+  } catch {}
+  if (status?.loggedIn === true)
+    return;
+  if (status?.loggedIn === false)
+    return `claude is not logged in to firegoat's config dir (${opts.env.CLAUDE_CONFIG_DIR}). Run: npx firegoat login`;
+  return `claude auth status gave no login state (exit ${r.status}): ${(r.stdout + r.stderr).trim().slice(0, 300)}`;
+}
+
+// src/which.ts
+import { accessSync, constants, statSync } from "node:fs";
+import { delimiter, join as join3, resolve } from "node:path";
+function resolveBin(bin) {
+  const executable = (p) => {
+    try {
+      accessSync(p, constants.X_OK);
+      return statSync(p).isFile();
+    } catch {
+      return false;
+    }
+  };
+  if (bin.includes("/"))
+    return executable(bin) ? resolve(bin) : undefined;
+  for (const dir of (process.env.PATH ?? "").split(delimiter)) {
+    if (dir && executable(join3(dir, bin)))
+      return join3(dir, bin);
+  }
+  return;
+}
+
+// src/build.ts
+var BUILD_TIMEOUT_MS = 600000;
+
+class BuildError extends Error {
+}
+function kindOf(b, dir) {
+  if (b.run)
+    return "run";
+  if (existsSync3(join4(dir, "package.json")))
+    return "npm";
+  if (existsSync3(join4(dir, "pyproject.toml")))
+    return "python";
+  throw new BuildError(`build.${b.name}: no package.json or pyproject.toml in ${dir}; add run: with a build command that writes to $FIREGOAT_OUT`);
+}
+function pythonBuildCommand() {
+  if (resolveBin("uv"))
+    return 'uv build --wheel --out-dir "$FIREGOAT_OUT"';
+  return 'python3 -m build --wheel --outdir "$FIREGOAT_OUT"';
+}
+var BUILD_COMMAND = {
+  npm: () => 'npm pack --ignore-scripts=false --pack-destination "$FIREGOAT_OUT"',
+  python: pythonBuildCommand
+};
+var ARTIFACT = { npm: ".tgz", python: ".whl" };
+async function sh(run, cwd, env, log, what) {
+  const r = await runShell({ run, cwd, env, timeoutMs: BUILD_TIMEOUT_MS, log });
+  if (r.code !== 0)
+    throw new BuildError(`${what} failed (${r.error ?? `exit ${r.code}`}): ${run}`);
+}
+async function buildOne(b, root, buildDir, env, log) {
+  const dir = resolve2(root, b.path);
+  if (!existsSync3(dir) || !statSync2(dir).isDirectory())
+    throw new BuildError(`build.${b.name}: ${dir} is not a directory`);
+  const kind = kindOf(b, dir);
+  const out = join4(buildDir, b.name);
+  mkdirSync2(out, { recursive: true });
+  const run = kind === "run" ? b.run : BUILD_COMMAND[kind]();
+  log.write(`## build.${b.name} (${kind}) in ${dir}
+`);
+  await sh(run, dir, { ...env, FIREGOAT_OUT: out }, log, `build.${b.name}`);
+  const files = readdirSync(out);
+  if (kind !== "run") {
+    const ext = ARTIFACT[kind];
+    const hits = files.filter((f) => f.endsWith(ext));
+    if (hits.length !== 1)
+      throw new BuildError(`build.${b.name}: expected one ${ext} in ${out}, found ${hits.length ? hits.join(", ") : "none"}`);
+  } else if (files.length === 0)
+    throw new BuildError(`build.${b.name}: run: wrote nothing to $FIREGOAT_OUT (${out})`);
+  return { name: b.name, kind, out };
+}
+async function buildAll(builds, root, buildDir, env) {
+  mkdirSync2(buildDir, { recursive: true });
+  const log = createWriteStream2(join4(buildDir, "build.txt"));
+  try {
+    const built = [];
+    for (const b of builds)
+      built.push(await buildOne(b, root, buildDir, env, log));
+    return { built };
+  } catch (e) {
+    if (e instanceof BuildError)
+      return { error: `${e.message} (log: ${join4(buildDir, "build.txt")})` };
+    throw e;
+  } finally {
+    await new Promise((r) => log.end(r));
+  }
+}
+async function install(built, runRoot, workdir, env, logPath) {
+  const result = { paths: {}, env: {} };
+  if (built.length === 0)
+    return result;
+  const log = createWriteStream2(logPath);
+  const path = [];
+  const artifacts = { npm: [], python: [] };
+  try {
+    for (const b of built) {
+      const copy = join4(runRoot, "packages", b.name);
+      cpSync(b.out, copy, { recursive: true });
+      const files = readdirSync(copy);
+      if (b.kind === "run") {
+        result.paths[b.name] = files.length === 1 ? join4(copy, files[0]) : copy;
+        path.push(copy);
+      } else {
+        const ext = ARTIFACT[b.kind];
+        const file = join4(copy, files.find((f) => f.endsWith(ext)));
+        result.paths[b.name] = file;
+        artifacts[b.kind].push(file);
+      }
+    }
+    if (artifacts.npm.length > 0) {
+      writeFileSync(join4(workdir, "package.json"), `${JSON.stringify({ name: "app", version: "0.0.0", private: true }, null, 2)}
+`);
+      await sh(`npm install --no-audit --no-fund ${artifacts.npm.map(q2).join(" ")}`, workdir, env, log, "npm install of the build");
+      path.push(join4(workdir, "node_modules", ".bin"));
+    }
+    if (artifacts.python.length > 0) {
+      const venv = join4(workdir, ".venv");
+      const wheels = artifacts.python.map(q2).join(" ");
+      await sh(resolveBin("uv") ? `uv venv .venv && uv pip install --python .venv/bin/python ${wheels}` : `python3 -m venv .venv && .venv/bin/pip install ${wheels}`, workdir, env, log, "pip install of the build");
+      path.push(join4(venv, "bin"));
+      result.env.VIRTUAL_ENV = venv;
+    }
+    result.env.PATH = [...path, env.PATH ?? ""].join(delimiter2);
+    return result;
+  } catch (e) {
+    if (e instanceof BuildError)
+      return { error: `${e.message} (log: ${logPath})` };
+    throw e;
+  } finally {
+    await new Promise((r) => log.end(r));
+  }
+}
+function q2(s) {
+  return `'${s.replaceAll("'", `'\\''`)}'`;
+}
 
 // src/exit.ts
 var EXIT = { pass: 0, fail: 1, notGraded: 2, usage: 3 };
@@ -7418,6 +7788,8 @@ function parseArgs(args) {
   const opts = {
     names: [],
     json: false,
+    host: false,
+    vars: {},
     claudeBin: process.env.FIREGOAT_CLAUDE_BIN || "claude"
   };
   for (let i = 0;i < args.length; i++) {
@@ -7435,7 +7807,16 @@ function parseArgs(args) {
     };
     if (flag === "--json")
       opts.json = true;
-    else if (flag === "--runs") {
+    else if (flag === "--host")
+      opts.host = true;
+    else if (flag === "--var") {
+      const kv = value();
+      const at = kv.indexOf("=");
+      const key = kv.slice(0, at);
+      if (at < 1 || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(key))
+        throw new UsageError(`--var needs KEY=value, got ${kv}`);
+      opts.vars[key] = kv.slice(at + 1);
+    } else if (flag === "--runs") {
       const v = value();
       const n = Number(v);
       if (!/^\d+$/.test(v) || n < 1)
@@ -7455,35 +7836,36 @@ function newRunId() {
   return `${ts}-${randomBytes(2).toString("hex")}`;
 }
 function gitignoreHasFiregoat(cwd) {
-  const file = join2(cwd, ".gitignore");
-  if (!existsSync2(file))
+  const file = join5(cwd, ".gitignore");
+  if (!existsSync4(file))
     return false;
   return readFileSync2(file, "utf8").split(`
 `).map((l) => l.trim()).some((l) => /^\/?\.firegoat\/?(\*\*?)?$/.test(l));
 }
-function resolveBin(bin) {
-  const executable = (p) => {
-    try {
-      accessSync(p, constants.X_OK);
-      return statSync(p).isFile();
-    } catch {
-      return false;
-    }
-  };
-  if (bin.includes("/"))
-    return executable(bin) ? resolve(bin) : undefined;
-  for (const dir of (process.env.PATH ?? "").split(delimiter)) {
-    if (dir && executable(join2(dir, bin)))
-      return join2(dir, bin);
-  }
-  return;
+function newRunRoot(label) {
+  const runRoot = realpathSync2(mkdtempSync(join5(tmpdir(), `firegoat-${label}-`)));
+  const workdir = join5(runRoot, "work");
+  mkdirSync3(workdir);
+  return { runRoot, workdir };
+}
+function isolate(iso, claudeBin, runRoot, workdir) {
+  if (iso.host)
+    return { profile: null };
+  const profile = sandboxProfile({
+    claudeBin,
+    runRoot,
+    configDir: claudeConfigDir(),
+    repo: iso.repo
+  });
+  const broken = probeSandbox({ profile, workdir, repoFile: iso.repoFile });
+  return broken ? { error: broken } : { profile };
 }
 async function runOne(opts) {
   const { test, n } = opts;
   const started = Date.now();
-  const dir = join2(opts.runDir, test.name, String(n));
-  mkdirSync(dir, { recursive: true });
-  const session = join2(dir, "session.jsonl");
+  const dir = join5(opts.runDir, test.name, String(n));
+  mkdirSync3(dir, { recursive: true });
+  const session = join5(dir, "session.jsonl");
   const summary = {
     test: test.name,
     n,
@@ -7502,14 +7884,23 @@ async function runOne(opts) {
     summary.duration_ms = Date.now() - started;
     return summary;
   };
-  const workdir = mkdtempSync(join2(tmpdir(), `firegoat-${test.name}-${n}-`));
+  const { runRoot, workdir } = newRunRoot(`${test.name}-${n}`);
   summary.workdir = workdir;
+  const sandbox = isolate(opts.iso, opts.claudeBin, runRoot, workdir);
+  if ("error" in sandbox)
+    return done("error", `not graded: ${sandbox.error}`);
+  const installed = await install(opts.built, runRoot, workdir, process.env, join5(dir, "setup.txt"));
+  if ("error" in installed)
+    return done("error", `not graded: setup: ${installed.error}`);
+  const values = { build: installed.paths, vars: opts.vars };
   const outcome = await runClaude({
+    env: { ...agentEnv(runRoot), ...installed.env },
+    profile: sandbox.profile,
     bin: opts.claudeBin,
-    prompt: test.prompt,
+    prompt: render(test.prompt, values),
     cwd: workdir,
     sessionPath: session,
-    stderrPath: join2(dir, "stderr.txt"),
+    stderrPath: join5(dir, "stderr.txt"),
     timeoutMs: test.timeout * 1000
   });
   if (outcome.result) {
@@ -7522,17 +7913,18 @@ async function runOne(opts) {
     opts.warn(`warning: ${test.name} has no checks, so its goal is unverified; counted as pass`);
     return done("pass", "no checks (goal unverified)");
   }
-  const log = createWriteStream2(join2(dir, "checks.txt"));
+  const log = createWriteStream3(join5(dir, "checks.txt"));
   const env = {
     ...process.env,
-    FIREGOAT_SESSION: resolve(session),
+    ...installed.env,
+    FIREGOAT_SESSION: resolve3(session),
     FIREGOAT_WORKDIR: workdir
   };
   const results = [];
   try {
     for (const c of test.checks) {
-      results.push(await runCheck({
-        run: c.run,
+      results.push(await runShell({
+        run: render(c.run, values),
         cwd: workdir,
         env,
         timeoutMs: CHECK_TIMEOUT_MS,
@@ -7577,25 +7969,77 @@ async function evalCommand(args, cwd) {
     tests = tests.filter((t) => opts.names.includes(t.name));
   }
   const runs = opts.runs ?? config.runs;
+  const vars = { ...config.vars, ...opts.vars };
+  const missing = [
+    ...new Set(tests.flatMap((t) => [t.prompt, ...t.checks.map((c) => c.run)]).flatMap(placeholders).filter((p) => p.kind === "vars" && !(p.key in vars)).map((p) => p.key))
+  ];
+  if (missing.length > 0) {
+    console.error(`firegoat eval: no value for ${missing.map((k) => `{{vars.${k}}}`).join(", ")}: pass --var ${missing[0]}=... or set it under vars: in firegoat.yaml`);
+    return EXIT.usage;
+  }
   const claudePath = resolveBin(opts.claudeBin);
   if (!claudePath) {
     console.error(`firegoat eval: not graded: claude binary ${JSON.stringify(opts.claudeBin)} not found. Install Claude Code (https://claude.com/claude-code) or pass --claude-bin PATH / FIREGOAT_CLAUDE_BIN.`);
     return EXIT.notGraded;
   }
-  mkdirSync(join2(root, ".firegoat"), { recursive: true });
+  if (opts.host)
+    warn("warning: --host: the agent runs without isolation and can read and write your files");
+  else if (process.platform !== "darwin" || !existsSync4(SANDBOX_EXEC)) {
+    console.error(`firegoat eval: isolation needs macOS (${SANDBOX_EXEC}); this is ${process.platform}. In CI or Linux, run inside a throwaway container and pass --host.`);
+    return EXIT.usage;
+  }
+  const iso = {
+    host: opts.host,
+    repo: root,
+    repoFile: join5(root, "firegoat.yaml")
+  };
+  mkdirSync3(claudeConfigDir(), { recursive: true });
+  const pre = newRunRoot("preflight");
+  const preSandbox = isolate(iso, claudePath, pre.runRoot, pre.workdir);
+  const loginProblem = "error" in preSandbox ? preSandbox.error : checkLogin({
+    bin: claudePath,
+    env: agentEnv(pre.runRoot),
+    profile: preSandbox.profile,
+    cwd: pre.workdir
+  });
+  if (loginProblem) {
+    console.error(`firegoat eval: not graded: ${loginProblem}`);
+    return EXIT.notGraded;
+  }
+  mkdirSync3(join5(root, ".firegoat"), { recursive: true });
   if (!gitignoreHasFiregoat(root))
     warn("hint: add .firegoat/ to .gitignore (it holds run traces)");
   const runId = newRunId();
-  const runDir = join2(root, ".firegoat", "runs", runId);
-  mkdirSync(runDir, { recursive: true });
-  const summaryPath = join2(runDir, "summary.json");
+  const runDir = join5(root, ".firegoat", "runs", runId);
+  mkdirSync3(runDir, { recursive: true });
+  const summaryPath = join5(runDir, "summary.json");
+  let built = [];
+  if (config.build.length > 0) {
+    say(`building ${config.build.map((b) => b.name).join(", ")}...`);
+    const b = await buildAll(config.build, root, join5(runDir, "build"), {
+      ...process.env,
+      ...vars
+    });
+    if ("error" in b) {
+      console.error(`firegoat eval: not graded: ${b.error}`);
+      return EXIT.notGraded;
+    }
+    built = b.built;
+  }
   const results = [];
   const writeSummary = () => {
     const totals = { pass: 0, fail: 0, error: 0 };
     for (const r of results)
       totals[r.status]++;
-    const doc = { runId, runsDir: runDir, totals, runs: results };
-    writeFileSync(summaryPath, `${JSON.stringify(doc, null, 2)}
+    const doc = {
+      runId,
+      runsDir: runDir,
+      isolation: opts.host ? "host" : "seatbelt",
+      builds: built.map((b) => ({ name: b.name, kind: b.kind })),
+      totals,
+      runs: results
+    };
+    writeFileSync2(summaryPath, `${JSON.stringify(doc, null, 2)}
 `);
     return doc;
   };
@@ -7603,7 +8047,16 @@ async function evalCommand(args, cwd) {
     for (let n = 1;n <= runs; n++) {
       let r;
       try {
-        r = await runOne({ test, n, runDir, claudeBin: claudePath, warn });
+        r = await runOne({
+          test,
+          n,
+          runDir,
+          claudeBin: claudePath,
+          iso,
+          built,
+          vars,
+          warn
+        });
       } catch (e) {
         r = {
           test: test.name,
@@ -7611,7 +8064,7 @@ async function evalCommand(args, cwd) {
           status: "error",
           reason: `not graded: firegoat crashed: ${e.message}`,
           workdir: null,
-          session: join2(runDir, test.name, String(n), "session.jsonl"),
+          session: join5(runDir, test.name, String(n), "session.jsonl"),
           checks: [],
           num_turns: null,
           cost: null,
@@ -7634,8 +8087,8 @@ ${pass} passed, ${fail} failed, ${error} not graded (${results.length} runs)`);
 }
 
 // src/init.ts
-import { cpSync, existsSync as existsSync3, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
-import { basename, join as join3 } from "node:path";
+import { cpSync as cpSync2, existsSync as existsSync5, readFileSync as readFileSync3, writeFileSync as writeFileSync3 } from "node:fs";
+import { basename as basename2, join as join6 } from "node:path";
 import { createInterface as createInterface2 } from "node:readline/promises";
 import { fileURLToPath } from "node:url";
 var INIT_USAGE = `Usage: firegoat init [--yes] [--force] [--skills]
@@ -7674,9 +8127,9 @@ function tomlSections(text) {
   return sections;
 }
 function detect(cwd) {
-  const readme = ["README.md", "README", "readme.md", "Readme.md"].find((f) => existsSync3(join3(cwd, f)));
-  const fallback = { name: basename(cwd), library: true, readme };
-  const pkgText = readText(join3(cwd, "package.json"));
+  const readme = ["README.md", "README", "readme.md", "Readme.md"].find((f) => existsSync5(join6(cwd, f)));
+  const fallback = { name: basename2(cwd), library: true, readme };
+  const pkgText = readText(join6(cwd, "package.json"));
   if (pkgText !== undefined) {
     try {
       const pkg = JSON.parse(pkgText);
@@ -7691,7 +8144,7 @@ function detect(cwd) {
       return { ecosystem: "npm", name, bin, library, readme, docsUrl };
     } catch {}
   }
-  const pyText = readText(join3(cwd, "pyproject.toml"));
+  const pyText = readText(join6(cwd, "pyproject.toml"));
   if (pyText !== undefined) {
     const s = tomlSections(pyText);
     const name = s.get("project")?.get("name") ?? s.get("tool.poetry")?.get("name") ?? fallback.name;
@@ -7701,12 +8154,12 @@ function detect(cwd) {
     const docsUrl = urls?.get("Documentation") ?? urls?.get("documentation") ?? urls?.get("Homepage") ?? urls?.get("homepage");
     return { ecosystem: "python", name, bin, library: true, readme, docsUrl };
   }
-  const goText = readText(join3(cwd, "go.mod"));
+  const goText = readText(join6(cwd, "go.mod"));
   if (goText !== undefined) {
     const name = goText.match(/^module\s+(\S+)/m)?.[1] ?? fallback.name;
     return { ecosystem: "go", name, library: true, readme };
   }
-  const cargoText = readText(join3(cwd, "Cargo.toml"));
+  const cargoText = readText(join6(cwd, "Cargo.toml"));
   if (cargoText !== undefined) {
     const name = tomlSections(cargoText).get("package")?.get("name");
     return {
@@ -7809,10 +8262,10 @@ function renderYaml(p, tests) {
 `;
 }
 function ensureGitignore(cwd) {
-  const file = join3(cwd, ".gitignore");
+  const file = join6(cwd, ".gitignore");
   const text = readText(file);
   if (text === undefined) {
-    writeFileSync2(file, `${GITIGNORE_LINE}
+    writeFileSync3(file, `${GITIGNORE_LINE}
 `);
     return `created .gitignore with ${GITIGNORE_LINE}`;
   }
@@ -7822,7 +8275,7 @@ function ensureGitignore(cwd) {
   const sep = text === "" || text.endsWith(`
 `) ? "" : `
 `;
-  writeFileSync2(file, `${text}${sep}${GITIGNORE_LINE}
+  writeFileSync3(file, `${text}${sep}${GITIGNORE_LINE}
 `);
   return `added ${GITIGNORE_LINE} to .gitignore`;
 }
@@ -7876,8 +8329,8 @@ ${INIT_USAGE}`);
         return EXIT.usage;
     }
   }
-  const configPath = join3(cwd, CONFIG_FILE);
-  const keepConfig = existsSync3(configPath) && !force;
+  const configPath = join6(cwd, CONFIG_FILE);
+  const keepConfig = existsSync5(configPath) && !force;
   if (keepConfig && !skills) {
     console.error(`firegoat init: ${CONFIG_FILE} already exists; not overwriting. Use --force to replace it.`);
     return EXIT.usage;
@@ -7888,7 +8341,7 @@ ${INIT_USAGE}`);
     const project = detect(cwd);
     const tests = templates(project);
     console.log(`detected: ${describe(project)}`);
-    writeFileSync2(configPath, renderYaml(project, tests));
+    writeFileSync3(configPath, renderYaml(project, tests));
     console.log(`wrote ${CONFIG_FILE} (${tests.length} test${tests.length === 1 ? "" : "s"}: ${tests.map((t) => t.name).join(", ")}); edit the prompts to fit your users`);
   }
   console.log(ensureGitignore(cwd));
@@ -7896,8 +8349,8 @@ ${INIT_USAGE}`);
   if (!skills && interactive)
     skills = await askYesNo(`Add the ${SKILL} skill to .claude/skills/ (helps your agent draft better evals)?`);
   if (skills) {
-    const dest = join3(cwd, ".claude", "skills", SKILL);
-    cpSync(skillSource(), dest, { recursive: true });
+    const dest = join6(cwd, ".claude", "skills", SKILL);
+    cpSync2(skillSource(), dest, { recursive: true });
     console.log(`copied ${SKILL} skill to .claude/skills/${SKILL}/`);
   } else {
     console.log(`hint: \`npx firegoat init --skills\` adds the ${SKILL} skill to .claude/skills/ (keeps your ${CONFIG_FILE})`);
@@ -7906,19 +8359,108 @@ ${INIT_USAGE}`);
   return 0;
 }
 
+// src/login.ts
+import { spawnSync as spawnSync3 } from "node:child_process";
+import { mkdirSync as mkdirSync4 } from "node:fs";
+function loginCommand(args) {
+  let claudeBin = process.env.FIREGOAT_CLAUDE_BIN || "claude";
+  for (let i = 0;i < args.length; i++) {
+    const a = args[i];
+    if (a === "--claude-bin" && args[i + 1] !== undefined)
+      claudeBin = args[++i];
+    else if (a.startsWith("--claude-bin="))
+      claudeBin = a.slice(13);
+    else {
+      console.error(`firegoat login: unknown argument: ${a}`);
+      return EXIT.usage;
+    }
+  }
+  const configDir = claudeConfigDir();
+  mkdirSync4(configDir, { recursive: true });
+  const env = {
+    ...process.env,
+    CLAUDE_CONFIG_DIR: configDir
+  };
+  delete env.ANTHROPIC_API_KEY;
+  console.error(`Signing claude in to ${configDir}. Evals run with this login, never your own ~/.claude.`);
+  const r = spawnSync3(claudeBin, ["auth", "login"], { env, stdio: "inherit" });
+  if (r.error) {
+    console.error(`firegoat login: could not run ${claudeBin}: ${r.error.message}`);
+    return EXIT.notGraded;
+  }
+  if (r.status !== 0) {
+    console.error(`firegoat login: claude auth login exited ${r.status}`);
+    return EXIT.notGraded;
+  }
+  console.log("logged in. next: npx firegoat eval");
+  return EXIT.pass;
+}
+
+// src/open.ts
+import { spawnSync as spawnSync4 } from "node:child_process";
+import { existsSync as existsSync6, readdirSync as readdirSync2 } from "node:fs";
+import { join as join7 } from "node:path";
+var OPENER = {
+  darwin: "open",
+  linux: "xdg-open"
+};
+function openCommand(args, cwd) {
+  let print = false;
+  const ids = [];
+  for (const a of args) {
+    if (a === "--print")
+      print = true;
+    else if (a.startsWith("-")) {
+      console.error(`firegoat open: unknown flag: ${a}`);
+      return EXIT.usage;
+    } else
+      ids.push(a);
+  }
+  if (ids.length > 1) {
+    console.error("firegoat open: takes at most one run id");
+    return EXIT.usage;
+  }
+  const base = join7(findConfigDir(cwd), ".firegoat", "runs");
+  const runs = existsSync6(base) ? readdirSync2(base).sort() : [];
+  const id = ids[0] ?? runs.at(-1);
+  if (!id) {
+    console.error(`firegoat open: no runs in ${base}. Run: npx firegoat eval`);
+    return EXIT.usage;
+  }
+  if (!runs.includes(id)) {
+    console.error(`firegoat open: no run ${JSON.stringify(id)} in ${base} (latest: ${runs.at(-1) ?? "none"})`);
+    return EXIT.usage;
+  }
+  const dir = join7(base, id);
+  console.log(dir);
+  const opener = OPENER[process.platform];
+  if (print || !opener)
+    return EXIT.pass;
+  const r = spawnSync4(opener, [dir], { stdio: "inherit" });
+  if (r.error || r.status !== 0) {
+    console.error(`firegoat open: ${opener} failed (${r.error?.message ?? `exit ${r.status}`}); the run is at the path above`);
+    return EXIT.notGraded;
+  }
+  return EXIT.pass;
+}
+
 // src/cli.ts
 var USAGE = `firegoat ${package_default.version}: local-first AX evals
 
 Usage:
   firegoat init [--yes] [--force] [--skills]   write firegoat.yaml with starter evals
-  firegoat eval [test..] [--runs N] [--json] [--claude-bin PATH]
-                                               run evals with claude in an empty workdir
+  firegoat login [--claude-bin PATH]          sign claude in to firegoat's own config dir (once)
+  firegoat eval [test..] [--runs N] [--var KEY=value].. [--json] [--host] [--claude-bin PATH]
+                                               run evals with claude in a sandboxed fresh workdir
+  firegoat open [runId] [--print]              open a run's folder (default: the latest)
   firegoat --version
 
 eval options:
   test..             only run these tests (by name)
   --runs N           runs per test (overrides runs: in firegoat.yaml)
+  --var KEY=value    fills {{vars.KEY}} in prompts and checks, and is in the env of build: commands
   --json             JSON summary on stdout, progress on stderr
+  --host             no sandbox (needed off macOS; use a throwaway container)
   --claude-bin PATH  claude binary to run (env: FIREGOAT_CLAUDE_BIN; default: claude on PATH)
 
 firegoat.yaml is looked up in the current dir, then parents up to the git root.
@@ -7943,8 +8485,12 @@ async function main(argv) {
       return 0;
     case "init":
       return initCommand(rest, process.cwd());
+    case "login":
+      return loginCommand(rest);
     case "eval":
       return evalCommand(rest, process.cwd());
+    case "open":
+      return openCommand(rest, process.cwd());
     default:
       console.error(`unknown command: ${cmd}
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.1.0",
+	"version": "0.3.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/team2027/firegoat-cli.git"

package/skills/firegoat-onboarding/SKILL.md

@@ -73,6 +73,12 @@ Schema (v1):
 ```yaml
 agents: [claude-code]      # only claude-code in v1
 runs: 1                    # runs per test, positive integer
+build:                     # optional: local packages preinstalled in the agent's workdir
+  sdk: ./packages/sdk      # package.json -> node_modules, pyproject.toml -> .venv on PATH
+  cli:                     # anything else: a command that writes outputs to $FIREGOAT_OUT
+    run: go build -o "$FIREGOAT_OUT/acme" ./cmd/acme     # outputs go on PATH
+vars:                      # optional: defaults for {{vars.KEY}}; `--var KEY=value` overrides
+  SHA: main
 tests:
   - name: quickstart       # letters, digits, '.', '_', '-'; unique
     prompt: >-
@@ -83,6 +89,9 @@ tests:
     timeout: 600           # optional, seconds
 ```
 
+Add `build:` when the repo ships a package or CLI whose local build the tests should use;
+then the prompt can say "The package is already installed from the build under test."
+
 If `firegoat.yaml` exists, merge: keep every existing test and top-level key untouched, append
 yours, and rename yours (`-2` suffix) if a name collides. Never delete or rewrite user tests.
 Quote prompts with `>-` folded blocks so colons and quotes stay valid YAML.