firegoat 0.2.0 → 0.4.0

package/README.md

@@ -2,16 +2,40 @@
 
 Local-first AX evals: how well coding agents use your SDK, CLI, API and docs.
 
-Status: early (v0.2). See `plan.md`.
+Status: early (v0.4). See `plan.md`.
 
 ## Use
 
 ```bash
 npx firegoat init            # writes firegoat.yaml with starter evals; --skills adds the onboarding skill
 npx firegoat login           # once per machine: signs claude in to ~/.firegoat/claude
-npx firegoat eval            # runs each test with your local `claude`, sandboxed, in a fresh empty folder
+npx firegoat eval            # runs each test with your local `claude`, sandboxed, in a fresh folder
+npx firegoat open            # opens the latest run's folder (--print to only print it)
 ```
 
+### Local builds: setup
+
+Plain shell, any language. `setup:` runs once per `eval` in the repo, on your machine; whatever it writes to `$FIREGOAT_OUT` is copied into each run's folder and put on the agent's PATH. `workdir_setup:` runs in each fresh workdir before the agent starts. The agent gets the build, never the source. Each takes one command or a list.
+
+```yaml
+# npm
+setup: npm pack --pack-destination "$FIREGOAT_OUT"
+workdir_setup: npm init -y >/dev/null && npm install "$FIREGOAT_OUT"/*.tgz
+
+# python
+setup: uv build --wheel --out-dir "$FIREGOAT_OUT"
+workdir_setup: uv venv && uv pip install "$FIREGOAT_OUT"/*.whl
+
+# a CLI binary: on PATH as is
+setup: go build -o "$FIREGOAT_OUT/acme" ./cmd/acme
+```
+
+Then say so in the prompt: "The package is already installed from the build under test."
+
+- `{{vars.KEY}}` in prompts and checks comes from `--var KEY=value` or `vars:` in `firegoat.yaml`. Vars are also env vars for `setup:`, `workdir_setup:` and checks, not for the agent. A missing var is a usage error; other `{{...}}` text is left alone.
+- Checks also get `$FIREGOAT_OUT` (the run's copy).
+- A failed `setup:` or `workdir_setup:` means not graded (exit 2); logs are `.firegoat/runs/<runId>/setup.txt` and `<test>/<n>/workdir_setup.txt`.
+
 ### Isolation
 
 On macOS, `eval` runs `claude` inside a Seatbelt (`sandbox-exec`) profile:

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.2.0",
+  version: "0.4.0",
   repository: {
     type: "git",
     url: "git+https://github.com/team2027/firegoat-cli.git"
@@ -7081,6 +7081,11 @@ function parseConfig(text, warn = console.error) {
     }
   });
   const runs = posInt(raw.runs, "runs", 1);
+  if (raw.build !== undefined)
+    fail("build", "was replaced by setup: in firegoat 0.4.0: a shell command (or list) that writes what the agent gets to $FIREGOAT_OUT");
+  const setup = commands(raw.setup, "setup");
+  const workdirSetup = commands(raw.workdir_setup, "workdir_setup");
+  const vars = parseVars(raw.vars);
   if (!Array.isArray(raw.tests) || raw.tests.length === 0)
     fail("tests", "must be a non-empty list");
   const names = new Set;
@@ -7089,7 +7094,7 @@ function parseConfig(text, warn = console.error) {
     if (!isRecord(t))
       fail(p, "must be a mapping");
     const name = str(t.name, `${p}.name`);
-    if (!/^[A-Za-z0-9._-]+$/.test(name))
+    if (!NAME.test(name))
       fail(`${p}.name`, "may only contain letters, digits, '.', '_' and '-'");
     if (names.has(name))
       fail(`${p}.name`, `duplicate test name ${JSON.stringify(name)}`);
@@ -7116,7 +7121,52 @@ function parseConfig(text, warn = console.error) {
       timeout: posInt(t.timeout, `${p}.timeout`, DEFAULT_TIMEOUT)
     };
   });
-  return { agents, runs, tests };
+  return {
+    agents,
+    runs,
+    setup,
+    workdirSetup,
+    vars,
+    tests
+  };
+}
+var NAME = /^[A-Za-z0-9._-]+$/;
+function commands(raw, path) {
+  if (raw === undefined)
+    return [];
+  if (typeof raw === "string")
+    return [str(raw, path)];
+  if (!Array.isArray(raw))
+    fail(path, "must be a shell command or a list of them");
+  return raw.map((c, i) => str(c, `${path}[${i}]`));
+}
+var VAR_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/;
+function parseVars(raw) {
+  if (raw === undefined)
+    return {};
+  if (!isRecord(raw))
+    fail("vars", "must be a mapping of KEY to value");
+  const out = {};
+  for (const [k, v] of Object.entries(raw)) {
+    if (!VAR_KEY.test(k))
+      fail(`vars.${k}`, "key must be letters, digits and _, not starting with a digit");
+    if (typeof v !== "string" && typeof v !== "number" && typeof v !== "boolean")
+      fail(`vars.${k}`, "must be a string, number or boolean");
+    out[k] = String(v);
+  }
+  return out;
+}
+var PLACEHOLDER = /\{\{\s*vars\.([A-Za-z0-9_]+)\s*\}\}/g;
+function varRefs(text) {
+  return [...text.matchAll(PLACEHOLDER)].map((m) => m[1]);
+}
+function render(text, vars) {
+  return text.replace(PLACEHOLDER, (_, key) => {
+    const v = vars[key];
+    if (v === undefined)
+      throw new ConfigError(`no value for {{vars.${key}}}`);
+    return v;
+  });
 }
 function loadConfig(dir, warn) {
   const file = `${dir}/${CONFIG_FILE}`;
@@ -7139,19 +7189,16 @@ function findConfigDir(cwd) {
 // src/eval.ts
 import { randomBytes } from "node:crypto";
 import {
-  accessSync,
-  constants,
-  createWriteStream as createWriteStream2,
+  createWriteStream as createWriteStream3,
   existsSync as existsSync3,
-  mkdirSync as mkdirSync2,
+  mkdirSync as mkdirSync3,
   mkdtempSync,
   readFileSync as readFileSync2,
   realpathSync as realpathSync2,
-  statSync,
   writeFileSync
 } from "node:fs";
 import { tmpdir } from "node:os";
-import { delimiter, join as join3, resolve } from "node:path";
+import { join as join5, resolve as resolve2 } from "node:path";
 
 // src/claude.ts
 import { spawn, spawnSync as spawnSync2 } from "node:child_process";
@@ -7185,6 +7232,7 @@ var TOOLCHAINS = [
   ".local/share/mise",
   ".local/share/fnm",
   ".local/share/claude",
+  ".local/share/uv",
   "go",
   "Library/pnpm"
 ];
@@ -7514,7 +7562,7 @@ async function runClaude(opts) {
     };
   return { ok: true, result };
 }
-async function runCheck(opts) {
+async function runShell(opts) {
   opts.log.write(`$ ${opts.run}
 `);
   const r = await spawnWatched({
@@ -7567,6 +7615,70 @@ function checkLogin(opts) {
 // src/exit.ts
 var EXIT = { pass: 0, fail: 1, notGraded: 2, usage: 3 };
 
+// src/setup.ts
+import { cpSync, createWriteStream as createWriteStream2, mkdirSync as mkdirSync2 } from "node:fs";
+import { delimiter, join as join3 } from "node:path";
+var SETUP_TIMEOUT_MS = 600000;
+async function runAll(commands, cwd, env, logPath, what) {
+  const log = createWriteStream2(logPath);
+  try {
+    for (const run of commands) {
+      const r = await runShell({
+        run,
+        cwd,
+        env,
+        timeoutMs: SETUP_TIMEOUT_MS,
+        log
+      });
+      if (r.code !== 0)
+        return `${what} failed (${r.error ?? `exit ${r.code}`}): ${run} (log: ${logPath})`;
+    }
+    return;
+  } finally {
+    await new Promise((r) => log.end(r));
+  }
+}
+async function setupOnce(opts) {
+  mkdirSync2(opts.out, { recursive: true });
+  return runAll(opts.commands, opts.root, { ...opts.env, FIREGOAT_OUT: opts.out }, opts.logPath, "setup");
+}
+async function prepareRun(opts) {
+  const out = join3(opts.runRoot, "out");
+  if (opts.out === null)
+    mkdirSync2(out);
+  else
+    cpSync(opts.out, out, { recursive: true });
+  const path = [out, opts.env.PATH ?? ""].join(delimiter);
+  const error = await runAll(opts.commands, opts.workdir, {
+    ...opts.env,
+    PATH: path,
+    FIREGOAT_OUT: out,
+    FIREGOAT_WORKDIR: opts.workdir
+  }, opts.logPath, "workdir_setup");
+  return error ? { error } : { out, path };
+}
+
+// src/which.ts
+import { accessSync, constants, statSync } from "node:fs";
+import { delimiter as delimiter2, join as join4, resolve } from "node:path";
+function resolveBin(bin) {
+  const executable = (p) => {
+    try {
+      accessSync(p, constants.X_OK);
+      return statSync(p).isFile();
+    } catch {
+      return false;
+    }
+  };
+  if (bin.includes("/"))
+    return executable(bin) ? resolve(bin) : undefined;
+  for (const dir of (process.env.PATH ?? "").split(delimiter2)) {
+    if (dir && executable(join4(dir, bin)))
+      return join4(dir, bin);
+  }
+  return;
+}
+
 // src/eval.ts
 var CHECK_TIMEOUT_MS = 120000;
 
@@ -7577,6 +7689,7 @@ function parseArgs(args) {
     names: [],
     json: false,
     host: false,
+    vars: {},
     claudeBin: process.env.FIREGOAT_CLAUDE_BIN || "claude"
   };
   for (let i = 0;i < args.length; i++) {
@@ -7596,7 +7709,14 @@ function parseArgs(args) {
       opts.json = true;
     else if (flag === "--host")
       opts.host = true;
-    else if (flag === "--runs") {
+    else if (flag === "--var") {
+      const kv = value();
+      const at = kv.indexOf("=");
+      const key = kv.slice(0, at);
+      if (at < 1 || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(key))
+        throw new UsageError(`--var needs KEY=value, got ${kv}`);
+      opts.vars[key] = kv.slice(at + 1);
+    } else if (flag === "--runs") {
       const v = value();
       const n = Number(v);
       if (!/^\d+$/.test(v) || n < 1)
@@ -7616,33 +7736,16 @@ function newRunId() {
   return `${ts}-${randomBytes(2).toString("hex")}`;
 }
 function gitignoreHasFiregoat(cwd) {
-  const file = join3(cwd, ".gitignore");
+  const file = join5(cwd, ".gitignore");
   if (!existsSync3(file))
     return false;
   return readFileSync2(file, "utf8").split(`
 `).map((l) => l.trim()).some((l) => /^\/?\.firegoat\/?(\*\*?)?$/.test(l));
 }
-function resolveBin(bin) {
-  const executable = (p) => {
-    try {
-      accessSync(p, constants.X_OK);
-      return statSync(p).isFile();
-    } catch {
-      return false;
-    }
-  };
-  if (bin.includes("/"))
-    return executable(bin) ? resolve(bin) : undefined;
-  for (const dir of (process.env.PATH ?? "").split(delimiter)) {
-    if (dir && executable(join3(dir, bin)))
-      return join3(dir, bin);
-  }
-  return;
-}
 function newRunRoot(label) {
-  const runRoot = realpathSync2(mkdtempSync(join3(tmpdir(), `firegoat-${label}-`)));
-  const workdir = join3(runRoot, "work");
-  mkdirSync2(workdir);
+  const runRoot = realpathSync2(mkdtempSync(join5(tmpdir(), `firegoat-${label}-`)));
+  const workdir = join5(runRoot, "work");
+  mkdirSync3(workdir);
   return { runRoot, workdir };
 }
 function isolate(iso, claudeBin, runRoot, workdir) {
@@ -7660,9 +7763,9 @@ function isolate(iso, claudeBin, runRoot, workdir) {
 async function runOne(opts) {
   const { test, n } = opts;
   const started = Date.now();
-  const dir = join3(opts.runDir, test.name, String(n));
-  mkdirSync2(dir, { recursive: true });
-  const session = join3(dir, "session.jsonl");
+  const dir = join5(opts.runDir, test.name, String(n));
+  mkdirSync3(dir, { recursive: true });
+  const session = join5(dir, "session.jsonl");
   const summary = {
     test: test.name,
     n,
@@ -7686,14 +7789,31 @@ async function runOne(opts) {
   const sandbox = isolate(opts.iso, opts.claudeBin, runRoot, workdir);
   if ("error" in sandbox)
     return done("error", `not graded: ${sandbox.error}`);
+  const hostEnv = { ...process.env, ...opts.vars };
+  let extra = {};
+  if (opts.out !== null || opts.workdirSetup.length > 0) {
+    const prepared = await prepareRun({
+      out: opts.out,
+      runRoot,
+      workdir,
+      commands: opts.workdirSetup,
+      env: hostEnv,
+      logPath: join5(dir, "workdir_setup.txt")
+    });
+    if ("error" in prepared)
+      return done("error", `not graded: ${prepared.error}`);
+    if (opts.out !== null)
+      extra = { PATH: prepared.path, FIREGOAT_OUT: prepared.out };
+  }
+  const agentPath = extra.PATH ? { PATH: extra.PATH } : {};
   const outcome = await runClaude({
-    env: agentEnv(runRoot),
+    env: { ...agentEnv(runRoot), ...agentPath },
     profile: sandbox.profile,
     bin: opts.claudeBin,
-    prompt: test.prompt,
+    prompt: render(test.prompt, opts.vars),
     cwd: workdir,
     sessionPath: session,
-    stderrPath: join3(dir, "stderr.txt"),
+    stderrPath: join5(dir, "stderr.txt"),
     timeoutMs: test.timeout * 1000
   });
   if (outcome.result) {
@@ -7706,17 +7826,19 @@ async function runOne(opts) {
     opts.warn(`warning: ${test.name} has no checks, so its goal is unverified; counted as pass`);
     return done("pass", "no checks (goal unverified)");
   }
-  const log = createWriteStream2(join3(dir, "checks.txt"));
+  const log = createWriteStream3(join5(dir, "checks.txt"));
   const env = {
     ...process.env,
-    FIREGOAT_SESSION: resolve(session),
+    ...hostEnv,
+    ...extra,
+    FIREGOAT_SESSION: resolve2(session),
     FIREGOAT_WORKDIR: workdir
   };
   const results = [];
   try {
     for (const c of test.checks) {
-      results.push(await runCheck({
-        run: c.run,
+      results.push(await runShell({
+        run: render(c.run, opts.vars),
         cwd: workdir,
         env,
         timeoutMs: CHECK_TIMEOUT_MS,
@@ -7761,6 +7883,14 @@ async function evalCommand(args, cwd) {
     tests = tests.filter((t) => opts.names.includes(t.name));
   }
   const runs = opts.runs ?? config.runs;
+  const vars = { ...config.vars, ...opts.vars };
+  const missing = [
+    ...new Set(tests.flatMap((t) => [t.prompt, ...t.checks.map((c) => c.run)]).flatMap(varRefs).filter((k) => !(k in vars)))
+  ];
+  if (missing.length > 0) {
+    console.error(`firegoat eval: no value for ${missing.map((k) => `{{vars.${k}}}`).join(", ")}: pass --var ${missing[0]}=... or set it under vars: in firegoat.yaml`);
+    return EXIT.usage;
+  }
   const claudePath = resolveBin(opts.claudeBin);
   if (!claudePath) {
     console.error(`firegoat eval: not graded: claude binary ${JSON.stringify(opts.claudeBin)} not found. Install Claude Code (https://claude.com/claude-code) or pass --claude-bin PATH / FIREGOAT_CLAUDE_BIN.`);
@@ -7775,9 +7905,9 @@ async function evalCommand(args, cwd) {
   const iso = {
     host: opts.host,
     repo: root,
-    repoFile: join3(root, "firegoat.yaml")
+    repoFile: join5(root, "firegoat.yaml")
   };
-  mkdirSync2(claudeConfigDir(), { recursive: true });
+  mkdirSync3(claudeConfigDir(), { recursive: true });
   const pre = newRunRoot("preflight");
   const preSandbox = isolate(iso, claudePath, pre.runRoot, pre.workdir);
   const loginProblem = "error" in preSandbox ? preSandbox.error : checkLogin({
@@ -7790,13 +7920,29 @@ async function evalCommand(args, cwd) {
     console.error(`firegoat eval: not graded: ${loginProblem}`);
     return EXIT.notGraded;
   }
-  mkdirSync2(join3(root, ".firegoat"), { recursive: true });
+  mkdirSync3(join5(root, ".firegoat"), { recursive: true });
   if (!gitignoreHasFiregoat(root))
     warn("hint: add .firegoat/ to .gitignore (it holds run traces)");
   const runId = newRunId();
-  const runDir = join3(root, ".firegoat", "runs", runId);
-  mkdirSync2(runDir, { recursive: true });
-  const summaryPath = join3(runDir, "summary.json");
+  const runDir = join5(root, ".firegoat", "runs", runId);
+  mkdirSync3(runDir, { recursive: true });
+  const summaryPath = join5(runDir, "summary.json");
+  let out = null;
+  if (config.setup.length > 0) {
+    say("setup...");
+    out = join5(runDir, "out");
+    const error = await setupOnce({
+      commands: config.setup,
+      root,
+      out,
+      env: { ...process.env, ...vars },
+      logPath: join5(runDir, "setup.txt")
+    });
+    if (error) {
+      console.error(`firegoat eval: not graded: ${error}`);
+      return EXIT.notGraded;
+    }
+  }
   const results = [];
   const writeSummary = () => {
     const totals = { pass: 0, fail: 0, error: 0 };
@@ -7823,6 +7969,9 @@ async function evalCommand(args, cwd) {
           runDir,
           claudeBin: claudePath,
           iso,
+          out,
+          workdirSetup: config.workdirSetup,
+          vars,
           warn
         });
       } catch (e) {
@@ -7832,7 +7981,7 @@ async function evalCommand(args, cwd) {
           status: "error",
           reason: `not graded: firegoat crashed: ${e.message}`,
           workdir: null,
-          session: join3(runDir, test.name, String(n), "session.jsonl"),
+          session: join5(runDir, test.name, String(n), "session.jsonl"),
           checks: [],
           num_turns: null,
           cost: null,
@@ -7855,8 +8004,8 @@ ${pass} passed, ${fail} failed, ${error} not graded (${results.length} runs)`);
 }
 
 // src/init.ts
-import { cpSync, existsSync as existsSync4, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
-import { basename as basename2, join as join4 } from "node:path";
+import { cpSync as cpSync2, existsSync as existsSync4, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
+import { basename as basename2, join as join6 } from "node:path";
 import { createInterface as createInterface2 } from "node:readline/promises";
 import { fileURLToPath } from "node:url";
 var INIT_USAGE = `Usage: firegoat init [--yes] [--force] [--skills]
@@ -7895,9 +8044,9 @@ function tomlSections(text) {
   return sections;
 }
 function detect(cwd) {
-  const readme = ["README.md", "README", "readme.md", "Readme.md"].find((f) => existsSync4(join4(cwd, f)));
+  const readme = ["README.md", "README", "readme.md", "Readme.md"].find((f) => existsSync4(join6(cwd, f)));
   const fallback = { name: basename2(cwd), library: true, readme };
-  const pkgText = readText(join4(cwd, "package.json"));
+  const pkgText = readText(join6(cwd, "package.json"));
   if (pkgText !== undefined) {
     try {
       const pkg = JSON.parse(pkgText);
@@ -7912,7 +8061,7 @@ function detect(cwd) {
       return { ecosystem: "npm", name, bin, library, readme, docsUrl };
     } catch {}
   }
-  const pyText = readText(join4(cwd, "pyproject.toml"));
+  const pyText = readText(join6(cwd, "pyproject.toml"));
   if (pyText !== undefined) {
     const s = tomlSections(pyText);
     const name = s.get("project")?.get("name") ?? s.get("tool.poetry")?.get("name") ?? fallback.name;
@@ -7922,12 +8071,12 @@ function detect(cwd) {
     const docsUrl = urls?.get("Documentation") ?? urls?.get("documentation") ?? urls?.get("Homepage") ?? urls?.get("homepage");
     return { ecosystem: "python", name, bin, library: true, readme, docsUrl };
   }
-  const goText = readText(join4(cwd, "go.mod"));
+  const goText = readText(join6(cwd, "go.mod"));
   if (goText !== undefined) {
     const name = goText.match(/^module\s+(\S+)/m)?.[1] ?? fallback.name;
     return { ecosystem: "go", name, library: true, readme };
   }
-  const cargoText = readText(join4(cwd, "Cargo.toml"));
+  const cargoText = readText(join6(cwd, "Cargo.toml"));
   if (cargoText !== undefined) {
     const name = tomlSections(cargoText).get("package")?.get("name");
     return {
@@ -8020,6 +8169,28 @@ function renderYaml(p, tests) {
     "# How many times to run each test.",
     "runs: 1",
     "",
+    "# Optional: test your local build instead of the published package.",
+    "# setup: runs once per eval, in this folder, on your machine. Whatever it writes to",
+    "# $FIREGOAT_OUT is copied into each run's folder and put on the agent's PATH.",
+    "# workdir_setup: runs in each fresh workdir before the agent starts. Then tell the",
+    '# agent in the prompt: "The package is already installed from the build under test."',
+    "#",
+    "# npm:",
+    '# setup: npm pack --pack-destination "$FIREGOAT_OUT"',
+    '# workdir_setup: npm init -y >/dev/null && npm install "$FIREGOAT_OUT"/*.tgz',
+    "#",
+    "# python:",
+    '# setup: uv build --wheel --out-dir "$FIREGOAT_OUT"',
+    '# workdir_setup: uv venv && uv pip install "$FIREGOAT_OUT"/*.whl',
+    "#",
+    "# a CLI binary (lands on PATH as is):",
+    '# setup: go build -o "$FIREGOAT_OUT/mycli" ./cmd/mycli',
+    "#",
+    "# Values for {{vars.KEY}} in prompts and checks, overridden by `--var KEY=value`;",
+    "# setup commands and checks also get them as env vars.",
+    "# vars:",
+    "#   SHA: main",
+    "",
     "tests:"
   ];
   for (const t of tests) {
@@ -8030,7 +8201,7 @@ function renderYaml(p, tests) {
 `;
 }
 function ensureGitignore(cwd) {
-  const file = join4(cwd, ".gitignore");
+  const file = join6(cwd, ".gitignore");
   const text = readText(file);
   if (text === undefined) {
     writeFileSync2(file, `${GITIGNORE_LINE}
@@ -8097,7 +8268,7 @@ ${INIT_USAGE}`);
         return EXIT.usage;
     }
   }
-  const configPath = join4(cwd, CONFIG_FILE);
+  const configPath = join6(cwd, CONFIG_FILE);
   const keepConfig = existsSync4(configPath) && !force;
   if (keepConfig && !skills) {
     console.error(`firegoat init: ${CONFIG_FILE} already exists; not overwriting. Use --force to replace it.`);
@@ -8117,8 +8288,8 @@ ${INIT_USAGE}`);
   if (!skills && interactive)
     skills = await askYesNo(`Add the ${SKILL} skill to .claude/skills/ (helps your agent draft better evals)?`);
   if (skills) {
-    const dest = join4(cwd, ".claude", "skills", SKILL);
-    cpSync(skillSource(), dest, { recursive: true });
+    const dest = join6(cwd, ".claude", "skills", SKILL);
+    cpSync2(skillSource(), dest, { recursive: true });
     console.log(`copied ${SKILL} skill to .claude/skills/${SKILL}/`);
   } else {
     console.log(`hint: \`npx firegoat init --skills\` adds the ${SKILL} skill to .claude/skills/ (keeps your ${CONFIG_FILE})`);
@@ -8129,7 +8300,7 @@ ${INIT_USAGE}`);
 
 // src/login.ts
 import { spawnSync as spawnSync3 } from "node:child_process";
-import { mkdirSync as mkdirSync3 } from "node:fs";
+import { mkdirSync as mkdirSync4 } from "node:fs";
 function loginCommand(args) {
   let claudeBin = process.env.FIREGOAT_CLAUDE_BIN || "claude";
   for (let i = 0;i < args.length; i++) {
@@ -8144,7 +8315,7 @@ function loginCommand(args) {
     }
   }
   const configDir = claudeConfigDir();
-  mkdirSync3(configDir, { recursive: true });
+  mkdirSync4(configDir, { recursive: true });
   const env = {
     ...process.env,
     CLAUDE_CONFIG_DIR: configDir
@@ -8164,19 +8335,69 @@ function loginCommand(args) {
   return EXIT.pass;
 }
 
+// src/open.ts
+import { spawnSync as spawnSync4 } from "node:child_process";
+import { existsSync as existsSync5, readdirSync } from "node:fs";
+import { join as join7 } from "node:path";
+var OPENER = {
+  darwin: "open",
+  linux: "xdg-open"
+};
+function openCommand(args, cwd) {
+  let print = false;
+  const ids = [];
+  for (const a of args) {
+    if (a === "--print")
+      print = true;
+    else if (a.startsWith("-")) {
+      console.error(`firegoat open: unknown flag: ${a}`);
+      return EXIT.usage;
+    } else
+      ids.push(a);
+  }
+  if (ids.length > 1) {
+    console.error("firegoat open: takes at most one run id");
+    return EXIT.usage;
+  }
+  const base = join7(findConfigDir(cwd), ".firegoat", "runs");
+  const runs = existsSync5(base) ? readdirSync(base).sort() : [];
+  const id = ids[0] ?? runs.at(-1);
+  if (!id) {
+    console.error(`firegoat open: no runs in ${base}. Run: npx firegoat eval`);
+    return EXIT.usage;
+  }
+  if (!runs.includes(id)) {
+    console.error(`firegoat open: no run ${JSON.stringify(id)} in ${base} (latest: ${runs.at(-1) ?? "none"})`);
+    return EXIT.usage;
+  }
+  const dir = join7(base, id);
+  console.log(dir);
+  const opener = OPENER[process.platform];
+  if (print || !opener)
+    return EXIT.pass;
+  const r = spawnSync4(opener, [dir], { stdio: "inherit" });
+  if (r.error || r.status !== 0) {
+    console.error(`firegoat open: ${opener} failed (${r.error?.message ?? `exit ${r.status}`}); the run is at the path above`);
+    return EXIT.notGraded;
+  }
+  return EXIT.pass;
+}
+
 // src/cli.ts
 var USAGE = `firegoat ${package_default.version}: local-first AX evals
 
 Usage:
   firegoat init [--yes] [--force] [--skills]   write firegoat.yaml with starter evals
   firegoat login [--claude-bin PATH]          sign claude in to firegoat's own config dir (once)
-  firegoat eval [test..] [--runs N] [--json] [--host] [--claude-bin PATH]
-                                               run evals with claude in a sandboxed empty workdir
+  firegoat eval [test..] [--runs N] [--var KEY=value].. [--json] [--host] [--claude-bin PATH]
+                                               run evals with claude in a sandboxed fresh workdir
+  firegoat open [runId] [--print]              open a run's folder (default: the latest)
   firegoat --version
 
 eval options:
   test..             only run these tests (by name)
   --runs N           runs per test (overrides runs: in firegoat.yaml)
+  --var KEY=value    fills {{vars.KEY}} in prompts and checks; env for setup, workdir_setup and checks
   --json             JSON summary on stdout, progress on stderr
   --host             no sandbox (needed off macOS; use a throwaway container)
   --claude-bin PATH  claude binary to run (env: FIREGOAT_CLAUDE_BIN; default: claude on PATH)
@@ -8207,6 +8428,8 @@ async function main(argv) {
       return loginCommand(rest);
     case "eval":
       return evalCommand(rest, process.cwd());
+    case "open":
+      return openCommand(rest, process.cwd());
     default:
       console.error(`unknown command: ${cmd}
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.2.0",
+	"version": "0.4.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/team2027/firegoat-cli.git"

package/skills/firegoat-onboarding/SKILL.md

@@ -73,6 +73,12 @@ Schema (v1):
 ```yaml
 agents: [claude-code]      # only claude-code in v1
 runs: 1                    # runs per test, positive integer
+setup: npm pack --pack-destination "$FIREGOAT_OUT"   # optional: once per eval, in the repo;
+                           # $FIREGOAT_OUT is copied into each run and put on PATH
+workdir_setup: npm init -y >/dev/null && npm install "$FIREGOAT_OUT"/*.tgz
+                           # optional: in each fresh workdir before the agent starts
+vars:                      # optional: defaults for {{vars.KEY}}; `--var KEY=value` overrides
+  SHA: main
 tests:
   - name: quickstart       # letters, digits, '.', '_', '-'; unique
     prompt: >-
@@ -83,6 +89,10 @@ tests:
     timeout: 600           # optional, seconds
 ```
 
+Add `setup:` / `workdir_setup:` when the tests should use the repo's local build (python:
+`uv build --wheel --out-dir "$FIREGOAT_OUT"` and `uv venv && uv pip install "$FIREGOAT_OUT"/*.whl`);
+then the prompt can say "The package is already installed from the build under test."
+
 If `firegoat.yaml` exists, merge: keep every existing test and top-level key untouched, append
 yours, and rename yours (`-2` suffix) if a name collides. Never delete or rewrite user tests.
 Quote prompts with `>-` folded blocks so colons and quotes stay valid YAML.