firegoat 0.3.0 → 0.5.0

package/README.md

@@ -2,36 +2,39 @@
 
 Local-first AX evals: how well coding agents use your SDK, CLI, API and docs.
 
-Status: early (v0.3). See `plan.md`.
+Status: early (v0.4). See `plan.md`.
 
 ## Use
 
 ```bash
-npx firegoat init            # writes firegoat.yaml with starter evals; --skills adds the onboarding skill
+npx firegoat init            # writes firegoat.yaml with starter evals; --skills adds the firegoat-onboarding skill (from team2027/skills)
 npx firegoat login           # once per machine: signs claude in to ~/.firegoat/claude
 npx firegoat eval            # runs each test with your local `claude`, sandboxed, in a fresh folder
 npx firegoat open            # opens the latest run's folder (--print to only print it)
 ```
 
-### Local builds
+### Local builds: setup
 
-`build:` builds your packages once per `eval` (on the host, from source) and preinstalls them into every run's fresh folder. The agent gets the build, never the source.
+Plain shell, any language. `setup:` runs once per `eval` in the repo, on your machine; whatever it writes to `$FIREGOAT_OUT` is copied into each run's folder and put on the agent's PATH. `workdir_setup:` runs in each fresh workdir before the agent starts. The agent gets the build, never the source. Each takes one command or a list.
 
 ```yaml
-build:
-  sdk: ./packages/sdk            # package.json: npm pack, then npm install into node_modules
-  py: ./python                   # pyproject.toml: wheel (uv build, or python3 -m build) into .venv, on PATH
-  cli:                           # anything else: your command writes outputs to $FIREGOAT_OUT,
-    run: go build -o "$FIREGOAT_OUT/acme" ./cmd/acme   # which go on the agent's PATH
-tests:
-  - name: quickstart
-    prompt: "Deploy commit {{vars.SHA}} with the acme CLI. The SDK is already installed; don't install it from the registry."
-    goal: ...
+# npm
+setup: npm pack --pack-destination "$FIREGOAT_OUT"
+workdir_setup: npm init -y >/dev/null && npm install "$FIREGOAT_OUT"/*.tgz
+
+# python
+setup: uv build --wheel --out-dir "$FIREGOAT_OUT"
+workdir_setup: uv venv && uv pip install "$FIREGOAT_OUT"/*.whl
+
+# a CLI binary: on PATH as is
+setup: go build -o "$FIREGOAT_OUT/acme" ./cmd/acme
 ```
 
-- `{{build.NAME}}` in a prompt or check is the path of that build's artifact inside the run's folder.
-- `{{vars.KEY}}` comes from `--var KEY=value` or `vars:` in `firegoat.yaml`; vars are also in the env of `build:` commands. A missing var is a usage error. Other `{{...}}` text is left alone.
-- A failed build or install means not graded (exit 2); logs are in `.firegoat/runs/<runId>/build/build.txt` and `<test>/<n>/setup.txt`.
+Then say so in the prompt: "The package is already installed from the build under test."
+
+- `{{vars.KEY}}` in prompts and checks comes from `--var KEY=value` or `vars:` in `firegoat.yaml`. Vars are also env vars for `setup:`, `workdir_setup:` and checks, not for the agent. A missing var is a usage error; other `{{...}}` text is left alone.
+- Checks also get `$FIREGOAT_OUT` (the run's copy).
+- A failed `setup:` or `workdir_setup:` means not graded (exit 2); logs are `.firegoat/runs/<runId>/setup.txt` and `<test>/<n>/workdir_setup.txt`.
 
 ### Isolation
 

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.3.0",
+  version: "0.5.0",
   repository: {
     type: "git",
     url: "git+https://github.com/team2027/firegoat-cli.git"
@@ -6968,8 +6968,7 @@ var package_default = {
     node: ">=18"
   },
   files: [
-    "dist",
-    "skills"
+    "dist"
   ],
   homepage: "https://github.com/team2027/firegoat-cli#readme",
   license: "MIT",
@@ -7081,7 +7080,10 @@ function parseConfig(text, warn = console.error) {
     }
   });
   const runs = posInt(raw.runs, "runs", 1);
-  const build = parseBuild(raw.build);
+  if (raw.build !== undefined)
+    fail("build", "was replaced by setup: in firegoat 0.4.0: a shell command (or list) that writes what the agent gets to $FIREGOAT_OUT");
+  const setup = commands(raw.setup, "setup");
+  const workdirSetup = commands(raw.workdir_setup, "workdir_setup");
   const vars = parseVars(raw.vars);
   if (!Array.isArray(raw.tests) || raw.tests.length === 0)
     fail("tests", "must be a non-empty list");
@@ -7118,40 +7120,24 @@ function parseConfig(text, warn = console.error) {
       timeout: posInt(t.timeout, `${p}.timeout`, DEFAULT_TIMEOUT)
     };
   });
-  const builds = new Set(build.map((b) => b.name));
-  tests.forEach((t, i) => {
-    const texts = [t.prompt, ...t.checks.map((c) => c.run)];
-    for (const ref of texts.flatMap(placeholders)) {
-      if (ref.kind === "build" && !builds.has(ref.key))
-        fail(`tests[${i}]`, `{{build.${ref.key}}} names no entry under build: (known: ${[...builds].join(", ") || "none"})`);
-    }
-  });
-  return { agents, runs, build, vars, tests };
+  return {
+    agents,
+    runs,
+    setup,
+    workdirSetup,
+    vars,
+    tests
+  };
 }
 var NAME = /^[A-Za-z0-9._-]+$/;
-function parseBuild(raw) {
+function commands(raw, path) {
   if (raw === undefined)
     return [];
-  if (!isRecord(raw))
-    fail("build", "must be a mapping of name to path or {path, run}");
-  return Object.entries(raw).map(([name, v]) => {
-    const p = `build.${name}`;
-    if (!NAME.test(name))
-      fail(p, "name may only contain letters, digits, '.', '_' and '-'");
-    if (typeof v === "string")
-      return { name, path: str(v, p) };
-    if (!isRecord(v))
-      fail(p, "must be a path or a mapping with path and/or run");
-    if (v.path === undefined && v.run === undefined)
-      fail(p, "needs path, run, or both");
-    const b = {
-      name,
-      path: v.path === undefined ? "." : str(v.path, `${p}.path`)
-    };
-    if (v.run !== undefined)
-      b.run = str(v.run, `${p}.run`);
-    return b;
-  });
+  if (typeof raw === "string")
+    return [str(raw, path)];
+  if (!Array.isArray(raw))
+    fail(path, "must be a shell command or a list of them");
+  return raw.map((c, i) => str(c, `${path}[${i}]`));
 }
 var VAR_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/;
 function parseVars(raw) {
@@ -7169,18 +7155,15 @@ function parseVars(raw) {
   }
   return out;
 }
-var PLACEHOLDER = /\{\{\s*(build|vars)\.([A-Za-z0-9._-]+)\s*\}\}/g;
-function placeholders(text) {
-  return [...text.matchAll(PLACEHOLDER)].map((m) => ({
-    kind: m[1],
-    key: m[2]
-  }));
+var PLACEHOLDER = /\{\{\s*vars\.([A-Za-z0-9_]+)\s*\}\}/g;
+function varRefs(text) {
+  return [...text.matchAll(PLACEHOLDER)].map((m) => m[1]);
 }
-function render(text, values) {
-  return text.replace(PLACEHOLDER, (_, kind, key) => {
-    const v = values[kind][key];
+function render(text, vars) {
+  return text.replace(PLACEHOLDER, (_, key) => {
+    const v = vars[key];
     if (v === undefined)
-      throw new ConfigError(`no value for {{${kind}.${key}}}`);
+      throw new ConfigError(`no value for {{vars.${key}}}`);
     return v;
   });
 }
@@ -7206,27 +7189,15 @@ function findConfigDir(cwd) {
 import { randomBytes } from "node:crypto";
 import {
   createWriteStream as createWriteStream3,
-  existsSync as existsSync4,
+  existsSync as existsSync3,
   mkdirSync as mkdirSync3,
   mkdtempSync,
   readFileSync as readFileSync2,
   realpathSync as realpathSync2,
-  writeFileSync as writeFileSync2
-} from "node:fs";
-import { tmpdir } from "node:os";
-import { join as join5, resolve as resolve3 } from "node:path";
-
-// src/build.ts
-import {
-  cpSync,
-  createWriteStream as createWriteStream2,
-  existsSync as existsSync3,
-  mkdirSync as mkdirSync2,
-  readdirSync,
-  statSync as statSync2,
   writeFileSync
 } from "node:fs";
-import { delimiter as delimiter2, join as join4, resolve as resolve2 } from "node:path";
+import { tmpdir } from "node:os";
+import { join as join5, resolve as resolve2 } from "node:path";
 
 // src/claude.ts
 import { spawn, spawnSync as spawnSync2 } from "node:child_process";
@@ -7640,9 +7611,55 @@ function checkLogin(opts) {
   return `claude auth status gave no login state (exit ${r.status}): ${(r.stdout + r.stderr).trim().slice(0, 300)}`;
 }
 
+// src/exit.ts
+var EXIT = { pass: 0, fail: 1, notGraded: 2, usage: 3 };
+
+// src/setup.ts
+import { cpSync, createWriteStream as createWriteStream2, mkdirSync as mkdirSync2 } from "node:fs";
+import { delimiter, join as join3 } from "node:path";
+var SETUP_TIMEOUT_MS = 600000;
+async function runAll(commands, cwd, env, logPath, what) {
+  const log = createWriteStream2(logPath);
+  try {
+    for (const run of commands) {
+      const r = await runShell({
+        run,
+        cwd,
+        env,
+        timeoutMs: SETUP_TIMEOUT_MS,
+        log
+      });
+      if (r.code !== 0)
+        return `${what} failed (${r.error ?? `exit ${r.code}`}): ${run} (log: ${logPath})`;
+    }
+    return;
+  } finally {
+    await new Promise((r) => log.end(r));
+  }
+}
+async function setupOnce(opts) {
+  mkdirSync2(opts.out, { recursive: true });
+  return runAll(opts.commands, opts.root, { ...opts.env, FIREGOAT_OUT: opts.out }, opts.logPath, "setup");
+}
+async function prepareRun(opts) {
+  const out = join3(opts.runRoot, "out");
+  if (opts.out === null)
+    mkdirSync2(out);
+  else
+    cpSync(opts.out, out, { recursive: true });
+  const path = [out, opts.env.PATH ?? ""].join(delimiter);
+  const error = await runAll(opts.commands, opts.workdir, {
+    ...opts.env,
+    PATH: path,
+    FIREGOAT_OUT: out,
+    FIREGOAT_WORKDIR: opts.workdir
+  }, opts.logPath, "workdir_setup");
+  return error ? { error } : { out, path };
+}
+
 // src/which.ts
 import { accessSync, constants, statSync } from "node:fs";
-import { delimiter, join as join3, resolve } from "node:path";
+import { delimiter as delimiter2, join as join4, resolve } from "node:path";
 function resolveBin(bin) {
   const executable = (p) => {
     try {
@@ -7654,131 +7671,13 @@ function resolveBin(bin) {
   };
   if (bin.includes("/"))
     return executable(bin) ? resolve(bin) : undefined;
-  for (const dir of (process.env.PATH ?? "").split(delimiter)) {
-    if (dir && executable(join3(dir, bin)))
-      return join3(dir, bin);
+  for (const dir of (process.env.PATH ?? "").split(delimiter2)) {
+    if (dir && executable(join4(dir, bin)))
+      return join4(dir, bin);
   }
   return;
 }
 
-// src/build.ts
-var BUILD_TIMEOUT_MS = 600000;
-
-class BuildError extends Error {
-}
-function kindOf(b, dir) {
-  if (b.run)
-    return "run";
-  if (existsSync3(join4(dir, "package.json")))
-    return "npm";
-  if (existsSync3(join4(dir, "pyproject.toml")))
-    return "python";
-  throw new BuildError(`build.${b.name}: no package.json or pyproject.toml in ${dir}; add run: with a build command that writes to $FIREGOAT_OUT`);
-}
-function pythonBuildCommand() {
-  if (resolveBin("uv"))
-    return 'uv build --wheel --out-dir "$FIREGOAT_OUT"';
-  return 'python3 -m build --wheel --outdir "$FIREGOAT_OUT"';
-}
-var BUILD_COMMAND = {
-  npm: () => 'npm pack --ignore-scripts=false --pack-destination "$FIREGOAT_OUT"',
-  python: pythonBuildCommand
-};
-var ARTIFACT = { npm: ".tgz", python: ".whl" };
-async function sh(run, cwd, env, log, what) {
-  const r = await runShell({ run, cwd, env, timeoutMs: BUILD_TIMEOUT_MS, log });
-  if (r.code !== 0)
-    throw new BuildError(`${what} failed (${r.error ?? `exit ${r.code}`}): ${run}`);
-}
-async function buildOne(b, root, buildDir, env, log) {
-  const dir = resolve2(root, b.path);
-  if (!existsSync3(dir) || !statSync2(dir).isDirectory())
-    throw new BuildError(`build.${b.name}: ${dir} is not a directory`);
-  const kind = kindOf(b, dir);
-  const out = join4(buildDir, b.name);
-  mkdirSync2(out, { recursive: true });
-  const run = kind === "run" ? b.run : BUILD_COMMAND[kind]();
-  log.write(`## build.${b.name} (${kind}) in ${dir}
-`);
-  await sh(run, dir, { ...env, FIREGOAT_OUT: out }, log, `build.${b.name}`);
-  const files = readdirSync(out);
-  if (kind !== "run") {
-    const ext = ARTIFACT[kind];
-    const hits = files.filter((f) => f.endsWith(ext));
-    if (hits.length !== 1)
-      throw new BuildError(`build.${b.name}: expected one ${ext} in ${out}, found ${hits.length ? hits.join(", ") : "none"}`);
-  } else if (files.length === 0)
-    throw new BuildError(`build.${b.name}: run: wrote nothing to $FIREGOAT_OUT (${out})`);
-  return { name: b.name, kind, out };
-}
-async function buildAll(builds, root, buildDir, env) {
-  mkdirSync2(buildDir, { recursive: true });
-  const log = createWriteStream2(join4(buildDir, "build.txt"));
-  try {
-    const built = [];
-    for (const b of builds)
-      built.push(await buildOne(b, root, buildDir, env, log));
-    return { built };
-  } catch (e) {
-    if (e instanceof BuildError)
-      return { error: `${e.message} (log: ${join4(buildDir, "build.txt")})` };
-    throw e;
-  } finally {
-    await new Promise((r) => log.end(r));
-  }
-}
-async function install(built, runRoot, workdir, env, logPath) {
-  const result = { paths: {}, env: {} };
-  if (built.length === 0)
-    return result;
-  const log = createWriteStream2(logPath);
-  const path = [];
-  const artifacts = { npm: [], python: [] };
-  try {
-    for (const b of built) {
-      const copy = join4(runRoot, "packages", b.name);
-      cpSync(b.out, copy, { recursive: true });
-      const files = readdirSync(copy);
-      if (b.kind === "run") {
-        result.paths[b.name] = files.length === 1 ? join4(copy, files[0]) : copy;
-        path.push(copy);
-      } else {
-        const ext = ARTIFACT[b.kind];
-        const file = join4(copy, files.find((f) => f.endsWith(ext)));
-        result.paths[b.name] = file;
-        artifacts[b.kind].push(file);
-      }
-    }
-    if (artifacts.npm.length > 0) {
-      writeFileSync(join4(workdir, "package.json"), `${JSON.stringify({ name: "app", version: "0.0.0", private: true }, null, 2)}
-`);
-      await sh(`npm install --no-audit --no-fund ${artifacts.npm.map(q2).join(" ")}`, workdir, env, log, "npm install of the build");
-      path.push(join4(workdir, "node_modules", ".bin"));
-    }
-    if (artifacts.python.length > 0) {
-      const venv = join4(workdir, ".venv");
-      const wheels = artifacts.python.map(q2).join(" ");
-      await sh(resolveBin("uv") ? `uv venv .venv && uv pip install --python .venv/bin/python ${wheels}` : `python3 -m venv .venv && .venv/bin/pip install ${wheels}`, workdir, env, log, "pip install of the build");
-      path.push(join4(venv, "bin"));
-      result.env.VIRTUAL_ENV = venv;
-    }
-    result.env.PATH = [...path, env.PATH ?? ""].join(delimiter2);
-    return result;
-  } catch (e) {
-    if (e instanceof BuildError)
-      return { error: `${e.message} (log: ${logPath})` };
-    throw e;
-  } finally {
-    await new Promise((r) => log.end(r));
-  }
-}
-function q2(s) {
-  return `'${s.replaceAll("'", `'\\''`)}'`;
-}
-
-// src/exit.ts
-var EXIT = { pass: 0, fail: 1, notGraded: 2, usage: 3 };
-
 // src/eval.ts
 var CHECK_TIMEOUT_MS = 120000;
 
@@ -7837,7 +7736,7 @@ function newRunId() {
 }
 function gitignoreHasFiregoat(cwd) {
   const file = join5(cwd, ".gitignore");
-  if (!existsSync4(file))
+  if (!existsSync3(file))
     return false;
   return readFileSync2(file, "utf8").split(`
 `).map((l) => l.trim()).some((l) => /^\/?\.firegoat\/?(\*\*?)?$/.test(l));
@@ -7889,15 +7788,28 @@ async function runOne(opts) {
   const sandbox = isolate(opts.iso, opts.claudeBin, runRoot, workdir);
   if ("error" in sandbox)
     return done("error", `not graded: ${sandbox.error}`);
-  const installed = await install(opts.built, runRoot, workdir, process.env, join5(dir, "setup.txt"));
-  if ("error" in installed)
-    return done("error", `not graded: setup: ${installed.error}`);
-  const values = { build: installed.paths, vars: opts.vars };
+  const hostEnv = { ...process.env, ...opts.vars };
+  let extra = {};
+  if (opts.out !== null || opts.workdirSetup.length > 0) {
+    const prepared = await prepareRun({
+      out: opts.out,
+      runRoot,
+      workdir,
+      commands: opts.workdirSetup,
+      env: hostEnv,
+      logPath: join5(dir, "workdir_setup.txt")
+    });
+    if ("error" in prepared)
+      return done("error", `not graded: ${prepared.error}`);
+    if (opts.out !== null)
+      extra = { PATH: prepared.path, FIREGOAT_OUT: prepared.out };
+  }
+  const agentPath = extra.PATH ? { PATH: extra.PATH } : {};
   const outcome = await runClaude({
-    env: { ...agentEnv(runRoot), ...installed.env },
+    env: { ...agentEnv(runRoot), ...agentPath },
     profile: sandbox.profile,
     bin: opts.claudeBin,
-    prompt: render(test.prompt, values),
+    prompt: render(test.prompt, opts.vars),
     cwd: workdir,
     sessionPath: session,
     stderrPath: join5(dir, "stderr.txt"),
@@ -7916,15 +7828,16 @@ async function runOne(opts) {
   const log = createWriteStream3(join5(dir, "checks.txt"));
   const env = {
     ...process.env,
-    ...installed.env,
-    FIREGOAT_SESSION: resolve3(session),
+    ...hostEnv,
+    ...extra,
+    FIREGOAT_SESSION: resolve2(session),
     FIREGOAT_WORKDIR: workdir
   };
   const results = [];
   try {
     for (const c of test.checks) {
       results.push(await runShell({
-        run: render(c.run, values),
+        run: render(c.run, opts.vars),
         cwd: workdir,
         env,
         timeoutMs: CHECK_TIMEOUT_MS,
@@ -7971,7 +7884,7 @@ async function evalCommand(args, cwd) {
   const runs = opts.runs ?? config.runs;
   const vars = { ...config.vars, ...opts.vars };
   const missing = [
-    ...new Set(tests.flatMap((t) => [t.prompt, ...t.checks.map((c) => c.run)]).flatMap(placeholders).filter((p) => p.kind === "vars" && !(p.key in vars)).map((p) => p.key))
+    ...new Set(tests.flatMap((t) => [t.prompt, ...t.checks.map((c) => c.run)]).flatMap(varRefs).filter((k) => !(k in vars)))
   ];
   if (missing.length > 0) {
     console.error(`firegoat eval: no value for ${missing.map((k) => `{{vars.${k}}}`).join(", ")}: pass --var ${missing[0]}=... or set it under vars: in firegoat.yaml`);
@@ -7984,7 +7897,7 @@ async function evalCommand(args, cwd) {
   }
   if (opts.host)
     warn("warning: --host: the agent runs without isolation and can read and write your files");
-  else if (process.platform !== "darwin" || !existsSync4(SANDBOX_EXEC)) {
+  else if (process.platform !== "darwin" || !existsSync3(SANDBOX_EXEC)) {
     console.error(`firegoat eval: isolation needs macOS (${SANDBOX_EXEC}); this is ${process.platform}. In CI or Linux, run inside a throwaway container and pass --host.`);
     return EXIT.usage;
   }
@@ -8013,18 +7926,21 @@ async function evalCommand(args, cwd) {
   const runDir = join5(root, ".firegoat", "runs", runId);
   mkdirSync3(runDir, { recursive: true });
   const summaryPath = join5(runDir, "summary.json");
-  let built = [];
-  if (config.build.length > 0) {
-    say(`building ${config.build.map((b) => b.name).join(", ")}...`);
-    const b = await buildAll(config.build, root, join5(runDir, "build"), {
-      ...process.env,
-      ...vars
+  let out = null;
+  if (config.setup.length > 0) {
+    say("setup...");
+    out = join5(runDir, "out");
+    const error = await setupOnce({
+      commands: config.setup,
+      root,
+      out,
+      env: { ...process.env, ...vars },
+      logPath: join5(runDir, "setup.txt")
     });
-    if ("error" in b) {
-      console.error(`firegoat eval: not graded: ${b.error}`);
+    if (error) {
+      console.error(`firegoat eval: not graded: ${error}`);
       return EXIT.notGraded;
     }
-    built = b.built;
   }
   const results = [];
   const writeSummary = () => {
@@ -8035,11 +7951,10 @@ async function evalCommand(args, cwd) {
       runId,
       runsDir: runDir,
       isolation: opts.host ? "host" : "seatbelt",
-      builds: built.map((b) => ({ name: b.name, kind: b.kind })),
       totals,
       runs: results
     };
-    writeFileSync2(summaryPath, `${JSON.stringify(doc, null, 2)}
+    writeFileSync(summaryPath, `${JSON.stringify(doc, null, 2)}
 `);
     return doc;
   };
@@ -8053,7 +7968,8 @@ async function evalCommand(args, cwd) {
           runDir,
           claudeBin: claudePath,
           iso,
-          built,
+          out,
+          workdirSetup: config.workdirSetup,
           vars,
           warn
         });
@@ -8087,17 +8003,27 @@ ${pass} passed, ${fail} failed, ${error} not graded (${results.length} runs)`);
 }
 
 // src/init.ts
-import { cpSync as cpSync2, existsSync as existsSync5, readFileSync as readFileSync3, writeFileSync as writeFileSync3 } from "node:fs";
+import { spawnSync as spawnSync3 } from "node:child_process";
+import { existsSync as existsSync4, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
 import { basename as basename2, join as join6 } from "node:path";
 import { createInterface as createInterface2 } from "node:readline/promises";
-import { fileURLToPath } from "node:url";
 var INIT_USAGE = `Usage: firegoat init [--yes] [--force] [--skills]
 
   --yes, -y     don't ask questions (default when stdin is not a terminal)
   --force, -f   overwrite an existing ${CONFIG_FILE}
-  --skills      copy the firegoat-onboarding skill into .claude/skills/
+  --skills      install the firegoat-onboarding and suggesting-eval-prompts skills
+                into .claude/skills/ (npx skills add team2027/skills)
 `;
-var SKILL = "firegoat-onboarding";
+var SKILLS = ["firegoat-onboarding", "suggesting-eval-prompts"];
+var SKILLS_ARGV = [
+  "skills",
+  "add",
+  "team2027/skills",
+  ...SKILLS.flatMap((s) => ["--skill", s]),
+  "--agent",
+  "claude-code",
+  "-y"
+];
 var GITIGNORE_LINE = ".firegoat/";
 function readText(file) {
   try {
@@ -8127,7 +8053,7 @@ function tomlSections(text) {
   return sections;
 }
 function detect(cwd) {
-  const readme = ["README.md", "README", "readme.md", "Readme.md"].find((f) => existsSync5(join6(cwd, f)));
+  const readme = ["README.md", "README", "readme.md", "Readme.md"].find((f) => existsSync4(join6(cwd, f)));
   const fallback = { name: basename2(cwd), library: true, readme };
   const pkgText = readText(join6(cwd, "package.json"));
   if (pkgText !== undefined) {
@@ -8252,6 +8178,28 @@ function renderYaml(p, tests) {
     "# How many times to run each test.",
     "runs: 1",
     "",
+    "# Optional: test your local build instead of the published package.",
+    "# setup: runs once per eval, in this folder, on your machine. Whatever it writes to",
+    "# $FIREGOAT_OUT is copied into each run's folder and put on the agent's PATH.",
+    "# workdir_setup: runs in each fresh workdir before the agent starts. Then tell the",
+    '# agent in the prompt: "The package is already installed from the build under test."',
+    "#",
+    "# npm:",
+    '# setup: npm pack --pack-destination "$FIREGOAT_OUT"',
+    '# workdir_setup: npm init -y >/dev/null && npm install "$FIREGOAT_OUT"/*.tgz',
+    "#",
+    "# python:",
+    '# setup: uv build --wheel --out-dir "$FIREGOAT_OUT"',
+    '# workdir_setup: uv venv && uv pip install "$FIREGOAT_OUT"/*.whl',
+    "#",
+    "# a CLI binary (lands on PATH as is):",
+    '# setup: go build -o "$FIREGOAT_OUT/mycli" ./cmd/mycli',
+    "#",
+    "# Values for {{vars.KEY}} in prompts and checks, overridden by `--var KEY=value`;",
+    "# setup commands and checks also get them as env vars.",
+    "# vars:",
+    "#   SHA: main",
+    "",
     "tests:"
   ];
   for (const t of tests) {
@@ -8265,7 +8213,7 @@ function ensureGitignore(cwd) {
   const file = join6(cwd, ".gitignore");
   const text = readText(file);
   if (text === undefined) {
-    writeFileSync3(file, `${GITIGNORE_LINE}
+    writeFileSync2(file, `${GITIGNORE_LINE}
 `);
     return `created .gitignore with ${GITIGNORE_LINE}`;
   }
@@ -8275,12 +8223,20 @@ function ensureGitignore(cwd) {
   const sep = text === "" || text.endsWith(`
 `) ? "" : `
 `;
-  writeFileSync3(file, `${text}${sep}${GITIGNORE_LINE}
+  writeFileSync2(file, `${text}${sep}${GITIGNORE_LINE}
 `);
   return `added ${GITIGNORE_LINE} to .gitignore`;
 }
-function skillSource() {
-  return fileURLToPath(new URL(`../skills/${SKILL}`, import.meta.url));
+function installSkills(cwd) {
+  const r = spawnSync3("npx", ["-y", ...SKILLS_ARGV], {
+    cwd,
+    stdio: "inherit"
+  });
+  if (r.error)
+    return r.error.message;
+  if (r.status !== 0)
+    return `exit ${r.status ?? "by signal"}`;
+  return;
 }
 async function askYesNo(question) {
   const rl = createInterface2({ input: process.stdin, output: process.stdout });
@@ -8330,7 +8286,7 @@ ${INIT_USAGE}`);
     }
   }
   const configPath = join6(cwd, CONFIG_FILE);
-  const keepConfig = existsSync5(configPath) && !force;
+  const keepConfig = existsSync4(configPath) && !force;
   if (keepConfig && !skills) {
     console.error(`firegoat init: ${CONFIG_FILE} already exists; not overwriting. Use --force to replace it.`);
     return EXIT.usage;
@@ -8341,26 +8297,29 @@ ${INIT_USAGE}`);
     const project = detect(cwd);
     const tests = templates(project);
     console.log(`detected: ${describe(project)}`);
-    writeFileSync3(configPath, renderYaml(project, tests));
+    writeFileSync2(configPath, renderYaml(project, tests));
     console.log(`wrote ${CONFIG_FILE} (${tests.length} test${tests.length === 1 ? "" : "s"}: ${tests.map((t) => t.name).join(", ")}); edit the prompts to fit your users`);
   }
   console.log(ensureGitignore(cwd));
   const interactive = !yes && process.stdin.isTTY && process.stdout.isTTY;
   if (!skills && interactive)
-    skills = await askYesNo(`Add the ${SKILL} skill to .claude/skills/ (helps your agent draft better evals)?`);
+    skills = await askYesNo(`Add the ${SKILLS.join(" and ")} skills to .claude/skills/ (help your agent draft better evals)?`);
   if (skills) {
-    const dest = join6(cwd, ".claude", "skills", SKILL);
-    cpSync2(skillSource(), dest, { recursive: true });
-    console.log(`copied ${SKILL} skill to .claude/skills/${SKILL}/`);
+    console.log(`installing skills: npx ${SKILLS_ARGV.join(" ")}`);
+    const failed = installSkills(cwd);
+    if (failed) {
+      console.error(`firegoat init: installing the skills failed (${failed}). Run it yourself: npx ${SKILLS_ARGV.join(" ")}`);
+      return EXIT.notGraded;
+    }
   } else {
-    console.log(`hint: \`npx firegoat init --skills\` adds the ${SKILL} skill to .claude/skills/ (keeps your ${CONFIG_FILE})`);
+    console.log(`hint: \`npx firegoat init --skills\` adds the ${SKILLS.join(" and ")} skills to .claude/skills/ (keeps your ${CONFIG_FILE})`);
   }
   console.log("next: npx firegoat eval");
   return 0;
 }
 
 // src/login.ts
-import { spawnSync as spawnSync3 } from "node:child_process";
+import { spawnSync as spawnSync4 } from "node:child_process";
 import { mkdirSync as mkdirSync4 } from "node:fs";
 function loginCommand(args) {
   let claudeBin = process.env.FIREGOAT_CLAUDE_BIN || "claude";
@@ -8383,7 +8342,7 @@ function loginCommand(args) {
   };
   delete env.ANTHROPIC_API_KEY;
   console.error(`Signing claude in to ${configDir}. Evals run with this login, never your own ~/.claude.`);
-  const r = spawnSync3(claudeBin, ["auth", "login"], { env, stdio: "inherit" });
+  const r = spawnSync4(claudeBin, ["auth", "login"], { env, stdio: "inherit" });
   if (r.error) {
     console.error(`firegoat login: could not run ${claudeBin}: ${r.error.message}`);
     return EXIT.notGraded;
@@ -8397,8 +8356,8 @@ function loginCommand(args) {
 }
 
 // src/open.ts
-import { spawnSync as spawnSync4 } from "node:child_process";
-import { existsSync as existsSync6, readdirSync as readdirSync2 } from "node:fs";
+import { spawnSync as spawnSync5 } from "node:child_process";
+import { existsSync as existsSync5, readdirSync } from "node:fs";
 import { join as join7 } from "node:path";
 var OPENER = {
   darwin: "open",
@@ -8421,7 +8380,7 @@ function openCommand(args, cwd) {
     return EXIT.usage;
   }
   const base = join7(findConfigDir(cwd), ".firegoat", "runs");
-  const runs = existsSync6(base) ? readdirSync2(base).sort() : [];
+  const runs = existsSync5(base) ? readdirSync(base).sort() : [];
   const id = ids[0] ?? runs.at(-1);
   if (!id) {
     console.error(`firegoat open: no runs in ${base}. Run: npx firegoat eval`);
@@ -8436,7 +8395,7 @@ function openCommand(args, cwd) {
   const opener = OPENER[process.platform];
   if (print || !opener)
     return EXIT.pass;
-  const r = spawnSync4(opener, [dir], { stdio: "inherit" });
+  const r = spawnSync5(opener, [dir], { stdio: "inherit" });
   if (r.error || r.status !== 0) {
     console.error(`firegoat open: ${opener} failed (${r.error?.message ?? `exit ${r.status}`}); the run is at the path above`);
     return EXIT.notGraded;
@@ -8458,7 +8417,7 @@ Usage:
 eval options:
   test..             only run these tests (by name)
   --runs N           runs per test (overrides runs: in firegoat.yaml)
-  --var KEY=value    fills {{vars.KEY}} in prompts and checks, and is in the env of build: commands
+  --var KEY=value    fills {{vars.KEY}} in prompts and checks; env for setup, workdir_setup and checks
   --json             JSON summary on stdout, progress on stderr
   --host             no sandbox (needed off macOS; use a throwaway container)
   --claude-bin PATH  claude binary to run (env: FIREGOAT_CLAUDE_BIN; default: claude on PATH)

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.3.0",
+	"version": "0.5.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/team2027/firegoat-cli.git"
@@ -21,8 +21,7 @@
 		"node": ">=18"
 	},
 	"files": [
-		"dist",
-		"skills"
+		"dist"
 	],
 	"homepage": "https://github.com/team2027/firegoat-cli#readme",
 	"license": "MIT",

package/skills/firegoat-onboarding/SKILL.md

@@ -1,114 +0,0 @@
----
-name: firegoat-onboarding
-description: Draft 2-3 firegoat evals (firegoat.yaml) for the current repo by reading its README, docs and package manifests, phrasing each test as a real user's problem with a gradeable goal and script checks. Use when the user says "write firegoat evals", "draft evals", "set up firegoat", "firegoat init", or asks for agent-experience (AX) evals of their SDK, CLI, API or docs.
----
-
-# Draft firegoat evals
-
-firegoat runs a coding agent (Claude Code) in an empty, isolated workdir with one prompt,
-then grades the run. Your job: write 2-3 tests into `firegoat.yaml` that tell the
-maintainer whether an agent can succeed with their product the way a new user would.
-
-## 1. Learn the product (read, don't run)
-
-Read, in this order, and stop when you can answer the questions below:
-- `README*`, then the docs dir (`docs/`, `website/`, `*.mdx`), quickstart/getting-started pages first.
-- Package manifests: `package.json` (`name`, `bin`, `exports`), `pyproject.toml`, `go.mod`,
-  `Cargo.toml`, `*.gemspec`, `Package.swift`. They tell you the install name and entry points.
-- `examples/`, `templates/`, `samples/`: the shapes users actually build.
-- An existing `firegoat.yaml`, if any: keep its tests (see step 5).
-
-Answer: What is it (SDK, CLI, API, framework)? Who uses it? What is the first thing a new
-user wants to have working? What is the public docs URL? Which credentials does it need
-(env var names only)?
-
-## 2. Pick 2-3 outcomes
-
-Choose the most important first-time user outcomes, e.g. "first successful API call",
-"the core feature end to end", "the thing users most often ask about". Prefer outcomes
-that leave observable proof. Skip admin, billing and anything needing a browser login
-unless credentials can come from env vars.
-
-## 3. Write each prompt as a user's problem
-
-A prompt is what a user would type to their agent, not a recipe. Each prompt has:
-- **What they want, not how.** No numbered implementation steps, no "use template X, add package Y".
-  The eval measures whether the agent finds its way with your docs.
-- **One concrete outcome**, small enough to finish in a few minutes.
-- **Observable proof**: print a marker to stdout, write a file, or take a screenshot.
-- **"Keep it minimal."** so the agent doesn't build a demo app.
-- **The docs URL as the only hint** (omit it if the test is about discovery).
-- **Credentials as env var names**, never values: "Credentials are in env vars ACME_API_KEY and ACME_API_URL."
-- If the test should use the build under test, say so: "The package is already installed
-  from the build under test. Do not install or upgrade it from the registry."
-
-## 4. Write the goal and checks
-
-`goal`: one sentence a judge could check from the transcript and workdir, e.g. "A minimal
-project runs and prints the answer, a numeric score and a reason."
-
-`checks`: shell scripts, run in the agent's workdir after the run. Exit code 0 = pass.
-Env: `FIREGOAT_WORKDIR` (the workdir) and `FIREGOAT_SESSION` (the agent's stream-json
-transcript, one JSON event per line). Checks should prove the outcome and catch cheating:
-- Output exists with the expected marker: `grep -rq hello-from-acme "$FIREGOAT_WORKDIR"`
-  or `test -s "$FIREGOAT_WORKDIR/screenshot.png"`.
-- The real thing was used, not faked: the transcript shows the SDK/CLI being called,
-  e.g. `grep -q '"command":"[^"]*acme ' "$FIREGOAT_SESSION"`.
-- Cleanup happened: the transcript shows the delete call after the create call.
-- The local build was used, not a registry install:
-  `! grep -Eq '"command":"[^"]*(pip install|npm (install|i) [^"]*acme@)' "$FIREGOAT_SESSION"`.
-- Only local docs were used (when that's the point): `! grep -q '"name":"WebFetch"' "$FIREGOAT_SESSION"`.
-
-Match tool-call JSON (`"command":"...`) rather than plain words, so the prompt text or the
-agent saying "I won't pip install" doesn't trip the check. Checks run in the agent's
-workdir, which does not contain the repo, so keep them self-contained: one-liners, or a
-`run: |` block for longer scripts. Never reference repo files by relative path.
-
-Do not emit `judge:` checks: v1 skips them with a warning.
-
-## 5. Write firegoat.yaml
-
-Schema (v1):
-
-```yaml
-agents: [claude-code]      # only claude-code in v1
-runs: 1                    # runs per test, positive integer
-build:                     # optional: local packages preinstalled in the agent's workdir
-  sdk: ./packages/sdk      # package.json -> node_modules, pyproject.toml -> .venv on PATH
-  cli:                     # anything else: a command that writes outputs to $FIREGOAT_OUT
-    run: go build -o "$FIREGOAT_OUT/acme" ./cmd/acme     # outputs go on PATH
-vars:                      # optional: defaults for {{vars.KEY}}; `--var KEY=value` overrides
-  SHA: main
-tests:
-  - name: quickstart       # letters, digits, '.', '_', '-'; unique
-    prompt: >-
-      ...
-    goal: ...
-    checks:                # optional
-      - run: grep -rq some-marker "$FIREGOAT_WORKDIR"
-    timeout: 600           # optional, seconds
-```
-
-Add `build:` when the repo ships a package or CLI whose local build the tests should use;
-then the prompt can say "The package is already installed from the build under test."
-
-If `firegoat.yaml` exists, merge: keep every existing test and top-level key untouched, append
-yours, and rename yours (`-2` suffix) if a name collides. Never delete or rewrite user tests.
-Quote prompts with `>-` folded blocks so colons and quotes stay valid YAML.
-
-Before finishing, re-read the file and check: it parses as YAML, every test has a non-empty
-`name`, `prompt` and `goal`, names are unique and match `^[A-Za-z0-9._-]+$`, and no credential
-values appear anywhere.
-
-## 6. Hand off
-
-Tell the user, in a few lines: which tests you added and why, which env vars they must export,
-and to run `npx firegoat eval`.
-
-## Examples
-
-See `references/examples.md` for good and bad tests with their YAML. Short version:
-
-Good: "Create a new app - a coffee timer. Run it on the simulator and get the screenshot."
-Bad: "Create a sandbox. https://docs.acme.dev" with no goal and no checks: every run passes,
-so it measures nothing.

package/skills/firegoat-onboarding/references/examples.md

@@ -1,114 +0,0 @@
-# Example firegoat tests
-
-Product names are fictional. Copy the shape, not the words.
-
-## Good
-
-### 1. Core feature end to end, printed proof
-
-An agent framework with scorers. The prompt names the outcome and what to print; the agent
-has to find how in the docs.
-
-```yaml
-tests:
-  - name: agent-with-scorer
-    prompt: >-
-      Build an agent with a scorer using https://docs.acme.dev/agents. Ask the agent
-      to explain recursion in one sentence, then run a scorer that grades whether the
-      answer is concise and accurate. Print the agent answer, the numeric score and
-      the scorer reason to stdout. Keep it minimal.
-    goal: >-
-      A minimal project runs and prints the agent's answer, a numeric score and the
-      scorer's reason.
-    checks:
-      - run: grep -Eq '"command":"[^"]*(node|npx|tsx|bun) ' "$FIREGOAT_SESSION"
-```
-
-### 2. Multi-step workflow, one realistic input
-
-```yaml
-tests:
-  - name: support-workflow
-    prompt: >-
-      Using https://docs.acme.dev/workflows, build a three-step workflow: extract the
-      issue type from a support message, have an agent draft a response, then add a
-      priority label. Run it once on a customer asking for a refund and print the
-      final result. Keep it minimal.
-    goal: >-
-      The workflow runs once on a refund request and prints a result containing the
-      issue type, a drafted response and a priority label.
-```
-
-### 3. Local build under test, cleanup, no registry install
-
-```yaml
-tests:
-  - name: sandbox-hello
-    prompt: >-
-      Using the Acme Python SDK, create a sandbox, run `echo hello-from-acme` inside
-      it, print the output, then delete the sandbox. The package is already installed
-      from the build under test. Do not install or upgrade it from PyPI.
-      Docs: https://docs.acme.dev/python. Credentials are in env vars ACME_API_KEY
-      and ACME_API_URL.
-    goal: >-
-      The script prints hello-from-acme from inside a sandbox and deletes the sandbox
-      at the end.
-    checks:
-      - run: grep -Eq '\.(delete|remove)\(' -r "$FIREGOAT_WORKDIR" --include='*.py'
-      - run: "! grep -Eq '\"command\":\"[^\"]*pip3? install[^\"]*acme' \"$FIREGOAT_SESSION\""
-    timeout: 900
-```
-
-### 4. Pure user need, discovery
-
-No docs URL, no package names: the test is whether the agent finds the right packages.
-
-```yaml
-tests:
-  - name: which-packages
-    prompt: >-
-      I'm building a mobile app with Acme. It needs a local database, maps, age
-      gating, localization, Sign in with Apple and a small on-device model. Does Acme
-      offer packages for these? Which ones would you use?
-    goal: >-
-      The answer names the right Acme package for each requirement and relies on the
-      docs shipped in the project, not web search.
-    checks:
-      - run: "! grep -q '\"name\":\"WebFetch\"' \"$FIREGOAT_SESSION\""
-```
-
-### 5. Shortest
-
-```yaml
-tests:
-  - name: coffee-timer
-    prompt: Create a new app - a coffee timer. Run it on the simulator and get the screenshot.
-    goal: A screenshot shows a working coffee timer app running on the simulator.
-    checks:
-      - run: find "$FIREGOAT_WORKDIR" -name '*.png' -size +1k | grep -q .
-```
-
-## Bad
-
-### No goal, no checks
-
-```text
-prompt: "Create a sandbox. https://docs.acme.dev"
-```
-
-Ungradeable: nothing says what success looks like, so every run "succeeds". Add an
-observable outcome ("run echo hello-from-acme inside it and print the output"), a goal
-and a check.
-
-### Implementation steps
-
-```text
-prompt: >-
-  1. Create a project from the acme-starter template.
-  2. Add the acme-maps package.
-  3. Render a map centered on Berlin.
-```
-
-This tests instruction-following, not whether an agent finds its way with your docs.
-Rewrite as the user's need: "Show a map centered on Berlin in a new Acme app. Keep it
-minimal. Take a screenshot."