firegoat 0.4.0 → 0.6.0

package/README.md

@@ -2,13 +2,14 @@
 
 Local-first AX evals: how well coding agents use your SDK, CLI, API and docs.
 
-Status: early (v0.4). See `plan.md`.
+Status: early (v0.6). See `plan.md`.
 
 ## Use
 
 ```bash
-npx firegoat init            # writes firegoat.yaml with starter evals; --skills adds the onboarding skill
-npx firegoat login           # once per machine: signs claude in to ~/.firegoat/claude
+npx firegoat init            # writes firegoat.yaml with starter evals, asks how the agent signs in;
+                             # --skills adds the firegoat-onboarding skill (from team2027/skills)
+npx firegoat claude login    # sign-in option 1, once per machine: claude in ~/.firegoat/claude
 npx firegoat eval            # runs each test with your local `claude`, sandboxed, in a fresh folder
 npx firegoat open            # opens the latest run's folder (--print to only print it)
 ```
@@ -36,14 +37,23 @@ Then say so in the prompt: "The package is already installed from the build unde
 - Checks also get `$FIREGOAT_OUT` (the run's copy).
 - A failed `setup:` or `workdir_setup:` means not graded (exit 2); logs are `.firegoat/runs/<runId>/setup.txt` and `<test>/<n>/workdir_setup.txt`.
 
+### How the agent signs in
+
+`firegoat init` asks (or pass `--auth`):
+
+1. **Login with Claude** (`--auth claude`, same as `npx firegoat claude login`): your Claude plan, in firegoat's own config dir `~/.firegoat/claude`, never your `~/.claude`.
+2. **Paste an Anthropic API key** (`--auth api-key`, piped on stdin or typed hidden): stored for firegoat only in `~/.firegoat/anthropic-api-key` (mode 0600), never in `firegoat.yaml` or the repo, and passed to the sandboxed claude as `ANTHROPIC_API_KEY`.
+
+`eval` prints which one it uses. In CI (`CI` set), `ANTHROPIC_API_KEY` from the env wins. Outside CI, a key in your shell is not passed to the agent, so it never silently replaces the Claude login. `firegoat login` still works as a deprecated alias of `firegoat claude login`; the name is reserved for firegoat.ai.
+
 ### Isolation
 
 On macOS, `eval` runs `claude` inside a Seatbelt (`sandbox-exec`) profile:
 
-- it signs in with `~/.firegoat/claude` (from `firegoat login`), never your `~/.claude`;
+- it signs in as chosen above, never with your `~/.claude`;
 - it cannot read your home dir or the repo, except toolchains (node, bun, cargo, ...) it needs to run;
 - it can write only in its run dir (`work/`, its own `TMPDIR` and package caches) and system temp;
-- `ANTHROPIC_API_KEY` is removed, so runs bill your subscription, not the API.
+- `ANTHROPIC_API_KEY` is set only for the API-key sign-in; firegoat checks claude's reported key source on every run.
 
 A probe checks the sandbox before every run; if it does not hold, the run is not graded. Network is open.
 

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.4.0",
+  version: "0.6.0",
   repository: {
     type: "git",
     url: "git+https://github.com/team2027/firegoat-cli.git"
@@ -6968,8 +6968,7 @@ var package_default = {
     node: ">=18"
   },
   files: [
-    "dist",
-    "skills"
+    "dist"
   ],
   homepage: "https://github.com/team2027/firegoat-cli#readme",
   license: "MIT",
@@ -6988,9 +6987,266 @@ var package_default = {
   packageManager: "bun@1.4.3"
 };
 
+// src/claude-login.ts
+import { spawnSync as spawnSync2 } from "node:child_process";
+import { mkdirSync as mkdirSync3 } from "node:fs";
+
+// src/auth.ts
+import {
+  chmodSync,
+  existsSync as existsSync2,
+  mkdirSync as mkdirSync2,
+  readFileSync,
+  writeFileSync
+} from "node:fs";
+import { join as join2 } from "node:path";
+
+// src/sandbox.ts
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, realpathSync } from "node:fs";
+import { homedir } from "node:os";
+import { basename, dirname, join } from "node:path";
+var SANDBOX_EXEC = "/usr/bin/sandbox-exec";
+function firegoatHome() {
+  return process.env.FIREGOAT_HOME || join(homedir(), ".firegoat");
+}
+function claudeConfigDir() {
+  return join(firegoatHome(), "claude");
+}
+var TOOLCHAINS = [
+  ".nvm",
+  ".fnm",
+  ".volta",
+  ".bun",
+  ".deno",
+  ".cargo",
+  ".rustup",
+  ".pyenv",
+  ".rbenv",
+  ".asdf",
+  ".sdkman",
+  ".local/share/mise",
+  ".local/share/fnm",
+  ".local/share/claude",
+  ".local/share/uv",
+  "go",
+  "Library/pnpm"
+];
+var TOOLCHAIN_SECRETS = [".cargo/credentials", ".cargo/credentials.toml"];
+function real(p) {
+  return realpathSync(p);
+}
+function under(p, dir) {
+  return p === dir || p.startsWith(`${dir}/`);
+}
+function readableToolDirs(home, claudeBin) {
+  const dirs = new Set;
+  const add = (p) => {
+    if (existsSync(p))
+      dirs.add(real(p));
+  };
+  for (const t of TOOLCHAINS)
+    add(join(home, t));
+  for (const p of (process.env.PATH ?? "").split(":")) {
+    if (!p.startsWith("/") || !existsSync(p))
+      continue;
+    const r = real(p);
+    if (!under(r, home) || r === home)
+      continue;
+    const prefix = dirname(r);
+    dirs.add(basename(r) === "bin" && prefix !== home && prefix !== join(home, ".local") ? prefix : r);
+  }
+  add(dirname(dirname(process.execPath)));
+  add(dirname(claudeBin));
+  add(dirname(real(claudeBin)));
+  return [...dirs];
+}
+var q = (s) => JSON.stringify(s);
+function sandboxProfile(opts) {
+  const home = real(homedir());
+  const reads = readableToolDirs(home, opts.claudeBin).map((d) => `(subpath ${q(d)})`);
+  return `(version 1)
+(allow default)
+
+(deny file-read* (subpath ${q(home)}))
+(allow file-read-metadata)
+(allow file-read*
+  ${reads.join(`
+  `)}
+  (subpath ${q(join(home, "Library/Keychains"))})
+  (regex ${q(`^${home}/Library/Preferences/com\\.apple\\.security`)})
+  (regex ${q(`^${home}/Library/Preferences/\\.GlobalPreferences`)}))
+(deny file-read* file-write*
+  ${TOOLCHAIN_SECRETS.map((s) => `(literal ${q(join(home, s))})`).join(`
+  `)})
+
+(deny file-write* (subpath "/"))
+(allow file-write*
+  (subpath "/dev")
+  (subpath "/private/tmp")
+  (subpath "/private/var/folders"))
+(allow file-read* file-write*
+  (subpath ${q(real(opts.configDir))})
+  (subpath ${q(real(opts.runRoot))}))
+
+(deny file-read* file-write* (subpath ${q(real(opts.repo))}))
+`;
+}
+function agentEnv(runRoot, apiKey) {
+  const env = { ...process.env };
+  delete env.ANTHROPIC_API_KEY;
+  if (apiKey)
+    env.ANTHROPIC_API_KEY = apiKey;
+  const cache = join(runRoot, "cache");
+  const tmp = join(runRoot, "tmp");
+  mkdirSync(cache, { recursive: true });
+  mkdirSync(tmp, { recursive: true });
+  return {
+    ...env,
+    CLAUDE_CONFIG_DIR: claudeConfigDir(),
+    TMPDIR: tmp,
+    XDG_CACHE_HOME: cache,
+    npm_config_cache: join(cache, "npm"),
+    npm_config_store_dir: join(cache, "pnpm-store"),
+    YARN_CACHE_FOLDER: join(cache, "yarn"),
+    BUN_INSTALL_CACHE_DIR: join(cache, "bun"),
+    PIP_CACHE_DIR: join(cache, "pip"),
+    UV_CACHE_DIR: join(cache, "uv"),
+    GOMODCACHE: join(cache, "gomod"),
+    GOCACHE: join(cache, "gobuild"),
+    CARGO_HOME: join(cache, "cargo")
+  };
+}
+function wrap(profile, cmd, args) {
+  if (profile === null)
+    return { cmd, args };
+  return { cmd: SANDBOX_EXEC, args: ["-p", profile, cmd, ...args] };
+}
+function probeSandbox(opts) {
+  const outside = join(homedir(), `.firegoat-escape-probe-${process.pid}-${Date.now()}`);
+  const script = 'touch "$1" 2>/dev/null && echo home-write; cat "$2" >/dev/null 2>&1 && echo repo-read; touch "$3" 2>/dev/null && echo work-write; rm -f "$1" "$3"';
+  const { cmd, args } = wrap(opts.profile, "/bin/sh", [
+    "-c",
+    script,
+    "probe",
+    outside,
+    opts.repoFile,
+    join(opts.workdir, ".firegoat-probe")
+  ]);
+  const r = spawnSync(cmd, args, { encoding: "utf8", cwd: opts.workdir });
+  if (r.error)
+    return `sandbox probe could not start: ${r.error.message}`;
+  const got = r.stdout.trim().split(`
+`).filter(Boolean);
+  if (got.includes("home-write") || got.includes("repo-read"))
+    return `sandbox does not hold (escaped: ${got.filter((g) => g !== "work-write").join(", ")}); refusing to run the agent`;
+  if (!got.includes("work-write"))
+    return `sandbox blocks writes in the workdir (stderr: ${r.stderr.trim() || "none"})`;
+  return;
+}
+
+// src/auth.ts
+var authFile = () => join2(firegoatHome(), "auth.json");
+var keyFile = () => join2(firegoatHome(), "anthropic-api-key");
+
+class AuthError extends Error {
+}
+function readMethod() {
+  const file = authFile();
+  if (!existsSync2(file))
+    return;
+  let doc;
+  try {
+    doc = JSON.parse(readFileSync(file, "utf8"));
+  } catch (e) {
+    throw new AuthError(`${file} is not valid JSON: ${e.message}`);
+  }
+  const method = doc?.method;
+  if (method !== "claude" && method !== "api-key")
+    throw new AuthError(`${file}: method must be "claude" or "api-key", got ${JSON.stringify(method)}. Run: npx firegoat init`);
+  return method;
+}
+function saveMethod(method) {
+  mkdirSync2(firegoatHome(), { recursive: true });
+  writeFileSync(authFile(), `${JSON.stringify({ method }, null, 2)}
+`);
+}
+function saveApiKey(apiKey) {
+  mkdirSync2(firegoatHome(), { recursive: true });
+  const file = keyFile();
+  writeFileSync(file, `${apiKey}
+`, { mode: 384 });
+  chmodSync(file, 384);
+  saveMethod("api-key");
+  return file;
+}
+function looksLikeAnthropicKey(s) {
+  return /^sk-ant-[A-Za-z0-9_-]{20,}$/.test(s);
+}
+function resolveAuth(env) {
+  if (env.CI && env.ANTHROPIC_API_KEY)
+    return {
+      method: "api-key",
+      label: "Anthropic API key from ANTHROPIC_API_KEY (CI)",
+      apiKey: env.ANTHROPIC_API_KEY
+    };
+  const method = readMethod() ?? "claude";
+  if (method === "claude")
+    return {
+      method,
+      label: `Claude login in ${claudeConfigDir()}`
+    };
+  const file = keyFile();
+  if (!existsSync2(file))
+    throw new AuthError(`auth is set to api-key but ${file} is missing. Run: npx firegoat init`);
+  const apiKey = readFileSync(file, "utf8").trim();
+  if (!apiKey)
+    throw new AuthError(`${file} is empty. Run: npx firegoat init`);
+  return { method, label: `Anthropic API key in ${file}`, apiKey };
+}
+
+// src/exit.ts
+var EXIT = { pass: 0, fail: 1, notGraded: 2, usage: 3 };
+
+// src/claude-login.ts
+function claudeLoginCommand(args) {
+  let claudeBin = process.env.FIREGOAT_CLAUDE_BIN || "claude";
+  for (let i = 0;i < args.length; i++) {
+    const a = args[i];
+    if (a === "--claude-bin" && args[i + 1] !== undefined)
+      claudeBin = args[++i];
+    else if (a.startsWith("--claude-bin="))
+      claudeBin = a.slice(13);
+    else {
+      console.error(`firegoat claude login: unknown argument: ${a}`);
+      return EXIT.usage;
+    }
+  }
+  const configDir = claudeConfigDir();
+  mkdirSync3(configDir, { recursive: true });
+  const env = {
+    ...process.env,
+    CLAUDE_CONFIG_DIR: configDir
+  };
+  delete env.ANTHROPIC_API_KEY;
+  console.error(`Signing claude in to ${configDir}. Evals run with this login, never your own ~/.claude.`);
+  const r = spawnSync2(claudeBin, ["auth", "login"], { env, stdio: "inherit" });
+  if (r.error) {
+    console.error(`firegoat claude login: could not run ${claudeBin}: ${r.error.message}`);
+    return EXIT.notGraded;
+  }
+  if (r.status !== 0) {
+    console.error(`firegoat claude login: claude auth login exited ${r.status}`);
+    return EXIT.notGraded;
+  }
+  saveMethod("claude");
+  console.log("logged in; evals now sign in with this Claude login. next: npx firegoat eval");
+  return EXIT.pass;
+}
+
 // src/config.ts
-import { existsSync, readFileSync } from "node:fs";
-import { dirname, join } from "node:path";
+import { existsSync as existsSync3, readFileSync as readFileSync2 } from "node:fs";
+import { dirname as dirname2, join as join3 } from "node:path";
 
 // node_modules/yaml/dist/index.js
 var composer = require_composer();
@@ -7170,17 +7426,17 @@ function render(text, vars) {
 }
 function loadConfig(dir, warn) {
   const file = `${dir}/${CONFIG_FILE}`;
-  if (!existsSync(file))
+  if (!existsSync3(file))
     throw new ConfigError(`${CONFIG_FILE} not found in ${dir}. Run: npx firegoat init`);
-  return parseConfig(readFileSync(file, "utf8"), warn);
+  return parseConfig(readFileSync2(file, "utf8"), warn);
 }
 function findConfigDir(cwd) {
   let dir = cwd;
   for (;; ) {
-    if (existsSync(join(dir, CONFIG_FILE)))
+    if (existsSync3(join3(dir, CONFIG_FILE)))
       return dir;
-    const parent = dirname(dir);
-    if (existsSync(join(dir, ".git")) || parent === dir)
+    const parent = dirname2(dir);
+    if (existsSync3(join3(dir, ".git")) || parent === dir)
       return cwd;
     dir = parent;
   }
@@ -7190,164 +7446,20 @@ function findConfigDir(cwd) {
 import { randomBytes } from "node:crypto";
 import {
   createWriteStream as createWriteStream3,
-  existsSync as existsSync3,
-  mkdirSync as mkdirSync3,
+  existsSync as existsSync4,
+  mkdirSync as mkdirSync5,
   mkdtempSync,
-  readFileSync as readFileSync2,
+  readFileSync as readFileSync3,
   realpathSync as realpathSync2,
-  writeFileSync
+  writeFileSync as writeFileSync2
 } from "node:fs";
 import { tmpdir } from "node:os";
-import { join as join5, resolve as resolve2 } from "node:path";
+import { join as join6, resolve as resolve2 } from "node:path";
 
 // src/claude.ts
-import { spawn, spawnSync as spawnSync2 } from "node:child_process";
+import { spawn, spawnSync as spawnSync3 } from "node:child_process";
 import { createWriteStream } from "node:fs";
 import { createInterface } from "node:readline";
-
-// src/sandbox.ts
-import { spawnSync } from "node:child_process";
-import { existsSync as existsSync2, mkdirSync, realpathSync } from "node:fs";
-import { homedir } from "node:os";
-import { basename, dirname as dirname2, join as join2 } from "node:path";
-var SANDBOX_EXEC = "/usr/bin/sandbox-exec";
-function firegoatHome() {
-  return process.env.FIREGOAT_HOME || join2(homedir(), ".firegoat");
-}
-function claudeConfigDir() {
-  return join2(firegoatHome(), "claude");
-}
-var TOOLCHAINS = [
-  ".nvm",
-  ".fnm",
-  ".volta",
-  ".bun",
-  ".deno",
-  ".cargo",
-  ".rustup",
-  ".pyenv",
-  ".rbenv",
-  ".asdf",
-  ".sdkman",
-  ".local/share/mise",
-  ".local/share/fnm",
-  ".local/share/claude",
-  ".local/share/uv",
-  "go",
-  "Library/pnpm"
-];
-var TOOLCHAIN_SECRETS = [".cargo/credentials", ".cargo/credentials.toml"];
-function real(p) {
-  return realpathSync(p);
-}
-function under(p, dir) {
-  return p === dir || p.startsWith(`${dir}/`);
-}
-function readableToolDirs(home, claudeBin) {
-  const dirs = new Set;
-  const add = (p) => {
-    if (existsSync2(p))
-      dirs.add(real(p));
-  };
-  for (const t of TOOLCHAINS)
-    add(join2(home, t));
-  for (const p of (process.env.PATH ?? "").split(":")) {
-    if (!p.startsWith("/") || !existsSync2(p))
-      continue;
-    const r = real(p);
-    if (!under(r, home) || r === home)
-      continue;
-    const prefix = dirname2(r);
-    dirs.add(basename(r) === "bin" && prefix !== home && prefix !== join2(home, ".local") ? prefix : r);
-  }
-  add(dirname2(dirname2(process.execPath)));
-  add(dirname2(claudeBin));
-  add(dirname2(real(claudeBin)));
-  return [...dirs];
-}
-var q = (s) => JSON.stringify(s);
-function sandboxProfile(opts) {
-  const home = real(homedir());
-  const reads = readableToolDirs(home, opts.claudeBin).map((d) => `(subpath ${q(d)})`);
-  return `(version 1)
-(allow default)
-
-(deny file-read* (subpath ${q(home)}))
-(allow file-read-metadata)
-(allow file-read*
-  ${reads.join(`
-  `)}
-  (subpath ${q(join2(home, "Library/Keychains"))})
-  (regex ${q(`^${home}/Library/Preferences/com\\.apple\\.security`)})
-  (regex ${q(`^${home}/Library/Preferences/\\.GlobalPreferences`)}))
-(deny file-read* file-write*
-  ${TOOLCHAIN_SECRETS.map((s) => `(literal ${q(join2(home, s))})`).join(`
-  `)})
-
-(deny file-write* (subpath "/"))
-(allow file-write*
-  (subpath "/dev")
-  (subpath "/private/tmp")
-  (subpath "/private/var/folders"))
-(allow file-read* file-write*
-  (subpath ${q(real(opts.configDir))})
-  (subpath ${q(real(opts.runRoot))}))
-
-(deny file-read* file-write* (subpath ${q(real(opts.repo))}))
-`;
-}
-function agentEnv(runRoot) {
-  const env = { ...process.env };
-  delete env.ANTHROPIC_API_KEY;
-  const cache = join2(runRoot, "cache");
-  const tmp = join2(runRoot, "tmp");
-  mkdirSync(cache, { recursive: true });
-  mkdirSync(tmp, { recursive: true });
-  return {
-    ...env,
-    CLAUDE_CONFIG_DIR: claudeConfigDir(),
-    TMPDIR: tmp,
-    XDG_CACHE_HOME: cache,
-    npm_config_cache: join2(cache, "npm"),
-    npm_config_store_dir: join2(cache, "pnpm-store"),
-    YARN_CACHE_FOLDER: join2(cache, "yarn"),
-    BUN_INSTALL_CACHE_DIR: join2(cache, "bun"),
-    PIP_CACHE_DIR: join2(cache, "pip"),
-    UV_CACHE_DIR: join2(cache, "uv"),
-    GOMODCACHE: join2(cache, "gomod"),
-    GOCACHE: join2(cache, "gobuild"),
-    CARGO_HOME: join2(cache, "cargo")
-  };
-}
-function wrap(profile, cmd, args) {
-  if (profile === null)
-    return { cmd, args };
-  return { cmd: SANDBOX_EXEC, args: ["-p", profile, cmd, ...args] };
-}
-function probeSandbox(opts) {
-  const outside = join2(homedir(), `.firegoat-escape-probe-${process.pid}-${Date.now()}`);
-  const script = 'touch "$1" 2>/dev/null && echo home-write; cat "$2" >/dev/null 2>&1 && echo repo-read; touch "$3" 2>/dev/null && echo work-write; rm -f "$1" "$3"';
-  const { cmd, args } = wrap(opts.profile, "/bin/sh", [
-    "-c",
-    script,
-    "probe",
-    outside,
-    opts.repoFile,
-    join2(opts.workdir, ".firegoat-probe")
-  ]);
-  const r = spawnSync(cmd, args, { encoding: "utf8", cwd: opts.workdir });
-  if (r.error)
-    return `sandbox probe could not start: ${r.error.message}`;
-  const got = r.stdout.trim().split(`
-`).filter(Boolean);
-  if (got.includes("home-write") || got.includes("repo-read"))
-    return `sandbox does not hold (escaped: ${got.filter((g) => g !== "work-write").join(", ")}); refusing to run the agent`;
-  if (!got.includes("work-write"))
-    return `sandbox blocks writes in the workdir (stderr: ${r.stderr.trim() || "none"})`;
-  return;
-}
-
-// src/claude.ts
 var CLAUDE_FLAGS = [
   "--output-format",
   "stream-json",
@@ -7512,8 +7624,8 @@ async function runClaude(opts) {
           badStart = `claude output did not start with a system/init stream-json line (got: ${line.slice(0, 120)})`;
           return "kill";
         }
-        if (msg.apiKeySource !== "none") {
-          badStart = `claude would use API key billing (apiKeySource=${JSON.stringify(msg.apiKeySource)}) instead of your Claude subscription. firegoat removes ANTHROPIC_API_KEY from the agent's env; check apiKeyHelper or other API key settings, then run \`claude /login\``;
+        if (msg.apiKeySource !== opts.keySource) {
+          badStart = opts.keySource === "none" ? `claude would use API key billing (apiKeySource=${JSON.stringify(msg.apiKeySource)}) instead of your Claude login. firegoat removes ANTHROPIC_API_KEY from the agent's env; check apiKeyHelper or other API key settings, then run \`npx firegoat claude login\`` : `claude did not use the configured API key (apiKeySource=${JSON.stringify(msg.apiKeySource)}, expected "ANTHROPIC_API_KEY")`;
           return "kill";
         }
         return;
@@ -7593,7 +7705,7 @@ async function runShell(opts) {
 }
 function checkLogin(opts) {
   const { cmd, args } = wrap(opts.profile, opts.bin, ["auth", "status"]);
-  const r = spawnSync2(cmd, args, {
+  const r = spawnSync3(cmd, args, {
     cwd: opts.cwd,
     env: opts.env,
     encoding: "utf8",
@@ -7608,16 +7720,13 @@ function checkLogin(opts) {
   if (status?.loggedIn === true)
     return;
   if (status?.loggedIn === false)
-    return `claude is not logged in to firegoat's config dir (${opts.env.CLAUDE_CONFIG_DIR}). Run: npx firegoat login`;
+    return `claude is not logged in to firegoat's config dir (${opts.env.CLAUDE_CONFIG_DIR}). Run: npx firegoat claude login`;
   return `claude auth status gave no login state (exit ${r.status}): ${(r.stdout + r.stderr).trim().slice(0, 300)}`;
 }
 
-// src/exit.ts
-var EXIT = { pass: 0, fail: 1, notGraded: 2, usage: 3 };
-
 // src/setup.ts
-import { cpSync, createWriteStream as createWriteStream2, mkdirSync as mkdirSync2 } from "node:fs";
-import { delimiter, join as join3 } from "node:path";
+import { cpSync, createWriteStream as createWriteStream2, mkdirSync as mkdirSync4 } from "node:fs";
+import { delimiter, join as join4 } from "node:path";
 var SETUP_TIMEOUT_MS = 600000;
 async function runAll(commands, cwd, env, logPath, what) {
   const log = createWriteStream2(logPath);
@@ -7639,13 +7748,13 @@ async function runAll(commands, cwd, env, logPath, what) {
   }
 }
 async function setupOnce(opts) {
-  mkdirSync2(opts.out, { recursive: true });
+  mkdirSync4(opts.out, { recursive: true });
   return runAll(opts.commands, opts.root, { ...opts.env, FIREGOAT_OUT: opts.out }, opts.logPath, "setup");
 }
 async function prepareRun(opts) {
-  const out = join3(opts.runRoot, "out");
+  const out = join4(opts.runRoot, "out");
   if (opts.out === null)
-    mkdirSync2(out);
+    mkdirSync4(out);
   else
     cpSync(opts.out, out, { recursive: true });
   const path = [out, opts.env.PATH ?? ""].join(delimiter);
@@ -7660,7 +7769,7 @@ async function prepareRun(opts) {
 
 // src/which.ts
 import { accessSync, constants, statSync } from "node:fs";
-import { delimiter as delimiter2, join as join4, resolve } from "node:path";
+import { delimiter as delimiter2, join as join5, resolve } from "node:path";
 function resolveBin(bin) {
   const executable = (p) => {
     try {
@@ -7673,8 +7782,8 @@ function resolveBin(bin) {
   if (bin.includes("/"))
     return executable(bin) ? resolve(bin) : undefined;
   for (const dir of (process.env.PATH ?? "").split(delimiter2)) {
-    if (dir && executable(join4(dir, bin)))
-      return join4(dir, bin);
+    if (dir && executable(join5(dir, bin)))
+      return join5(dir, bin);
   }
   return;
 }
@@ -7736,16 +7845,16 @@ function newRunId() {
   return `${ts}-${randomBytes(2).toString("hex")}`;
 }
 function gitignoreHasFiregoat(cwd) {
-  const file = join5(cwd, ".gitignore");
-  if (!existsSync3(file))
+  const file = join6(cwd, ".gitignore");
+  if (!existsSync4(file))
     return false;
-  return readFileSync2(file, "utf8").split(`
+  return readFileSync3(file, "utf8").split(`
 `).map((l) => l.trim()).some((l) => /^\/?\.firegoat\/?(\*\*?)?$/.test(l));
 }
 function newRunRoot(label) {
-  const runRoot = realpathSync2(mkdtempSync(join5(tmpdir(), `firegoat-${label}-`)));
-  const workdir = join5(runRoot, "work");
-  mkdirSync3(workdir);
+  const runRoot = realpathSync2(mkdtempSync(join6(tmpdir(), `firegoat-${label}-`)));
+  const workdir = join6(runRoot, "work");
+  mkdirSync5(workdir);
   return { runRoot, workdir };
 }
 function isolate(iso, claudeBin, runRoot, workdir) {
@@ -7763,9 +7872,9 @@ function isolate(iso, claudeBin, runRoot, workdir) {
 async function runOne(opts) {
   const { test, n } = opts;
   const started = Date.now();
-  const dir = join5(opts.runDir, test.name, String(n));
-  mkdirSync3(dir, { recursive: true });
-  const session = join5(dir, "session.jsonl");
+  const dir = join6(opts.runDir, test.name, String(n));
+  mkdirSync5(dir, { recursive: true });
+  const session = join6(dir, "session.jsonl");
   const summary = {
     test: test.name,
     n,
@@ -7798,7 +7907,7 @@ async function runOne(opts) {
       workdir,
       commands: opts.workdirSetup,
       env: hostEnv,
-      logPath: join5(dir, "workdir_setup.txt")
+      logPath: join6(dir, "workdir_setup.txt")
     });
     if ("error" in prepared)
       return done("error", `not graded: ${prepared.error}`);
@@ -7807,13 +7916,17 @@ async function runOne(opts) {
   }
   const agentPath = extra.PATH ? { PATH: extra.PATH } : {};
   const outcome = await runClaude({
-    env: { ...agentEnv(runRoot), ...agentPath },
+    env: {
+      ...agentEnv(runRoot, opts.auth.method === "api-key" ? opts.auth.apiKey : undefined),
+      ...agentPath
+    },
+    keySource: opts.auth.method === "api-key" ? "ANTHROPIC_API_KEY" : "none",
     profile: sandbox.profile,
     bin: opts.claudeBin,
     prompt: render(test.prompt, opts.vars),
     cwd: workdir,
     sessionPath: session,
-    stderrPath: join5(dir, "stderr.txt"),
+    stderrPath: join6(dir, "stderr.txt"),
     timeoutMs: test.timeout * 1000
   });
   if (outcome.result) {
@@ -7826,7 +7939,7 @@ async function runOne(opts) {
     opts.warn(`warning: ${test.name} has no checks, so its goal is unverified; counted as pass`);
     return done("pass", "no checks (goal unverified)");
   }
-  const log = createWriteStream3(join5(dir, "checks.txt"));
+  const log = createWriteStream3(join6(dir, "checks.txt"));
   const env = {
     ...process.env,
     ...hostEnv,
@@ -7898,45 +8011,58 @@ async function evalCommand(args, cwd) {
   }
   if (opts.host)
     warn("warning: --host: the agent runs without isolation and can read and write your files");
-  else if (process.platform !== "darwin" || !existsSync3(SANDBOX_EXEC)) {
+  else if (process.platform !== "darwin" || !existsSync4(SANDBOX_EXEC)) {
     console.error(`firegoat eval: isolation needs macOS (${SANDBOX_EXEC}); this is ${process.platform}. In CI or Linux, run inside a throwaway container and pass --host.`);
     return EXIT.usage;
   }
   const iso = {
     host: opts.host,
     repo: root,
-    repoFile: join5(root, "firegoat.yaml")
+    repoFile: join6(root, "firegoat.yaml")
   };
-  mkdirSync3(claudeConfigDir(), { recursive: true });
+  let auth;
+  try {
+    auth = resolveAuth(process.env);
+  } catch (e) {
+    if (e instanceof AuthError) {
+      console.error(`firegoat eval: ${e.message}`);
+      return EXIT.usage;
+    }
+    throw e;
+  }
+  say(`auth: ${auth.label}`);
+  if (auth.method === "claude" && process.env.ANTHROPIC_API_KEY)
+    warn("note: ANTHROPIC_API_KEY in your shell is not passed to the agent (it is used only in CI, or after `firegoat init --auth api-key`)");
+  mkdirSync5(claudeConfigDir(), { recursive: true });
   const pre = newRunRoot("preflight");
   const preSandbox = isolate(iso, claudePath, pre.runRoot, pre.workdir);
-  const loginProblem = "error" in preSandbox ? preSandbox.error : checkLogin({
+  const loginProblem = "error" in preSandbox ? preSandbox.error : auth.method === "claude" ? checkLogin({
     bin: claudePath,
     env: agentEnv(pre.runRoot),
     profile: preSandbox.profile,
     cwd: pre.workdir
-  });
+  }) : undefined;
   if (loginProblem) {
     console.error(`firegoat eval: not graded: ${loginProblem}`);
     return EXIT.notGraded;
   }
-  mkdirSync3(join5(root, ".firegoat"), { recursive: true });
+  mkdirSync5(join6(root, ".firegoat"), { recursive: true });
   if (!gitignoreHasFiregoat(root))
     warn("hint: add .firegoat/ to .gitignore (it holds run traces)");
   const runId = newRunId();
-  const runDir = join5(root, ".firegoat", "runs", runId);
-  mkdirSync3(runDir, { recursive: true });
-  const summaryPath = join5(runDir, "summary.json");
+  const runDir = join6(root, ".firegoat", "runs", runId);
+  mkdirSync5(runDir, { recursive: true });
+  const summaryPath = join6(runDir, "summary.json");
   let out = null;
   if (config.setup.length > 0) {
     say("setup...");
-    out = join5(runDir, "out");
+    out = join6(runDir, "out");
     const error = await setupOnce({
       commands: config.setup,
       root,
       out,
       env: { ...process.env, ...vars },
-      logPath: join5(runDir, "setup.txt")
+      logPath: join6(runDir, "setup.txt")
     });
     if (error) {
       console.error(`firegoat eval: not graded: ${error}`);
@@ -7952,10 +8078,11 @@ async function evalCommand(args, cwd) {
       runId,
       runsDir: runDir,
       isolation: opts.host ? "host" : "seatbelt",
+      auth: auth.method,
       totals,
       runs: results
     };
-    writeFileSync(summaryPath, `${JSON.stringify(doc, null, 2)}
+    writeFileSync2(summaryPath, `${JSON.stringify(doc, null, 2)}
 `);
     return doc;
   };
@@ -7969,6 +8096,7 @@ async function evalCommand(args, cwd) {
           runDir,
           claudeBin: claudePath,
           iso,
+          auth,
           out,
           workdirSetup: config.workdirSetup,
           vars,
@@ -7981,7 +8109,7 @@ async function evalCommand(args, cwd) {
           status: "error",
           reason: `not graded: firegoat crashed: ${e.message}`,
           workdir: null,
-          session: join5(runDir, test.name, String(n), "session.jsonl"),
+          session: join6(runDir, test.name, String(n), "session.jsonl"),
           checks: [],
           num_turns: null,
           cost: null,
@@ -8004,21 +8132,36 @@ ${pass} passed, ${fail} failed, ${error} not graded (${results.length} runs)`);
 }
 
 // src/init.ts
-import { cpSync as cpSync2, existsSync as existsSync4, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
-import { basename as basename2, join as join6 } from "node:path";
+import { spawnSync as spawnSync4 } from "node:child_process";
+import { existsSync as existsSync5, readFileSync as readFileSync4, writeFileSync as writeFileSync3 } from "node:fs";
+import { basename as basename2, join as join7 } from "node:path";
 import { createInterface as createInterface2 } from "node:readline/promises";
-import { fileURLToPath } from "node:url";
-var INIT_USAGE = `Usage: firegoat init [--yes] [--force] [--skills]
+import { Writable } from "node:stream";
+var INIT_USAGE = `Usage: firegoat init [--yes] [--force] [--skills] [--auth claude|api-key]
 
   --yes, -y     don't ask questions (default when stdin is not a terminal)
   --force, -f   overwrite an existing ${CONFIG_FILE}
-  --skills      copy the firegoat-onboarding skill into .claude/skills/
+  --skills      install the firegoat-onboarding and suggesting-eval-prompts skills
+                into .claude/skills/ (npx skills add team2027/skills)
+  --auth claude   how the agent signs in: run \`firegoat claude login\` (your Claude plan)
+  --auth api-key  store an Anthropic API key for firegoat only (~/.firegoat, mode 0600),
+                  read from stdin when piped, else asked for without echo
+                Without --auth, init asks (unless --yes). In CI, ANTHROPIC_API_KEY in env wins.
 `;
-var SKILL = "firegoat-onboarding";
+var SKILLS = ["firegoat-onboarding", "suggesting-eval-prompts"];
+var SKILLS_ARGV = [
+  "skills",
+  "add",
+  "team2027/skills",
+  ...SKILLS.flatMap((s) => ["--skill", s]),
+  "--agent",
+  "claude-code",
+  "-y"
+];
 var GITIGNORE_LINE = ".firegoat/";
 function readText(file) {
   try {
-    return readFileSync3(file, "utf8");
+    return readFileSync4(file, "utf8");
   } catch {
     return;
   }
@@ -8044,9 +8187,9 @@ function tomlSections(text) {
   return sections;
 }
 function detect(cwd) {
-  const readme = ["README.md", "README", "readme.md", "Readme.md"].find((f) => existsSync4(join6(cwd, f)));
+  const readme = ["README.md", "README", "readme.md", "Readme.md"].find((f) => existsSync5(join7(cwd, f)));
   const fallback = { name: basename2(cwd), library: true, readme };
-  const pkgText = readText(join6(cwd, "package.json"));
+  const pkgText = readText(join7(cwd, "package.json"));
   if (pkgText !== undefined) {
     try {
       const pkg = JSON.parse(pkgText);
@@ -8061,7 +8204,7 @@ function detect(cwd) {
       return { ecosystem: "npm", name, bin, library, readme, docsUrl };
     } catch {}
   }
-  const pyText = readText(join6(cwd, "pyproject.toml"));
+  const pyText = readText(join7(cwd, "pyproject.toml"));
   if (pyText !== undefined) {
     const s = tomlSections(pyText);
     const name = s.get("project")?.get("name") ?? s.get("tool.poetry")?.get("name") ?? fallback.name;
@@ -8071,12 +8214,12 @@ function detect(cwd) {
     const docsUrl = urls?.get("Documentation") ?? urls?.get("documentation") ?? urls?.get("Homepage") ?? urls?.get("homepage");
     return { ecosystem: "python", name, bin, library: true, readme, docsUrl };
   }
-  const goText = readText(join6(cwd, "go.mod"));
+  const goText = readText(join7(cwd, "go.mod"));
   if (goText !== undefined) {
     const name = goText.match(/^module\s+(\S+)/m)?.[1] ?? fallback.name;
     return { ecosystem: "go", name, library: true, readme };
   }
-  const cargoText = readText(join6(cwd, "Cargo.toml"));
+  const cargoText = readText(join7(cwd, "Cargo.toml"));
   if (cargoText !== undefined) {
     const name = tomlSections(cargoText).get("package")?.get("name");
     return {
@@ -8201,10 +8344,10 @@ function renderYaml(p, tests) {
 `;
 }
 function ensureGitignore(cwd) {
-  const file = join6(cwd, ".gitignore");
+  const file = join7(cwd, ".gitignore");
   const text = readText(file);
   if (text === undefined) {
-    writeFileSync2(file, `${GITIGNORE_LINE}
+    writeFileSync3(file, `${GITIGNORE_LINE}
 `);
     return `created .gitignore with ${GITIGNORE_LINE}`;
   }
@@ -8214,12 +8357,83 @@ function ensureGitignore(cwd) {
   const sep = text === "" || text.endsWith(`
 `) ? "" : `
 `;
-  writeFileSync2(file, `${text}${sep}${GITIGNORE_LINE}
+  writeFileSync3(file, `${text}${sep}${GITIGNORE_LINE}
 `);
   return `added ${GITIGNORE_LINE} to .gitignore`;
 }
-function skillSource() {
-  return fileURLToPath(new URL(`../skills/${SKILL}`, import.meta.url));
+function installSkills(cwd) {
+  const r = spawnSync4("npx", ["-y", ...SKILLS_ARGV], {
+    cwd,
+    stdio: "inherit"
+  });
+  if (r.error)
+    return r.error.message;
+  if (r.status !== 0)
+    return `exit ${r.status ?? "by signal"}`;
+  return;
+}
+async function ask(question) {
+  const rl = createInterface2({ input: process.stdin, output: process.stdout });
+  try {
+    return (await rl.question(question)).trim();
+  } finally {
+    rl.close();
+  }
+}
+async function readSecret(question) {
+  if (!process.stdin.isTTY) {
+    const chunks = [];
+    for await (const c of process.stdin)
+      chunks.push(c);
+    return Buffer.concat(chunks).toString("utf8").trim();
+  }
+  process.stdout.write(question);
+  const silent = new Writable({ write: (_c, _e, cb) => cb() });
+  const rl = createInterface2({
+    input: process.stdin,
+    output: silent,
+    terminal: true
+  });
+  try {
+    return (await rl.question("")).trim();
+  } finally {
+    rl.close();
+    process.stdout.write(`
+`);
+  }
+}
+async function chooseAuth() {
+  let current;
+  try {
+    current = resolveAuth({}).label;
+  } catch (e) {
+    current = e.message;
+  }
+  console.log(`How should the agent sign in? (now: ${current})
+  1) Login with Claude (your Claude plan)
+  2) Paste an Anthropic API key`);
+  for (;; ) {
+    const a = await ask("choose [1/2]: ");
+    if (a === "1")
+      return "claude";
+    if (a === "2")
+      return "api-key";
+    console.log("type 1 or 2");
+  }
+}
+async function setUpAuth(method) {
+  if (method === "claude") {
+    const code = claudeLoginCommand([]);
+    return code === 0 ? undefined : code;
+  }
+  const key = await readSecret("Anthropic API key (input hidden): ");
+  if (!looksLikeAnthropicKey(key)) {
+    console.error("firegoat init: that does not look like an Anthropic API key (sk-ant-...); nothing stored");
+    return EXIT.usage;
+  }
+  const file = saveApiKey(key);
+  console.log(`stored the API key in ${file} (mode 0600, not in the repo); evals pass it to the agent as ANTHROPIC_API_KEY`);
+  return;
 }
 async function askYesNo(question) {
   const rl = createInterface2({ input: process.stdin, output: process.stdout });
@@ -8244,7 +8458,18 @@ async function initCommand(args, cwd) {
   let yes = false;
   let force = false;
   let skills = false;
-  for (const a of args) {
+  let auth;
+  for (let i = 0;i < args.length; i++) {
+    const a = args[i];
+    if (a === "--auth" || a.startsWith("--auth=")) {
+      const v = a === "--auth" ? args[++i] : a.slice(7);
+      if (v !== "claude" && v !== "api-key") {
+        console.error(`firegoat init: --auth must be claude or api-key, got ${v ?? "nothing"}`);
+        return EXIT.usage;
+      }
+      auth = v;
+      continue;
+    }
     switch (a) {
       case "--yes":
       case "-y":
@@ -8268,9 +8493,9 @@ ${INIT_USAGE}`);
         return EXIT.usage;
     }
   }
-  const configPath = join6(cwd, CONFIG_FILE);
-  const keepConfig = existsSync4(configPath) && !force;
-  if (keepConfig && !skills) {
+  const configPath = join7(cwd, CONFIG_FILE);
+  const keepConfig = existsSync5(configPath) && !force;
+  if (keepConfig && !skills && !auth) {
     console.error(`firegoat init: ${CONFIG_FILE} already exists; not overwriting. Use --force to replace it.`);
     return EXIT.usage;
   }
@@ -8280,65 +8505,38 @@ ${INIT_USAGE}`);
     const project = detect(cwd);
     const tests = templates(project);
     console.log(`detected: ${describe(project)}`);
-    writeFileSync2(configPath, renderYaml(project, tests));
+    writeFileSync3(configPath, renderYaml(project, tests));
     console.log(`wrote ${CONFIG_FILE} (${tests.length} test${tests.length === 1 ? "" : "s"}: ${tests.map((t) => t.name).join(", ")}); edit the prompts to fit your users`);
   }
   console.log(ensureGitignore(cwd));
   const interactive = !yes && process.stdin.isTTY && process.stdout.isTTY;
+  if (!auth && interactive)
+    auth = await chooseAuth();
+  if (auth) {
+    const failed = await setUpAuth(auth);
+    if (failed !== undefined)
+      return failed;
+  }
   if (!skills && interactive)
-    skills = await askYesNo(`Add the ${SKILL} skill to .claude/skills/ (helps your agent draft better evals)?`);
+    skills = await askYesNo(`Add the ${SKILLS.join(" and ")} skills to .claude/skills/ (help your agent draft better evals)?`);
   if (skills) {
-    const dest = join6(cwd, ".claude", "skills", SKILL);
-    cpSync2(skillSource(), dest, { recursive: true });
-    console.log(`copied ${SKILL} skill to .claude/skills/${SKILL}/`);
+    console.log(`installing skills: npx ${SKILLS_ARGV.join(" ")}`);
+    const failed = installSkills(cwd);
+    if (failed) {
+      console.error(`firegoat init: installing the skills failed (${failed}). Run it yourself: npx ${SKILLS_ARGV.join(" ")}`);
+      return EXIT.notGraded;
+    }
   } else {
-    console.log(`hint: \`npx firegoat init --skills\` adds the ${SKILL} skill to .claude/skills/ (keeps your ${CONFIG_FILE})`);
+    console.log(`hint: \`npx firegoat init --skills\` adds the ${SKILLS.join(" and ")} skills to .claude/skills/ (keeps your ${CONFIG_FILE})`);
   }
   console.log("next: npx firegoat eval");
   return 0;
 }
 
-// src/login.ts
-import { spawnSync as spawnSync3 } from "node:child_process";
-import { mkdirSync as mkdirSync4 } from "node:fs";
-function loginCommand(args) {
-  let claudeBin = process.env.FIREGOAT_CLAUDE_BIN || "claude";
-  for (let i = 0;i < args.length; i++) {
-    const a = args[i];
-    if (a === "--claude-bin" && args[i + 1] !== undefined)
-      claudeBin = args[++i];
-    else if (a.startsWith("--claude-bin="))
-      claudeBin = a.slice(13);
-    else {
-      console.error(`firegoat login: unknown argument: ${a}`);
-      return EXIT.usage;
-    }
-  }
-  const configDir = claudeConfigDir();
-  mkdirSync4(configDir, { recursive: true });
-  const env = {
-    ...process.env,
-    CLAUDE_CONFIG_DIR: configDir
-  };
-  delete env.ANTHROPIC_API_KEY;
-  console.error(`Signing claude in to ${configDir}. Evals run with this login, never your own ~/.claude.`);
-  const r = spawnSync3(claudeBin, ["auth", "login"], { env, stdio: "inherit" });
-  if (r.error) {
-    console.error(`firegoat login: could not run ${claudeBin}: ${r.error.message}`);
-    return EXIT.notGraded;
-  }
-  if (r.status !== 0) {
-    console.error(`firegoat login: claude auth login exited ${r.status}`);
-    return EXIT.notGraded;
-  }
-  console.log("logged in. next: npx firegoat eval");
-  return EXIT.pass;
-}
-
 // src/open.ts
-import { spawnSync as spawnSync4 } from "node:child_process";
-import { existsSync as existsSync5, readdirSync } from "node:fs";
-import { join as join7 } from "node:path";
+import { spawnSync as spawnSync5 } from "node:child_process";
+import { existsSync as existsSync6, readdirSync } from "node:fs";
+import { join as join8 } from "node:path";
 var OPENER = {
   darwin: "open",
   linux: "xdg-open"
@@ -8359,8 +8557,8 @@ function openCommand(args, cwd) {
     console.error("firegoat open: takes at most one run id");
     return EXIT.usage;
   }
-  const base = join7(findConfigDir(cwd), ".firegoat", "runs");
-  const runs = existsSync5(base) ? readdirSync(base).sort() : [];
+  const base = join8(findConfigDir(cwd), ".firegoat", "runs");
+  const runs = existsSync6(base) ? readdirSync(base).sort() : [];
   const id = ids[0] ?? runs.at(-1);
   if (!id) {
     console.error(`firegoat open: no runs in ${base}. Run: npx firegoat eval`);
@@ -8370,12 +8568,12 @@ function openCommand(args, cwd) {
     console.error(`firegoat open: no run ${JSON.stringify(id)} in ${base} (latest: ${runs.at(-1) ?? "none"})`);
     return EXIT.usage;
   }
-  const dir = join7(base, id);
+  const dir = join8(base, id);
   console.log(dir);
   const opener = OPENER[process.platform];
   if (print || !opener)
     return EXIT.pass;
-  const r = spawnSync4(opener, [dir], { stdio: "inherit" });
+  const r = spawnSync5(opener, [dir], { stdio: "inherit" });
   if (r.error || r.status !== 0) {
     console.error(`firegoat open: ${opener} failed (${r.error?.message ?? `exit ${r.status}`}); the run is at the path above`);
     return EXIT.notGraded;
@@ -8387,8 +8585,9 @@ function openCommand(args, cwd) {
 var USAGE = `firegoat ${package_default.version}: local-first AX evals
 
 Usage:
-  firegoat init [--yes] [--force] [--skills]   write firegoat.yaml with starter evals
-  firegoat login [--claude-bin PATH]          sign claude in to firegoat's own config dir (once)
+  firegoat init [--yes] [--force] [--skills] [--auth claude|api-key]
+                                               write firegoat.yaml, choose how the agent signs in
+  firegoat claude login [--claude-bin PATH]   sign claude in to firegoat's own config dir (once)
   firegoat eval [test..] [--runs N] [--var KEY=value].. [--json] [--host] [--claude-bin PATH]
                                                run evals with claude in a sandboxed fresh workdir
   firegoat open [runId] [--print]              open a run's folder (default: the latest)
@@ -8424,8 +8623,14 @@ async function main(argv) {
       return 0;
     case "init":
       return initCommand(rest, process.cwd());
+    case "claude":
+      if (rest[0] === "login")
+        return claudeLoginCommand(rest.slice(1));
+      console.error(`usage: firegoat claude login [--claude-bin PATH]`);
+      return EXIT.usage;
     case "login":
-      return loginCommand(rest);
+      console.error("firegoat login is deprecated for the Claude sign-in: use `firegoat claude login`");
+      return claudeLoginCommand(rest);
     case "eval":
       return evalCommand(rest, process.cwd());
     case "open":

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.4.0",
+	"version": "0.6.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/team2027/firegoat-cli.git"
@@ -21,8 +21,7 @@
 		"node": ">=18"
 	},
 	"files": [
-		"dist",
-		"skills"
+		"dist"
 	],
 	"homepage": "https://github.com/team2027/firegoat-cli#readme",
 	"license": "MIT",

package/skills/firegoat-onboarding/SKILL.md

@@ -1,115 +0,0 @@
----
-name: firegoat-onboarding
-description: Draft 2-3 firegoat evals (firegoat.yaml) for the current repo by reading its README, docs and package manifests, phrasing each test as a real user's problem with a gradeable goal and script checks. Use when the user says "write firegoat evals", "draft evals", "set up firegoat", "firegoat init", or asks for agent-experience (AX) evals of their SDK, CLI, API or docs.
----
-
-# Draft firegoat evals
-
-firegoat runs a coding agent (Claude Code) in an empty, isolated workdir with one prompt,
-then grades the run. Your job: write 2-3 tests into `firegoat.yaml` that tell the
-maintainer whether an agent can succeed with their product the way a new user would.
-
-## 1. Learn the product (read, don't run)
-
-Read, in this order, and stop when you can answer the questions below:
-- `README*`, then the docs dir (`docs/`, `website/`, `*.mdx`), quickstart/getting-started pages first.
-- Package manifests: `package.json` (`name`, `bin`, `exports`), `pyproject.toml`, `go.mod`,
-  `Cargo.toml`, `*.gemspec`, `Package.swift`. They tell you the install name and entry points.
-- `examples/`, `templates/`, `samples/`: the shapes users actually build.
-- An existing `firegoat.yaml`, if any: keep its tests (see step 5).
-
-Answer: What is it (SDK, CLI, API, framework)? Who uses it? What is the first thing a new
-user wants to have working? What is the public docs URL? Which credentials does it need
-(env var names only)?
-
-## 2. Pick 2-3 outcomes
-
-Choose the most important first-time user outcomes, e.g. "first successful API call",
-"the core feature end to end", "the thing users most often ask about". Prefer outcomes
-that leave observable proof. Skip admin, billing and anything needing a browser login
-unless credentials can come from env vars.
-
-## 3. Write each prompt as a user's problem
-
-A prompt is what a user would type to their agent, not a recipe. Each prompt has:
-- **What they want, not how.** No numbered implementation steps, no "use template X, add package Y".
-  The eval measures whether the agent finds its way with your docs.
-- **One concrete outcome**, small enough to finish in a few minutes.
-- **Observable proof**: print a marker to stdout, write a file, or take a screenshot.
-- **"Keep it minimal."** so the agent doesn't build a demo app.
-- **The docs URL as the only hint** (omit it if the test is about discovery).
-- **Credentials as env var names**, never values: "Credentials are in env vars ACME_API_KEY and ACME_API_URL."
-- If the test should use the build under test, say so: "The package is already installed
-  from the build under test. Do not install or upgrade it from the registry."
-
-## 4. Write the goal and checks
-
-`goal`: one sentence a judge could check from the transcript and workdir, e.g. "A minimal
-project runs and prints the answer, a numeric score and a reason."
-
-`checks`: shell scripts, run in the agent's workdir after the run. Exit code 0 = pass.
-Env: `FIREGOAT_WORKDIR` (the workdir) and `FIREGOAT_SESSION` (the agent's stream-json
-transcript, one JSON event per line). Checks should prove the outcome and catch cheating:
-- Output exists with the expected marker: `grep -rq hello-from-acme "$FIREGOAT_WORKDIR"`
-  or `test -s "$FIREGOAT_WORKDIR/screenshot.png"`.
-- The real thing was used, not faked: the transcript shows the SDK/CLI being called,
-  e.g. `grep -q '"command":"[^"]*acme ' "$FIREGOAT_SESSION"`.
-- Cleanup happened: the transcript shows the delete call after the create call.
-- The local build was used, not a registry install:
-  `! grep -Eq '"command":"[^"]*(pip install|npm (install|i) [^"]*acme@)' "$FIREGOAT_SESSION"`.
-- Only local docs were used (when that's the point): `! grep -q '"name":"WebFetch"' "$FIREGOAT_SESSION"`.
-
-Match tool-call JSON (`"command":"...`) rather than plain words, so the prompt text or the
-agent saying "I won't pip install" doesn't trip the check. Checks run in the agent's
-workdir, which does not contain the repo, so keep them self-contained: one-liners, or a
-`run: |` block for longer scripts. Never reference repo files by relative path.
-
-Do not emit `judge:` checks: v1 skips them with a warning.
-
-## 5. Write firegoat.yaml
-
-Schema (v1):
-
-```yaml
-agents: [claude-code]      # only claude-code in v1
-runs: 1                    # runs per test, positive integer
-setup: npm pack --pack-destination "$FIREGOAT_OUT"   # optional: once per eval, in the repo;
-                           # $FIREGOAT_OUT is copied into each run and put on PATH
-workdir_setup: npm init -y >/dev/null && npm install "$FIREGOAT_OUT"/*.tgz
-                           # optional: in each fresh workdir before the agent starts
-vars:                      # optional: defaults for {{vars.KEY}}; `--var KEY=value` overrides
-  SHA: main
-tests:
-  - name: quickstart       # letters, digits, '.', '_', '-'; unique
-    prompt: >-
-      ...
-    goal: ...
-    checks:                # optional
-      - run: grep -rq some-marker "$FIREGOAT_WORKDIR"
-    timeout: 600           # optional, seconds
-```
-
-Add `setup:` / `workdir_setup:` when the tests should use the repo's local build (python:
-`uv build --wheel --out-dir "$FIREGOAT_OUT"` and `uv venv && uv pip install "$FIREGOAT_OUT"/*.whl`);
-then the prompt can say "The package is already installed from the build under test."
-
-If `firegoat.yaml` exists, merge: keep every existing test and top-level key untouched, append
-yours, and rename yours (`-2` suffix) if a name collides. Never delete or rewrite user tests.
-Quote prompts with `>-` folded blocks so colons and quotes stay valid YAML.
-
-Before finishing, re-read the file and check: it parses as YAML, every test has a non-empty
-`name`, `prompt` and `goal`, names are unique and match `^[A-Za-z0-9._-]+$`, and no credential
-values appear anywhere.
-
-## 6. Hand off
-
-Tell the user, in a few lines: which tests you added and why, which env vars they must export,
-and to run `npx firegoat eval`.
-
-## Examples
-
-See `references/examples.md` for good and bad tests with their YAML. Short version:
-
-Good: "Create a new app - a coffee timer. Run it on the simulator and get the screenshot."
-Bad: "Create a sandbox. https://docs.acme.dev" with no goal and no checks: every run passes,
-so it measures nothing.

package/skills/firegoat-onboarding/references/examples.md

@@ -1,114 +0,0 @@
-# Example firegoat tests
-
-Product names are fictional. Copy the shape, not the words.
-
-## Good
-
-### 1. Core feature end to end, printed proof
-
-An agent framework with scorers. The prompt names the outcome and what to print; the agent
-has to find how in the docs.
-
-```yaml
-tests:
-  - name: agent-with-scorer
-    prompt: >-
-      Build an agent with a scorer using https://docs.acme.dev/agents. Ask the agent
-      to explain recursion in one sentence, then run a scorer that grades whether the
-      answer is concise and accurate. Print the agent answer, the numeric score and
-      the scorer reason to stdout. Keep it minimal.
-    goal: >-
-      A minimal project runs and prints the agent's answer, a numeric score and the
-      scorer's reason.
-    checks:
-      - run: grep -Eq '"command":"[^"]*(node|npx|tsx|bun) ' "$FIREGOAT_SESSION"
-```
-
-### 2. Multi-step workflow, one realistic input
-
-```yaml
-tests:
-  - name: support-workflow
-    prompt: >-
-      Using https://docs.acme.dev/workflows, build a three-step workflow: extract the
-      issue type from a support message, have an agent draft a response, then add a
-      priority label. Run it once on a customer asking for a refund and print the
-      final result. Keep it minimal.
-    goal: >-
-      The workflow runs once on a refund request and prints a result containing the
-      issue type, a drafted response and a priority label.
-```
-
-### 3. Local build under test, cleanup, no registry install
-
-```yaml
-tests:
-  - name: sandbox-hello
-    prompt: >-
-      Using the Acme Python SDK, create a sandbox, run `echo hello-from-acme` inside
-      it, print the output, then delete the sandbox. The package is already installed
-      from the build under test. Do not install or upgrade it from PyPI.
-      Docs: https://docs.acme.dev/python. Credentials are in env vars ACME_API_KEY
-      and ACME_API_URL.
-    goal: >-
-      The script prints hello-from-acme from inside a sandbox and deletes the sandbox
-      at the end.
-    checks:
-      - run: grep -Eq '\.(delete|remove)\(' -r "$FIREGOAT_WORKDIR" --include='*.py'
-      - run: "! grep -Eq '\"command\":\"[^\"]*pip3? install[^\"]*acme' \"$FIREGOAT_SESSION\""
-    timeout: 900
-```
-
-### 4. Pure user need, discovery
-
-No docs URL, no package names: the test is whether the agent finds the right packages.
-
-```yaml
-tests:
-  - name: which-packages
-    prompt: >-
-      I'm building a mobile app with Acme. It needs a local database, maps, age
-      gating, localization, Sign in with Apple and a small on-device model. Does Acme
-      offer packages for these? Which ones would you use?
-    goal: >-
-      The answer names the right Acme package for each requirement and relies on the
-      docs shipped in the project, not web search.
-    checks:
-      - run: "! grep -q '\"name\":\"WebFetch\"' \"$FIREGOAT_SESSION\""
-```
-
-### 5. Shortest
-
-```yaml
-tests:
-  - name: coffee-timer
-    prompt: Create a new app - a coffee timer. Run it on the simulator and get the screenshot.
-    goal: A screenshot shows a working coffee timer app running on the simulator.
-    checks:
-      - run: find "$FIREGOAT_WORKDIR" -name '*.png' -size +1k | grep -q .
-```
-
-## Bad
-
-### No goal, no checks
-
-```text
-prompt: "Create a sandbox. https://docs.acme.dev"
-```
-
-Ungradeable: nothing says what success looks like, so every run "succeeds". Add an
-observable outcome ("run echo hello-from-acme inside it and print the output"), a goal
-and a check.
-
-### Implementation steps
-
-```text
-prompt: >-
-  1. Create a project from the acme-starter template.
-  2. Add the acme-maps package.
-  3. Render a map centered on Berlin.
-```
-
-This tests instruction-following, not whether an agent finds its way with your docs.
-Rewrite as the user's need: "Show a map centered on Berlin in a new Acme app. Keep it
-minimal. Take a screenshot."