firegoat 0.9.1 → 0.9.3

package/README.md

@@ -47,7 +47,7 @@ tests:
       - run: grep -q '"command":"acme deploy' "$FIREGOAT_SESSION"
 ```
 
-Whatever `setup` writes to `$FIREGOAT_OUT` is first on the agent's `PATH`.
+Whatever `setup` writes to `$FIREGOAT_OUT` is first on the agent's `PATH`. That only helps for binaries called by bare name: if users run your CLI via `npx`/`uvx`, pack and install it like the npm example, or `npx` fetches the registry release.
 
 ### Can an agent find the right package from my docs alone?
 
@@ -85,7 +85,7 @@ npx firegoat eval --var SHA=$(git rev-parse HEAD)
 ## What eval does
 
 - claude runs on its own login (or `ANTHROPIC_API_KEY`, if set) with `CLAUDE_CONFIG_DIR=.firegoat/.claude` and `--setting-sources "" --strict-mcp-config --disable-slash-commands`: none of your settings, CLAUDE.md, plugins, skills or MCP servers. claude saves its session there, in its own format.
-- No sandbox: the agent's folder is inside `.firegoat/runs/`, and it can reach the rest of your machine. `.firegoat/` ignores itself in git.
+- No sandbox: the agent's folder is inside `.firegoat/runs/`, and it can reach the rest of your machine. In `.firegoat/`, git ignores `runs/` and `.claude/`; keep test fixtures (scripts, sample data) in `.firegoat/fixtures/` and commit them with `firegoat.yaml`.
 - Script checks run in the agent's folder afterwards and set the exit code. `end_goal` is not graded by firegoat: it's for you, or your coding agent, to judge from `run.json`, the session and the folder.
 - Output per run, in `.firegoat/runs/<runId>/<test>/<n>/`: `run.json` (status, prompt, end_goal, check results, paths), `work/`, logs. `summary.json` covers the whole eval; `--json` prints it.
 

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.9.1",
+  version: "0.9.3",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -7167,7 +7167,7 @@ function loadConfig(dir) {
 import { randomBytes } from "node:crypto";
 import {
   createWriteStream as createWriteStream2,
-  existsSync as existsSync3,
+  existsSync as existsSync2,
   mkdirSync as mkdirSync2,
   readdirSync,
   writeFileSync as writeFileSync2
@@ -7328,15 +7328,15 @@ function checkLogin(opts) {
 var EXIT = { pass: 0, fail: 1, notGraded: 2, usage: 3 };
 
 // src/firegoat-dir.ts
-import { existsSync as existsSync2, mkdirSync, writeFileSync } from "node:fs";
+import { mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
+var GITIGNORE = `runs/
+.claude/
+`;
 function ensureFiregoatDir(root) {
   const dir = join(root, ".firegoat");
   mkdirSync(dir, { recursive: true });
-  const ignore = join(dir, ".gitignore");
-  if (!existsSync2(ignore))
-    writeFileSync(ignore, `*
-`);
+  writeFileSync(join(dir, ".gitignore"), GITIGNORE);
   return dir;
 }
 
@@ -7367,10 +7367,11 @@ ${EXIT_CODES}`;
 }
 var INIT_HELP = `Usage: firegoat init [--force] [--no-skills]
 
-In the current folder: creates .firegoat/ (ignored by git through its own .gitignore)
-and a starter firegoat.yaml, installs the firegoat-onboarding and
-suggesting-eval-prompts skills into .claude/skills/ (npx skills add firegoat-ai/skills;
-asks first in a terminal), then prints a prompt to paste into your coding agent.
+In the current folder: creates .firegoat/ and a starter firegoat.yaml, installs the
+firegoat-onboarding and suggesting-eval-prompts skills into .claude/skills/ (npx skills
+add firegoat-ai/skills; asks first in a terminal), then prints a prompt to paste into
+your coding agent. In .firegoat/, git ignores runs/ and .claude/; keep test fixtures
+(helper scripts, sample data) in .firegoat/fixtures/ and commit them.
 
   --force       replace an existing firegoat.yaml
   --no-skills   don't install the skills
@@ -7391,7 +7392,8 @@ firegoat.yaml reference:
         - run: ...            afterwards; exit 0 = pass; they set eval's exit code
       timeout: 600            optional, seconds
 
-  Unknown keys are errors. setup, workdir_setup and checks get the vars as env vars,
+  Unknown keys are errors. Keep helper files out of firegoat.yaml: put them in
+  .firegoat/fixtures/ and copy them in setup or workdir_setup. setup, workdir_setup and checks get the vars as env vars,
   plus $FIREGOAT_OUT and $FIREGOAT_WORKDIR; checks also get $FIREGOAT_SESSION (the
   claude session file, one JSON event per line).
 
@@ -7546,15 +7548,24 @@ function parseArgs(args) {
 }
 function findSession(configDir, sessionId) {
   const projects = join3(configDir, "projects");
-  if (!existsSync3(projects))
+  if (!existsSync2(projects))
     return null;
   for (const d of readdirSync(projects)) {
     const f = join3(projects, d, `${sessionId}.jsonl`);
-    if (existsSync3(f))
+    if (existsSync2(f))
       return f;
   }
   return null;
 }
+function usage(r) {
+  const parts = [];
+  if (r.cost !== null)
+    parts.push(`$${r.cost.toFixed(2)}`);
+  if (r.num_turns !== null)
+    parts.push(`${r.num_turns} turns`);
+  parts.push(`${Math.round(r.duration_ms / 1000)}s`);
+  return parts.join(", ");
+}
 function newRunId() {
   const ts = new Date().toISOString().replace(/\.\d+Z$/, "Z").replace(/[-:]/g, "");
   return `${ts}-${randomBytes(2).toString("hex")}`;
@@ -7688,6 +7699,7 @@ async function evalCommand(args, root) {
   const firegoatDir = ensureFiregoatDir(root);
   const configDir = join3(firegoatDir, ".claude");
   mkdirSync2(configDir, { recursive: true });
+  say(process.env.ANTHROPIC_API_KEY ? "auth: ANTHROPIC_API_KEY (claude bills the API key; unset it to use your claude login)" : "auth: your claude login");
   const loginProblem = checkLogin({
     bin: claudePath,
     env: claudeEnv(configDir),
@@ -7731,7 +7743,7 @@ async function evalCommand(args, root) {
         vars
       });
       results.push(r);
-      say(`${r.status.toUpperCase()} ${r.test}#${r.n} ${r.reason}`);
+      say(`${r.status.toUpperCase()} ${r.test}#${r.n} ${r.reason} (${usage(r)})`);
     }
   const totals = { pass: 0, fail: 0, error: 0 };
   for (const r of results)
@@ -7739,8 +7751,9 @@ async function evalCommand(args, root) {
   const doc = { runId, runsDir: runDir, totals, runs: results };
   writeFileSync2(join3(runDir, "summary.json"), `${JSON.stringify(doc, null, 2)}
 `);
+  const cost = results.reduce((sum, r) => sum + (r.cost ?? 0), 0);
   say(`
-${totals.pass} passed, ${totals.fail} failed, ${totals.error} not graded (${results.length} runs)`);
+${totals.pass} passed, ${totals.fail} failed, ${totals.error} not graded (${results.length} runs, $${cost.toFixed(2)})`);
   say(`runs: ${runDir} (end_goal is for you to judge: see run.json per run)`);
   if (opts.json)
     console.log(JSON.stringify(doc, null, 2));
@@ -7749,7 +7762,7 @@ ${totals.pass} passed, ${totals.fail} failed, ${totals.error} not graded (${resu
 
 // src/init.ts
 import { spawnSync as spawnSync2 } from "node:child_process";
-import { existsSync as existsSync4, writeFileSync as writeFileSync3 } from "node:fs";
+import { existsSync as existsSync3, writeFileSync as writeFileSync3 } from "node:fs";
 import { basename, join as join4 } from "node:path";
 import { createInterface } from "node:readline/promises";
 var SKILLS_ARGV = [
@@ -7785,8 +7798,14 @@ runs: 1
 # setup: uv build --wheel --out-dir "$FIREGOAT_OUT"
 # workdir_setup: uv venv && uv pip install "$FIREGOAT_OUT"/*.whl
 #
-# a CLI binary:
+# a CLI binary users call by bare name (on PATH as is):
 # setup: go build -o "$FIREGOAT_OUT/mycli" ./cmd/mycli
+# (an npm/pypi CLI that users run via npx/uvx: pack + install it as above instead;
+# a binary on PATH does not stop "npx mycli" from fetching the registry release)
+#
+# Helper files (scripts, sample data) go in .firegoat/fixtures/ (committed), not inline here.
+# setup runs in this folder, so: setup: cp -R .firegoat/fixtures/. "$FIREGOAT_OUT"
+# then e.g. workdir_setup: cp "$FIREGOAT_OUT/wizard.mjs" .
 
 tests:
   - name: quickstart
@@ -7849,7 +7868,7 @@ ${INIT_HELP}`);
   }
   ensureFiregoatDir(cwd);
   const configPath = join4(cwd, CONFIG_FILE);
-  if (existsSync4(configPath) && !force)
+  if (existsSync3(configPath) && !force)
     console.log(`kept ${CONFIG_FILE} (--force replaces it)`);
   else {
     writeFileSync3(configPath, template(basename(cwd)));
@@ -7865,18 +7884,28 @@ ${INIT_HELP}`);
     const r = spawnSync2("npx", ["-y", ...SKILLS_ARGV], {
       cwd,
       env,
-      stdio: "inherit"
+      stdio: interactive ? "inherit" : "pipe",
+      encoding: "utf8"
     });
     if (r.error || r.status !== 0) {
+      if (!interactive)
+        process.stderr.write(`${r.stdout ?? ""}${r.stderr ?? ""}`);
       console.error(`firegoat init: installing the skills failed (${r.error?.message ?? `exit ${r.status}`}). Run it yourself: npx ${SKILLS_ARGV.join(" ")}`);
       return EXIT.notGraded;
     }
+    console.log("installed skills firegoat-onboarding, suggesting-eval-prompts in .claude/skills/");
+    const ignored = spawnSync2("git", ["check-ignore", "-q", ".claude/skills"], {
+      cwd
+    });
+    if (ignored.status === 0)
+      console.error("warning: git ignores .claude/skills, so teammates and CI won't get the skills. Commit them with: git add -f .claude/skills");
   }
-  console.log(`
+  console.log(interactive ? `
 Paste this into your coding agent:
 
 ${AGENT_PROMPT}
-`);
+` : `
+Prompt for your coding agent: ${AGENT_PROMPT}`);
   if (interactive && copyToClipboard(AGENT_PROMPT))
     console.log("\x1B[32mPrompt copied!\x1B[0m");
   return EXIT.pass;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.9.1",
+	"version": "0.9.3",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/firegoat-ai/firegoat-cli.git"