firegoat 0.4.0 → 0.5.0

package/README.md

@@ -7,7 +7,7 @@ Status: early (v0.4). See `plan.md`.
 ## Use
 
 ```bash
-npx firegoat init            # writes firegoat.yaml with starter evals; --skills adds the onboarding skill
+npx firegoat init            # writes firegoat.yaml with starter evals; --skills adds the firegoat-onboarding skill (from team2027/skills)
 npx firegoat login           # once per machine: signs claude in to ~/.firegoat/claude
 npx firegoat eval            # runs each test with your local `claude`, sandboxed, in a fresh folder
 npx firegoat open            # opens the latest run's folder (--print to only print it)

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.4.0",
+  version: "0.5.0",
   repository: {
     type: "git",
     url: "git+https://github.com/team2027/firegoat-cli.git"
@@ -6968,8 +6968,7 @@ var package_default = {
     node: ">=18"
   },
   files: [
-    "dist",
-    "skills"
+    "dist"
   ],
   homepage: "https://github.com/team2027/firegoat-cli#readme",
   license: "MIT",
@@ -8004,17 +8003,27 @@ ${pass} passed, ${fail} failed, ${error} not graded (${results.length} runs)`);
 }
 
 // src/init.ts
-import { cpSync as cpSync2, existsSync as existsSync4, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
+import { spawnSync as spawnSync3 } from "node:child_process";
+import { existsSync as existsSync4, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "node:fs";
 import { basename as basename2, join as join6 } from "node:path";
 import { createInterface as createInterface2 } from "node:readline/promises";
-import { fileURLToPath } from "node:url";
 var INIT_USAGE = `Usage: firegoat init [--yes] [--force] [--skills]
 
   --yes, -y     don't ask questions (default when stdin is not a terminal)
   --force, -f   overwrite an existing ${CONFIG_FILE}
-  --skills      copy the firegoat-onboarding skill into .claude/skills/
+  --skills      install the firegoat-onboarding and suggesting-eval-prompts skills
+                into .claude/skills/ (npx skills add team2027/skills)
 `;
-var SKILL = "firegoat-onboarding";
+var SKILLS = ["firegoat-onboarding", "suggesting-eval-prompts"];
+var SKILLS_ARGV = [
+  "skills",
+  "add",
+  "team2027/skills",
+  ...SKILLS.flatMap((s) => ["--skill", s]),
+  "--agent",
+  "claude-code",
+  "-y"
+];
 var GITIGNORE_LINE = ".firegoat/";
 function readText(file) {
   try {
@@ -8218,8 +8227,16 @@ function ensureGitignore(cwd) {
 `);
   return `added ${GITIGNORE_LINE} to .gitignore`;
 }
-function skillSource() {
-  return fileURLToPath(new URL(`../skills/${SKILL}`, import.meta.url));
+function installSkills(cwd) {
+  const r = spawnSync3("npx", ["-y", ...SKILLS_ARGV], {
+    cwd,
+    stdio: "inherit"
+  });
+  if (r.error)
+    return r.error.message;
+  if (r.status !== 0)
+    return `exit ${r.status ?? "by signal"}`;
+  return;
 }
 async function askYesNo(question) {
   const rl = createInterface2({ input: process.stdin, output: process.stdout });
@@ -8286,20 +8303,23 @@ ${INIT_USAGE}`);
   console.log(ensureGitignore(cwd));
   const interactive = !yes && process.stdin.isTTY && process.stdout.isTTY;
   if (!skills && interactive)
-    skills = await askYesNo(`Add the ${SKILL} skill to .claude/skills/ (helps your agent draft better evals)?`);
+    skills = await askYesNo(`Add the ${SKILLS.join(" and ")} skills to .claude/skills/ (help your agent draft better evals)?`);
   if (skills) {
-    const dest = join6(cwd, ".claude", "skills", SKILL);
-    cpSync2(skillSource(), dest, { recursive: true });
-    console.log(`copied ${SKILL} skill to .claude/skills/${SKILL}/`);
+    console.log(`installing skills: npx ${SKILLS_ARGV.join(" ")}`);
+    const failed = installSkills(cwd);
+    if (failed) {
+      console.error(`firegoat init: installing the skills failed (${failed}). Run it yourself: npx ${SKILLS_ARGV.join(" ")}`);
+      return EXIT.notGraded;
+    }
   } else {
-    console.log(`hint: \`npx firegoat init --skills\` adds the ${SKILL} skill to .claude/skills/ (keeps your ${CONFIG_FILE})`);
+    console.log(`hint: \`npx firegoat init --skills\` adds the ${SKILLS.join(" and ")} skills to .claude/skills/ (keeps your ${CONFIG_FILE})`);
   }
   console.log("next: npx firegoat eval");
   return 0;
 }
 
 // src/login.ts
-import { spawnSync as spawnSync3 } from "node:child_process";
+import { spawnSync as spawnSync4 } from "node:child_process";
 import { mkdirSync as mkdirSync4 } from "node:fs";
 function loginCommand(args) {
   let claudeBin = process.env.FIREGOAT_CLAUDE_BIN || "claude";
@@ -8322,7 +8342,7 @@ function loginCommand(args) {
   };
   delete env.ANTHROPIC_API_KEY;
   console.error(`Signing claude in to ${configDir}. Evals run with this login, never your own ~/.claude.`);
-  const r = spawnSync3(claudeBin, ["auth", "login"], { env, stdio: "inherit" });
+  const r = spawnSync4(claudeBin, ["auth", "login"], { env, stdio: "inherit" });
   if (r.error) {
     console.error(`firegoat login: could not run ${claudeBin}: ${r.error.message}`);
     return EXIT.notGraded;
@@ -8336,7 +8356,7 @@ function loginCommand(args) {
 }
 
 // src/open.ts
-import { spawnSync as spawnSync4 } from "node:child_process";
+import { spawnSync as spawnSync5 } from "node:child_process";
 import { existsSync as existsSync5, readdirSync } from "node:fs";
 import { join as join7 } from "node:path";
 var OPENER = {
@@ -8375,7 +8395,7 @@ function openCommand(args, cwd) {
   const opener = OPENER[process.platform];
   if (print || !opener)
     return EXIT.pass;
-  const r = spawnSync4(opener, [dir], { stdio: "inherit" });
+  const r = spawnSync5(opener, [dir], { stdio: "inherit" });
   if (r.error || r.status !== 0) {
     console.error(`firegoat open: ${opener} failed (${r.error?.message ?? `exit ${r.status}`}); the run is at the path above`);
     return EXIT.notGraded;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.4.0",
+	"version": "0.5.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/team2027/firegoat-cli.git"
@@ -21,8 +21,7 @@
 		"node": ">=18"
 	},
 	"files": [
-		"dist",
-		"skills"
+		"dist"
 	],
 	"homepage": "https://github.com/team2027/firegoat-cli#readme",
 	"license": "MIT",

package/skills/firegoat-onboarding/SKILL.md

@@ -1,115 +0,0 @@
----
-name: firegoat-onboarding
-description: Draft 2-3 firegoat evals (firegoat.yaml) for the current repo by reading its README, docs and package manifests, phrasing each test as a real user's problem with a gradeable goal and script checks. Use when the user says "write firegoat evals", "draft evals", "set up firegoat", "firegoat init", or asks for agent-experience (AX) evals of their SDK, CLI, API or docs.
----
-
-# Draft firegoat evals
-
-firegoat runs a coding agent (Claude Code) in an empty, isolated workdir with one prompt,
-then grades the run. Your job: write 2-3 tests into `firegoat.yaml` that tell the
-maintainer whether an agent can succeed with their product the way a new user would.
-
-## 1. Learn the product (read, don't run)
-
-Read, in this order, and stop when you can answer the questions below:
-- `README*`, then the docs dir (`docs/`, `website/`, `*.mdx`), quickstart/getting-started pages first.
-- Package manifests: `package.json` (`name`, `bin`, `exports`), `pyproject.toml`, `go.mod`,
-  `Cargo.toml`, `*.gemspec`, `Package.swift`. They tell you the install name and entry points.
-- `examples/`, `templates/`, `samples/`: the shapes users actually build.
-- An existing `firegoat.yaml`, if any: keep its tests (see step 5).
-
-Answer: What is it (SDK, CLI, API, framework)? Who uses it? What is the first thing a new
-user wants to have working? What is the public docs URL? Which credentials does it need
-(env var names only)?
-
-## 2. Pick 2-3 outcomes
-
-Choose the most important first-time user outcomes, e.g. "first successful API call",
-"the core feature end to end", "the thing users most often ask about". Prefer outcomes
-that leave observable proof. Skip admin, billing and anything needing a browser login
-unless credentials can come from env vars.
-
-## 3. Write each prompt as a user's problem
-
-A prompt is what a user would type to their agent, not a recipe. Each prompt has:
-- **What they want, not how.** No numbered implementation steps, no "use template X, add package Y".
-  The eval measures whether the agent finds its way with your docs.
-- **One concrete outcome**, small enough to finish in a few minutes.
-- **Observable proof**: print a marker to stdout, write a file, or take a screenshot.
-- **"Keep it minimal."** so the agent doesn't build a demo app.
-- **The docs URL as the only hint** (omit it if the test is about discovery).
-- **Credentials as env var names**, never values: "Credentials are in env vars ACME_API_KEY and ACME_API_URL."
-- If the test should use the build under test, say so: "The package is already installed
-  from the build under test. Do not install or upgrade it from the registry."
-
-## 4. Write the goal and checks
-
-`goal`: one sentence a judge could check from the transcript and workdir, e.g. "A minimal
-project runs and prints the answer, a numeric score and a reason."
-
-`checks`: shell scripts, run in the agent's workdir after the run. Exit code 0 = pass.
-Env: `FIREGOAT_WORKDIR` (the workdir) and `FIREGOAT_SESSION` (the agent's stream-json
-transcript, one JSON event per line). Checks should prove the outcome and catch cheating:
-- Output exists with the expected marker: `grep -rq hello-from-acme "$FIREGOAT_WORKDIR"`
-  or `test -s "$FIREGOAT_WORKDIR/screenshot.png"`.
-- The real thing was used, not faked: the transcript shows the SDK/CLI being called,
-  e.g. `grep -q '"command":"[^"]*acme ' "$FIREGOAT_SESSION"`.
-- Cleanup happened: the transcript shows the delete call after the create call.
-- The local build was used, not a registry install:
-  `! grep -Eq '"command":"[^"]*(pip install|npm (install|i) [^"]*acme@)' "$FIREGOAT_SESSION"`.
-- Only local docs were used (when that's the point): `! grep -q '"name":"WebFetch"' "$FIREGOAT_SESSION"`.
-
-Match tool-call JSON (`"command":"...`) rather than plain words, so the prompt text or the
-agent saying "I won't pip install" doesn't trip the check. Checks run in the agent's
-workdir, which does not contain the repo, so keep them self-contained: one-liners, or a
-`run: |` block for longer scripts. Never reference repo files by relative path.
-
-Do not emit `judge:` checks: v1 skips them with a warning.
-
-## 5. Write firegoat.yaml
-
-Schema (v1):
-
-```yaml
-agents: [claude-code]      # only claude-code in v1
-runs: 1                    # runs per test, positive integer
-setup: npm pack --pack-destination "$FIREGOAT_OUT"   # optional: once per eval, in the repo;
-                           # $FIREGOAT_OUT is copied into each run and put on PATH
-workdir_setup: npm init -y >/dev/null && npm install "$FIREGOAT_OUT"/*.tgz
-                           # optional: in each fresh workdir before the agent starts
-vars:                      # optional: defaults for {{vars.KEY}}; `--var KEY=value` overrides
-  SHA: main
-tests:
-  - name: quickstart       # letters, digits, '.', '_', '-'; unique
-    prompt: >-
-      ...
-    goal: ...
-    checks:                # optional
-      - run: grep -rq some-marker "$FIREGOAT_WORKDIR"
-    timeout: 600           # optional, seconds
-```
-
-Add `setup:` / `workdir_setup:` when the tests should use the repo's local build (python:
-`uv build --wheel --out-dir "$FIREGOAT_OUT"` and `uv venv && uv pip install "$FIREGOAT_OUT"/*.whl`);
-then the prompt can say "The package is already installed from the build under test."
-
-If `firegoat.yaml` exists, merge: keep every existing test and top-level key untouched, append
-yours, and rename yours (`-2` suffix) if a name collides. Never delete or rewrite user tests.
-Quote prompts with `>-` folded blocks so colons and quotes stay valid YAML.
-
-Before finishing, re-read the file and check: it parses as YAML, every test has a non-empty
-`name`, `prompt` and `goal`, names are unique and match `^[A-Za-z0-9._-]+$`, and no credential
-values appear anywhere.
-
-## 6. Hand off
-
-Tell the user, in a few lines: which tests you added and why, which env vars they must export,
-and to run `npx firegoat eval`.
-
-## Examples
-
-See `references/examples.md` for good and bad tests with their YAML. Short version:
-
-Good: "Create a new app - a coffee timer. Run it on the simulator and get the screenshot."
-Bad: "Create a sandbox. https://docs.acme.dev" with no goal and no checks: every run passes,
-so it measures nothing.

package/skills/firegoat-onboarding/references/examples.md

@@ -1,114 +0,0 @@
-# Example firegoat tests
-
-Product names are fictional. Copy the shape, not the words.
-
-## Good
-
-### 1. Core feature end to end, printed proof
-
-An agent framework with scorers. The prompt names the outcome and what to print; the agent
-has to find how in the docs.
-
-```yaml
-tests:
-  - name: agent-with-scorer
-    prompt: >-
-      Build an agent with a scorer using https://docs.acme.dev/agents. Ask the agent
-      to explain recursion in one sentence, then run a scorer that grades whether the
-      answer is concise and accurate. Print the agent answer, the numeric score and
-      the scorer reason to stdout. Keep it minimal.
-    goal: >-
-      A minimal project runs and prints the agent's answer, a numeric score and the
-      scorer's reason.
-    checks:
-      - run: grep -Eq '"command":"[^"]*(node|npx|tsx|bun) ' "$FIREGOAT_SESSION"
-```
-
-### 2. Multi-step workflow, one realistic input
-
-```yaml
-tests:
-  - name: support-workflow
-    prompt: >-
-      Using https://docs.acme.dev/workflows, build a three-step workflow: extract the
-      issue type from a support message, have an agent draft a response, then add a
-      priority label. Run it once on a customer asking for a refund and print the
-      final result. Keep it minimal.
-    goal: >-
-      The workflow runs once on a refund request and prints a result containing the
-      issue type, a drafted response and a priority label.
-```
-
-### 3. Local build under test, cleanup, no registry install
-
-```yaml
-tests:
-  - name: sandbox-hello
-    prompt: >-
-      Using the Acme Python SDK, create a sandbox, run `echo hello-from-acme` inside
-      it, print the output, then delete the sandbox. The package is already installed
-      from the build under test. Do not install or upgrade it from PyPI.
-      Docs: https://docs.acme.dev/python. Credentials are in env vars ACME_API_KEY
-      and ACME_API_URL.
-    goal: >-
-      The script prints hello-from-acme from inside a sandbox and deletes the sandbox
-      at the end.
-    checks:
-      - run: grep -Eq '\.(delete|remove)\(' -r "$FIREGOAT_WORKDIR" --include='*.py'
-      - run: "! grep -Eq '\"command\":\"[^\"]*pip3? install[^\"]*acme' \"$FIREGOAT_SESSION\""
-    timeout: 900
-```
-
-### 4. Pure user need, discovery
-
-No docs URL, no package names: the test is whether the agent finds the right packages.
-
-```yaml
-tests:
-  - name: which-packages
-    prompt: >-
-      I'm building a mobile app with Acme. It needs a local database, maps, age
-      gating, localization, Sign in with Apple and a small on-device model. Does Acme
-      offer packages for these? Which ones would you use?
-    goal: >-
-      The answer names the right Acme package for each requirement and relies on the
-      docs shipped in the project, not web search.
-    checks:
-      - run: "! grep -q '\"name\":\"WebFetch\"' \"$FIREGOAT_SESSION\""
-```
-
-### 5. Shortest
-
-```yaml
-tests:
-  - name: coffee-timer
-    prompt: Create a new app - a coffee timer. Run it on the simulator and get the screenshot.
-    goal: A screenshot shows a working coffee timer app running on the simulator.
-    checks:
-      - run: find "$FIREGOAT_WORKDIR" -name '*.png' -size +1k | grep -q .
-```
-
-## Bad
-
-### No goal, no checks
-
-```text
-prompt: "Create a sandbox. https://docs.acme.dev"
-```
-
-Ungradeable: nothing says what success looks like, so every run "succeeds". Add an
-observable outcome ("run echo hello-from-acme inside it and print the output"), a goal
-and a check.
-
-### Implementation steps
-
-```text
-prompt: >-
-  1. Create a project from the acme-starter template.
-  2. Add the acme-maps package.
-  3. Render a map centered on Berlin.
-```
-
-This tests instruction-following, not whether an agent finds its way with your docs.
-Rewrite as the user's need: "Show a map centered on Berlin in a new Acme app. Keep it
-minimal. Take a screenshot."