firegoat 0.9.3 → 0.10.0

package/README.md

@@ -2,7 +2,7 @@
 
 Local AX evals: how well coding agents use your SDK, CLI, API and docs. firegoat gives your own `claude` one prompt in a fresh folder, saves the session, and runs your checks.
 
-Status: early (v0.9). See `plan.md`.
+Status: early (v0.10). See `plan.md`.
 
 ## Quickstart
 
@@ -30,7 +30,7 @@ tests:
       and print the event id. Credentials are in ACME_API_KEY. Keep it minimal.
     end_goal: A script sends one event with the SDK's client and prints its id.
     checks:
-      - run: grep -qE 'evt_[A-Za-z0-9]+' "$FIREGOAT_SESSION"
+      - run: grep -qE 'evt_[A-Za-z0-9]+' "$FIREGOAT_TRANSCRIPT"
 ```
 
 `setup` builds your package once; `workdir_setup` installs that build into each fresh folder, so the agent uses your local code, not the registry.
@@ -44,7 +44,7 @@ tests:
     prompt: Deploy the hello-world template to a preview environment with the acme CLI and print the URL.
     end_goal: The agent finds the right acme commands and ends with a working preview URL.
     checks:
-      - run: grep -q '"command":"acme deploy' "$FIREGOAT_SESSION"
+      - run: grep -q '\$ acme deploy' "$FIREGOAT_TRANSCRIPT"
 ```
 
 Whatever `setup` writes to `$FIREGOAT_OUT` is first on the agent's `PATH`. That only helps for binaries called by bare name: if users run your CLI via `npx`/`uvx`, pack and install it like the npm example, or `npx` fetches the registry release.
@@ -87,7 +87,7 @@ npx firegoat eval --var SHA=$(git rev-parse HEAD)
 - claude runs on its own login (or `ANTHROPIC_API_KEY`, if set) with `CLAUDE_CONFIG_DIR=.firegoat/.claude` and `--setting-sources "" --strict-mcp-config --disable-slash-commands`: none of your settings, CLAUDE.md, plugins, skills or MCP servers. claude saves its session there, in its own format.
 - No sandbox: the agent's folder is inside `.firegoat/runs/`, and it can reach the rest of your machine. In `.firegoat/`, git ignores `runs/` and `.claude/`; keep test fixtures (scripts, sample data) in `.firegoat/fixtures/` and commit them with `firegoat.yaml`.
 - Script checks run in the agent's folder afterwards and set the exit code. `end_goal` is not graded by firegoat: it's for you, or your coding agent, to judge from `run.json`, the session and the folder.
-- Output per run, in `.firegoat/runs/<runId>/<test>/<n>/`: `run.json` (status, prompt, end_goal, check results, paths), `work/`, logs. `summary.json` covers the whole eval; `--json` prints it.
+- Output per run, in `.firegoat/runs/<runId>/<test>/<n>/`: `transcript.txt` (each tool call with its command and short output, the final answer, files made; checks get it as `$FIREGOAT_TRANSCRIPT`), `run.json` (status, prompt, end_goal, check results, paths), `work/`, logs. Read the transcript to judge `end_goal`; claude's full session file is ~50x larger. `summary.json` covers the whole eval; `--json` prints it.
 
 `firegoat.yaml` keys: `runs`, `setup`, `workdir_setup`, `vars`, and `tests` (`name`, `prompt`, `end_goal`, `checks`, `timeout`). Unknown keys are errors. Full reference: `npx firegoat init --help`.
 

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.9.3",
+  version: "0.10.0",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -7169,10 +7169,10 @@ import {
   createWriteStream as createWriteStream2,
   existsSync as existsSync2,
   mkdirSync as mkdirSync2,
-  readdirSync,
-  writeFileSync as writeFileSync2
+  readdirSync as readdirSync2,
+  writeFileSync as writeFileSync3
 } from "node:fs";
-import { delimiter as delimiter2, join as join3 } from "node:path";
+import { delimiter as delimiter2, join as join4 } from "node:path";
 
 // src/claude.ts
 import { spawn, spawnSync } from "node:child_process";
@@ -7239,6 +7239,7 @@ async function runClaude(opts) {
   });
   if (exited.error)
     return {
+      events: [],
       ok: false,
       reason: `could not start ${opts.bin}: ${exited.error.message}`
     };
@@ -7250,16 +7251,21 @@ async function runClaude(opts) {
       return;
     }
   });
-  const init = events[0];
-  if (init?.type !== "system" || init.subtype !== "init")
+  const init = events.find((e) => e?.type === "system" && e.subtype === "init");
+  if (!init) {
+    const said = stdout.trim().split(`
+`)[0]?.slice(0, 300);
     return {
+      events,
       ok: false,
-      reason: `claude did not start a session (exit ${exited.code ?? "by signal"}), see stderr.txt`
+      reason: `claude did not start a session (exit ${exited.code ?? "by signal"})${said ? `: ${said}` : ", see stderr.txt"}`
     };
+  }
   const sessionId = String(init.session_id);
   const result = events.findLast((e) => e?.type === "result");
   if (exited.timedOut)
     return {
+      events,
       ok: false,
       reason: `timed out after ${opts.timeoutMs / 1000}s`,
       result,
@@ -7267,18 +7273,20 @@ async function runClaude(opts) {
     };
   if (!result)
     return {
+      events,
       ok: false,
       reason: `claude exited ${exited.code ?? "by signal"} without a result, see stderr.txt`,
       sessionId
     };
   if (result.is_error === true || result.subtype !== "success")
     return {
+      events,
       ok: false,
-      reason: `claude result is an error (subtype=${result.subtype})`,
+      reason: `claude result is an error (subtype=${result.subtype})${result.result ? `: ${result.result.slice(0, 300)}` : ""}`,
       result,
       sessionId
     };
-  return { ok: true, result, sessionId };
+  return { events, ok: true, result, sessionId };
 }
 async function runShell(opts) {
   opts.log.write(`$ ${opts.run}
@@ -7394,8 +7402,10 @@ firegoat.yaml reference:
 
   Unknown keys are errors. Keep helper files out of firegoat.yaml: put them in
   .firegoat/fixtures/ and copy them in setup or workdir_setup. setup, workdir_setup and checks get the vars as env vars,
-  plus $FIREGOAT_OUT and $FIREGOAT_WORKDIR; checks also get $FIREGOAT_SESSION (the
-  claude session file, one JSON event per line).
+  plus $FIREGOAT_OUT and $FIREGOAT_WORKDIR. Checks also get $FIREGOAT_TRANSCRIPT, a short
+  plain-text transcript ("[1] $ <command>" per shell call, "[2] Write <path>", then
+  each output), and
+  $FIREGOAT_SESSION (claude's full session file, JSON lines).
 
 Example:
   setup: npm pack --pack-destination "$FIREGOAT_OUT"
@@ -7409,9 +7419,9 @@ Example:
         A script sends one event with the SDK's client (not raw HTTP) and prints its
         id, without installing acme from the registry.
       checks:
-        - run: grep -qE 'evt_[A-Za-z0-9]+' "$FIREGOAT_SESSION"
+        - run: grep -qE 'evt_[A-Za-z0-9]+' "$FIREGOAT_TRANSCRIPT"
         - run: |
-            ! grep -Eq 'npm (install|i) [^"]*acme' "$FIREGOAT_SESSION"
+            ! grep -Eq '$ .*npm (install|i|add) .*acme' "$FIREGOAT_TRANSCRIPT"
 
 Exit codes: 0 done, 2 the skills install failed, 3 usage error.`;
 var EVAL_HELP = `Usage: firegoat eval [test..] [--runs N] [--var KEY=value].. [--json]
@@ -7437,12 +7447,15 @@ Output: .firegoat/runs/<runId>/
   summary.json            every run, plus totals
   <test>/<n>/run.json     one run: status and reason (from the checks), prompt,
                           end_goal, checks [{run, code}], and paths: workdir,
-                          session (claude's session file), stderr, checksLog
+                          transcript, session (claude's session file), stderr,
+                          checksLog
+  <test>/<n>/transcript.txt  what the agent did: each tool call with its command,
+                          short output, its final answer, files it made
   setup.txt, <test>/<n>/workdir_setup.txt, checks.txt   logs
 
 Grading: firegoat runs only the script checks; they set the exit code. end_goal is
-yours to judge. The session is one JSON event per line: "assistant" messages hold
-"tool_use" items (the agent's commands), "user" messages their output.
+yours to judge: read transcript.txt first (a few KB); the session file has everything
+(often 50x larger).
 
 Examples:
   npx firegoat eval --json                  all tests, JSON summary on stdout
@@ -7476,14 +7489,106 @@ async function runSteps(opts) {
   }
 }
 
+// src/transcript.ts
+import { readdirSync, statSync, writeFileSync as writeFileSync2 } from "node:fs";
+import { join as join2, relative } from "node:path";
+var OUTPUT_LINES = 15;
+var OUTPUT_CHARS = 1500;
+var TEXT_CHARS = 1500;
+var SKIP_DIRS = new Set(["node_modules", ".git", ".venv", "__pycache__"]);
+function truncate(text, maxLines, maxChars) {
+  let out = text.length > maxChars ? text.slice(0, maxChars) : text;
+  const lines = out.split(`
+`);
+  if (lines.length > maxLines)
+    out = lines.slice(0, maxLines).join(`
+`);
+  return out.length < text.length ? `${out}
+… (${text.length - out.length} more chars)` : out;
+}
+function indent(text) {
+  return text.split(`
+`).map((l) => `    ${l}`).join(`
+`);
+}
+function describeCall(name, input) {
+  if (typeof input.command === "string")
+    return `$ ${input.command}`;
+  for (const key of ["file_path", "path", "url", "pattern", "query"])
+    if (typeof input[key] === "string")
+      return `${name} ${input[key]}`;
+  return `${name} ${truncate(JSON.stringify(input), 1, 300)}`;
+}
+function resultText(content) {
+  if (typeof content === "string")
+    return content;
+  if (Array.isArray(content))
+    return content.map((c) => typeof c.text === "string" ? c.text : `[${c.type}]`).join(`
+`);
+  return "";
+}
+function workdirFiles(dir) {
+  const out = [];
+  const walk = (d) => {
+    for (const name of readdirSync(d)) {
+      if (SKIP_DIRS.has(name))
+        continue;
+      const p = join2(d, name);
+      if (statSync(p).isDirectory())
+        walk(p);
+      else
+        out.push(relative(dir, p));
+    }
+  };
+  walk(dir);
+  return out.sort();
+}
+function writeTranscript(opts) {
+  const out = [`# ${opts.title}`, "", "## prompt", opts.prompt, "", "## steps"];
+  let step = 0;
+  let final = "";
+  let lastText = -1;
+  for (const e of opts.events) {
+    const message = e?.message;
+    const content = Array.isArray(message?.content) ? message.content : [];
+    if (e?.type === "assistant")
+      for (const b of content) {
+        if (b.type === "tool_use")
+          out.push(`[${++step}] ${describeCall(String(b.name), b.input ?? {})}`);
+        else if (b.type === "text" && typeof b.text === "string") {
+          lastText = out.length;
+          out.push(`agent: ${truncate(b.text, 30, TEXT_CHARS)}`);
+        }
+      }
+    else if (e?.type === "user")
+      for (const b of content) {
+        if (b.type !== "tool_result")
+          continue;
+        const text = resultText(b.content).trimEnd();
+        const status = b.is_error === true ? "error" : "ok";
+        out.push(text ? `    → ${status}
+${indent(truncate(text, OUTPUT_LINES, OUTPUT_CHARS))}` : `    → ${status}`);
+      }
+    else if (e?.type === "result" && typeof e.result === "string")
+      final = e.result;
+  }
+  if (lastText === out.length - 1 && final && out[lastText] === `agent: ${truncate(final, 30, TEXT_CHARS)}`)
+    out.pop();
+  out.push("", "## final answer", final || "(none)");
+  out.push("", "## files in the workdir", ...workdirFiles(opts.workdir));
+  writeFileSync2(opts.path, `${out.join(`
+`)}
+`);
+}
+
 // src/which.ts
-import { accessSync, constants, statSync } from "node:fs";
-import { delimiter, join as join2, resolve } from "node:path";
+import { accessSync, constants, statSync as statSync2 } from "node:fs";
+import { delimiter, join as join3, resolve } from "node:path";
 function resolveBin(bin) {
   const executable = (p) => {
     try {
       accessSync(p, constants.X_OK);
-      return statSync(p).isFile();
+      return statSync2(p).isFile();
     } catch {
       return false;
     }
@@ -7491,8 +7596,8 @@ function resolveBin(bin) {
   if (bin.includes("/"))
     return executable(bin) ? resolve(bin) : undefined;
   for (const dir of (process.env.PATH ?? "").split(delimiter)) {
-    if (dir && executable(join2(dir, bin)))
-      return join2(dir, bin);
+    if (dir && executable(join3(dir, bin)))
+      return join3(dir, bin);
   }
   return;
 }
@@ -7547,11 +7652,11 @@ function parseArgs(args) {
   return opts;
 }
 function findSession(configDir, sessionId) {
-  const projects = join3(configDir, "projects");
+  const projects = join4(configDir, "projects");
   if (!existsSync2(projects))
     return null;
-  for (const d of readdirSync(projects)) {
-    const f = join3(projects, d, `${sessionId}.jsonl`);
+  for (const d of readdirSync2(projects)) {
+    const f = join4(projects, d, `${sessionId}.jsonl`);
     if (existsSync2(f))
       return f;
   }
@@ -7573,8 +7678,8 @@ function newRunId() {
 async function runOne(opts) {
   const { test, n } = opts;
   const started = Date.now();
-  const dir = join3(opts.runDir, test.name, String(n));
-  const workdir = join3(dir, "work");
+  const dir = join4(opts.runDir, test.name, String(n));
+  const workdir = join4(dir, "work");
   mkdirSync2(workdir, { recursive: true });
   const r = {
     test: test.name,
@@ -7587,8 +7692,9 @@ async function runOne(opts) {
     dir,
     workdir,
     session: null,
-    stderr: join3(dir, "stderr.txt"),
-    checksLog: join3(dir, "checks.txt"),
+    transcript: join4(dir, "transcript.txt"),
+    stderr: join4(dir, "stderr.txt"),
+    checksLog: join4(dir, "checks.txt"),
     num_turns: null,
     cost: null,
     duration_ms: 0
@@ -7597,7 +7703,7 @@ async function runOne(opts) {
     r.status = status;
     r.reason = reason;
     r.duration_ms = Date.now() - started;
-    writeFileSync2(join3(dir, "run.json"), `${JSON.stringify(r, null, 2)}
+    writeFileSync3(join4(dir, "run.json"), `${JSON.stringify(r, null, 2)}
 `);
     return r;
   };
@@ -7608,7 +7714,7 @@ async function runOne(opts) {
       commands: opts.workdirSetup,
       cwd: workdir,
       env: stepEnv,
-      logPath: join3(dir, "workdir_setup.txt")
+      logPath: join4(dir, "workdir_setup.txt")
     });
     if (error)
       return done("error", `not graded: ${error}`);
@@ -7625,6 +7731,13 @@ async function runOne(opts) {
   await new Promise((res) => stderr.end(res));
   if (outcome.sessionId)
     r.session = findSession(opts.configDir, outcome.sessionId);
+  writeTranscript({
+    path: r.transcript,
+    title: `${test.name}#${n}`,
+    prompt: r.prompt,
+    events: outcome.events,
+    workdir
+  });
   r.num_turns = outcome.result?.num_turns ?? null;
   r.cost = outcome.result?.total_cost_usd ?? null;
   if (!outcome.ok)
@@ -7637,7 +7750,11 @@ async function runOne(opts) {
       r.checks.push(await runShell({
         run: render(c.run, opts.vars),
         cwd: workdir,
-        env: { ...stepEnv, FIREGOAT_SESSION: r.session },
+        env: {
+          ...stepEnv,
+          FIREGOAT_SESSION: r.session,
+          FIREGOAT_TRANSCRIPT: r.transcript
+        },
         timeoutMs: CHECK_TIMEOUT_MS,
         log
       }));
@@ -7697,7 +7814,7 @@ async function evalCommand(args, root) {
   if (!claudePath)
     return fatal(EXIT.notGraded, `not graded: claude binary ${JSON.stringify(opts.claudeBin)} not found. Install Claude Code (https://claude.com/claude-code) or pass --claude-bin PATH / FIREGOAT_CLAUDE_BIN.`);
   const firegoatDir = ensureFiregoatDir(root);
-  const configDir = join3(firegoatDir, ".claude");
+  const configDir = join4(firegoatDir, ".claude");
   mkdirSync2(configDir, { recursive: true });
   say(process.env.ANTHROPIC_API_KEY ? "auth: ANTHROPIC_API_KEY (claude bills the API key; unset it to use your claude login)" : "auth: your claude login");
   const loginProblem = checkLogin({
@@ -7708,8 +7825,8 @@ async function evalCommand(args, root) {
   if (loginProblem)
     return fatal(EXIT.notGraded, `not graded: ${loginProblem}`);
   const runId = newRunId();
-  const runDir = join3(firegoatDir, "runs", runId);
-  const out = join3(runDir, "out");
+  const runDir = join4(firegoatDir, "runs", runId);
+  const out = join4(runDir, "out");
   mkdirSync2(out, { recursive: true });
   const stepEnv = {
     ...process.env,
@@ -7724,7 +7841,7 @@ async function evalCommand(args, root) {
       commands: config.setup,
       cwd: root,
       env: stepEnv,
-      logPath: join3(runDir, "setup.txt")
+      logPath: join4(runDir, "setup.txt")
     });
     if (error)
       return fatal(EXIT.notGraded, `not graded: ${error}`);
@@ -7749,7 +7866,7 @@ async function evalCommand(args, root) {
   for (const r of results)
     totals[r.status]++;
   const doc = { runId, runsDir: runDir, totals, runs: results };
-  writeFileSync2(join3(runDir, "summary.json"), `${JSON.stringify(doc, null, 2)}
+  writeFileSync3(join4(runDir, "summary.json"), `${JSON.stringify(doc, null, 2)}
 `);
   const cost = results.reduce((sum, r) => sum + (r.cost ?? 0), 0);
   say(`
@@ -7762,8 +7879,8 @@ ${totals.pass} passed, ${totals.fail} failed, ${totals.error} not graded (${resu
 
 // src/init.ts
 import { spawnSync as spawnSync2 } from "node:child_process";
-import { existsSync as existsSync3, writeFileSync as writeFileSync3 } from "node:fs";
-import { basename, join as join4 } from "node:path";
+import { existsSync as existsSync3, writeFileSync as writeFileSync4 } from "node:fs";
+import { basename, join as join5 } from "node:path";
 import { createInterface } from "node:readline/promises";
 var SKILLS_ARGV = [
   "skills",
@@ -7867,11 +7984,11 @@ ${INIT_HELP}`);
     }
   }
   ensureFiregoatDir(cwd);
-  const configPath = join4(cwd, CONFIG_FILE);
+  const configPath = join5(cwd, CONFIG_FILE);
   if (existsSync3(configPath) && !force)
     console.log(`kept ${CONFIG_FILE} (--force replaces it)`);
   else {
-    writeFileSync3(configPath, template(basename(cwd)));
+    writeFileSync4(configPath, template(basename(cwd)));
     console.log(`wrote ${CONFIG_FILE}: edit the quickstart test to fit your users`);
   }
   const interactive = process.stdin.isTTY && process.stdout.isTTY;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.9.3",
+	"version": "0.10.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/firegoat-ai/firegoat-cli.git"