firegoat 0.26.0 → 0.27.0

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.26.0",
+  version: "0.27.0",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -7157,7 +7157,7 @@ function read(stdout, exited, timeoutMs) {
       reportedModel,
       version,
       ok: false,
-      reason: `claude result is an error (subtype=${result.subtype})${result.result ? `: ${result.result.slice(0, 300)}` : ""}`,
+      reason: `claude reported an error${result.subtype === undefined || result.subtype === "success" ? "" : ` (${result.subtype})`}${result.result ? `: ${result.result.slice(0, 300)}` : ""}`,
       sessionId
     };
   return { ...trace, reportedModel, version, ok: true, sessionId };
@@ -7231,6 +7231,7 @@ var claude = {
   }),
   authLabel: () => process.env.ANTHROPIC_API_KEY ? "ANTHROPIC_API_KEY (claude bills the API key; unset it to use your claude login)" : "your claude login",
   checkLogin,
+  isFinal: (line) => parse(line)?.type === "result",
   progress,
   read,
   sessionsDir: (configDir) => join(configDir, "projects"),
@@ -7549,6 +7550,10 @@ var codex = {
   }),
   authLabel: () => `your codex login (${join3(userCodexHome(), "auth.json")})${process.env.CODEX_API_KEY ? "; CODEX_API_KEY is set and passes through" : ""}`,
   checkLogin: checkLogin2,
+  isFinal: (line) => {
+    const type = parse2(line)?.type;
+    return type === "turn.completed" || type === "turn.failed";
+  },
   progress: progress2,
   read: read2,
   sessionsDir: (configDir) => join3(configDir, "sessions"),
@@ -7653,7 +7658,7 @@ function steps3(events) {
 function resultError(result) {
   const errors = Array.isArray(result.errors) ? result.errors.filter((x) => typeof x === "string").join("; ") : "";
   const said = errors || result.result || "";
-  return `grok result is an error (subtype=${result.subtype})${said ? `: ${said.slice(0, 300)}` : ""}`;
+  return `grok reported an error${result.subtype === undefined || result.subtype === "success" ? "" : ` (${result.subtype})`}${said ? `: ${said.slice(0, 300)}` : ""}`;
 }
 function read3(stdout, exited, timeoutMs) {
   const events = stdout.split(`
@@ -7836,6 +7841,7 @@ var grok = {
   }),
   authLabel: () => `your grok login (${USER_AUTH})`,
   checkLogin: checkLogin3,
+  isFinal: (line) => parse3(line)?.type === "result",
   progress: progress3,
   read: read3,
   sessionsDir: (configDir) => join4(configDir, "sessions"),
@@ -8516,6 +8522,26 @@ var CLAUDE_AUTH = /^CLAUDE_CODE_(OAUTH_TOKEN|OAUTH_REFRESH_TOKEN|USE_[A-Z_]+|SKI
 function hostEnv() {
   return Object.fromEntries(Object.entries(process.env).filter(([k]) => !NPM_INJECTED.test(k) && !k.startsWith("FIREGOAT_") && (!CLAUDE_SESSION.test(k) || CLAUDE_AUTH.test(k))));
 }
+var GRACE_MS = 5000;
+var liveGroups = new Set;
+var signalsHooked = false;
+function hookSignals() {
+  if (signalsHooked)
+    return;
+  signalsHooked = true;
+  const onSignal = (signal) => {
+    for (const g of liveGroups)
+      try {
+        process.kill(-g, "SIGKILL");
+      } catch {}
+    process.kill(process.pid, signal);
+  };
+  for (const s of ["SIGINT", "SIGTERM", "SIGHUP"])
+    process.once(s, onSignal);
+}
+function forgetGroup(pgid) {
+  liveGroups.delete(pgid);
+}
 function processTree(pid) {
   const children = new Map;
   const ps = execFileSync("ps", ["-A", "-o", "pid=,ppid="], {
@@ -8533,7 +8559,11 @@ function processTree(pid) {
     tree.push(...children.get(tree[i]) ?? []);
   return tree;
 }
-function killTree(pid) {
+function killTree(pid, group) {
+  if (group)
+    try {
+      process.kill(-pid, "SIGKILL");
+    } catch {}
   let tree = [pid];
   let problem;
   try {
@@ -8549,18 +8579,27 @@ function killTree(pid) {
 }
 function run(opts) {
   return new Promise((resolve) => {
+    const detached = opts.detached === true;
+    if (detached)
+      hookSignals();
     const child = spawn(opts.cmd, opts.args, {
       cwd: opts.cwd,
       env: opts.env,
-      stdio: ["ignore", "pipe", "pipe"]
+      stdio: ["ignore", "pipe", "pipe"],
+      detached
     });
+    if (child.pid !== undefined) {
+      if (detached)
+        liveGroups.add(child.pid);
+      opts.onSpawn?.(child.pid);
+    }
     let timedOut = false;
     let settled = false;
     const timer = setTimeout(() => {
       timedOut = true;
       if (child.pid === undefined)
         return;
-      const problem = killTree(child.pid);
+      const problem = killTree(child.pid, detached);
       if (problem)
         opts.onStderr(Buffer.from(`firegoat: timed out; could not list ${opts.cmd}'s child processes (${problem}), killed only ${opts.cmd}
 `));
@@ -8574,6 +8613,7 @@ function run(opts) {
     };
     child.stdout.on("data", opts.onStdout);
     child.stderr.on("data", opts.onStderr);
+    opts.finalSeen?.then(() => setTimeout(() => done({ code: null, timedOut: false, ended: true }), GRACE_MS).unref());
     child.on("error", (error) => done({ code: null, timedOut, error }));
     child.on("close", (code) => done({ code, timedOut }));
     child.on("exit", (code) => setTimeout(() => {
@@ -8666,41 +8706,54 @@ function gitRepo(dir) {
 async function runAgent(harness, opts) {
   let stdout = "";
   let pending = "";
+  let group;
+  let markFinal = () => {};
+  const finalSeen = new Promise((r) => {
+    markFinal = r;
+  });
   const exited = await run({
     cmd: opts.bin,
     args: harness.args(opts),
     cwd: opts.cwd,
     env: opts.env,
     timeoutMs: opts.timeoutMs,
+    detached: true,
+    finalSeen,
+    onSpawn: (pid) => {
+      group = pid;
+    },
     onStdout: (c) => {
       const text = c.toString();
       stdout += text;
-      if (!opts.onProgress)
-        return;
       pending += text;
       const lines = pending.split(`
 `);
       pending = lines.pop() ?? "";
       for (const line of lines) {
-        const p = harness.progress(line);
+        if (harness.isFinal(line))
+          markFinal();
+        const p = opts.onProgress && harness.progress(line);
         if (p)
-          opts.onProgress(p);
+          opts.onProgress?.(p);
       }
     },
     onStderr: (c) => opts.stderr.write(c)
   });
   if (exited.error)
     return {
-      ok: false,
-      reason: `could not start ${opts.bin}: ${exited.error.message}`,
-      steps: [],
-      final: "",
-      reportedModel: null,
-      version: null,
-      turns: null,
-      cost: null
+      group,
+      outcome: {
+        ok: false,
+        reason: `could not start ${opts.bin}: ${exited.error.message}`,
+        steps: [],
+        final: "",
+        reportedModel: null,
+        version: null,
+        turns: null,
+        cost: null
+      }
     };
-  return harness.read(stdout, exited, opts.timeoutMs);
+  return { group, outcome: harness.read(stdout, exited, opts.timeoutMs) };
 }
 
 // src/help.ts
@@ -8778,8 +8831,10 @@ firegoat.yaml reference:
   install: <sh> | [<sh>, ...] In each fresh folder, before the agent starts; a
                               failure: that run is not graded
   cleanup: <sh> | [<sh>, ...] In each run's folder once the run is over: after the
-                              checks, and also after an error or timeout. Stops
-                              what the agent left running (servers, daemons).
+                              checks, and also after an error or timeout. For
+                              what lives outside the run (e.g. seeded data);
+                              processes the agent left running are stopped by
+                              firegoat after cleanup (see eval --help).
                               Every step runs; a failure is a warning (run.json
                               cleanup, <test>/<n>/cleanup.txt), never the verdict
   vars: {KEY: value}          Defaults for {{vars.KEY}} in prompts, checks and
@@ -8891,7 +8946,6 @@ relative paths in it are from the run's folder, claude's cwd:
 Example:
   build: npm pack --pack-destination "$FIREGOAT_OUT"
   install: npm init -y >/dev/null && npm install "$FIREGOAT_OUT"/*.tgz
-  cleanup: pkill -f "$FIREGOAT_WORKDIR" || true
   tests:
     - name: first-call
       prompt: >-
@@ -8973,6 +9027,17 @@ top-level). cleanup: runs whenever the folder exists, also after a failed instal
 a claude error or a timeout; a failing cleanup step prints a warning and never
 changes the run's status or the exit code.
 
+The agent's run ends with its final result (claude's and grok's result line,
+codex's turn.completed or turn.failed): a harness that hasn't exited 5s later
+(a server it started in the background keeps it alive) is left running, so the
+checks can still reach that server. After cleanup, firegoat stops whatever is
+still running from the run (SIGTERM, then SIGKILL after 2s): the harness's
+process group, every process below it, and every process whose working
+directory is inside the run's folder (lsof on macOS, /proc on Linux). They are
+listed in run.json leftover, with one warning per run; never the verdict. Not
+found: a daemon that left the group and changed its working directory out of
+the run's folder.
+
 claude runs on its own login (or ANTHROPIC_API_KEY if set, as claude always does)
 with a fresh config dir, ~/.firegoat/runs/<hash>/.claude: none of your settings,
 CLAUDE.md, plugins, skills or MCP servers (harness_args: in firegoat.yaml adds
@@ -9087,8 +9152,9 @@ Output: .firegoat/runs/<runId>/
                           the harness reported running, or null), harnessArgs
                           (harness_args passed), prompt, end_goal,
                           checks [{run, code}], cleanup [{run, code}],
-                          and paths: dir (this folder), workdir, home (the
-                          run's HOME; both real paths, under ~/.firegoat),
+                          leftover [{pid, command}] (processes stopped after
+                          cleanup), and paths: dir (this folder), workdir, home
+                          (the run's HOME; both real paths, under ~/.firegoat),
                           transcript, session (the harness's session file), stderr,
                           checksLog, cleanupLog; startedAt; upload ({status: "uploaded",
                           url} | {status: "failed", error} | {status:
@@ -9153,6 +9219,121 @@ Example:
 
 Exit codes: 0 sent, 2 the service failed (or rate limit), 3 usage error.`;
 
+// src/leftover.ts
+import { spawn as spawn2 } from "node:child_process";
+import { readdirSync as readdirSync4, readlinkSync as readlinkSync2, realpathSync as realpathSync2 } from "node:fs";
+var TERM_WAIT_MS = 2000;
+function output(cmd, args) {
+  return new Promise((resolve, reject) => {
+    const child = spawn2(cmd, args, { stdio: ["ignore", "pipe", "pipe"] });
+    let out = "";
+    let err = "";
+    child.stdout.on("data", (c) => {
+      out += c;
+    });
+    child.stderr.on("data", (c) => {
+      err += c;
+    });
+    child.on("error", (e) => reject(new Error(`could not run ${cmd}: ${e.message}`)));
+    child.on("close", (code) => code === 0 || out !== "" ? resolve(out) : reject(new Error(`${cmd} exited ${code}: ${err.trim().slice(0, 300)}`)));
+  });
+}
+async function processes() {
+  const ps = await output("ps", [
+    "-A",
+    "-o",
+    "pid=,ppid=,pgid=,stat=,command="
+  ]);
+  const procs = [];
+  for (const line of ps.split(`
+`)) {
+    const m = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s*(.*)$/.exec(line);
+    if (!m || m[4]?.startsWith("Z"))
+      continue;
+    procs.push({
+      pid: Number(m[1]),
+      ppid: Number(m[2]),
+      pgid: Number(m[3]),
+      command: m[5] ?? ""
+    });
+  }
+  return procs;
+}
+async function cwdInside(dir) {
+  const inside = (p) => p === dir || p.startsWith(`${dir}/`);
+  if (process.platform === "linux") {
+    const pids = [];
+    for (const name of readdirSync4("/proc")) {
+      if (!/^\d+$/.test(name))
+        continue;
+      try {
+        if (inside(readlinkSync2(`/proc/${name}/cwd`)))
+          pids.push(Number(name));
+      } catch {}
+    }
+    return pids;
+  }
+  if (process.platform === "darwin") {
+    const pids = [];
+    let pid = 0;
+    for (const line of (await output("lsof", ["-d", "cwd", "-Fpn", "-w"])).split(`
+`)) {
+      if (line.startsWith("p"))
+        pid = Number(line.slice(1));
+      else if (line.startsWith("n") && inside(line.slice(1)))
+        pids.push(pid);
+    }
+    return pids;
+  }
+  throw new Error(`finding processes by cwd is not supported on ${process.platform}`);
+}
+function alive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function signal(pids, group, sig) {
+  for (const p of group === undefined ? pids : [...pids, -group])
+    try {
+      process.kill(p, sig);
+    } catch {}
+}
+async function stopLeftovers(opts) {
+  const workdir = realpathSync2(opts.workdir);
+  const [procs, inWorkdir] = await Promise.all([
+    processes(),
+    cwdInside(workdir)
+  ]);
+  const found = new Set(inWorkdir);
+  if (opts.group !== undefined) {
+    for (const p of procs)
+      if (p.pgid === opts.group)
+        found.add(p.pid);
+  }
+  for (let grew = true;grew; ) {
+    grew = false;
+    for (const p of procs)
+      if (found.has(p.ppid) && !found.has(p.pid)) {
+        found.add(p.pid);
+        grew = true;
+      }
+  }
+  found.delete(process.pid);
+  const leftover = procs.filter((p) => found.has(p.pid)).map((p) => ({ pid: p.pid, command: printable(p.command, 200) }));
+  const pids = leftover.map((l) => l.pid);
+  signal(pids, opts.group, "SIGTERM");
+  const deadline = Date.now() + TERM_WAIT_MS;
+  while (pids.some(alive) && Date.now() < deadline)
+    await new Promise((r) => setTimeout(r, 100));
+  signal(pids.filter(alive), opts.group, "SIGKILL");
+  if (opts.group !== undefined)
+    forgetGroup(opts.group);
+  return leftover;
+}
+
 // src/steps.ts
 import { createWriteStream } from "node:fs";
 var STEP_TIMEOUT_MS = 600000;
@@ -9287,7 +9468,7 @@ async function sendTelemetry(event, exitCode, startedAt) {
 }
 
 // src/transcript.ts
-import { readdirSync as readdirSync4, statSync as statSync2, writeFileSync as writeFileSync3 } from "node:fs";
+import { readdirSync as readdirSync5, statSync as statSync2, writeFileSync as writeFileSync3 } from "node:fs";
 import { join as join8, relative } from "node:path";
 var OUTPUT_LINES = 15;
 var OUTPUT_CHARS = 1500;
@@ -9337,7 +9518,7 @@ function describeToolCall(step, c) {
 function workdirFiles(dir) {
   const out = [];
   const walk = (d) => {
-    for (const name of readdirSync4(d)) {
+    for (const name of readdirSync5(d)) {
       if (SKIP_DIRS.has(name))
         continue;
       const p = join8(d, name);
@@ -9420,7 +9601,7 @@ function runTag(harness, model) {
   return parts.length > 0 ? ` [${parts.join(" ")}]` : "";
 }
 function notGradedWhy(reason) {
-  const said = /claude result is an error \(subtype=[^)]*\): (.*)$/s.exec(reason);
+  const said = /claude reported an error(?: \([^)]*\))?: (.*)$/s.exec(reason);
   if (said)
     return `claude said: "${said[1]}"`;
   return reason.replace(/^not graded: /, "");
@@ -10077,6 +10258,7 @@ async function runOne(opts) {
     checksLog: join10(dir, "checks.txt"),
     cleanup: [],
     cleanupLog: join10(dir, "cleanup.txt"),
+    leftover: [],
     num_turns: null,
     cost: null,
     duration_ms: 0,
@@ -10094,7 +10276,8 @@ async function runOne(opts) {
     FIREGOAT_REPO: opts.repo
   };
   const steps = renderRunSteps(test, opts, opts.vars, fill);
-  const [status, reason] = await attempt(opts, r, steps, env, stepEnv, envFile);
+  const agent = {};
+  const [status, reason] = await attempt(opts, r, steps, env, stepEnv, envFile, agent);
   if (steps.cleanup.length > 0)
     r.cleanup = await runAllSteps({
       commands: steps.cleanup,
@@ -10102,6 +10285,7 @@ async function runOne(opts) {
       env: stepEnv,
       logPath: r.cleanupLog
     });
+  r.leftover = await stopLeftovers({ group: agent.group, workdir });
   r.status = status;
   r.reason = reason;
   r.duration_ms = Date.now() - started;
@@ -10111,7 +10295,7 @@ async function runOne(opts) {
 function withoutOldPwd(env) {
   return Object.fromEntries(Object.entries(env).filter(([k]) => k !== "OLDPWD"));
 }
-async function attempt(opts, r, steps, env, stepEnv, homeEnvFile) {
+async function attempt(opts, r, steps, env, stepEnv, homeEnvFile, agent) {
   const { test, n } = opts;
   const { dir, workdir } = r;
   if (steps.install.length > 0) {
@@ -10127,7 +10311,7 @@ async function attempt(opts, r, steps, env, stepEnv, homeEnvFile) {
   }
   const stderr = createWriteStream2(r.stderr);
   const { harness } = opts;
-  const outcome = await runAgent(harness, {
+  const { outcome, group } = await runAgent(harness, {
     bin: opts.bin,
     prompt: r.prompt,
     cwd: workdir,
@@ -10150,6 +10334,7 @@ async function attempt(opts, r, steps, env, stepEnv, homeEnvFile) {
     extraArgs: r.harnessArgs,
     onProgress: opts.onProgress
   });
+  agent.group = group;
   if (outcome.version !== null)
     evalStats({ claudeVersion: claudeVersion(outcome.version) });
   await new Promise((res) => stderr.end(res));
@@ -10538,6 +10723,9 @@ async function evalCommand(args, root) {
     for (const c of r.cleanup)
       if (c.code !== 0)
         console.error(`firegoat eval: Warning: ${r.test}#${r.n} cleanup failed (${c.error ?? `exit ${c.code}`}): ${c.run} (log: ${r.cleanupLog})`);
+  for (const r of results)
+    if (r.leftover.length > 0)
+      console.error(`firegoat eval: Warning: ${r.test}#${r.n} left ${r.leftover.length} process${r.leftover.length === 1 ? "" : "es"} running, stopped after cleanup: ${r.leftover.map((l) => `${l.pid} ${l.command}`).join("; ")} (run.json leftover)`);
   if (uploadsFailed > 0)
     console.error(`firegoat eval: ${uploadsFailed} upload${uploadsFailed === 1 ? "" : "s"} failed (each run's upload.error in summary.json)`);
   if (opts.json)
@@ -10822,12 +11010,12 @@ firegoat ${package_default.version}` : `firegoat ${package_default.version}`,
 }
 
 // src/init.ts
-import { spawn as spawn3, spawnSync as spawnSync4 } from "node:child_process";
+import { spawn as spawn4, spawnSync as spawnSync4 } from "node:child_process";
 import { existsSync as existsSync6, writeFileSync as writeFileSync5 } from "node:fs";
 import { basename as basename4, join as join11 } from "node:path";
 
 // src/login.ts
-import { spawn as spawn2 } from "node:child_process";
+import { spawn as spawn3 } from "node:child_process";
 var TIMEOUT_MS3 = 30000;
 var MAX_EXPIRES_S = 900;
 var MAX_INTERVAL_S = 60;
@@ -10844,7 +11032,7 @@ function str3(o, k, step) {
 function openBrowser(url) {
   const cmd = process.platform === "darwin" ? "open" : "xdg-open";
   try {
-    const child = spawn2(cmd, [url], {
+    const child = spawn3(cmd, [url], {
       env: hostEnv(),
       stdio: "ignore",
       detached: true
@@ -11168,10 +11356,12 @@ runs: 1
 # install: cp -R "$FIREGOAT_OUT/starter-app/." .
 
 # Optional: cleanup: runs in each run's folder once the run is over (after the
-# checks, also after an error or timeout), e.g. to stop a server the agent left
-# running. A failing cleanup step is a warning, never the verdict.
+# checks, also after an error or timeout), for what lives outside the run, e.g.
+# data install: seeded in a shared service. Servers the agent left running need
+# none: firegoat stops them after cleanup. A failing cleanup step is a warning,
+# never the verdict.
 # cleanup:
-#   - pkill -f "$FIREGOAT_WORKDIR" || true
+#   - acme db drop "eval-{{run.id}}-{{run.n}}" || true
 #
 # A test can add its own install: (after this one) and cleanup: (before this one).
 
@@ -11273,7 +11463,7 @@ class Timeline {
 }
 function runQuiet(cmd, args, opts) {
   return new Promise((resolve) => {
-    const child = spawn3(cmd, args, {
+    const child = spawn4(cmd, args, {
       ...opts,
       stdio: ["ignore", "pipe", "pipe"]
     });

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.26.0",
+	"version": "0.27.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/firegoat-ai/firegoat-cli.git"