firegoat 0.29.0 → 0.30.0

package/README.md

@@ -22,6 +22,8 @@ In a terminal it asks where to install the firegoat-onboarding skill (global by
 npx firegoat@latest eval
 ```
 
+Run by an agent, eval ends with a results block to paste as-is (also in `.firegoat/runs/<runId>/results.txt`): ask it to show you the results table.
+
 ## Example
 
 ```yaml

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.29.0",
+  version: "0.30.0",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -9203,6 +9203,14 @@ says "N uploads failed" on stderr. A file over 20 MiB fails that run's upload.
 Runs without a session file are not uploaded. FIREGOAT_BASE_URL points elsewhere
 (default https://firegoat.ai/evals). See firegoat login --help.
 
+Results block: without a terminal (an agent, a pipe, CI) and without --json, eval
+ends with the terminal's final timeline as one plain block (no color, at most 100
+columns): the logo header, a row per run (status, a ramp of one block per agent turn
+ending ▲ pass, ╳ fail or ╌ not graded, checks passed, cost, turns, time), its failed
+checks and trace URL, the totals and the tier ("GOATED · tier S" when all pass). It
+sits between two lines of ═, after a "Results (paste the block ...)" line; the last
+line says where it is saved, results.txt. Show it to your user as-is in a code block.
+
 Output: .firegoat/runs/<runId>/
   out                     A symlink to $FIREGOAT_OUT,
                           ~/.firegoat/runs/<hash>/<random>/out
@@ -9211,6 +9219,7 @@ Output: .firegoat/runs/<runId>/
                           repo), harnesses, upload {origin, email} (null when
                           not uploading), runs (every run.json), then the
                           verdict: totals, tier, status and exitCode
+  results.txt             The results block, also with --json or in a terminal
   ../index.jsonl          One line per eval: runId, startedAt, dir, branch,
                           commit, harnesses, passed, failed, notGraded, cost (null when
                           any run's cost is unknown), uploaded, uploadFailed
@@ -10313,17 +10322,9 @@ function plainUI(write) {
     }
   };
 }
-function ttyUI(out, opts) {
-  const p = painter(opts.color);
-  let rows = [];
-  let build = null;
-  let authText = "";
-  const notes = [];
-  let frame = 0;
-  let drawn = 0;
-  let finished = null;
-  const width = () => Math.min(out.columns || 80, 100);
-  const cut = (text, max) => text.length > max ? `${text.slice(0, max - 1)}…` : text;
+var cut = (text, max) => text.length > max ? `${text.slice(0, max - 1)}…` : text;
+function timeline(t, p, w) {
+  const { rows, build, finished, frame } = t;
   const statusWord = (r) => {
     switch (r.state) {
       case "pass":
@@ -10352,77 +10353,109 @@ function ttyUI(out, opts) {
         return p(C.gray, "○");
     }
   };
-  const lines = () => {
-    const w = width();
-    const rule = p(C.gray, ` ${"─".repeat(w - 2)}`);
-    const c = counts(rows);
-    const doneCount = c.pass + c.fail + c.error;
-    const badge = finished ? `DONE · RUN ${finished.runId.slice(-4)}` : `RUNNING · ${doneCount}/${rows.length}`;
-    const out = [
-      ...logoHeader(p, `firegoat ${opts.version} eval · ${opts.project}`, p(finished ? C.green : C.ember, badge), w),
-      rule
-    ];
-    if (authText)
-      out.push(p(C.gray, `  Auth  ${authText}`));
-    for (const n of notes)
-      out.push(p(C.gray, `  ${cut(n, w - 2)}`));
-    if (build) {
-      const mark = build.state === "ok" ? p(C.green, "✓") : build.state === "fail" ? p(C.red, "✗") : p(C.ember, SPIN[frame % SPIN.length]);
-      out.push(`  ${build.state === "running" ? p(C.ember, "◐") : p(C.green, "●")}  Build${" ".repeat(Math.max(1, w - 20))}${mark}${build.seconds !== undefined ? ` ${build.seconds}s` : ""}`);
-    }
-    const column = (width) => Math.min(24, Math.max(0, ...rows.map(width)));
-    const harnessCols = column((r) => r.harness?.length ?? 0);
-    const modelCols = column((r) => r.model?.length ?? 0);
-    rows.forEach((r, i) => {
-      out.push(p(C.gray, "  │"));
-      const num = String(i + 1).padStart(2, "0");
-      const name = cut(`${r.test}${r.n > 1 ? `#${r.n}` : ""}`, 16).padEnd(16);
-      const harness = harnessCols > 0 ? ` ${p(C.cyan, cut(r.harness ?? "", harnessCols).padEnd(harnessCols))}` : "";
-      const model = modelCols > 0 ? ` ${p(C.cyan, cut(r.model ?? "", modelCols).padEnd(modelCols))}` : "";
-      let detail = "";
-      if (r.state === "running")
-        detail = p(C.gray, `Turn ${r.turns}`);
-      else if (r.state === "pass" || r.state === "fail")
-        detail = `${checksPassed(r).padEnd(11)} ${p(C.gray, usage(r))}`;
-      else if (r.state === "error")
-        detail = p(C.gray, usage(r));
-      out.push(`  ${glyph(r)}  ${p(C.gray, num)}  ${p(C.bold, name)}${harness}${model} ${p(C.ember, ramp(r))} ${statusWord(r)}  ${detail}`);
-      if (r.state === "running" && r.command)
-        out.push(p(C.gray, `  ┊        └ $ ${cut(r.command, w - 16)}`));
-      if (r.state === "fail")
-        for (const ck of (r.checks ?? []).filter((x) => x.code !== 0)) {
-          out.push(`  ${p(C.gray, "│")}        ${p(C.red, "✗")} ${cut(ck.run, w - 14)}`);
-          out.push(p(C.gray, `  │          ${ck.error ?? `exit ${ck.code}`}`));
-        }
-      if (r.state === "error")
-        out.push(p(C.amber, `           ${cut(notGradedWhy(r.reason ?? ""), w - 12)}`));
-      if (r.upload === "uploading")
-        out.push(p(C.gray, "  │        ↑ Uploading…"));
-      else if (r.upload?.status === "uploaded")
-        out.push(`  ${p(C.gray, "│")}        ${p(C.cyan, `↑ ${r.upload.url}`)}`);
-      else if (r.upload)
-        out.push(p(C.amber, `  │        ${cut(r.upload.status === "failed" ? `↑ Upload failed: ${r.upload.error}` : `↑ Not uploaded: ${r.upload.reason}`, w - 12)}`));
-    });
-    out.push(rule);
-    if (!finished) {
-      const track = Math.max(10, w - 34);
-      const at = rows.length ? Math.round(doneCount / rows.length * (track - 1)) : 0;
-      const progress = `${"·".repeat(at)}▶${"·".repeat(Math.max(0, track - at - 1))}`;
-      out.push(`  ${p(C.ember, progress)}  ${p(C.gray, `${doneCount}/${rows.length} · ${totalCost(rows)} so far`)}`);
-      return out;
-    }
-    const tr = tier(rows);
-    const goated = c.pass === rows.length && rows.length > 0;
-    const tally = `${c.pass} PASS · ${c.fail} FAIL · ${c.error} NOT GRADED`;
-    const left = goated ? `${rows.length}/${rows.length} runs passed their checks` : tally;
-    const right = goated ? `GOATED · tier ${tr}` : tr ? `Tier ${tr}` : "";
-    out.push(`  ${goated ? p(C.green, left) : left}${" ".repeat(Math.max(2, w - left.length - right.length - 3))}${goated ? p(C.ember, right) : right}`);
-    const turns = totalTurns(rows);
-    out.push(p(C.gray, `  Total ${totalCost(rows)}${turns === null ? "" : ` · ${turns} turns`} · ${duration(finished.seconds)}`));
-    out.push(p(C.gray, `  Runs  ${shortPath(finished.runsDir)}/`));
-    out.push(`  Checks are a score, not the verdict: judge each end_goal from ${p(C.cyan, "<test>/<n>/transcript.txt")}`);
+  const rule = p(C.gray, ` ${"─".repeat(w - 2)}`);
+  const c = counts(rows);
+  const doneCount = c.pass + c.fail + c.error;
+  const badge = finished ? `DONE · RUN ${finished.runId.slice(-4)}` : `RUNNING · ${doneCount}/${rows.length}`;
+  const out = [
+    ...logoHeader(p, `firegoat ${t.version} eval · ${t.project}`, p(finished ? C.green : C.ember, badge), w),
+    rule
+  ];
+  if (t.authText)
+    out.push(p(C.gray, `  Auth  ${t.authText}`));
+  for (const n of t.notes)
+    out.push(p(C.gray, `  ${cut(n, w - 2)}`));
+  if (build) {
+    const mark = build.state === "ok" ? p(C.green, "✓") : build.state === "fail" ? p(C.red, "✗") : p(C.ember, SPIN[frame % SPIN.length]);
+    out.push(`  ${build.state === "running" ? p(C.ember, "◐") : p(C.green, "●")}  Build${" ".repeat(Math.max(1, w - 20))}${mark}${build.seconds !== undefined ? ` ${build.seconds}s` : ""}`);
+  }
+  const column = (width) => Math.min(24, Math.max(0, ...rows.map(width)));
+  const harnessCols = column((r) => r.harness?.length ?? 0);
+  const modelCols = column((r) => r.model?.length ?? 0);
+  const detailCols = w - 1 - (47 + (harnessCols > 0 ? harnessCols + 1 : 0) + (modelCols > 0 ? modelCols + 1 : 0));
+  rows.forEach((r, i) => {
+    out.push(p(C.gray, "  │"));
+    const num = String(i + 1).padStart(2, "0");
+    const name = cut(`${r.test}${r.n > 1 ? `#${r.n}` : ""}`, 16).padEnd(16);
+    const harness = harnessCols > 0 ? ` ${p(C.cyan, cut(r.harness ?? "", harnessCols).padEnd(harnessCols))}` : "";
+    const model = modelCols > 0 ? ` ${p(C.cyan, cut(r.model ?? "", modelCols).padEnd(modelCols))}` : "";
+    let detail = "";
+    if (r.state === "running")
+      detail = p(C.gray, `Turn ${r.turns}`);
+    else if (r.state === "pass" || r.state === "fail")
+      detail = `${checksPassed(r).padEnd(11)} ${p(C.gray, cut(usage(r), Math.max(1, detailCols - 12)))}`;
+    else if (r.state === "error")
+      detail = p(C.gray, cut(usage(r), Math.max(1, detailCols)));
+    out.push(`  ${glyph(r)}  ${p(C.gray, num)}  ${p(C.bold, name)}${harness}${model} ${p(C.ember, ramp(r))} ${statusWord(r)}  ${detail}`);
+    if (r.state === "running" && r.command)
+      out.push(p(C.gray, `  ┊        └ $ ${cut(r.command, w - 16)}`));
+    if (r.state === "fail")
+      for (const ck of (r.checks ?? []).filter((x) => x.code !== 0)) {
+        out.push(`  ${p(C.gray, "│")}        ${p(C.red, "✗")} ${cut(ck.run, w - 14)}`);
+        out.push(p(C.gray, `  │          ${cut(ck.error ?? `exit ${ck.code}`, w - 14)}`));
+      }
+    if (r.state === "error")
+      out.push(p(C.amber, `           ${cut(notGradedWhy(r.reason ?? ""), w - 12)}`));
+    if (r.upload === "uploading")
+      out.push(p(C.gray, "  │        ↑ Uploading…"));
+    else if (r.upload?.status === "uploaded")
+      out.push(`  ${p(C.gray, "│")}        ${p(C.cyan, `↑ ${r.upload.url}`)}`);
+    else if (r.upload)
+      out.push(p(C.amber, `  │        ${cut(r.upload.status === "failed" ? `↑ Upload failed: ${r.upload.error}` : `↑ Not uploaded: ${r.upload.reason}`, w - 12)}`));
+  });
+  out.push(rule);
+  if (!finished) {
+    const track = Math.max(10, w - 34);
+    const at = rows.length ? Math.round(doneCount / rows.length * (track - 1)) : 0;
+    const progress = `${"·".repeat(at)}▶${"·".repeat(Math.max(0, track - at - 1))}`;
+    out.push(`  ${p(C.ember, progress)}  ${p(C.gray, `${doneCount}/${rows.length} · ${totalCost(rows)} so far`)}`);
     return out;
-  };
+  }
+  const tr = tier(rows);
+  const goated = c.pass === rows.length && rows.length > 0;
+  const tally = `${c.pass} PASS · ${c.fail} FAIL · ${c.error} NOT GRADED`;
+  const left = goated ? `${rows.length}/${rows.length} runs passed their checks` : tally;
+  const right = goated ? `GOATED · tier ${tr}` : tr ? `Tier ${tr}` : "";
+  out.push(`  ${goated ? p(C.green, left) : left}${" ".repeat(Math.max(2, w - left.length - right.length - 3))}${goated ? p(C.ember, right) : right}`);
+  const turns = totalTurns(rows);
+  out.push(p(C.gray, `  Total ${totalCost(rows)}${turns === null ? "" : ` · ${turns} turns`} · ${duration(finished.seconds)}`));
+  out.push(p(C.gray, `  Runs  ${shortPath(finished.runsDir)}/`));
+  out.push(`  Checks are a score, not the verdict: judge each end_goal from ${p(C.cyan, "<test>/<n>/transcript.txt")}`);
+  return out;
+}
+var RESULTS_WIDTH = 100;
+function resultsBlock(t) {
+  return timeline({
+    project: t.project,
+    version: t.version,
+    rows: t.rows,
+    build: null,
+    authText: "",
+    notes: [],
+    finished: { runId: t.runId, runsDir: t.runsDir, seconds: t.seconds },
+    frame: 0
+  }, painter(false), RESULTS_WIDTH);
+}
+function ttyUI(out, opts) {
+  const p = painter(opts.color);
+  let rows = [];
+  let build = null;
+  let authText = "";
+  const notes = [];
+  let frame = 0;
+  let drawn = 0;
+  let finished = null;
+  const width = () => Math.min(out.columns || 80, 100);
+  const lines = () => timeline({
+    project: opts.project,
+    version: opts.version,
+    rows,
+    build,
+    authText,
+    notes,
+    finished,
+    frame
+  }, p, width());
   const rowsOf = (line) => {
     const visible = line.replace(ANSI, "");
     return Math.max(1, Math.ceil(visible.length / (out.columns || 80)));
@@ -11043,6 +11076,16 @@ function uploadedSummary(origin, results, runDir) {
   for (const u of urls)
     console.log(`  ${u}`);
 }
+var RESULTS_EDGE = "═".repeat(RESULTS_WIDTH);
+function printResults(block, file) {
+  console.log("");
+  console.log("Results (paste the block between the ═ lines as-is, in a code block):");
+  console.log(RESULTS_EDGE);
+  for (const line of block)
+    console.log(line);
+  console.log(RESULTS_EDGE);
+  console.log(`Results block saved to ${file}`);
+}
 async function evalCommand(args, root) {
   if (args.includes("--help") || args.includes("-h")) {
     console.log(EVAL_HELP);
@@ -11333,12 +11376,8 @@ async function evalCommand(args, root) {
     uploadFailed: uploadsFailed
   })}
 `);
-  ui.done({
-    runId,
-    runsDir: runDir,
-    rows,
-    seconds: Math.round((Date.now() - evalStarted) / 1000)
-  });
+  const seconds = Math.round((Date.now() - evalStarted) / 1000);
+  ui.done({ runId, runsDir: runDir, rows, seconds });
   if (upload && !opts.json)
     uploadedSummary(upload.origin, results, runDir);
   for (const r of results)
@@ -11350,6 +11389,20 @@ async function evalCommand(args, root) {
       console.error(`firegoat eval: Warning: ${r.test}#${r.n} left ${r.leftover.length} process${r.leftover.length === 1 ? "" : "es"} running, stopped after cleanup: ${r.leftover.map((l) => `${l.pid} ${l.command}`).join("; ")} (run.json leftover)`);
   if (uploadsFailed > 0)
     console.error(`firegoat eval: ${uploadsFailed} upload${uploadsFailed === 1 ? "" : "s"} failed (each run's upload.error in summary.json)`);
+  const block = resultsBlock({
+    project: basename2(root),
+    version: package_default.version,
+    rows,
+    runId,
+    runsDir: runDir,
+    seconds
+  });
+  const resultsFile = join10(runDir, "results.txt");
+  writeFileSync5(resultsFile, `${block.join(`
+`)}
+`);
+  if (!opts.json && !stream.isTTY)
+    printResults(block, resultsFile);
   if (opts.json)
     console.log(JSON.stringify(doc, null, 2));
   if (status === "error")
@@ -12049,7 +12102,7 @@ function copyToClipboard(text) {
   });
 }
 var LOGIN_HINT = "Runs stay local: npx firegoat login uploads them to firegoat.ai";
-function cut(s, n) {
+function cut2(s, n) {
   return s.length <= n ? s : `${s.slice(0, Math.max(0, n - 1))}~`;
 }
 
@@ -12064,8 +12117,8 @@ class Timeline {
   line(mark, n, label, status = "") {
     const head = `  ${mark}  ${this.paint("90", String(n).padStart(2, "0"))}  `;
     if (!status)
-      return head + cut(label, this.width - 9);
-    return `${head}${label.padEnd(28)}${this.paint("90", cut(status, this.width - 37))}`;
+      return head + cut2(label, this.width - 9);
+    return `${head}${label.padEnd(28)}${this.paint("90", cut2(status, this.width - 37))}`;
   }
   done(n, label, status) {
     return this.line(this.paint("32", "●"), n, label, status);
@@ -12088,7 +12141,7 @@ class Timeline {
     return select(block, options.length, (current) => [
       this.line(this.paint("38;5;208", "◉"), n, question),
       ...options.map(([label, note], i) => {
-        const text = cut(`${label.padEnd(cols)}${note}`, room);
+        const text = cut2(`${label.padEnd(cols)}${note}`, room);
         return i === current ? `${Timeline.INDENT}${this.paint("38;5;208", `● ${text}`)}` : this.paint("90", `${Timeline.INDENT}○ ${text}`);
       }),
       this.paint("90", `${Timeline.INDENT}Up/down to move, enter to pick`)

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.29.0",
+	"version": "0.30.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/firegoat-ai/firegoat-cli.git"