firegoat 0.30.0 → 0.31.0

package/README.md

@@ -8,7 +8,7 @@ So far we've got the CLI. It lets anyone use a simplified version of our product
 
 ## What you get
 
-When you run `firegoat eval`, you get visibility into how agents use your product: whether they fail or succeed, and how much time they take doing it. You can point them at your local codebase or at your cloud product, and see how Claude acts on the simple tasks your product aims to do.
+When you run `firegoat eval`, you get visibility into how agents use your product: whether they fail or succeed, and how much time and how many tokens they take doing it. You can point them at your local codebase or at your cloud product, and see how Claude acts on the simple tasks your product aims to do. Give harder tests more `reward_points` and eval tells you the tokens spent per point earned, per harness and model.
 
 ## Start
 

package/dist/cli.js

@@ -6947,7 +6947,7 @@ var require_public_api = __commonJS(function(exports) {
 // package.json
 var package_default = {
   name: "firegoat",
-  version: "0.30.0",
+  version: "0.31.0",
   repository: {
     type: "git",
     url: "git+https://github.com/firegoat-ai/firegoat-cli.git"
@@ -7039,6 +7039,230 @@ var $visitAsync = visit.visitAsync;
 import { spawnSync } from "node:child_process";
 import { existsSync, readdirSync } from "node:fs";
 import { join } from "node:path";
+
+// src/process.ts
+import { execFileSync, spawn } from "node:child_process";
+var NPM_INJECTED = /^(npm_config_|npm_lifecycle_|npm_package_)|^(npm_execpath|npm_node_execpath|npm_command|INIT_CWD)$/;
+var CLAUDE_SESSION = /^(CLAUDECODE|CLAUDE_CODE_.+|CLAUDE_PID|CLAUDE_ENV_FILE|CLAUDE_SESSION_ID|CLAUDE_PROJECT_DIR|CLAUDE_EFFORT|CLAUDE_AGENT_SDK_.+|CLAUDE_PLUGIN_ROOT|CLAUDE_PLUGIN_DATA|CLAUDE_SKILL_DIR)$/;
+var CLAUDE_AUTH = /^CLAUDE_CODE_(OAUTH_TOKEN|OAUTH_REFRESH_TOKEN|USE_[A-Z_]+|SKIP_[A-Z]+_AUTH|CLIENT_CERT|CLIENT_KEY|CLIENT_KEY_PASSPHRASE)$/;
+function hostEnv() {
+  return Object.fromEntries(Object.entries(process.env).filter(([k]) => !NPM_INJECTED.test(k) && !k.startsWith("FIREGOAT_") && (!CLAUDE_SESSION.test(k) || CLAUDE_AUTH.test(k))));
+}
+var GRACE_MS = 5000;
+var liveGroups = new Set;
+var signalsHooked = false;
+function hookSignals() {
+  if (signalsHooked)
+    return;
+  signalsHooked = true;
+  const onSignal = (signal) => {
+    for (const g of liveGroups)
+      try {
+        process.kill(-g, "SIGKILL");
+      } catch {}
+    process.kill(process.pid, signal);
+  };
+  for (const s of ["SIGINT", "SIGTERM", "SIGHUP"])
+    process.once(s, onSignal);
+}
+function forgetGroup(pgid) {
+  liveGroups.delete(pgid);
+}
+function processTree(pid) {
+  const children = new Map;
+  const ps = execFileSync("ps", ["-A", "-o", "pid=,ppid="], {
+    encoding: "utf8"
+  });
+  for (const line of ps.split(`
+`)) {
+    const [child, parent] = line.trim().split(/\s+/).map(Number);
+    if (!child || parent === undefined)
+      continue;
+    children.set(parent, [...children.get(parent) ?? [], child]);
+  }
+  const tree = [pid];
+  for (let i = 0;i < tree.length; i++)
+    tree.push(...children.get(tree[i]) ?? []);
+  return tree;
+}
+function killTree(pid, group) {
+  if (group)
+    try {
+      process.kill(-pid, "SIGKILL");
+    } catch {}
+  let tree = [pid];
+  let problem;
+  try {
+    tree = processTree(pid);
+  } catch (e) {
+    problem = e.message;
+  }
+  for (const p of tree)
+    try {
+      process.kill(p, "SIGKILL");
+    } catch {}
+  return problem;
+}
+function run(opts) {
+  return new Promise((resolve) => {
+    const detached = opts.detached === true;
+    if (detached)
+      hookSignals();
+    const child = spawn(opts.cmd, opts.args, {
+      cwd: opts.cwd,
+      env: opts.env,
+      stdio: ["ignore", "pipe", "pipe"],
+      detached
+    });
+    if (child.pid !== undefined) {
+      if (detached)
+        liveGroups.add(child.pid);
+      opts.onSpawn?.(child.pid);
+    }
+    let timedOut = false;
+    let settled = false;
+    const timer = setTimeout(() => {
+      timedOut = true;
+      if (child.pid === undefined)
+        return;
+      const problem = killTree(child.pid, detached);
+      if (problem)
+        opts.onStderr(Buffer.from(`firegoat: timed out; could not list ${opts.cmd}'s child processes (${problem}), killed only ${opts.cmd}
+`));
+    }, opts.timeoutMs);
+    const done = (r) => {
+      if (settled)
+        return;
+      settled = true;
+      clearTimeout(timer);
+      resolve(r);
+    };
+    child.stdout.on("data", opts.onStdout);
+    child.stderr.on("data", opts.onStderr);
+    opts.finalSeen?.then(() => setTimeout(() => done({ code: null, timedOut: false, ended: true }), GRACE_MS).unref());
+    child.on("error", (error) => done({ code: null, timedOut, error }));
+    child.on("close", (code) => done({ code, timedOut }));
+    child.on("exit", (code) => setTimeout(() => {
+      child.stdout.destroy();
+      child.stderr.destroy();
+      done({ code, timedOut });
+    }, 2000).unref());
+  });
+}
+async function runShell(opts) {
+  opts.log.write(`$ ${opts.run}
+`);
+  const r = await run({
+    cmd: "/bin/sh",
+    args: ["-c", opts.run],
+    cwd: opts.cwd,
+    env: opts.env,
+    timeoutMs: opts.timeoutMs,
+    onStdout: (c) => opts.log.write(c),
+    onStderr: (c) => opts.log.write(c)
+  });
+  let error;
+  if (r.error)
+    error = `could not start /bin/sh: ${r.error.message}`;
+  else if (r.timedOut)
+    error = `timed out after ${opts.timeoutMs / 1000}s`;
+  else if (r.code === null)
+    error = "killed by a signal";
+  opts.log.write(`[exit ${r.code ?? "none"}${error ? `: ${error}` : ""}]
+
+`);
+  return error ? { run: opts.run, code: r.code, error } : { run: opts.run, code: r.code };
+}
+
+// src/harness.ts
+function tokens(parts = {
+  input: null,
+  output: null,
+  cacheRead: null,
+  cacheWrite: null
+}) {
+  const { input, output, cacheRead, cacheWrite } = parts;
+  const total = input === null || output === null || cacheRead === null || cacheWrite === null ? null : input + output + cacheRead + cacheWrite;
+  return { input, output, cacheRead, cacheWrite, total };
+}
+function modelUsageTokens(modelUsage) {
+  if (typeof modelUsage !== "object" || modelUsage === null)
+    return tokens();
+  const entries = Object.values(modelUsage);
+  if (entries.length === 0)
+    return tokens();
+  const sum = (key) => {
+    let n = 0;
+    for (const e of entries) {
+      const v = e?.[key];
+      if (typeof v !== "number" || !Number.isInteger(v) || v < 0)
+        return null;
+      n += v;
+    }
+    return n;
+  };
+  return tokens({
+    input: sum("inputTokens"),
+    output: sum("outputTokens"),
+    cacheRead: sum("cacheReadInputTokens"),
+    cacheWrite: sum("cacheCreationInputTokens")
+  });
+}
+async function runAgent(harness, opts) {
+  let stdout = "";
+  let pending = "";
+  let group;
+  let markFinal = () => {};
+  const finalSeen = new Promise((r) => {
+    markFinal = r;
+  });
+  const exited = await run({
+    cmd: opts.bin,
+    args: harness.args(opts),
+    cwd: opts.cwd,
+    env: opts.env,
+    timeoutMs: opts.timeoutMs,
+    detached: true,
+    finalSeen,
+    onSpawn: (pid) => {
+      group = pid;
+    },
+    onStdout: (c) => {
+      const text = c.toString();
+      stdout += text;
+      pending += text;
+      const lines = pending.split(`
+`);
+      pending = lines.pop() ?? "";
+      for (const line of lines) {
+        if (harness.isFinal(line))
+          markFinal();
+        const p = opts.onProgress && harness.progress(line);
+        if (p)
+          opts.onProgress?.(p);
+      }
+    },
+    onStderr: (c) => opts.stderr.write(c)
+  });
+  if (exited.error)
+    return {
+      group,
+      outcome: {
+        ok: false,
+        reason: `could not start ${opts.bin}: ${exited.error.message}`,
+        steps: [],
+        final: "",
+        reportedModel: null,
+        version: null,
+        turns: null,
+        cost: null,
+        tokens: tokens()
+      }
+    };
+  return { group, outcome: harness.read(stdout, exited, opts.timeoutMs) };
+}
+
+// src/harnesses/claude.ts
 var FLAGS = [
   "--output-format",
   "stream-json",
@@ -7050,6 +7274,17 @@ var FLAGS = [
   "--disable-slash-commands"
 ];
 var SET = "firegoat sets it for every run";
+function sumTurns(results) {
+  if (results.length === 0)
+    return null;
+  let n = 0;
+  for (const r of results) {
+    if (typeof r.num_turns !== "number")
+      return null;
+    n += r.num_turns;
+  }
+  return n;
+}
 function parse(line) {
   try {
     const e = JSON.parse(line);
@@ -7105,12 +7340,14 @@ function read(stdout, exited, timeoutMs) {
   for (const e of events)
     if (e?.type === "result" && typeof e.result === "string")
       final = e.result;
-  const result = events.findLast((e) => e?.type === "result");
+  const results = events.filter((e) => e?.type === "result");
+  const result = results.at(-1);
   const trace = {
     steps: steps(events),
     final,
-    turns: result?.num_turns ?? null,
-    cost: result?.total_cost_usd ?? null
+    turns: sumTurns(results),
+    cost: result?.total_cost_usd ?? null,
+    tokens: modelUsageTokens(result?.modelUsage)
   };
   const init = events.find((e) => e?.type === "system" && e.subtype === "init");
   if (!init) {
@@ -7269,142 +7506,6 @@ import { execFile } from "node:child_process";
 import { mkdirSync, writeFileSync } from "node:fs";
 import { join as join2 } from "node:path";
 import { promisify } from "node:util";
-
-// src/process.ts
-import { execFileSync, spawn } from "node:child_process";
-var NPM_INJECTED = /^(npm_config_|npm_lifecycle_|npm_package_)|^(npm_execpath|npm_node_execpath|npm_command|INIT_CWD)$/;
-var CLAUDE_SESSION = /^(CLAUDECODE|CLAUDE_CODE_.+|CLAUDE_PID|CLAUDE_ENV_FILE|CLAUDE_SESSION_ID|CLAUDE_PROJECT_DIR|CLAUDE_EFFORT|CLAUDE_AGENT_SDK_.+|CLAUDE_PLUGIN_ROOT|CLAUDE_PLUGIN_DATA|CLAUDE_SKILL_DIR)$/;
-var CLAUDE_AUTH = /^CLAUDE_CODE_(OAUTH_TOKEN|OAUTH_REFRESH_TOKEN|USE_[A-Z_]+|SKIP_[A-Z]+_AUTH|CLIENT_CERT|CLIENT_KEY|CLIENT_KEY_PASSPHRASE)$/;
-function hostEnv() {
-  return Object.fromEntries(Object.entries(process.env).filter(([k]) => !NPM_INJECTED.test(k) && !k.startsWith("FIREGOAT_") && (!CLAUDE_SESSION.test(k) || CLAUDE_AUTH.test(k))));
-}
-var GRACE_MS = 5000;
-var liveGroups = new Set;
-var signalsHooked = false;
-function hookSignals() {
-  if (signalsHooked)
-    return;
-  signalsHooked = true;
-  const onSignal = (signal) => {
-    for (const g of liveGroups)
-      try {
-        process.kill(-g, "SIGKILL");
-      } catch {}
-    process.kill(process.pid, signal);
-  };
-  for (const s of ["SIGINT", "SIGTERM", "SIGHUP"])
-    process.once(s, onSignal);
-}
-function forgetGroup(pgid) {
-  liveGroups.delete(pgid);
-}
-function processTree(pid) {
-  const children = new Map;
-  const ps = execFileSync("ps", ["-A", "-o", "pid=,ppid="], {
-    encoding: "utf8"
-  });
-  for (const line of ps.split(`
-`)) {
-    const [child, parent] = line.trim().split(/\s+/).map(Number);
-    if (!child || parent === undefined)
-      continue;
-    children.set(parent, [...children.get(parent) ?? [], child]);
-  }
-  const tree = [pid];
-  for (let i = 0;i < tree.length; i++)
-    tree.push(...children.get(tree[i]) ?? []);
-  return tree;
-}
-function killTree(pid, group) {
-  if (group)
-    try {
-      process.kill(-pid, "SIGKILL");
-    } catch {}
-  let tree = [pid];
-  let problem;
-  try {
-    tree = processTree(pid);
-  } catch (e) {
-    problem = e.message;
-  }
-  for (const p of tree)
-    try {
-      process.kill(p, "SIGKILL");
-    } catch {}
-  return problem;
-}
-function run(opts) {
-  return new Promise((resolve) => {
-    const detached = opts.detached === true;
-    if (detached)
-      hookSignals();
-    const child = spawn(opts.cmd, opts.args, {
-      cwd: opts.cwd,
-      env: opts.env,
-      stdio: ["ignore", "pipe", "pipe"],
-      detached
-    });
-    if (child.pid !== undefined) {
-      if (detached)
-        liveGroups.add(child.pid);
-      opts.onSpawn?.(child.pid);
-    }
-    let timedOut = false;
-    let settled = false;
-    const timer = setTimeout(() => {
-      timedOut = true;
-      if (child.pid === undefined)
-        return;
-      const problem = killTree(child.pid, detached);
-      if (problem)
-        opts.onStderr(Buffer.from(`firegoat: timed out; could not list ${opts.cmd}'s child processes (${problem}), killed only ${opts.cmd}
-`));
-    }, opts.timeoutMs);
-    const done = (r) => {
-      if (settled)
-        return;
-      settled = true;
-      clearTimeout(timer);
-      resolve(r);
-    };
-    child.stdout.on("data", opts.onStdout);
-    child.stderr.on("data", opts.onStderr);
-    opts.finalSeen?.then(() => setTimeout(() => done({ code: null, timedOut: false, ended: true }), GRACE_MS).unref());
-    child.on("error", (error) => done({ code: null, timedOut, error }));
-    child.on("close", (code) => done({ code, timedOut }));
-    child.on("exit", (code) => setTimeout(() => {
-      child.stdout.destroy();
-      child.stderr.destroy();
-      done({ code, timedOut });
-    }, 2000).unref());
-  });
-}
-async function runShell(opts) {
-  opts.log.write(`$ ${opts.run}
-`);
-  const r = await run({
-    cmd: "/bin/sh",
-    args: ["-c", opts.run],
-    cwd: opts.cwd,
-    env: opts.env,
-    timeoutMs: opts.timeoutMs,
-    onStdout: (c) => opts.log.write(c),
-    onStderr: (c) => opts.log.write(c)
-  });
-  let error;
-  if (r.error)
-    error = `could not start /bin/sh: ${r.error.message}`;
-  else if (r.timedOut)
-    error = `timed out after ${opts.timeoutMs / 1000}s`;
-  else if (r.code === null)
-    error = "killed by a signal";
-  opts.log.write(`[exit ${r.code ?? "none"}${error ? `: ${error}` : ""}]
-
-`);
-  return error ? { run: opts.run, code: r.code, error } : { run: opts.run, code: r.code };
-}
-
-// src/home.ts
 function homeEnv(home) {
   return {
     HOME: home,
@@ -7551,6 +7652,33 @@ function errorMessage(raw) {
   } catch {}
   return raw;
 }
+function usageTokens(events) {
+  const usages = events.filter((e) => e?.type === "turn.completed").map((e) => e?.usage ?? {});
+  if (usages.length === 0)
+    return tokens();
+  const sum = (key) => {
+    let n = 0;
+    for (const u of usages) {
+      const v = u[key];
+      if (typeof v !== "number" || !Number.isInteger(v) || v < 0)
+        return null;
+      n += v;
+    }
+    return n;
+  };
+  const input = sum("input_tokens");
+  const cacheRead = sum("cached_input_tokens");
+  const cacheWrite = sum("cache_write_input_tokens");
+  const uncached = input === null || cacheRead === null || cacheWrite === null ? null : input - cacheRead - cacheWrite;
+  if (uncached !== null && uncached < 0)
+    throw new Error(`codex reported input_tokens ${input} below cached_input_tokens ${cacheRead} + cache_write_input_tokens ${cacheWrite}: its usage is no longer what firegoat reads (src/harnesses/codex.ts usageTokens)`);
+  return tokens({
+    input: uncached,
+    output: sum("output_tokens"),
+    cacheRead,
+    cacheWrite
+  });
+}
 function read2(stdout, exited, timeoutMs) {
   const events = stdout.split(`
 `).filter((l) => l.trim() !== "").map(parse2);
@@ -7560,7 +7688,8 @@ function read2(stdout, exited, timeoutMs) {
     reportedModel: null,
     version: null,
     turns: null,
-    cost: null
+    cost: null,
+    tokens: usageTokens(events)
   };
   const thread = events.find((e) => e?.type === "thread.started");
   if (!thread) {
@@ -7833,7 +7962,8 @@ function read3(stdout, exited, timeoutMs) {
     steps: steps3(events),
     final: typeof result?.result === "string" ? result.result : "",
     turns: result?.num_turns ?? null,
-    cost: result?.total_cost_usd ?? null
+    cost: result?.total_cost_usd ?? null,
+    tokens: modelUsageTokens(result?.modelUsage)
   };
   const init = events.find((e) => e?.type === "system" && e.subtype === "init");
   const reportedModel = typeof init?.model === "string" && init.model !== "unknown" ? init.model : null;
@@ -8110,6 +8240,7 @@ function parseConfig(text) {
       "end_goal",
       "checks",
       "timeout",
+      "reward_points",
       "env",
       "install",
       "cleanup",
@@ -8140,6 +8271,7 @@ function parseConfig(text) {
       end_goal: str2(t.end_goal, `${p}.end_goal`),
       checks,
       timeout: posInt(t.timeout, `${p}.timeout`, DEFAULT_TIMEOUT),
+      rewardPoints: posInt(t.reward_points, `${p}.reward_points`, 1),
       env: { ...env, ...parseEnv(t.env, `${p}.env`) },
       install: commands(t.install, `${p}.install`, RUN_PLACEHOLDERS),
       cleanup: commands(t.cleanup, `${p}.cleanup`, RUN_PLACEHOLDERS),
@@ -8743,60 +8875,6 @@ function gitRepo(dir) {
   };
 }
 
-// src/harness.ts
-async function runAgent(harness, opts) {
-  let stdout = "";
-  let pending = "";
-  let group;
-  let markFinal = () => {};
-  const finalSeen = new Promise((r) => {
-    markFinal = r;
-  });
-  const exited = await run({
-    cmd: opts.bin,
-    args: harness.args(opts),
-    cwd: opts.cwd,
-    env: opts.env,
-    timeoutMs: opts.timeoutMs,
-    detached: true,
-    finalSeen,
-    onSpawn: (pid) => {
-      group = pid;
-    },
-    onStdout: (c) => {
-      const text = c.toString();
-      stdout += text;
-      pending += text;
-      const lines = pending.split(`
-`);
-      pending = lines.pop() ?? "";
-      for (const line of lines) {
-        if (harness.isFinal(line))
-          markFinal();
-        const p = opts.onProgress && harness.progress(line);
-        if (p)
-          opts.onProgress?.(p);
-      }
-    },
-    onStderr: (c) => opts.stderr.write(c)
-  });
-  if (exited.error)
-    return {
-      group,
-      outcome: {
-        ok: false,
-        reason: `could not start ${opts.bin}: ${exited.error.message}`,
-        steps: [],
-        final: "",
-        reportedModel: null,
-        version: null,
-        turns: null,
-        cost: null
-      }
-    };
-  return { group, outcome: harness.read(stdout, exited, opts.timeoutMs) };
-}
-
 // src/help.ts
 var EXIT_CODES = `Exit codes:
   0  Every run's script checks passed
@@ -8944,6 +9022,8 @@ firegoat.yaml reference:
       checks:                 Optional shell commands, run in the agent's folder
         - run: ...            Afterwards; exit 0 = pass; they set eval's exit code
       timeout: 600            Optional, seconds
+      reward_points: 3        Optional positive integer (default 1): what a run
+                              earns when its checks pass; harder end goals get more
       env: {NAME: value}      Optional, on top of the top-level env:
       install: <sh> | [...]   Optional, runs after the top-level install
       cleanup: <sh> | [...]   Optional, runs before the top-level cleanup
@@ -9171,7 +9251,8 @@ firegoat.yaml, fixtures and build: stay in the current folder.
                      cleanup); env for build, install, checks and cleanup
   --json             The summary as JSON on stdout (or {error, exitCode} if eval
                      stops early); progress goes to stderr. The verdict comes
-                     last, after runs: totals {pass, fail, error}, tier
+                     last, after runs and the reward points (Reward, below):
+                     totals {pass, fail, error}, tier
                      (S/A/B/C, as printed; null when no run was graded), status
                      ("pass", "fail" or "error" = not graded) and exitCode
   --no-upload        Keep runs local even when logged in to firegoat.ai
@@ -9206,23 +9287,42 @@ Runs without a session file are not uploaded. FIREGOAT_BASE_URL points elsewhere
 Results block: without a terminal (an agent, a pipe, CI) and without --json, eval
 ends with the terminal's final timeline as one plain block (no color, at most 100
 columns): the logo header, a row per run (status, a ramp of one block per agent turn
-ending ▲ pass, ╳ fail or ╌ not graded, checks passed, cost, turns, time), its failed
-checks and trace URL, the totals and the tier ("GOATED · tier S" when all pass). It
+ending ▲ pass, ╳ fail or ╌ not graded, checks passed, cost, turns, tokens, time), its
+failed checks and trace URL, the totals, the tier ("GOATED · tier S" when all pass),
+"Reward points 7/9 · 12k tokens per point", and with more than one harness × model a
+table of each one's points, tokens per point, turns and time. It
 sits between two lines of ═, after a "Results (paste the block ...)" line; the last
 line says where it is saved, results.txt. Show it to your user as-is in a code block.
 
+Reward: value per token. A run earns its test's reward_points when its checks pass,
+else 0; not graded runs earn 0 and still count in the max. tokens = input (uncached)
++ output + cacheRead + cacheWrite, as the harness reports them, never estimated:
+claude-code and grok-cli from the result line's modelUsage (all models, subagents
+too), codex-cli from turn.completed usage (its cached and cache-written tokens are
+taken out of input_tokens). Tokens per reward point = the tokens of every run,
+passed or not, / the points earned; null when no points were earned or any run's
+tokens are unknown. Tokenizers differ (claude, gpt, grok), so token counts across
+harnesses are approximate.
+
 Output: .firegoat/runs/<runId>/
   out                     A symlink to $FIREGOAT_OUT,
                           ~/.firegoat/runs/<hash>/<random>/out
   summary.json            firegoatVersion, runId, runsDir, dir (where eval
                           ran), git {branch, commit, dirty} (no git outside a
                           repo), harnesses, upload {origin, email} (null when
-                          not uploading), runs (every run.json), then the
-                          verdict: totals, tier, status and exitCode
+                          not uploading), runs (every run.json),
+                          rewardPoints, rewardPointsMax, tokens (each part
+                          summed; null when any run's is), tokensPerRewardPoint
+                          (null with 0 points or unknown tokens), groups (per
+                          harness × model: harness, model, runs, the same four,
+                          turns, durationMs), then the verdict: totals, tier,
+                          status and exitCode
   results.txt             The results block, also with --json or in a terminal
   ../index.jsonl          One line per eval: runId, startedAt, dir, branch,
                           commit, harnesses, passed, failed, notGraded, cost (null when
-                          any run's cost is unknown), uploaded, uploadFailed
+                          any run's cost is unknown), rewardPoints,
+                          rewardPointsMax, tokens (the total, or null),
+                          uploaded, uploadFailed
   <test>/<n>/run.json     One run: status and reason (from the checks), harness,
                           model (asked for, or null) and reportedModel (what
                           the harness reported running, or null), harnessArgs
@@ -9237,7 +9337,11 @@ Output: .firegoat/runs/<runId>/
                           own file, else <test>/<n>/session.jsonl made from the
                           stream) and sessionId (its id, the trace's), stderr,
                           checksLog, cleanupLog; turns and cost (null when the
-                          harness reports none), durationMs, startedAt; upload ({status: "uploaded",
+                          harness reports none), rewardPoints (earned),
+                          rewardPointsMax (the test's reward_points), tokens
+                          {input, output, cacheRead, cacheWrite, total} (null
+                          parts when the harness reports none), durationMs,
+                          startedAt; upload ({status: "uploaded",
                           url} | {status: "failed", error} | {status:
                           "skipped", reason}, null when not uploading)
   <test>/<n>/transcript.txt  What the agent did, below
@@ -9415,6 +9519,46 @@ async function stopLeftovers(opts) {
   return leftover;
 }
 
+// src/reward.ts
+function earned(status, rewardPointsMax) {
+  return status === "pass" ? rewardPointsMax : 0;
+}
+function sum(values) {
+  let n = 0;
+  for (const v of values) {
+    if (v === null)
+      return null;
+    n += v;
+  }
+  return n;
+}
+function tally(runs) {
+  const rewardPoints = runs.reduce((s, r) => s + r.rewardPoints, 0);
+  const t = tokens({
+    input: sum(runs.map((r) => r.tokens.input)),
+    output: sum(runs.map((r) => r.tokens.output)),
+    cacheRead: sum(runs.map((r) => r.tokens.cacheRead)),
+    cacheWrite: sum(runs.map((r) => r.tokens.cacheWrite))
+  });
+  return {
+    rewardPoints,
+    rewardPointsMax: runs.reduce((s, r) => s + r.rewardPointsMax, 0),
+    tokens: t,
+    tokensPerRewardPoint: rewardPoints === 0 || t.total === null ? null : Math.round(t.total / rewardPoints)
+  };
+}
+function byGroup(runs) {
+  const groups = [];
+  for (const r of runs) {
+    const g = groups.find((x) => x.harness === r.harness && x.model === r.model);
+    if (g)
+      g.runs.push(r);
+    else
+      groups.push({ harness: r.harness, model: r.model, runs: [r] });
+  }
+  return groups;
+}
+
 // src/session.ts
 import { randomUUID } from "node:crypto";
 import { writeFileSync as writeFileSync3 } from "node:fs";
@@ -10230,16 +10374,56 @@ function ramp(r) {
   const end = r.state === "pass" ? "▲" : r.state === "fail" ? "╳" : r.state === "error" ? "╌" : r.state === "running" ? "●" : "";
   return (blocks + end).padEnd(8);
 }
+var plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
+function compactTokens(n) {
+  const unit = (v, suffix) => `${v < 9.95 ? (Math.round(v * 10) / 10).toString() : Math.round(v)}${suffix}`;
+  if (n < 1000)
+    return String(n);
+  if (n < 999500)
+    return unit(n / 1000, "k");
+  return unit(n / 1e6, "M");
+}
 function usageParts(r) {
   const parts = [];
   if (typeof r.cost === "number")
     parts.push(`$${r.cost.toFixed(2)}`);
   if (typeof r.reportedTurns === "number")
-    parts.push(`${r.reportedTurns} turns`);
+    parts.push(plural(r.reportedTurns, "turn"));
+  if (r.tokens.total !== null)
+    parts.push(`${compactTokens(r.tokens.total)} tokens`);
   if (r.seconds !== undefined)
     parts.push(`${r.seconds}s`);
   return parts;
 }
+function rewardLine(t, rows) {
+  const points = `Reward points ${t.rewardPoints}/${t.rewardPointsMax}`;
+  const unknown = rows.filter((r) => r.tokens.total === null).length;
+  if (unknown > 0)
+    return `${points} · tokens unknown for ${plural(unknown, "run")}`;
+  if (t.tokensPerRewardPoint === null)
+    return `${points} · no points, so no tokens per point`;
+  return `${points} · ${compactTokens(t.tokensPerRewardPoint)} tokens per point`;
+}
+function comparison(rows) {
+  const groups = byGroup(rows.map((r) => ({ ...r, harness: r.harnessName })));
+  const label = (g) => `${g.harness} ${g.model ?? "(default model)"}`;
+  const width = Math.min(40, Math.max(5, ...groups.map((g) => label(g).length)));
+  const line = (cells) => `  ${cells[0].padEnd(width)}  ${cells[1].padEnd(8)}  ${cells[2].padEnd(10)}  ${cells[3].padEnd(6)}  ${cells[4]}`;
+  return [
+    line(["Group", "Points", "Tokens/pt", "Turns", "Time"]),
+    ...groups.map((g) => {
+      const t = tally(g.runs);
+      const turns = totalTurns(g.runs);
+      return line([
+        cut(label(g), width),
+        `${t.rewardPoints}/${t.rewardPointsMax}`,
+        g.runs.some((r) => r.tokens.total === null) ? "unknown" : t.tokensPerRewardPoint === null ? "no points" : compactTokens(t.tokensPerRewardPoint),
+        turns === null ? "unknown" : String(turns),
+        duration(g.runs.reduce((s, r) => s + (r.seconds ?? 0), 0))
+      ]);
+    })
+  ];
+}
 function usage(r) {
   return usageParts(r).join("  ");
 }
@@ -10413,12 +10597,16 @@ function timeline(t, p, w) {
   }
   const tr = tier(rows);
   const goated = c.pass === rows.length && rows.length > 0;
-  const tally = `${c.pass} PASS · ${c.fail} FAIL · ${c.error} NOT GRADED`;
-  const left = goated ? `${rows.length}/${rows.length} runs passed their checks` : tally;
+  const passFail = `${c.pass} PASS · ${c.fail} FAIL · ${c.error} NOT GRADED`;
+  const left = goated ? `${rows.length}/${rows.length} runs passed their checks` : passFail;
   const right = goated ? `GOATED · tier ${tr}` : tr ? `Tier ${tr}` : "";
   out.push(`  ${goated ? p(C.green, left) : left}${" ".repeat(Math.max(2, w - left.length - right.length - 3))}${goated ? p(C.ember, right) : right}`);
   const turns = totalTurns(rows);
-  out.push(p(C.gray, `  Total ${totalCost(rows)}${turns === null ? "" : ` · ${turns} turns`} · ${duration(finished.seconds)}`));
+  const reward = tally(rows);
+  out.push(p(C.gray, `  Total ${totalCost(rows)}${turns === null ? "" : ` · ${plural(turns, "turn")}`}${reward.tokens.total === null ? "" : ` · ${compactTokens(reward.tokens.total)} tokens`} · ${duration(finished.seconds)}`));
+  out.push(`  ${rewardLine(reward, rows)}`);
+  if (byGroup(rows.map((r) => ({ ...r, harness: r.harnessName }))).length > 1)
+    out.push(...comparison(rows).map((l) => p(C.gray, l)));
   out.push(p(C.gray, `  Runs  ${shortPath(finished.runsDir)}/`));
   out.push(`  Checks are a score, not the verdict: judge each end_goal from ${p(C.cyan, "<test>/<n>/transcript.txt")}`);
   return out;
@@ -10906,6 +11094,9 @@ async function runOne(opts) {
     leftover: [],
     turns: null,
     cost: null,
+    rewardPoints: 0,
+    rewardPointsMax: test.rewardPoints,
+    tokens: tokens(),
     durationMs: 0,
     startedAt: new Date(started).toISOString(),
     upload: null
@@ -10937,6 +11128,7 @@ async function runOne(opts) {
   r.leftover = await stopLeftovers({ group: agent.group, workdir });
   r.status = status;
   r.reason = reason;
+  r.rewardPoints = earned(status, r.rewardPointsMax);
   r.durationMs = Date.now() - started;
   writeRunJson(r);
   return r;
@@ -11027,6 +11219,7 @@ async function attempt(opts, r, steps, env, stepEnv, homeEnvFile, agent) {
   });
   r.turns = outcome.turns;
   r.cost = outcome.cost;
+  r.tokens = outcome.tokens;
   if (!outcome.ok)
     return ["error", `not graded: ${outcome.reason}`];
   if (!r.session)
@@ -11249,8 +11442,12 @@ async function evalCommand(args, root) {
   const rows = plan.map((p) => ({
     ...p,
     harness: showHarness ? p.harness : null,
+    harnessName: p.harness,
     state: "queued",
-    turns: 0
+    turns: 0,
+    rewardPoints: 0,
+    rewardPointsMax: tests.find((t) => t.name === p.test).rewardPoints,
+    tokens: tokens()
   }));
   if (config.build.length > 0) {
     const buildStarted = Date.now();
@@ -11308,6 +11505,8 @@ async function evalCommand(args, root) {
       reason: r.reason,
       checks: r.checks,
       cost: r.cost,
+      rewardPoints: r.rewardPoints,
+      tokens: r.tokens,
       seconds: Math.round(r.durationMs / 1000),
       command: undefined
     });
@@ -11345,6 +11544,7 @@ async function evalCommand(args, root) {
     fail: EXIT.fail,
     pass: EXIT.pass
   }[status];
+  const reward = tally(results);
   const doc = {
     firegoatVersion: package_default.version,
     runId,
@@ -11354,6 +11554,15 @@ async function evalCommand(args, root) {
     harnesses: harnessNames,
     upload: upload && { origin: upload.origin, email: upload.email },
     runs: results,
+    ...reward,
+    groups: byGroup(results).map((g) => ({
+      harness: g.harness,
+      model: g.model,
+      runs: g.runs.length,
+      ...tally(g.runs),
+      turns: g.runs.every((r) => r.turns !== null) ? g.runs.reduce((s, r) => s + r.turns, 0) : null,
+      durationMs: g.runs.reduce((s, r) => s + r.durationMs, 0)
+    })),
     totals,
     tier: tier(rows),
     status,
@@ -11372,6 +11581,9 @@ async function evalCommand(args, root) {
     failed: totals.fail,
     notGraded: totals.error,
     cost: results.every((r) => r.cost !== null) ? results.reduce((sum, r) => sum + r.cost, 0) : null,
+    rewardPoints: reward.rewardPoints,
+    rewardPointsMax: reward.rewardPointsMax,
+    tokens: reward.tokens.total,
     uploaded,
     uploadFailed: uploadsFailed
   })}
@@ -12086,6 +12298,9 @@ tests:
           grep -Eiq '^agent: .*(you.*goat|goat.*you)' "$FIREGOAT_TRANSCRIPT"
     # Seconds before the agent run is stopped.
     timeout: 600
+    # What a run earns when its checks pass (default 1): give harder end goals
+    # more. eval prints tokens per point earned, the value per token.
+    # reward_points: 3
 `;
 }
 var AGENT_PROMPT = "Scan my project and setup 2-3 agent experience prompts to test general usage of the app. Use firegoat skills";

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "firegoat",
-	"version": "0.30.0",
+	"version": "0.31.0",
 	"repository": {
 		"type": "git",
 		"url": "git+https://github.com/firegoat-ai/firegoat-cli.git"