@forwardimpact/libeval 0.1.60 → 0.1.62

package/bin/fit-benchmark.js

@@ -2,152 +2,11 @@
 
 import "@forwardimpact/libpreflight/node22";
 
-import { realpathSync } from "node:fs";
 import { createCli } from "@forwardimpact/libcli";
 import { createDefaultRuntime } from "@forwardimpact/libutil/runtime";
 import { createLogger } from "@forwardimpact/libtelemetry";
 
-import { runBenchmarkRunCommand } from "../src/commands/benchmark-run.js";
-import { runBenchmarkInvariantsCommand } from "../src/commands/benchmark-invariants.js";
-import { runBenchmarkReportCommand } from "../src/commands/benchmark-report.js";
-import {
-  BENCHMARK_AGENT_MODEL,
-  LEAD_MODEL,
-} from "@forwardimpact/libutil/models";
-
-export const definition = {
-  name: "fit-benchmark",
-  description:
-    "Run coding-agent task families, grade hidden tests, and aggregate pass@k across runs.",
-  commands: [
-    {
-      name: "run",
-      args: [],
-      handler: runBenchmarkRunCommand,
-      description:
-        "Run every task in a family for N runs and emit one result record per (task, runIndex).",
-      options: {
-        family: {
-          type: "string",
-          description: "Path or git URL to a task family",
-        },
-        output: {
-          type: "string",
-          description:
-            "Run-output directory (created if missing, default: benchmark-runs)",
-        },
-        runs: {
-          type: "string",
-          description: "Runs per task (integer ≥ 1, default: 5)",
-        },
-        "agent-model": {
-          type: "string",
-          description: `Claude model for the agent-under-test (default: ${BENCHMARK_AGENT_MODEL})`,
-        },
-        "lead-model": {
-          type: "string",
-          description: `Claude model for the lead role (default: ${LEAD_MODEL})`,
-        },
-        "judge-model": {
-          type: "string",
-          description: `Claude model for the judge (default: ${LEAD_MODEL})`,
-        },
-        "agent-profile": {
-          type: "string",
-          description: "Agent-under-test profile name",
-        },
-        "judge-profile": {
-          type: "string",
-          description: "Judge profile name",
-        },
-        "max-turns": {
-          type: "string",
-          description:
-            "Agent-under-test turn budget (default: 50, 0 = unlimited)",
-        },
-        "allowed-tools": {
-          type: "string",
-          description:
-            "Comma-separated tool allowlist for the agent-under-test (default: Bash,Read,Glob,Grep,Write,Edit,Agent,TodoWrite)",
-        },
-      },
-    },
-    {
-      name: "invariants",
-      args: [],
-      handler: runBenchmarkInvariantsCommand,
-      description:
-        "Check a single task's invariants against a post-run workdir without invoking an agent.",
-      options: {
-        family: {
-          type: "string",
-          description: "Path or git URL to a task family",
-        },
-        task: {
-          type: "string",
-          description: "Task id (directory name under tasks/)",
-        },
-        workdir: {
-          type: "string",
-          description:
-            "Post-run directory; <workdir>/cwd/ is the agent CWD invariants run against",
-        },
-        output: {
-          type: "string",
-          description: "Output file (defaults to stdout; one JSONL line)",
-        },
-      },
-    },
-    {
-      name: "report",
-      args: [],
-      handler: runBenchmarkReportCommand,
-      description:
-        "Aggregate result records into pass@k via the OpenAI HumanEval estimator.",
-      options: {
-        input: {
-          type: "string",
-          description:
-            "Run-output directory containing results.jsonl (default: benchmark-runs)",
-        },
-        k: {
-          type: "string",
-          description: "Comma-separated k values (default: 1,3,5)",
-        },
-        format: {
-          type: "string",
-          description: "Output format (json|text, default: json)",
-        },
-      },
-    },
-  ],
-  globalOptions: {
-    help: { type: "boolean", short: "h", description: "Show this help" },
-    version: { type: "boolean", description: "Show version" },
-    json: { type: "boolean", description: "Output help as JSON" },
-  },
-  examples: [
-    "fit-benchmark run --family=./families/coding",
-    `fit-benchmark run --family=./families/coding --runs=10 --agent-model=${BENCHMARK_AGENT_MODEL}`,
-    "fit-benchmark invariants --family=./families/coding --task=todo-api --workdir=./benchmark-runs/runs/todo-api/0",
-    "fit-benchmark report --format=text",
-    "fit-benchmark report --input=./runs/today --k=1,3,5 --format=text",
-  ],
-  documentation: [
-    {
-      title: "Run a Benchmark",
-      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/index.md",
-      description:
-        "Author a coding-task family, run a benchmark across multiple runs, and read the pass@k report.",
-    },
-    {
-      title: "Automate with GitHub Actions",
-      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/ci-workflow/index.md",
-      description:
-        "Run benchmarks in CI with the forwardimpact/fit-benchmark action.",
-    },
-  ],
-};
+import { definition } from "../src/commands/benchmark-definition.js";
 
 const runtime = createDefaultRuntime();
 const logger = createLogger("benchmark", runtime);
@@ -178,12 +37,8 @@ async function main() {
   runtime.proc.exit(envelope.ok ? 0 : (envelope.code ?? 1));
 }
 
-// Run main only when invoked as a CLI. Importing for tests (e.g. parity)
-// should not execute the entry point.
-if (import.meta.url === `file://${realpathSync(process.argv[1])}`) {
-  main().catch((error) => {
-    logger.exception("main", error);
-    createCli(definition, { runtime }).error(error.message);
-    process.exit(1);
-  });
-}
+main().catch((error) => {
+  logger.exception("main", error);
+  createCli(definition, { runtime }).error(error.message);
+  process.exit(1);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.60",
+  "version": "0.1.62",
   "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
   "keywords": [
     "eval",

package/src/commands/benchmark-definition.js

@@ -0,0 +1,147 @@
+/**
+ * `fit-benchmark` CLI definition. Lives in `src/` so the bin stays an
+ * execute-on-import entry point — launcher packages import the bin to run
+ * it — while tests import the definition without running the CLI.
+ */
+
+import { runBenchmarkRunCommand } from "./benchmark-run.js";
+import { runBenchmarkInvariantsCommand } from "./benchmark-invariants.js";
+import { runBenchmarkReportCommand } from "./benchmark-report.js";
+import {
+  BENCHMARK_AGENT_MODEL,
+  LEAD_MODEL,
+} from "@forwardimpact/libutil/models";
+
+export const definition = {
+  name: "fit-benchmark",
+  description:
+    "Run coding-agent task families, grade hidden tests, and aggregate pass@k across runs.",
+  commands: [
+    {
+      name: "run",
+      args: [],
+      handler: runBenchmarkRunCommand,
+      description:
+        "Run every task in a family for N runs and emit one result record per (task, runIndex).",
+      options: {
+        family: {
+          type: "string",
+          description: "Path or git URL to a task family",
+        },
+        output: {
+          type: "string",
+          description:
+            "Run-output directory (created if missing, default: benchmark-runs)",
+        },
+        runs: {
+          type: "string",
+          description: "Runs per task (integer ≥ 1, default: 5)",
+        },
+        "agent-model": {
+          type: "string",
+          description: `Claude model for the agent-under-test (default: ${BENCHMARK_AGENT_MODEL})`,
+        },
+        "lead-model": {
+          type: "string",
+          description: `Claude model for the lead role (default: ${LEAD_MODEL})`,
+        },
+        "judge-model": {
+          type: "string",
+          description: `Claude model for the judge (default: ${LEAD_MODEL})`,
+        },
+        "agent-profile": {
+          type: "string",
+          description: "Agent-under-test profile name",
+        },
+        "judge-profile": {
+          type: "string",
+          description: "Judge profile name",
+        },
+        "max-turns": {
+          type: "string",
+          description:
+            "Agent-under-test turn budget (default: 50, 0 = unlimited)",
+        },
+        "allowed-tools": {
+          type: "string",
+          description:
+            "Comma-separated tool allowlist for the agent-under-test (default: Bash,Read,Glob,Grep,Write,Edit,Agent,TodoWrite)",
+        },
+      },
+    },
+    {
+      name: "invariants",
+      args: [],
+      handler: runBenchmarkInvariantsCommand,
+      description:
+        "Check a single task's invariants against a post-run workdir without invoking an agent.",
+      options: {
+        family: {
+          type: "string",
+          description: "Path or git URL to a task family",
+        },
+        task: {
+          type: "string",
+          description: "Task id (directory name under tasks/)",
+        },
+        workdir: {
+          type: "string",
+          description:
+            "Post-run directory; <workdir>/cwd/ is the agent CWD invariants run against",
+        },
+        output: {
+          type: "string",
+          description: "Output file (defaults to stdout; one JSONL line)",
+        },
+      },
+    },
+    {
+      name: "report",
+      args: [],
+      handler: runBenchmarkReportCommand,
+      description:
+        "Aggregate result records into pass@k via the OpenAI HumanEval estimator.",
+      options: {
+        input: {
+          type: "string",
+          description:
+            "Run-output directory containing results.jsonl (default: benchmark-runs)",
+        },
+        k: {
+          type: "string",
+          description: "Comma-separated k values (default: 1,3,5)",
+        },
+        format: {
+          type: "string",
+          description: "Output format (json|text, default: json)",
+        },
+      },
+    },
+  ],
+  globalOptions: {
+    help: { type: "boolean", short: "h", description: "Show this help" },
+    version: { type: "boolean", description: "Show version" },
+    json: { type: "boolean", description: "Output help as JSON" },
+  },
+  examples: [
+    "fit-benchmark run --family=./families/coding",
+    `fit-benchmark run --family=./families/coding --runs=10 --agent-model=${BENCHMARK_AGENT_MODEL}`,
+    "fit-benchmark invariants --family=./families/coding --task=todo-api --workdir=./benchmark-runs/runs/todo-api/0",
+    "fit-benchmark report --format=text",
+    "fit-benchmark report --input=./runs/today --k=1,3,5 --format=text",
+  ],
+  documentation: [
+    {
+      title: "Run a Benchmark",
+      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/index.md",
+      description:
+        "Author a coding-task family, run a benchmark across multiple runs, and read the pass@k report.",
+    },
+    {
+      title: "Automate with GitHub Actions",
+      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/ci-workflow/index.md",
+      description:
+        "Run benchmarks in CI with the forwardimpact/fit-benchmark action.",
+    },
+  ],
+};

package/src/trace-collector.js

@@ -218,25 +218,24 @@ export class TraceCollector {
   }
 
   /**
+   * Accumulate a result event into the running summary. Facilitated and
+   * supervised sessions emit one result event per runner invocation, so a
+   * single trace can carry several — cost, duration, turn, and token
+   * figures sum across all of them. `result` reflects the latest event;
+   * `isError` is true once any event errored.
    * @param {object} event
    */
   handleResult(event) {
+    const prev = this.result ?? EMPTY_RESULT;
+
     this.result = {
       result: event.subtype ?? "unknown",
-      isError: event.is_error ?? false,
-      totalCostUsd: event.total_cost_usd ?? 0,
-      durationMs: event.duration_ms ?? 0,
-      numTurns: event.num_turns ?? 0,
-      tokenUsage: event.usage
-        ? {
-            inputTokens: event.usage.input_tokens ?? 0,
-            outputTokens: event.usage.output_tokens ?? 0,
-            cacheReadInputTokens: event.usage.cache_read_input_tokens ?? 0,
-            cacheCreationInputTokens:
-              event.usage.cache_creation_input_tokens ?? 0,
-          }
-        : null,
-      modelUsage: event.modelUsage ?? null,
+      isError: prev.isError || (event.is_error ?? false),
+      totalCostUsd: prev.totalCostUsd + (event.total_cost_usd ?? 0),
+      durationMs: prev.durationMs + (event.duration_ms ?? 0),
+      numTurns: prev.numTurns + (event.num_turns ?? 0),
+      tokenUsage: sumTokenUsage(prev.tokenUsage, normalizeUsage(event.usage)),
+      modelUsage: event.modelUsage ?? prev.modelUsage,
     };
   }
 
@@ -303,7 +302,9 @@ export class TraceCollector {
    * Format the trailing result summary line. When an orchestrator
    * summary is present (supervised / facilitated mode), the headline word is
    * the supervisor's verdict ("success" / "failure") rather than the SDK's
-   * per-runner subtype, so the footer aligns with the CI exit code.
+   * per-runner subtype, so the footer aligns with the CI exit code. Turn,
+   * cost, and duration figures are the accumulated totals across every
+   * result event in the trace, not the last event's.
    * @returns {string}
    */
   #formatResultTail() {
@@ -318,6 +319,50 @@ export class TraceCollector {
   }
 }
 
+/** Identity element for result-event accumulation in handleResult. */
+const EMPTY_RESULT = {
+  isError: false,
+  totalCostUsd: 0,
+  durationMs: 0,
+  numTurns: 0,
+  tokenUsage: null,
+  modelUsage: null,
+};
+
+/**
+ * Normalize an SDK snake_case usage block to camelCase token fields.
+ * @param {object|null|undefined} usage
+ * @returns {object|null}
+ */
+function normalizeUsage(usage) {
+  if (!usage) return null;
+  return {
+    inputTokens: usage.input_tokens ?? 0,
+    outputTokens: usage.output_tokens ?? 0,
+    cacheReadInputTokens: usage.cache_read_input_tokens ?? 0,
+    cacheCreationInputTokens: usage.cache_creation_input_tokens ?? 0,
+  };
+}
+
+/**
+ * Sum two token-usage records field-by-field. Either side may be null
+ * (a result event without usage); the sum is null only when both are.
+ * @param {object|null} a
+ * @param {object|null} b
+ * @returns {object|null}
+ */
+function sumTokenUsage(a, b) {
+  if (!a) return b;
+  if (!b) return a;
+  return {
+    inputTokens: a.inputTokens + b.inputTokens,
+    outputTokens: a.outputTokens + b.outputTokens,
+    cacheReadInputTokens: a.cacheReadInputTokens + b.cacheReadInputTokens,
+    cacheCreationInputTokens:
+      a.cacheCreationInputTokens + b.cacheCreationInputTokens,
+  };
+}
+
 /**
  * Format milliseconds into a human-readable duration.
  * @param {number} ms - Duration in milliseconds

package/src/trace-query.js

@@ -278,38 +278,20 @@ export class TraceQuery {
 
   /**
    * Token usage and cost breakdown per assistant turn, plus totals.
+   *
+   * Token totals prefer the summary's result-event usage — the SDK's
+   * authoritative ledger, accumulated across every result event in the
+   * trace — over per-turn sums, whose stream-time snapshots double-count
+   * re-emitted messages. Traces without a result event (truncated or
+   * in-flight) fall back to the per-turn sums.
    * @returns {object}
    */
   stats() {
-    let totalInput = 0;
-    let totalOutput = 0;
-    let totalCacheRead = 0;
-    let totalCacheCreate = 0;
-    const perTurn = [];
-
-    for (const turn of this.turns) {
-      if (turn.role !== "assistant" || !turn.usage) continue;
-      const u = turn.usage;
-      totalInput += u.inputTokens ?? 0;
-      totalOutput += u.outputTokens ?? 0;
-      totalCacheRead += u.cacheReadInputTokens ?? 0;
-      totalCacheCreate += u.cacheCreationInputTokens ?? 0;
-
-      perTurn.push({
-        index: turn.index,
-        inputTokens: u.inputTokens ?? 0,
-        outputTokens: u.outputTokens ?? 0,
-        cacheReadInputTokens: u.cacheReadInputTokens ?? 0,
-        cacheCreationInputTokens: u.cacheCreationInputTokens ?? 0,
-      });
-    }
-
+    const { perTurn, totals: turnTotals } = perTurnUsage(this.turns);
+    const tokenTotals = this.summary.tokenUsage ?? turnTotals;
     return {
       totals: {
-        inputTokens: totalInput,
-        outputTokens: totalOutput,
-        cacheReadInputTokens: totalCacheRead,
-        cacheCreationInputTokens: totalCacheCreate,
+        ...tokenTotals,
         totalCostUsd: this.summary.totalCostUsd ?? 0,
         durationMs: this.summary.durationMs ?? 0,
       },
@@ -318,6 +300,38 @@ export class TraceQuery {
   }
 }
 
+/**
+ * Sum per-turn assistant usage and build the per-turn breakdown rows.
+ * @param {object[]} turns
+ * @returns {{perTurn: object[], totals: object}}
+ */
+function perTurnUsage(turns) {
+  const totals = {
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheReadInputTokens: 0,
+    cacheCreationInputTokens: 0,
+  };
+  const perTurn = [];
+
+  for (const turn of turns) {
+    if (turn.role !== "assistant" || !turn.usage) continue;
+    const row = {
+      index: turn.index,
+      inputTokens: turn.usage.inputTokens ?? 0,
+      outputTokens: turn.usage.outputTokens ?? 0,
+      cacheReadInputTokens: turn.usage.cacheReadInputTokens ?? 0,
+      cacheCreationInputTokens: turn.usage.cacheCreationInputTokens ?? 0,
+    };
+    totals.inputTokens += row.inputTokens;
+    totals.outputTokens += row.outputTokens;
+    totals.cacheReadInputTokens += row.cacheReadInputTokens;
+    totals.cacheCreationInputTokens += row.cacheCreationInputTokens;
+    perTurn.push(row);
+  }
+  return { perTurn, totals };
+}
+
 /**
  * @param {object} turn
  * @param {string|undefined} role