@forwardimpact/libeval 0.1.61 → 0.1.63

package/bin/fit-benchmark.js

@@ -2,152 +2,11 @@
 
 import "@forwardimpact/libpreflight/node22";
 
-import { realpathSync } from "node:fs";
 import { createCli } from "@forwardimpact/libcli";
 import { createDefaultRuntime } from "@forwardimpact/libutil/runtime";
 import { createLogger } from "@forwardimpact/libtelemetry";
 
-import { runBenchmarkRunCommand } from "../src/commands/benchmark-run.js";
-import { runBenchmarkInvariantsCommand } from "../src/commands/benchmark-invariants.js";
-import { runBenchmarkReportCommand } from "../src/commands/benchmark-report.js";
-import {
-  BENCHMARK_AGENT_MODEL,
-  LEAD_MODEL,
-} from "@forwardimpact/libutil/models";
-
-export const definition = {
-  name: "fit-benchmark",
-  description:
-    "Run coding-agent task families, grade hidden tests, and aggregate pass@k across runs.",
-  commands: [
-    {
-      name: "run",
-      args: [],
-      handler: runBenchmarkRunCommand,
-      description:
-        "Run every task in a family for N runs and emit one result record per (task, runIndex).",
-      options: {
-        family: {
-          type: "string",
-          description: "Path or git URL to a task family",
-        },
-        output: {
-          type: "string",
-          description:
-            "Run-output directory (created if missing, default: benchmark-runs)",
-        },
-        runs: {
-          type: "string",
-          description: "Runs per task (integer ≥ 1, default: 5)",
-        },
-        "agent-model": {
-          type: "string",
-          description: `Claude model for the agent-under-test (default: ${BENCHMARK_AGENT_MODEL})`,
-        },
-        "lead-model": {
-          type: "string",
-          description: `Claude model for the lead role (default: ${LEAD_MODEL})`,
-        },
-        "judge-model": {
-          type: "string",
-          description: `Claude model for the judge (default: ${LEAD_MODEL})`,
-        },
-        "agent-profile": {
-          type: "string",
-          description: "Agent-under-test profile name",
-        },
-        "judge-profile": {
-          type: "string",
-          description: "Judge profile name",
-        },
-        "max-turns": {
-          type: "string",
-          description:
-            "Agent-under-test turn budget (default: 50, 0 = unlimited)",
-        },
-        "allowed-tools": {
-          type: "string",
-          description:
-            "Comma-separated tool allowlist for the agent-under-test (default: Bash,Read,Glob,Grep,Write,Edit,Agent,TodoWrite)",
-        },
-      },
-    },
-    {
-      name: "invariants",
-      args: [],
-      handler: runBenchmarkInvariantsCommand,
-      description:
-        "Check a single task's invariants against a post-run workdir without invoking an agent.",
-      options: {
-        family: {
-          type: "string",
-          description: "Path or git URL to a task family",
-        },
-        task: {
-          type: "string",
-          description: "Task id (directory name under tasks/)",
-        },
-        workdir: {
-          type: "string",
-          description:
-            "Post-run directory; <workdir>/cwd/ is the agent CWD invariants run against",
-        },
-        output: {
-          type: "string",
-          description: "Output file (defaults to stdout; one JSONL line)",
-        },
-      },
-    },
-    {
-      name: "report",
-      args: [],
-      handler: runBenchmarkReportCommand,
-      description:
-        "Aggregate result records into pass@k via the OpenAI HumanEval estimator.",
-      options: {
-        input: {
-          type: "string",
-          description:
-            "Run-output directory containing results.jsonl (default: benchmark-runs)",
-        },
-        k: {
-          type: "string",
-          description: "Comma-separated k values (default: 1,3,5)",
-        },
-        format: {
-          type: "string",
-          description: "Output format (json|text, default: json)",
-        },
-      },
-    },
-  ],
-  globalOptions: {
-    help: { type: "boolean", short: "h", description: "Show this help" },
-    version: { type: "boolean", description: "Show version" },
-    json: { type: "boolean", description: "Output help as JSON" },
-  },
-  examples: [
-    "fit-benchmark run --family=./families/coding",
-    `fit-benchmark run --family=./families/coding --runs=10 --agent-model=${BENCHMARK_AGENT_MODEL}`,
-    "fit-benchmark invariants --family=./families/coding --task=todo-api --workdir=./benchmark-runs/runs/todo-api/0",
-    "fit-benchmark report --format=text",
-    "fit-benchmark report --input=./runs/today --k=1,3,5 --format=text",
-  ],
-  documentation: [
-    {
-      title: "Run a Benchmark",
-      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/index.md",
-      description:
-        "Author a coding-task family, run a benchmark across multiple runs, and read the pass@k report.",
-    },
-    {
-      title: "Automate with GitHub Actions",
-      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/ci-workflow/index.md",
-      description:
-        "Run benchmarks in CI with the forwardimpact/fit-benchmark action.",
-    },
-  ],
-};
+import { definition } from "../src/commands/benchmark-definition.js";
 
 const runtime = createDefaultRuntime();
 const logger = createLogger("benchmark", runtime);
@@ -178,12 +37,8 @@ async function main() {
   runtime.proc.exit(envelope.ok ? 0 : (envelope.code ?? 1));
 }
 
-// Run main only when invoked as a CLI. Importing for tests (e.g. parity)
-// should not execute the entry point.
-if (import.meta.url === `file://${realpathSync(process.argv[1])}`) {
-  main().catch((error) => {
-    logger.exception("main", error);
-    createCli(definition, { runtime }).error(error.message);
-    process.exit(1);
-  });
-}
+main().catch((error) => {
+  logger.exception("main", error);
+  createCli(definition, { runtime }).error(error.message);
+  process.exit(1);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.61",
+  "version": "0.1.63",
   "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
   "keywords": [
     "eval",

package/src/agent-runner.js

@@ -10,6 +10,32 @@ import { AGENT_MODEL } from "@forwardimpact/libutil/models";
 
 const DEFAULT_ALLOWED_TOOLS = ["Bash", "Read", "Glob", "Grep", "Write", "Edit"];
 
+/**
+ * Did the session actually invoke the model? A genuine run always bills
+ * tokens (the system prompt alone is thousands of input tokens) and costs
+ * more than zero. A `result` message with `subtype: "success"` but zero
+ * token usage and zero cost means the model was never reached — the
+ * canonical signature of a Claude Code init/auth failure (e.g. an invalid
+ * `ANTHROPIC_API_KEY`), which the SDK otherwise reports as a clean success.
+ *
+ * If the SDK gave us neither a `usage` object nor `total_cost_usd`, don't
+ * second-guess the subtype — trust the reported success.
+ * @param {object|null} result - The SDK `result` message, or null.
+ * @returns {boolean}
+ */
+function modelDidWork(result) {
+  if (!result) return false;
+  const { usage, total_cost_usd: cost } = result;
+  if (usage == null && cost == null) return true;
+  const tokens = usage
+    ? (usage.input_tokens ?? 0) +
+      (usage.output_tokens ?? 0) +
+      (usage.cache_creation_input_tokens ?? 0) +
+      (usage.cache_read_input_tokens ?? 0)
+    : 0;
+  return tokens > 0 || (cost ?? 0) > 0;
+}
+
 // fit-eval and kata-action run headless in CI/CD with no human to answer
 // permission prompts. The SDK is always launched in bypass mode — not
 // overridable — so a future caller can't accidentally reduce permissions.
@@ -148,6 +174,7 @@ export class AgentRunner {
   async #consumeQuery(iterator) {
     let text = "";
     let stopReason = null;
+    let resultMessage = null;
     let error = null;
     let aborted = false;
 
@@ -157,6 +184,7 @@ export class AgentRunner {
         if (message.type === "result") {
           text = message.result ?? "";
           stopReason = message.subtype;
+          resultMessage = message;
         }
       }
     } catch (err) {
@@ -167,8 +195,23 @@ export class AgentRunner {
       }
     }
 
+    // A "success" subtype is necessary but not sufficient: the SDK reports a
+    // failed init (e.g. an invalid API key) as success with zero model work.
+    // Require evidence the model actually ran, and surface a clear error when
+    // it didn't, so the masked failure can't be reported as a green run.
+    const reportedSuccess = stopReason === "success";
+    const success =
+      reportedSuccess &&
+      resultMessage?.is_error !== true &&
+      modelDidWork(resultMessage);
+    if (reportedSuccess && !success && !error) {
+      error = new Error(
+        "agent reported success but performed no model work (zero token usage) — likely a Claude Code init or authentication failure",
+      );
+    }
+
     return {
-      success: stopReason === "success",
+      success,
       text,
       sessionId: this.sessionId,
       error,

package/src/commands/benchmark-definition.js

@@ -0,0 +1,147 @@
+/**
+ * `fit-benchmark` CLI definition. Lives in `src/` so the bin stays an
+ * execute-on-import entry point — launcher packages import the bin to run
+ * it — while tests import the definition without running the CLI.
+ */
+
+import { runBenchmarkRunCommand } from "./benchmark-run.js";
+import { runBenchmarkInvariantsCommand } from "./benchmark-invariants.js";
+import { runBenchmarkReportCommand } from "./benchmark-report.js";
+import {
+  BENCHMARK_AGENT_MODEL,
+  LEAD_MODEL,
+} from "@forwardimpact/libutil/models";
+
+export const definition = {
+  name: "fit-benchmark",
+  description:
+    "Run coding-agent task families, grade hidden tests, and aggregate pass@k across runs.",
+  commands: [
+    {
+      name: "run",
+      args: [],
+      handler: runBenchmarkRunCommand,
+      description:
+        "Run every task in a family for N runs and emit one result record per (task, runIndex).",
+      options: {
+        family: {
+          type: "string",
+          description: "Path or git URL to a task family",
+        },
+        output: {
+          type: "string",
+          description:
+            "Run-output directory (created if missing, default: benchmark-runs)",
+        },
+        runs: {
+          type: "string",
+          description: "Runs per task (integer ≥ 1, default: 5)",
+        },
+        "agent-model": {
+          type: "string",
+          description: `Claude model for the agent-under-test (default: ${BENCHMARK_AGENT_MODEL})`,
+        },
+        "lead-model": {
+          type: "string",
+          description: `Claude model for the lead role (default: ${LEAD_MODEL})`,
+        },
+        "judge-model": {
+          type: "string",
+          description: `Claude model for the judge (default: ${LEAD_MODEL})`,
+        },
+        "agent-profile": {
+          type: "string",
+          description: "Agent-under-test profile name",
+        },
+        "judge-profile": {
+          type: "string",
+          description: "Judge profile name",
+        },
+        "max-turns": {
+          type: "string",
+          description:
+            "Agent-under-test turn budget (default: 50, 0 = unlimited)",
+        },
+        "allowed-tools": {
+          type: "string",
+          description:
+            "Comma-separated tool allowlist for the agent-under-test (default: Bash,Read,Glob,Grep,Write,Edit,Agent,TodoWrite)",
+        },
+      },
+    },
+    {
+      name: "invariants",
+      args: [],
+      handler: runBenchmarkInvariantsCommand,
+      description:
+        "Check a single task's invariants against a post-run workdir without invoking an agent.",
+      options: {
+        family: {
+          type: "string",
+          description: "Path or git URL to a task family",
+        },
+        task: {
+          type: "string",
+          description: "Task id (directory name under tasks/)",
+        },
+        workdir: {
+          type: "string",
+          description:
+            "Post-run directory; <workdir>/cwd/ is the agent CWD invariants run against",
+        },
+        output: {
+          type: "string",
+          description: "Output file (defaults to stdout; one JSONL line)",
+        },
+      },
+    },
+    {
+      name: "report",
+      args: [],
+      handler: runBenchmarkReportCommand,
+      description:
+        "Aggregate result records into pass@k via the OpenAI HumanEval estimator.",
+      options: {
+        input: {
+          type: "string",
+          description:
+            "Run-output directory containing results.jsonl (default: benchmark-runs)",
+        },
+        k: {
+          type: "string",
+          description: "Comma-separated k values (default: 1,3,5)",
+        },
+        format: {
+          type: "string",
+          description: "Output format (json|text, default: json)",
+        },
+      },
+    },
+  ],
+  globalOptions: {
+    help: { type: "boolean", short: "h", description: "Show this help" },
+    version: { type: "boolean", description: "Show version" },
+    json: { type: "boolean", description: "Output help as JSON" },
+  },
+  examples: [
+    "fit-benchmark run --family=./families/coding",
+    `fit-benchmark run --family=./families/coding --runs=10 --agent-model=${BENCHMARK_AGENT_MODEL}`,
+    "fit-benchmark invariants --family=./families/coding --task=todo-api --workdir=./benchmark-runs/runs/todo-api/0",
+    "fit-benchmark report --format=text",
+    "fit-benchmark report --input=./runs/today --k=1,3,5 --format=text",
+  ],
+  documentation: [
+    {
+      title: "Run a Benchmark",
+      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/index.md",
+      description:
+        "Author a coding-task family, run a benchmark across multiple runs, and read the pass@k report.",
+    },
+    {
+      title: "Automate with GitHub Actions",
+      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/ci-workflow/index.md",
+      description:
+        "Run benchmarks in CI with the forwardimpact/fit-benchmark action.",
+    },
+  ],
+};

package/src/commands/run.js

@@ -140,5 +140,7 @@ export async function runRunCommand(ctx) {
     await new Promise((r) => fileStream.end(r));
   }
 
-  return result.success ? { ok: true } : { ok: false, code: 1, error: "" };
+  return result.success
+    ? { ok: true }
+    : { ok: false, code: 1, error: result.error?.message ?? "" };
 }