@forwardimpact/libeval 0.1.36 → 0.1.39

package/bin/fit-benchmark.js

@@ -34,15 +34,26 @@ export const definition = {
         },
         output: {
           type: "string",
-          description: "Run-output directory (created if missing)",
+          description:
+            "Run-output directory (created if missing, default: benchmark-runs)",
         },
         runs: {
           type: "string",
-          description: "Runs per task (integer ≥ 1, default 1)",
+          description: "Runs per task (integer ≥ 1, default: 5)",
+        },
+        "agent-model": {
+          type: "string",
+          description:
+            "Claude model for the agent-under-test (default: claude-sonnet-4-6)",
+        },
+        "supervisor-model": {
+          type: "string",
+          description:
+            "Claude model for the supervisor (default: claude-opus-4-7)",
         },
-        model: {
+        "judge-model": {
           type: "string",
-          description: "Claude model id (default: claude-opus-4-7[1m])",
+          description: "Claude model for the judge (default: claude-opus-4-7)",
         },
         "agent-profile": {
           type: "string",
@@ -57,6 +68,11 @@ export const definition = {
           description:
             "Agent-under-test turn budget (default: 50, 0 = unlimited)",
         },
+        "allowed-tools": {
+          type: "string",
+          description:
+            "Comma-separated tool allowlist for the agent-under-test (default: Bash,Read,Glob,Grep,Write,Edit,Agent,TodoWrite)",
+        },
       },
     },
     {
@@ -92,7 +108,8 @@ export const definition = {
       options: {
         input: {
           type: "string",
-          description: "Run-output directory containing results.jsonl",
+          description:
+            "Run-output directory containing results.jsonl (default: benchmark-runs)",
         },
         k: {
           type: "string",
@@ -111,8 +128,10 @@ export const definition = {
     json: { type: "boolean", description: "Output help as JSON" },
   },
   examples: [
-    "fit-benchmark run --family=./families/coding --output=./runs/2026-05-11 --runs=5",
-    "fit-benchmark score --family=./families/coding --task=todo-api --workdir=./runs/2026-05-11/runs/todo-api/0",
+    "fit-benchmark run --family=./families/coding",
+    "fit-benchmark run --family=./families/coding --runs=10 --agent-model=claude-sonnet-4-6",
+    "fit-benchmark score --family=./families/coding --task=todo-api --workdir=./benchmark-runs/runs/todo-api/0",
+    "fit-benchmark report --format=text",
     "fit-benchmark report --input=./runs/2026-05-11 --k=1,3,5 --format=text",
   ],
   documentation: [
@@ -122,6 +141,12 @@ export const definition = {
       description:
         "Author a coding-task family, run a benchmark across multiple runs, and read the pass@k report.",
     },
+    {
+      title: "Automate with GitHub Actions",
+      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/ci-workflow/index.md",
+      description:
+        "Run benchmarks in CI with the forwardimpact/fit-benchmark action.",
+    },
   ],
 };
 

package/bin/fit-eval.js

@@ -41,7 +41,11 @@ const definition = {
           type: "string",
           description: "Additional text appended to the task",
         },
-        model: { type: "string", description: "Claude model (default: opus)" },
+        "agent-model": {
+          type: "string",
+          description:
+            "Claude model for the agent (default: claude-opus-4-7[1m])",
+        },
         "max-turns": {
           type: "string",
           description: "Max agentic turns (default: 50, 0 = unlimited)",
@@ -84,7 +88,16 @@ const definition = {
           type: "string",
           description: "Additional text appended to the task",
         },
-        model: { type: "string", description: "Claude model (default: opus)" },
+        "agent-model": {
+          type: "string",
+          description:
+            "Claude model for the agent (default: claude-opus-4-7[1m])",
+        },
+        "supervisor-model": {
+          type: "string",
+          description:
+            "Claude model for the supervisor (default: claude-opus-4-7[1m])",
+        },
         "max-turns": {
           type: "string",
           description: "Max agentic turns (default: 20, 0 = unlimited)",
@@ -136,7 +149,15 @@ const definition = {
           type: "string",
           description: "Additional text appended to the task",
         },
-        model: { type: "string", description: "Claude model (default: opus)" },
+        "agent-model": {
+          type: "string",
+          description: "Claude model for agents (default: claude-opus-4-7[1m])",
+        },
+        "facilitator-model": {
+          type: "string",
+          description:
+            "Claude model for the facilitator (default: claude-opus-4-7[1m])",
+        },
         "max-turns": {
           type: "string",
           description: "Max agentic turns (default: 20, 0 = unlimited)",

package/bin/fit-trace.js

@@ -25,6 +25,7 @@ import {
   runFilterCommand,
   runSplitCommand,
 } from "../src/commands/trace.js";
+import { runAssertCommand } from "../src/commands/assert.js";
 
 // `bun build --compile` injects FIT_TRACE_VERSION via --define, eliminating
 // the readFileSync branch in the compiled binary (which would ENOENT against
@@ -199,6 +200,41 @@ const definition = {
         },
       },
     },
+    {
+      name: "assert",
+      args: "<test-name> <file>",
+      description:
+        "Shell-friendly assertion — outputs structured JSON for scoring hooks",
+      options: {
+        grep: {
+          type: "string",
+          description:
+            "Pass if extended regex matches file content (case-insensitive)",
+        },
+        query: {
+          type: "string",
+          description:
+            "Pass if JMESPath expression against JSON/NDJSON yields a truthy result",
+        },
+        exists: {
+          type: "boolean",
+          description: "Pass if file exists",
+        },
+        "cites-job": {
+          type: "string",
+          description:
+            "Pass if <file> contains the canonical citation from a <job> tag in the given JTBD file",
+        },
+        not: {
+          type: "boolean",
+          description: "Invert the assertion",
+        },
+        message: {
+          type: "string",
+          description: "Custom failure message",
+        },
+      },
+    },
   ],
   globalOptions: {
     help: { type: "boolean", short: "h", description: "Show this help" },
@@ -220,6 +256,11 @@ const definition = {
     "fit-trace search structured.json 'error|fail' --context 1",
     "fit-trace filter structured.json --tool Bash --error",
     "fit-trace turn structured.json 3",
+    "fit-trace assert has-heading --grep '^## Problem' spec.md",
+    "fit-trace assert no-leak --not --grep 'password' output.log",
+    "fit-trace assert file-present --exists path/to/spec.md",
+    "fit-trace assert cites-jtbd --cites-job jtbd-excerpt.md spec.md",
+    "fit-trace assert used-edit --query \"[?type=='assistant'].message.content[] | [?name=='Edit']\" trace.ndjson",
   ],
   documentation: [
     {
@@ -265,6 +306,7 @@ const COMMANDS = {
   turn: runTurnCommand,
   filter: runFilterCommand,
   split: runSplitCommand,
+  assert: runAssertCommand,
 };
 
 async function main() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.36",
+  "version": "0.1.39",
   "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
   "keywords": [
     "eval",
@@ -53,6 +53,7 @@
     "@forwardimpact/libcli": "^0.1.0",
     "@forwardimpact/libconfig": "^0.1.0",
     "@forwardimpact/libtelemetry": "^0.1.22",
+    "jmespath": "^0.16.0",
     "zod": "^4.4.3"
   },
   "devDependencies": {

package/src/benchmark/apm-installer.js

@@ -1,14 +1,13 @@
 /**
- * ApmInstaller — materialises the family's pre-staged `.claude/` tree into a
- * single staging directory, computes the manifest fingerprint, and is invoked
- * once per family install. Per-task copy happens later in WorkdirManager.
- *
- * v1 trusts the family's checked-in `.claude/` (P1); the lockfile is hashed
- * verbatim, not interpreted.
+ * ApmInstaller — runs `apm install --target claude` in the family root to
+ * materialise skills and agents, copies the resulting `.claude/` into a
+ * staging directory, and computes the manifest fingerprint from the lockfile.
+ * Per-task copy happens later in WorkdirManager.
  */
 
+import { spawn } from "node:child_process";
 import { createHash } from "node:crypto";
-import { access, cp, rm } from "node:fs/promises";
+import { access, cp, mkdir, readFile, rm } from "node:fs/promises";
 import { join } from "node:path";
 
 /**
@@ -20,20 +19,83 @@ export async function installApm(family, outputDir) {
   const stagingDir = join(outputDir, ".apm-staging");
   const stagedClaude = join(stagingDir, ".claude");
   const sourceClaude = join(family.rootPath, ".claude");
+  const apmYml = join(family.rootPath, "apm.yml");
 
+  const hasApm = await access(apmYml)
+    .then(() => true)
+    .catch(() => false);
+
+  if (hasApm) {
+    await runApmInstall(family.rootPath);
+    try {
+      await access(sourceClaude);
+    } catch {
+      throw new Error(
+        `apm install did not produce .claude/ at ${sourceClaude}; check the family's apm.yml`,
+      );
+    }
+  }
+
+  await rm(stagingDir, { recursive: true, force: true });
+  const hasClaudeDir = await access(sourceClaude)
+    .then(() => true)
+    .catch(() => false);
+  if (hasClaudeDir) {
+    await cp(sourceClaude, stagedClaude, { recursive: true });
+  } else {
+    await mkdir(stagedClaude, { recursive: true });
+  }
+
+  // Stage the family-local judge profile outside .claude/ so it is available
+  // to the judge but never copied into the agent-under-test's CWD.
+  const judgeSource = join(family.rootPath, "judge.md");
+  const judgeProfilesDir = join(stagingDir, "judge-profiles");
   try {
-    await access(sourceClaude);
+    await access(judgeSource);
+    await mkdir(judgeProfilesDir, { recursive: true });
+    await cp(judgeSource, join(judgeProfilesDir, "judge.md"));
+  } catch {}
+
+  const lockPath = join(family.rootPath, "apm.lock.yaml");
+  let skillSetHash = "";
+  try {
+    const lockBytes = await readFile(lockPath);
+    skillSetHash =
+      "sha256:" +
+      createHash("sha256").update(normalizeLf(lockBytes)).digest("hex");
   } catch {
-    throw new Error(
-      `task family missing .claude/ at ${sourceClaude}; family must check in a pre-staged skills/agents tree (design decision P1)`,
-    );
+    // No lockfile — family doesn't use skill packs.
   }
 
-  await rm(stagingDir, { recursive: true, force: true });
-  await cp(sourceClaude, stagedClaude, { recursive: true });
+  return { stagingDir, skillSetHash, judgeProfilesDir };
+}
 
-  const skillSetHash =
-    "sha256:" + createHash("sha256").update(family.apmLockBytes).digest("hex");
+function normalizeLf(buf) {
+  const out = [];
+  for (let i = 0; i < buf.length; i++) {
+    if (buf[i] === 0x0d && i + 1 < buf.length && buf[i + 1] === 0x0a) continue;
+    out.push(buf[i]);
+  }
+  return Buffer.from(out);
+}
 
-  return { stagingDir, skillSetHash };
+function runApmInstall(cwd) {
+  return new Promise((res, rej) => {
+    const child = spawn("apm", ["install", "--target", "claude"], {
+      cwd,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    let stderr = "";
+    child.stdout.on("data", () => {});
+    child.stderr.on("data", (d) => {
+      stderr += d.toString();
+    });
+    child.on("error", (e) => {
+      rej(new Error(`failed to spawn apm: ${e.message}`));
+    });
+    child.on("close", (code) => {
+      if (code === 0) res();
+      else rej(new Error(`apm install exited ${code}: ${stderr}`));
+    });
+  });
 }

package/src/benchmark/env-loader.js

@@ -0,0 +1,146 @@
+/**
+ * Env-loader — auto-discover `.env` / `.env.local` files in a task family
+ * and its tasks, load them into `process.env`, and render the merged result
+ * into each agent CWD.
+ *
+ * Discovery paths (loaded in this order, first value per key wins):
+ *   1. process.env  (CI secrets, shell env — never overwritten)
+ *   2. <family>/.env.local
+ *   3. <family>/.env
+ *   4. tasks/<id>/.env.local
+ *   5. tasks/<id>/.env
+ *
+ * Every discovered env file — family or task — is loaded into process.env
+ * AND rendered (with resolved values) into the agent working directory.
+ */
+
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+const ENV_FILES = [".env.local", ".env"];
+
+/**
+ * Parse a `.env` file into an array of {key, value} pairs.
+ * Handles KEY=VALUE, # comments, blank lines, and single/double-quoted values.
+ * @param {string} content
+ * @returns {Array<{key: string, value: string}>}
+ */
+export function parseEnvFile(content) {
+  const entries = [];
+  for (const raw of content.split("\n")) {
+    const line = raw.trim();
+    if (!line || line.startsWith("#")) continue;
+    const eq = line.indexOf("=");
+    if (eq === -1) continue;
+    const key = line.slice(0, eq).trim();
+    if (!key) continue;
+    let value = line.slice(eq + 1).trim();
+    if (
+      (value.startsWith('"') && value.endsWith('"')) ||
+      (value.startsWith("'") && value.endsWith("'"))
+    ) {
+      value = value.slice(1, -1);
+    }
+    entries.push({ key, value });
+  }
+  return entries;
+}
+
+/**
+ * Read and parse an env file, returning [] if the file does not exist.
+ * @param {string} filePath
+ * @returns {Promise<Array<{key: string, value: string}>>}
+ */
+async function readEnvFile(filePath) {
+  try {
+    const content = await readFile(filePath, "utf8");
+    return parseEnvFile(content);
+  } catch (e) {
+    if (e.code === "ENOENT") return [];
+    throw e;
+  }
+}
+
+/**
+ * Load entries into process.env. Existing keys are never overwritten.
+ * @param {Array<{key: string, value: string}>} entries
+ * @returns {string[]} var names that were loaded
+ */
+function applyToProcessEnv(entries) {
+  const names = [];
+  for (const { key, value } of entries) {
+    names.push(key);
+    if (process.env[key] === undefined) {
+      process.env[key] = value;
+    }
+  }
+  return names;
+}
+
+/**
+ * Load one env file: apply to process.env, record keys in the merged map.
+ * @param {string} dir
+ * @param {string} file
+ * @param {Set<string>} names
+ * @param {Map<string, Map<string, true>>} merged
+ */
+async function loadOneEnvFile(dir, file, names, merged) {
+  const entries = await readEnvFile(join(dir, file));
+  if (entries.length === 0) return;
+  for (const name of applyToProcessEnv(entries)) names.add(name);
+  if (!merged.has(file)) merged.set(file, new Map());
+  const fileMap = merged.get(file);
+  for (const { key } of entries) {
+    if (!fileMap.has(key)) fileMap.set(key, true);
+  }
+}
+
+/**
+ * Scan directories for env files, load into process.env, and collect
+ * a merged key manifest per filename.
+ * @param {string[]} dirs
+ * @returns {Promise<{names: Set<string>, merged: Map<string, Map<string, true>>}>}
+ */
+async function collectEnvEntries(dirs) {
+  const names = new Set();
+  const merged = new Map();
+  for (const dir of dirs) {
+    for (const file of ENV_FILES) {
+      await loadOneEnvFile(dir, file, names, merged);
+    }
+  }
+  return { names, merged };
+}
+
+/**
+ * Write resolved env files into the agent CWD and warn about empty values.
+ * @param {Map<string, Map<string, true>>} merged
+ * @param {string} agentCwd
+ */
+async function renderEnvFiles(merged, agentCwd) {
+  for (const [file, keyMap] of merged) {
+    const keys = [...keyMap.keys()];
+    const resolved = keys.map((key) => `${key}=${process.env[key] ?? ""}`);
+    await writeFile(join(agentCwd, file), resolved.join("\n") + "\n");
+    const empty = keys.filter((key) => !process.env[key]);
+    if (empty.length > 0) {
+      process.stderr.write(
+        `libeval: env warning: ${file} declares vars with no value: ${empty.join(", ")}\n`,
+      );
+    }
+  }
+}
+
+/**
+ * Discover `.env` / `.env.local` in one or more directories, load them
+ * into process.env, and render the resolved values into the agent CWD.
+ *
+ * @param {string[]} dirs - Directories to scan (family root, task dir, etc.)
+ * @param {string} agentCwd - Agent working directory to render into.
+ * @returns {Promise<string[]>} All var names discovered (for redaction).
+ */
+export async function loadEnv(dirs, agentCwd) {
+  const { names, merged } = await collectEnvEntries(dirs);
+  await renderEnvFiles(merged, agentCwd);
+  return [...names];
+}

package/src/benchmark/judge.js

@@ -6,7 +6,7 @@
  *
  * Template variables available in `judge.task.md`:
  *
- *   {{AGENT_INSTRUCTIONS}}  — contents of instructions.md
+ *   {{AGENT_INSTRUCTIONS}}  — contents of agent.task.md
  *   {{AGENT_PROFILE}}       — agent profile body (empty string if none)
  *   {{AGENT_TRACE_PATH}}    — path to agent.ndjson
  *   {{SCORING_RESULT}}      — JSON scoring object
@@ -37,7 +37,7 @@ import { createRedactor } from "../redaction.js";
 
 /**
  * @typedef {object} JudgeContext
- * @property {string} agentInstructions - Contents of instructions.md.
+ * @property {string} agentInstructions - Contents of agent.task.md.
  * @property {string} agentProfile - Agent profile body (empty string if none).
  * @property {string} skillSetHash - SHA-256 fingerprint from apm.lock.yaml.
  */
@@ -47,7 +47,7 @@ import { createRedactor } from "../redaction.js";
  * @param {import("./task-family.js").Task} task
  * @param {import("./workdir.js").Workdir} workdir
  * @param {import("./scorer.js").ScoringResult} scoring
- * @param {{query: Function, model: string, judgeProfile?: string}} deps
+ * @param {{query: Function, model: string, judgeProfile?: string, profilesDir?: string}} deps
  * @param {JudgeContext} [context]
  * @returns {Promise<JudgeVerdict>}
  */
@@ -71,6 +71,7 @@ export async function runJudge(task, workdir, scoring, deps, context) {
     output,
     model: deps.model,
     judgeProfile: deps.judgeProfile,
+    profilesDir: deps.profilesDir,
     maxTurns: 25,
     redactor: createRedactor(),
   });

package/src/benchmark/report.js

@@ -178,24 +178,46 @@ function renderFullReport(report, kValues) {
 function renderSummary(report) {
   const { totals } = report;
   const passing = report.tasks.filter((t) => t.c > 0 && t.c === t.n).length;
+  const icon = statusIcon(passing === totals.tasks);
   const lines = [
     "# Benchmark Report",
     "",
-    `**Result: ${passing}/${totals.tasks} tasks passing** | ${totals.runs} runs${totals.skipped ? ` | ${totals.skipped} skipped` : ""}`,
+    `${icon} **${passing}/${totals.tasks} tasks passing** | ${totals.runs} runs${totals.skipped ? ` | ${totals.skipped} skipped` : ""}`,
   ];
+
+  const headers = [];
+  const values = [];
+  if (totals.costUsd != null) {
+    headers.push("Cost");
+    values.push(formatCost(totals.costUsd));
+  }
+  if (totals.medianDurationMs != null) {
+    headers.push("Median Duration");
+    values.push(formatDuration(totals.medianDurationMs));
+  }
+  if (totals.medianTurns != null) {
+    headers.push("Median Turns");
+    values.push(String(totals.medianTurns));
+  }
+  if (headers.length) {
+    lines.push("");
+    lines.push(`| ${headers.join(" | ")} |`);
+    lines.push(`| ${headers.map(() => "---").join(" | ")} |`);
+    lines.push(`| ${values.join(" | ")} |`);
+  }
+
   const meta = [];
-  if (totals.model) meta.push(`Model: \`${totals.model}\``);
+  if (totals.model) {
+    meta.push(`Agent: \`${totals.model.agent}\``);
+    meta.push(`Supervisor: \`${totals.model.supervisor}\``);
+    meta.push(`Judge: \`${totals.model.judge}\``);
+  }
   if (totals.skillSetHash) meta.push(`Skill set: \`${totals.skillSetHash}\``);
   if (totals.familyRevision) meta.push(`Family: \`${totals.familyRevision}\``);
-  if (meta.length) lines.push(meta.join(" | "));
-
-  const stats = [];
-  if (totals.costUsd != null) stats.push(`Cost: ${formatCost(totals.costUsd)}`);
-  if (totals.medianDurationMs != null)
-    stats.push(`Median duration: ${formatDuration(totals.medianDurationMs)}`);
-  if (totals.medianTurns != null)
-    stats.push(`Median turns: ${totals.medianTurns}`);
-  if (stats.length) lines.push(stats.join(" | "));
+  if (meta.length) {
+    lines.push("");
+    lines.push(meta.join(" | "));
+  }
 
   lines.push("");
   return lines.join("\n");
@@ -229,13 +251,13 @@ function renderTotalsLine(report) {
 
 function renderTaskDetail(task) {
   const runs = task.runs ?? [];
-  const status = task.c === task.n ? "PASS" : "FAIL";
+  const icon = statusIcon(task.c === task.n);
   const singleRun = runs.length === 1;
 
   const lines = [
     `### ${task.taskId}`,
     "",
-    `**${status} — ${task.c}/${task.n} runs passed**`,
+    `${icon} **${task.c}/${task.n} runs passed**`,
   ];
 
   lines.push("", renderRunsTable(runs));
@@ -267,16 +289,16 @@ function renderRunsTable(runs) {
     const scoringCell = r.preflightError
       ? "preflight error"
       : r.scoring
-        ? r.scoring.verdict
+        ? statusIcon(r.scoring.verdict === "pass")
         : "—";
     const judgeCell = r.preflightError
       ? "—"
       : r.judgeVerdict
-        ? r.judgeVerdict.verdict
+        ? statusIcon(r.judgeVerdict.verdict === "pass")
         : "—";
     rows.push([
       String(r.runIndex),
-      r.verdict.toUpperCase(),
+      statusIcon(r.verdict === "pass"),
       scoringCell,
       judgeCell,
       formatCost(r.costUsd),
@@ -317,7 +339,7 @@ function collectScoringRows(runs) {
       rows.push({
         run: r.runIndex,
         check: escapeCell(String(d.test ?? "(unnamed)")),
-        result: d.pass ? "PASS" : "FAIL",
+        result: statusIcon(d.pass),
         message: escapeCell(String(d.message ?? "")),
       });
     }
@@ -365,6 +387,10 @@ function renderErrors(runs) {
 // Formatting helpers
 // ---------------------------------------------------------------------------
 
+function statusIcon(pass) {
+  return pass ? "✅" : "❌";
+}
+
 function formatPassAt(v) {
   if (v == null) return "—";
   if (typeof v === "object" && "error" in v) return v.error;

package/src/benchmark/result.js

@@ -29,7 +29,7 @@ const JUDGE_VERDICT_SHAPE = z.object({
 
 const PROFILES_SHAPE = z.object({
   agent: z.union([z.string(), z.null()]),
-  supervisor: z.null(),
+  supervisor: z.union([z.string(), z.null()]),
   judge: z.union([z.string(), z.null()]),
 });
 
@@ -46,7 +46,11 @@ const COMMON_FIELDS = {
   costUsd: z.number(),
   turns: z.number().int().min(0),
   profiles: PROFILES_SHAPE,
-  model: z.string(),
+  model: z.object({
+    agent: z.string(),
+    supervisor: z.string().optional(),
+    judge: z.string().optional(),
+  }),
   skillSetHash: z.string(),
   familyRevision: z.string(),
   durationMs: z.number().int().min(0),
@@ -61,8 +65,9 @@ const HAPPY_RECORD = z.object({
   ...COMMON_FIELDS,
   scoring: SCORING_SHAPE,
   submission: z.string(),
-  judgeVerdict: JUDGE_VERDICT_SHAPE,
+  judgeVerdict: JUDGE_VERDICT_SHAPE.optional(),
   agentTracePath: z.string(),
+  supervisorTracePath: z.string(),
   judgeTracePath: z.string(),
   agentError: AGENT_ERROR_SHAPE.optional(),
   preflightError: z.undefined().optional(),
@@ -76,6 +81,7 @@ const PREFLIGHT_RECORD = z.object({
   // them in WorkdirManager.start) so the record is uniform across branches
   // and downstream consumers can reference them without conditional fields.
   agentTracePath: z.string(),
+  supervisorTracePath: z.string(),
   judgeTracePath: z.string(),
   scoring: z.undefined().optional(),
   submission: z.undefined().optional(),