@forwardimpact/libeval 0.1.35 → 0.1.38

package/bin/fit-benchmark.js

@@ -34,15 +34,26 @@ export const definition = {
         },
         output: {
           type: "string",
-          description: "Run-output directory (created if missing)",
+          description:
+            "Run-output directory (created if missing, default: benchmark-runs)",
         },
         runs: {
           type: "string",
-          description: "Runs per task (integer ≥ 1, default 1)",
+          description: "Runs per task (integer ≥ 1, default: 5)",
+        },
+        "agent-model": {
+          type: "string",
+          description:
+            "Claude model for the agent-under-test (default: claude-sonnet-4-6)",
         },
-        model: {
+        "supervisor-model": {
           type: "string",
-          description: "Claude model id (default: claude-opus-4-7[1m])",
+          description:
+            "Claude model for the supervisor (default: claude-opus-4-7)",
+        },
+        "judge-model": {
+          type: "string",
+          description: "Claude model for the judge (default: claude-opus-4-7)",
         },
         "agent-profile": {
           type: "string",
@@ -92,7 +103,8 @@ export const definition = {
       options: {
         input: {
           type: "string",
-          description: "Run-output directory containing results.jsonl",
+          description:
+            "Run-output directory containing results.jsonl (default: benchmark-runs)",
         },
         k: {
           type: "string",
@@ -111,8 +123,10 @@ export const definition = {
     json: { type: "boolean", description: "Output help as JSON" },
   },
   examples: [
-    "fit-benchmark run --family=./families/coding --output=./runs/2026-05-11 --runs=5",
-    "fit-benchmark score --family=./families/coding --task=todo-api --workdir=./runs/2026-05-11/runs/todo-api/0",
+    "fit-benchmark run --family=./families/coding",
+    "fit-benchmark run --family=./families/coding --runs=10 --agent-model=claude-sonnet-4-6",
+    "fit-benchmark score --family=./families/coding --task=todo-api --workdir=./benchmark-runs/runs/todo-api/0",
+    "fit-benchmark report --format=text",
     "fit-benchmark report --input=./runs/2026-05-11 --k=1,3,5 --format=text",
   ],
   documentation: [
@@ -122,6 +136,12 @@ export const definition = {
       description:
         "Author a coding-task family, run a benchmark across multiple runs, and read the pass@k report.",
     },
+    {
+      title: "Automate with GitHub Actions",
+      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/ci-workflow/index.md",
+      description:
+        "Run benchmarks in CI with the forwardimpact/fit-benchmark action.",
+    },
   ],
 };
 

package/bin/fit-eval.js

@@ -41,7 +41,11 @@ const definition = {
           type: "string",
           description: "Additional text appended to the task",
         },
-        model: { type: "string", description: "Claude model (default: opus)" },
+        "agent-model": {
+          type: "string",
+          description:
+            "Claude model for the agent (default: claude-opus-4-7[1m])",
+        },
         "max-turns": {
           type: "string",
           description: "Max agentic turns (default: 50, 0 = unlimited)",
@@ -84,7 +88,16 @@ const definition = {
           type: "string",
           description: "Additional text appended to the task",
         },
-        model: { type: "string", description: "Claude model (default: opus)" },
+        "agent-model": {
+          type: "string",
+          description:
+            "Claude model for the agent (default: claude-opus-4-7[1m])",
+        },
+        "supervisor-model": {
+          type: "string",
+          description:
+            "Claude model for the supervisor (default: claude-opus-4-7[1m])",
+        },
         "max-turns": {
           type: "string",
           description: "Max agentic turns (default: 20, 0 = unlimited)",
@@ -136,7 +149,15 @@ const definition = {
           type: "string",
           description: "Additional text appended to the task",
         },
-        model: { type: "string", description: "Claude model (default: opus)" },
+        "agent-model": {
+          type: "string",
+          description: "Claude model for agents (default: claude-opus-4-7[1m])",
+        },
+        "facilitator-model": {
+          type: "string",
+          description:
+            "Claude model for the facilitator (default: claude-opus-4-7[1m])",
+        },
         "max-turns": {
           type: "string",
           description: "Max agentic turns (default: 20, 0 = unlimited)",

package/bin/fit-trace.js

@@ -25,6 +25,7 @@ import {
   runFilterCommand,
   runSplitCommand,
 } from "../src/commands/trace.js";
+import { runAssertCommand } from "../src/commands/assert.js";
 
 // `bun build --compile` injects FIT_TRACE_VERSION via --define, eliminating
 // the readFileSync branch in the compiled binary (which would ENOENT against
@@ -199,6 +200,41 @@ const definition = {
         },
       },
     },
+    {
+      name: "assert",
+      args: "<test-name> <file>",
+      description:
+        "Shell-friendly assertion — outputs structured JSON for scoring hooks",
+      options: {
+        grep: {
+          type: "string",
+          description:
+            "Pass if extended regex matches file content (case-insensitive)",
+        },
+        query: {
+          type: "string",
+          description:
+            "Pass if JMESPath expression against JSON/NDJSON yields a truthy result",
+        },
+        exists: {
+          type: "boolean",
+          description: "Pass if file exists",
+        },
+        "cites-job": {
+          type: "string",
+          description:
+            "Pass if <file> contains the canonical citation from a <job> tag in the given JTBD file",
+        },
+        not: {
+          type: "boolean",
+          description: "Invert the assertion",
+        },
+        message: {
+          type: "string",
+          description: "Custom failure message",
+        },
+      },
+    },
   ],
   globalOptions: {
     help: { type: "boolean", short: "h", description: "Show this help" },
@@ -220,6 +256,11 @@ const definition = {
     "fit-trace search structured.json 'error|fail' --context 1",
     "fit-trace filter structured.json --tool Bash --error",
     "fit-trace turn structured.json 3",
+    "fit-trace assert has-heading --grep '^## Problem' spec.md",
+    "fit-trace assert no-leak --not --grep 'password' output.log",
+    "fit-trace assert file-present --exists path/to/spec.md",
+    "fit-trace assert cites-jtbd --cites-job jtbd-excerpt.md spec.md",
+    "fit-trace assert used-edit --query \"[?type=='assistant'].message.content[] | [?name=='Edit']\" trace.ndjson",
   ],
   documentation: [
     {
@@ -265,6 +306,7 @@ const COMMANDS = {
   turn: runTurnCommand,
   filter: runFilterCommand,
   split: runSplitCommand,
+  assert: runAssertCommand,
 };
 
 async function main() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.35",
+  "version": "0.1.38",
   "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
   "keywords": [
     "eval",
@@ -53,6 +53,7 @@
     "@forwardimpact/libcli": "^0.1.0",
     "@forwardimpact/libconfig": "^0.1.0",
     "@forwardimpact/libtelemetry": "^0.1.22",
+    "jmespath": "^0.16.0",
     "zod": "^4.4.3"
   },
   "devDependencies": {

package/src/benchmark/apm-installer.js

@@ -1,14 +1,13 @@
 /**
- * ApmInstaller — materialises the family's pre-staged `.claude/` tree into a
- * single staging directory, computes the manifest fingerprint, and is invoked
- * once per family install. Per-task copy happens later in WorkdirManager.
- *
- * v1 trusts the family's checked-in `.claude/` (P1); the lockfile is hashed
- * verbatim, not interpreted.
+ * ApmInstaller — runs `apm install --target claude` in the family root to
+ * materialise skills and agents, copies the resulting `.claude/` into a
+ * staging directory, and computes the manifest fingerprint from the lockfile.
+ * Per-task copy happens later in WorkdirManager.
  */
 
+import { spawn } from "node:child_process";
 import { createHash } from "node:crypto";
-import { access, cp, rm } from "node:fs/promises";
+import { access, cp, mkdir, readFile, rm } from "node:fs/promises";
 import { join } from "node:path";
 
 /**
@@ -21,19 +20,66 @@ export async function installApm(family, outputDir) {
   const stagedClaude = join(stagingDir, ".claude");
   const sourceClaude = join(family.rootPath, ".claude");
 
+  await runApmInstall(family.rootPath);
+
   try {
     await access(sourceClaude);
   } catch {
     throw new Error(
-      `task family missing .claude/ at ${sourceClaude}; family must check in a pre-staged skills/agents tree (design decision P1)`,
+      `apm install did not produce .claude/ at ${sourceClaude}; check the family's apm.yml`,
     );
   }
 
   await rm(stagingDir, { recursive: true, force: true });
   await cp(sourceClaude, stagedClaude, { recursive: true });
 
+  // Stage the family-local judge profile outside .claude/ so it is available
+  // to the judge but never copied into the agent-under-test's CWD.
+  const judgeSource = join(family.rootPath, "judge.md");
+  const judgeProfilesDir = join(stagingDir, "judge-profiles");
+  try {
+    await access(judgeSource);
+    await mkdir(judgeProfilesDir, { recursive: true });
+    await cp(judgeSource, join(judgeProfilesDir, "judge.md"));
+  } catch {}
+
+  const lockPath = join(family.rootPath, "apm.lock.yaml");
+  const lockBytes = await readFile(lockPath).catch(() => {
+    throw new Error(`apm install did not produce apm.lock.yaml at ${lockPath}`);
+  });
   const skillSetHash =
-    "sha256:" + createHash("sha256").update(family.apmLockBytes).digest("hex");
+    "sha256:" +
+    createHash("sha256").update(normalizeLf(lockBytes)).digest("hex");
+
+  return { stagingDir, skillSetHash, judgeProfilesDir };
+}
+
+function normalizeLf(buf) {
+  const out = [];
+  for (let i = 0; i < buf.length; i++) {
+    if (buf[i] === 0x0d && i + 1 < buf.length && buf[i + 1] === 0x0a) continue;
+    out.push(buf[i]);
+  }
+  return Buffer.from(out);
+}
 
-  return { stagingDir, skillSetHash };
+function runApmInstall(cwd) {
+  return new Promise((res, rej) => {
+    const child = spawn("apm", ["install", "--target", "claude"], {
+      cwd,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    let stderr = "";
+    child.stdout.on("data", () => {});
+    child.stderr.on("data", (d) => {
+      stderr += d.toString();
+    });
+    child.on("error", (e) => {
+      rej(new Error(`failed to spawn apm: ${e.message}`));
+    });
+    child.on("close", (code) => {
+      if (code === 0) res();
+      else rej(new Error(`apm install exited ${code}: ${stderr}`));
+    });
+  });
 }

package/src/benchmark/judge.js

@@ -1,9 +1,20 @@
 /**
  * Benchmark adapter for the libeval `Judge`. Templates the family's
- * `judge.task.md` ({{SCORING}} / {{AGENT_TRACE_PATH}} substitution), runs the
- * judge against the post-run agent CWD, and returns the verdict in the
- * benchmark's `pass`/`fail` vocabulary (mapped from libeval's
- * `success`/`failure`).
+ * `judge.task.md` with structured context variables, runs the judge against
+ * the post-run agent CWD, and returns the verdict in the benchmark's
+ * `pass`/`fail` vocabulary (mapped from libeval's `success`/`failure`).
+ *
+ * Template variables available in `judge.task.md`:
+ *
+ *   {{AGENT_INSTRUCTIONS}}  — contents of agent.task.md
+ *   {{AGENT_PROFILE}}       — agent profile body (empty string if none)
+ *   {{AGENT_TRACE_PATH}}    — path to agent.ndjson
+ *   {{SCORING_RESULT}}      — JSON scoring object
+ *   {{SKILL_SET_HASH}}      — SHA-256 from apm.lock.yaml
+ *   {{TASK_ID}}             — task name (directory under tasks/)
+ *   {{TASK_DIR}}            — agent working directory path
+ *
+ * Legacy alias: {{SCORING}} is accepted as an alias for {{SCORING_RESULT}}.
  *
  * The judge verdict is captured from the orchestration context's
  * `concluded` flag directly — no trace parsing on the happy path.
@@ -24,19 +35,34 @@ import { createRedactor } from "../redaction.js";
  * @property {string} summary
  */
 
+/**
+ * @typedef {object} JudgeContext
+ * @property {string} agentInstructions - Contents of agent.task.md.
+ * @property {string} agentProfile - Agent profile body (empty string if none).
+ * @property {string} skillSetHash - SHA-256 fingerprint from apm.lock.yaml.
+ */
+
 /**
  * Run the judge over a completed task run.
  * @param {import("./task-family.js").Task} task
  * @param {import("./workdir.js").Workdir} workdir
  * @param {import("./scorer.js").ScoringResult} scoring
- * @param {{query: Function, model: string, judgeProfile?: string}} deps
+ * @param {{query: Function, model: string, judgeProfile?: string, profilesDir?: string}} deps
+ * @param {JudgeContext} [context]
  * @returns {Promise<JudgeVerdict>}
  */
-export async function runJudge(task, workdir, scoring, deps) {
+export async function runJudge(task, workdir, scoring, deps, context) {
   const template = await readFile(task.paths.judge, "utf8");
+  const scoringJson = JSON.stringify(scoring, null, 2);
   const taskText = template
-    .replaceAll("{{SCORING}}", JSON.stringify(scoring, null, 2))
-    .replaceAll("{{AGENT_TRACE_PATH}}", workdir.agentTracePath);
+    .replaceAll("{{SCORING_RESULT}}", scoringJson)
+    .replaceAll("{{SCORING}}", scoringJson)
+    .replaceAll("{{AGENT_TRACE_PATH}}", workdir.agentTracePath)
+    .replaceAll("{{AGENT_INSTRUCTIONS}}", context?.agentInstructions ?? "")
+    .replaceAll("{{AGENT_PROFILE}}", context?.agentProfile ?? "")
+    .replaceAll("{{SKILL_SET_HASH}}", context?.skillSetHash ?? "")
+    .replaceAll("{{TASK_ID}}", task.id)
+    .replaceAll("{{TASK_DIR}}", workdir.cwd);
 
   const output = createWriteStream(workdir.judgeTracePath);
   const judge = createJudge({
@@ -45,6 +71,7 @@ export async function runJudge(task, workdir, scoring, deps) {
     output,
     model: deps.model,
     judgeProfile: deps.judgeProfile,
+    profilesDir: deps.profilesDir,
     maxTurns: 25,
     redactor: createRedactor(),
   });