@forwardimpact/libeval 0.1.31 → 0.1.33

package/README.md

@@ -12,3 +12,23 @@ reproducible evidence.
 ```js
 import { createTraceCollector, createTraceQuery, createAgentRunner } from '@forwardimpact/libeval';
 ```
+
+## Trace redaction
+
+`fit-eval run`, `fit-eval supervise`, and `fit-eval facilitate` redact
+secrets in trace artifacts before they reach disk. Two layers compose:
+
+- **Env-var allowlist**, defaulting to `ANTHROPIC_API_KEY`, `GH_TOKEN`,
+  `GITHUB_TOKEN`. The runtime values of these vars are replaced with
+  `[REDACTED:env:NAME]` wherever they appear in tool inputs, tool
+  outputs, assistant text, or orchestrator summaries. Override the list
+  with `LIBEVAL_REDACTION_ENV_VARS=NAME1,NAME2,…` (replaces, not extends).
+- **Credential-shape patterns**, covering Anthropic API keys (`sk-ant-`),
+  GitHub PATs (`ghp_`), installation tokens (`ghs_`), OAuth tokens
+  (`gho_`), and fine-grained PATs (`github_pat_`). Pattern hits become
+  `[REDACTED:pattern:KIND]`.
+
+Redaction is on by default. To disable, set `LIBEVAL_REDACTION_DISABLED=1`
+— a stderr warning fires once per run. Never set this in CI on a public
+repository: workflow artifacts there are downloadable through the
+retention window.

package/bin/fit-benchmark.js

@@ -0,0 +1,167 @@
+#!/usr/bin/env node
+
+import { readFileSync } from "node:fs";
+import { createCli } from "@forwardimpact/libcli";
+import { createLogger } from "@forwardimpact/libtelemetry";
+
+import { runBenchmarkRunCommand } from "../src/commands/benchmark-run.js";
+import { runBenchmarkScoreCommand } from "../src/commands/benchmark-score.js";
+import { runBenchmarkReportCommand } from "../src/commands/benchmark-report.js";
+
+// `bun build --compile` injects FIT_BENCHMARK_VERSION via --define, eliminating
+// the readFileSync branch in the compiled binary (which would ENOENT against
+// the bunfs virtual mount). Source execution falls through to package.json.
+const VERSION =
+  process.env.FIT_BENCHMARK_VERSION ||
+  JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"))
+    .version;
+
+export const definition = {
+  name: "fit-benchmark",
+  version: VERSION,
+  description:
+    "Run coding-agent task families, grade hidden tests, and aggregate pass@k across runs.",
+  commands: [
+    {
+      name: "run",
+      args: "",
+      description:
+        "Run every task in a family for N runs and emit one result record per (task, runIndex).",
+      options: {
+        family: {
+          type: "string",
+          description: "Path or git URL to a task family",
+        },
+        output: {
+          type: "string",
+          description: "Run-output directory (created if missing)",
+        },
+        runs: {
+          type: "string",
+          description: "Runs per task (integer ≥ 1, default 1)",
+        },
+        model: {
+          type: "string",
+          description: "Claude model id (default: claude-opus-4-7[1m])",
+        },
+        "agent-profile": {
+          type: "string",
+          description: "Agent-under-test profile name",
+        },
+        "judge-profile": {
+          type: "string",
+          description: "Judge profile name",
+        },
+        "max-turns": {
+          type: "string",
+          description:
+            "Agent-under-test turn budget (default: 50, 0 = unlimited)",
+        },
+      },
+    },
+    {
+      name: "score",
+      args: "",
+      description:
+        "Score a single task against a post-run workdir without invoking an agent.",
+      options: {
+        family: {
+          type: "string",
+          description: "Path or git URL to a task family",
+        },
+        task: {
+          type: "string",
+          description: "METR-style task id (task_family_name/task_name)",
+        },
+        workdir: {
+          type: "string",
+          description:
+            "Post-run directory; <workdir>/cwd/ is the agent CWD scoring runs against",
+        },
+        output: {
+          type: "string",
+          description: "Output file (defaults to stdout; one JSONL line)",
+        },
+      },
+    },
+    {
+      name: "report",
+      args: "",
+      description:
+        "Aggregate result records into pass@k via the OpenAI HumanEval estimator.",
+      options: {
+        input: {
+          type: "string",
+          description: "Run-output directory containing results.jsonl",
+        },
+        k: {
+          type: "string",
+          description: "Comma-separated k values (default: 1,3,5)",
+        },
+        format: {
+          type: "string",
+          description: "Output format (json|text, default: json)",
+        },
+      },
+    },
+  ],
+  globalOptions: {
+    help: { type: "boolean", short: "h", description: "Show this help" },
+    version: { type: "boolean", description: "Show version" },
+    json: { type: "boolean", description: "Output help as JSON" },
+  },
+  examples: [
+    "fit-benchmark run --family=./families/coding --output=./runs/2026-05-11 --runs=5",
+    "fit-benchmark score --family=./families/coding --task=coding/todo-api --workdir=./runs/2026-05-11/runs/coding__todo-api/0",
+    "fit-benchmark report --input=./runs/2026-05-11 --k=1,3,5 --format=text",
+  ],
+  documentation: [
+    {
+      title: "Run a Benchmark",
+      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/index.md",
+      description:
+        "Author a coding-task family, run a benchmark across multiple runs, and read the pass@k report.",
+    },
+  ],
+};
+
+const cli = createCli(definition);
+const logger = createLogger("benchmark");
+
+const COMMANDS = {
+  run: runBenchmarkRunCommand,
+  score: runBenchmarkScoreCommand,
+  report: runBenchmarkReportCommand,
+};
+
+async function main() {
+  const parsed = cli.parse(process.argv.slice(2));
+  if (!parsed) process.exit(0);
+
+  const { values, positionals } = parsed;
+
+  if (positionals.length === 0) {
+    cli.usageError("no command specified");
+    process.exit(2);
+  }
+
+  const [command, ...args] = positionals;
+  const handler = COMMANDS[command];
+
+  if (!handler) {
+    cli.usageError(`unknown command "${command}"`);
+    process.exit(2);
+  }
+
+  await handler(values, args);
+}
+
+// Run main only when invoked as a CLI. Importing for tests (e.g. parity)
+// should not execute the entry point.
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main().catch((error) => {
+    logger.exception("main", error);
+    cli.error(error.message);
+    process.exit(1);
+  });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.31",
+  "version": "0.1.33",
   "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
   "keywords": [
     "eval",
@@ -32,11 +32,13 @@
   "exports": {
     ".": "./src/index.js",
     "./bin/fit-eval.js": "./bin/fit-eval.js",
-    "./bin/fit-trace.js": "./bin/fit-trace.js"
+    "./bin/fit-trace.js": "./bin/fit-trace.js",
+    "./bin/fit-benchmark.js": "./bin/fit-benchmark.js"
   },
   "bin": {
     "fit-eval": "./bin/fit-eval.js",
-    "fit-trace": "./bin/fit-trace.js"
+    "fit-trace": "./bin/fit-trace.js",
+    "fit-benchmark": "./bin/fit-benchmark.js"
   },
   "files": [
     "src/**/*.js",

package/src/agent-runner.js

@@ -54,7 +54,9 @@ export class AgentRunner {
     if (!deps.cwd) throw new Error("cwd is required");
     if (!deps.query) throw new Error("query is required");
     if (!deps.output) throw new Error("output is required");
+    if (!deps.redactor) throw new Error("redactor is required");
     Object.assign(this, applyDefaults(deps));
+    this.redactor = deps.redactor;
     this.sessionId = null;
     this.buffer = [];
     /** @type {AbortController|null} */
@@ -203,12 +205,16 @@ export class AgentRunner {
    * @param {{pendingBatch: string[], assistantTextCount: number}} state
    */
   #recordLine(message, state) {
-    const line = JSON.stringify(message);
+    const redacted = this.redactor.redactValue(message);
+    const line = JSON.stringify(redacted);
     this.output.write(line + "\n");
     this.buffer.push(line);
     if (this.onLine) this.onLine(line);
     if (this.onBatch) state.pendingBatch.push(line);
 
+    // Session-id / text-block tracking reads the ORIGINAL message —
+    // these fields are not secret carriers, and the trackers rely on
+    // shape, not string contents.
     if (message.type === "system" && message.subtype === "init") {
       this.sessionId = message.session_id;
     }

package/src/benchmark/apm-installer.js

@@ -0,0 +1,39 @@
+/**
+ * ApmInstaller — materialises the family's pre-staged `.claude/` tree into a
+ * single staging directory, computes the manifest fingerprint, and is invoked
+ * once per family install. Per-task copy happens later in WorkdirManager.
+ *
+ * v1 trusts the family's checked-in `.claude/` (P1); the lockfile is hashed
+ * verbatim, not interpreted.
+ */
+
+import { createHash } from "node:crypto";
+import { access, cp, rm } from "node:fs/promises";
+import { join } from "node:path";
+
+/**
+ * @param {import("./task-family.js").TaskFamily} family
+ * @param {string} outputDir - The benchmark run's output directory.
+ * @returns {Promise<{stagingDir: string, skillSetHash: string}>}
+ */
+export async function installApm(family, outputDir) {
+  const stagingDir = join(outputDir, ".apm-staging");
+  const stagedClaude = join(stagingDir, ".claude");
+  const sourceClaude = join(family.rootPath, ".claude");
+
+  try {
+    await access(sourceClaude);
+  } catch {
+    throw new Error(
+      `task family missing .claude/ at ${sourceClaude}; family must check in a pre-staged skills/agents tree (design decision P1)`,
+    );
+  }
+
+  await rm(stagingDir, { recursive: true, force: true });
+  await cp(sourceClaude, stagedClaude, { recursive: true });
+
+  const skillSetHash =
+    "sha256:" + createHash("sha256").update(family.apmLockBytes).digest("hex");
+
+  return { stagingDir, skillSetHash };
+}

package/src/benchmark/judge.js

@@ -0,0 +1,146 @@
+/**
+ * Benchmark adapter for the libeval `Judge`. Templates the family's
+ * `judge.task.md` ({{SCORING}} / {{AGENT_TRACE_PATH}} substitution), runs the
+ * judge against the post-run agent CWD, and returns the verdict in the
+ * benchmark's `pass`/`fail` vocabulary (mapped from libeval's
+ * `success`/`failure`).
+ *
+ * The judge verdict is captured from the orchestration context's
+ * `concluded` flag directly — no trace parsing on the happy path.
+ * `parseConcludeFromTrace` is preserved for offline analysis and as a
+ * fallback when the runtime ctx isn't available (e.g. re-grading a
+ * historical run from its judge.ndjson file).
+ */
+
+import { createReadStream, createWriteStream } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { createInterface } from "node:readline";
+import { createJudge } from "../judge.js";
+import { createRedactor } from "../redaction.js";
+
+/**
+ * @typedef {object} JudgeVerdict
+ * @property {"pass" | "fail"} verdict
+ * @property {string} summary
+ */
+
+/**
+ * Run the judge over a completed task run.
+ * @param {import("./task-family.js").Task} task
+ * @param {import("./workdir.js").Workdir} workdir
+ * @param {import("./scorer.js").ScoringResult} scoring
+ * @param {{query: Function, model: string, judgeProfile?: string}} deps
+ * @returns {Promise<JudgeVerdict>}
+ */
+export async function runJudge(task, workdir, scoring, deps) {
+  const template = await readFile(task.paths.judge, "utf8");
+  const taskText = template
+    .replaceAll("{{SCORING}}", JSON.stringify(scoring, null, 2))
+    .replaceAll("{{AGENT_TRACE_PATH}}", workdir.agentTracePath);
+
+  const output = createWriteStream(workdir.judgeTracePath);
+  const judge = createJudge({
+    cwd: workdir.cwd,
+    query: deps.query,
+    output,
+    model: deps.model,
+    judgeProfile: deps.judgeProfile,
+    maxTurns: 5,
+    redactor: createRedactor(),
+  });
+
+  let outcome;
+  try {
+    outcome = await judge.run(taskText);
+  } finally {
+    await new Promise((r) => output.end(r));
+  }
+
+  if (outcome.verdict === null) {
+    return { verdict: "fail", summary: "judge did not conclude" };
+  }
+  return {
+    verdict: outcome.verdict === "success" ? "pass" : "fail",
+    summary: outcome.summary ?? "",
+  };
+}
+
+/**
+ * Parse the last judge-source (or supervisor-source, for backward compat
+ * with pre-Judge-class traces) `Conclude` tool call from an NDJSON trace
+ * and map the verdict (`success → pass`, `failure → fail`). Preserved for
+ * offline analysis; not used on the runtime happy path.
+ * @param {string} tracePath
+ * @returns {Promise<JudgeVerdict | null>}
+ */
+export async function parseConcludeFromTrace(tracePath) {
+  const stream = createReadStream(tracePath);
+  const rl = createInterface({ input: stream, crlfDelay: Infinity });
+  let last = null;
+  for await (const line of rl) {
+    const candidate = extractConcludeInput(line);
+    if (candidate) last = candidate;
+  }
+  if (!last) return null;
+  return {
+    verdict: last.verdict === "success" ? "pass" : "fail",
+    summary: last.summary ?? "",
+  };
+}
+
+/**
+ * Return the `Conclude` tool input if the line carries a judge-source or
+ * supervisor-source assistant message ending in a `Conclude` tool_use
+ * block; null otherwise.
+ * @param {string} line
+ * @returns {{verdict: string, summary?: string} | null}
+ */
+function extractConcludeInput(line) {
+  const trimmed = line.trim();
+  if (!trimmed) return null;
+  let event;
+  try {
+    event = JSON.parse(trimmed);
+  } catch {
+    return null;
+  }
+  const wrapped =
+    event.event && typeof event.source === "string"
+      ? { source: event.source, inner: event.event }
+      : { source: null, inner: event };
+  if (
+    wrapped.source !== null &&
+    wrapped.source !== "judge" &&
+    wrapped.source !== "supervisor"
+  ) {
+    return null;
+  }
+  if (wrapped.inner.type !== "assistant") return null;
+  const content = wrapped.inner.message?.content ?? wrapped.inner.content;
+  if (!Array.isArray(content)) return null;
+  let found = null;
+  for (const block of content) {
+    if (
+      block.type === "tool_use" &&
+      isConcludeToolName(block.name) &&
+      block.input
+    ) {
+      found = block.input;
+    }
+  }
+  return found;
+}
+
+/**
+ * The Claude Agent SDK reports MCP tool names as
+ * `mcp__<server>__<tool>` when the model invokes them — the orchestration
+ * `Conclude` arrives as `mcp__orchestration__Conclude`. Pre-baked
+ * supervisor traces (and the libeval-internal envelopes) sometimes carry
+ * the bare `Conclude` name. Accept both forms so the parser is robust to
+ * trace source.
+ */
+function isConcludeToolName(name) {
+  if (typeof name !== "string") return false;
+  if (name === "Conclude") return true;
+  return name.endsWith("__Conclude");
+}

package/src/benchmark/report.js

@@ -0,0 +1,161 @@
+/**
+ * ReportAggregator — read a run-output directory's `results.jsonl`, group
+ * records by `taskId`, and compute pass@k via the OpenAI HumanEval
+ * unbiased estimator: `1 - C(n-c, k) / C(n, k)`.
+ *
+ * Records that fail schema validation are skipped with a stderr warning
+ * (counted under `totals.skipped`) so a corrupt line cannot abort the
+ * whole report.
+ */
+
+import { createReadStream } from "node:fs";
+import { join } from "node:path";
+import { createInterface } from "node:readline";
+
+import { validateResultRecord } from "./result.js";
+
+/**
+ * @typedef {object} TaskReport
+ * @property {string} taskId
+ * @property {number} n - Total runs.
+ * @property {number} c - Passing runs.
+ * @property {Record<string|number, number|null>} passAtK
+ */
+
+/**
+ * @param {{inputDir: string, kValues: number[]}} opts
+ * @returns {Promise<{tasks: TaskReport[], totals: {tasks: number, runs: number, skipped: number}}>}
+ */
+export async function aggregate({ inputDir, kValues }) {
+  const records = await loadRecords(inputDir);
+  const grouped = groupByTask(records.records);
+  const tasks = [];
+  let runs = 0;
+  for (const [taskId, group] of grouped) {
+    const n = group.length;
+    const c = group.filter((r) => r.verdict === "pass").length;
+    runs += n;
+    const passAtK = {};
+    for (const k of kValues) passAtK[k] = passAtKValue(n, c, k);
+    tasks.push({ taskId, n, c, passAtK });
+  }
+  tasks.sort((a, b) =>
+    a.taskId < b.taskId ? -1 : a.taskId > b.taskId ? 1 : 0,
+  );
+  return {
+    tasks,
+    totals: { tasks: tasks.length, runs, skipped: records.skipped },
+  };
+}
+
+/**
+ * Render an aggregate report as a Markdown table. Columns: taskId | n | c |
+ * pass@k1 | pass@k2 ... — one column per kValues entry, in the same order.
+ * @param {Awaited<ReturnType<typeof aggregate>>} report
+ * @param {number[]} kValues
+ * @returns {string}
+ */
+export function renderTextReport(report, kValues) {
+  const header = ["taskId", "n", "c", ...kValues.map((k) => `pass@${k}`)];
+  const rows = [header, header.map(() => "---")];
+  for (const t of report.tasks) {
+    rows.push([
+      t.taskId,
+      String(t.n),
+      String(t.c),
+      ...kValues.map((k) => formatPassAt(t.passAtK[k])),
+    ]);
+  }
+  const lines = rows.map((r) => `| ${r.join(" | ")} |`);
+  lines.push("");
+  lines.push(
+    `Totals — tasks: ${report.totals.tasks}, runs: ${report.totals.runs}, skipped: ${report.totals.skipped}`,
+  );
+  return lines.join("\n");
+}
+
+function formatPassAt(v) {
+  if (v == null) return "—";
+  if (typeof v === "object" && "error" in v) return v.error;
+  return Number(v).toFixed(4);
+}
+
+async function loadRecords(inputDir) {
+  const path = join(inputDir, "results.jsonl");
+  const stream = createReadStream(path);
+  const rl = createInterface({ input: stream, crlfDelay: Infinity });
+  const records = [];
+  let skipped = 0;
+  for await (const line of rl) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    let record;
+    try {
+      record = JSON.parse(trimmed);
+    } catch (e) {
+      process.stderr.write(
+        `benchmark report: skipped malformed JSON line — ${e.message}\n`,
+      );
+      skipped++;
+      continue;
+    }
+    try {
+      validateResultRecord(record);
+    } catch (e) {
+      process.stderr.write(
+        `benchmark report: skipped record failing schema — ${describeError(e)}\n`,
+      );
+      skipped++;
+      continue;
+    }
+    records.push(record);
+  }
+  return { records, skipped };
+}
+
+function describeError(e) {
+  if (e && Array.isArray(e.issues)) {
+    return e.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ");
+  }
+  return e.message ?? String(e);
+}
+
+function groupByTask(records) {
+  const out = new Map();
+  for (const r of records) {
+    if (!out.has(r.taskId)) out.set(r.taskId, []);
+    out.get(r.taskId).push(r);
+  }
+  return out;
+}
+
+/**
+ * pass@k = 1 - C(n - c, k) / C(n, k). Compute with BigInt to avoid
+ * floating-point loss on large n.
+ * @param {number} n
+ * @param {number} c
+ * @param {number} k
+ * @returns {number | {error: string}}
+ */
+function passAtKValue(n, c, k) {
+  if (k > n) return { error: "k > n" };
+  if (n - c < k) return 1;
+  const total = binomial(BigInt(n), BigInt(k));
+  const fail = binomial(BigInt(n - c), BigInt(k));
+  // Compute the ratio as a single division so we avoid `1 - x` which
+  // accumulates IEEE-754 error (e.g. 1 - 0.6 = 0.39999...).
+  const passing = total - fail;
+  return Number(passing) / Number(total);
+}
+
+function binomial(n, k) {
+  if (k < 0n || k > n) return 0n;
+  if (k === 0n || k === n) return 1n;
+  let kk = k;
+  if (kk > n - kk) kk = n - kk;
+  let result = 1n;
+  for (let i = 0n; i < kk; i++) {
+    result = (result * (n - i)) / (i + 1n);
+  }
+  return result;
+}

package/src/benchmark/result.js

@@ -0,0 +1,108 @@
+/**
+ * Result-record schemas and runtime validators.
+ *
+ * Two schemas live here:
+ *   - RESULT_RECORD_SCHEMA — one record per (task, runIndex) from a full
+ *     benchmark run. Has a happy branch (scoring + judge present) and a
+ *     pre-flight-failure branch (scoring/judgeVerdict/submission absent).
+ *   - SCORING_RECORD_SCHEMA — narrower output of `benchmark-score` (P7):
+ *     ad-hoc grading without a full lifecycle.
+ *
+ * Validation is throw-on-mismatch so the runner can wrap every JSONL append
+ * in a guard and reject schema drift at write time.
+ */
+
+import { z } from "zod";
+
+const VERDICT_ENUM = z.enum(["pass", "fail"]);
+
+const SCORING_SHAPE = z.object({
+  verdict: VERDICT_ENUM,
+  details: z.array(z.unknown()),
+  exitCode: z.number().int(),
+});
+
+const JUDGE_VERDICT_SHAPE = z.object({
+  verdict: VERDICT_ENUM,
+  summary: z.string(),
+});
+
+const PROFILES_SHAPE = z.object({
+  agent: z.union([z.string(), z.null()]),
+  supervisor: z.null(),
+  judge: z.union([z.string(), z.null()]),
+});
+
+const PREFLIGHT_ERROR_SHAPE = z.object({
+  phase: z.string(),
+  message: z.string(),
+  exitCode: z.number().int(),
+});
+
+const COMMON_FIELDS = {
+  taskId: z.string().min(1),
+  runIndex: z.number().int().min(0),
+  verdict: VERDICT_ENUM,
+  costUsd: z.number(),
+  turns: z.number().int().min(0),
+  profiles: PROFILES_SHAPE,
+  model: z.string(),
+  skillSetHash: z.string(),
+  familyRevision: z.string(),
+  durationMs: z.number().int().min(0),
+};
+
+const AGENT_ERROR_SHAPE = z.object({
+  message: z.string(),
+  aborted: z.boolean(),
+});
+
+const HAPPY_RECORD = z.object({
+  ...COMMON_FIELDS,
+  scoring: SCORING_SHAPE,
+  submission: z.string(),
+  judgeVerdict: JUDGE_VERDICT_SHAPE,
+  agentTracePath: z.string(),
+  judgeTracePath: z.string(),
+  agentError: AGENT_ERROR_SHAPE.optional(),
+  preflightError: z.undefined().optional(),
+});
+
+const PREFLIGHT_RECORD = z.object({
+  ...COMMON_FIELDS,
+  costUsd: z.literal(0),
+  preflightError: PREFLIGHT_ERROR_SHAPE,
+  // Trace paths are populated even on preflight failure (the runner allocates
+  // them in WorkdirManager.start) so the record is uniform across branches
+  // and downstream consumers can reference them without conditional fields.
+  agentTracePath: z.string(),
+  judgeTracePath: z.string(),
+  scoring: z.undefined().optional(),
+  submission: z.undefined().optional(),
+  judgeVerdict: z.undefined().optional(),
+  agentError: z.undefined().optional(),
+});
+
+export const RESULT_RECORD_SCHEMA = z.union([HAPPY_RECORD, PREFLIGHT_RECORD]);
+
+export const SCORING_RECORD_SCHEMA = z.object({
+  taskId: z.string().min(1),
+  scoring: SCORING_SHAPE,
+  exitCode: z.number().int(),
+});
+
+/**
+ * Throw on schema mismatch.
+ * @param {object} record
+ */
+export function validateResultRecord(record) {
+  RESULT_RECORD_SCHEMA.parse(record);
+}
+
+/**
+ * Throw on schema mismatch.
+ * @param {object} record
+ */
+export function validateScoringRecord(record) {
+  SCORING_RECORD_SCHEMA.parse(record);
+}