@forwardimpact/libharness 0.1.22 → 1.1.0

package/src/commands/assert.js

@@ -0,0 +1,153 @@
+import { basename } from "node:path";
+import jmespath from "jmespath";
+
+/**
+ * Evaluate an assertion and return the structured result.
+ * @param {object} values - { grep?: string, query?: string, exists?: boolean, not?: boolean, message?: string }
+ * @param {string[]} args - [testName, file]
+ * @param {object} fsSync - Sync filesystem surface (`runtime.fsSync`): `existsSync`, `readFileSync`.
+ * @returns {{ test: string, pass: boolean, message?: string }}
+ */
+// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: assertion dispatch by type
+export function evaluateAssertion(values, args, fsSync) {
+  const testName = args[0];
+  if (!testName) throw new Error("assert: missing test name");
+
+  const file = args[1];
+  const modes = [
+    values.grep,
+    values.query,
+    values.exists,
+    values["cites-job"],
+  ].filter((v) => v !== undefined && v !== false);
+  if (modes.length === 0) {
+    throw new Error(
+      "assert: specify one of --grep, --query, --exists, or --cites-job",
+    );
+  }
+  if (modes.length > 1) {
+    throw new Error(
+      "assert: specify only one of --grep, --query, --exists, or --cites-job",
+    );
+  }
+
+  let result;
+  if (values.exists) {
+    if (!file) throw new Error("assert: missing file argument");
+    result = assertExists(file, fsSync);
+  } else if (values.grep) {
+    if (!file) throw new Error("assert: missing file argument for --grep");
+    result = assertGrep(values.grep, file, fsSync);
+  } else if (values["cites-job"]) {
+    if (!file) throw new Error("assert: missing file argument for --cites-job");
+    result = assertCitesJob(values["cites-job"], file, fsSync);
+  } else {
+    if (!file) throw new Error("assert: missing file argument for --query");
+    result = assertQuery(values.query, file, fsSync);
+  }
+
+  if (values.not) {
+    result.pass = !result.pass;
+    if (result.pass) {
+      delete result.message;
+    } else {
+      result.message =
+        result.message ?? `inverted assertion failed for ${basename(file)}`;
+    }
+  }
+
+  if (!result.pass && values.message) {
+    result.message = values.message;
+  }
+
+  const output = { test: testName, pass: result.pass };
+  if (result.message) output.message = result.message;
+  return output;
+}
+
+/**
+ * Run an assertion, write JSON to stdout, and return a failure envelope when
+ * the assertion does not pass.
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
+ */
+export async function runAssertCommand(ctx) {
+  const runtime = ctx.deps.runtime;
+  const args = [ctx.args["test-name"], ctx.args.file];
+  let result;
+  try {
+    result = evaluateAssertion(ctx.options, args, runtime.fsSync);
+  } catch (err) {
+    return { ok: false, code: 1, error: err.message };
+  }
+  runtime.proc.stdout.write(JSON.stringify(result) + "\n");
+  return result.pass ? { ok: true } : { ok: false, code: 1, error: "" };
+}
+
+function assertExists(file, fsSync) {
+  if (fsSync.existsSync(file)) return { pass: true };
+  return { pass: false, message: `${file} not found` };
+}
+
+function assertGrep(pattern, file, fsSync) {
+  const content = fsSync.readFileSync(file, "utf8");
+  const re = new RegExp(pattern, "im");
+  if (re.test(content)) return { pass: true };
+  return {
+    pass: false,
+    message: `pattern "${pattern}" not found in ${basename(file)}`,
+  };
+}
+
+function assertQuery(expression, file, fsSync) {
+  const content = fsSync.readFileSync(file, "utf8");
+  const data = parseJsonOrNdjson(content);
+  const result = jmespath.search(data, expression);
+  const truthy =
+    result !== null &&
+    result !== undefined &&
+    result !== false &&
+    (Array.isArray(result) ? result.length > 0 : true);
+  if (truthy) return { pass: true };
+  return {
+    pass: false,
+    message: `query returned ${JSON.stringify(result)}`,
+  };
+}
+
+const JOB_TAG_RE = /<job\s+user="([^"]*)"\s+goal="([^"]*)">/;
+
+function assertCitesJob(jobFile, file, fsSync) {
+  const jobContent = fsSync.readFileSync(jobFile, "utf8");
+  const match = JOB_TAG_RE.exec(jobContent);
+  if (!match) {
+    return {
+      pass: false,
+      message: `no <job> tag found in ${basename(jobFile)}`,
+    };
+  }
+  const citation = `${match[1]}: ${match[2]}`;
+  const content = fsSync.readFileSync(file, "utf8");
+  if (content.includes(citation)) return { pass: true };
+  return { pass: false, message: `missing "${citation}"` };
+}
+
+function parseJsonOrNdjson(content) {
+  try {
+    return JSON.parse(content);
+  } catch {
+    // Fall through to NDJSON
+  }
+  const lines = [];
+  for (const raw of content.split("\n")) {
+    const trimmed = raw.trim();
+    if (!trimmed) continue;
+    try {
+      lines.push(JSON.parse(trimmed));
+    } catch {
+      // skip unparseable lines
+    }
+  }
+  if (lines.length === 0) throw new Error("assert: no valid JSON in file");
+  return lines;
+}

package/src/commands/benchmark-definition.js

@@ -0,0 +1,175 @@
+/**
+ * `fit-benchmark` CLI definition. Lives in `src/` so the bin stays an
+ * execute-on-import entry point — launcher packages import the bin to run
+ * it — while tests import the definition without running the CLI.
+ */
+
+import { runBenchmarkRunCommand } from "./benchmark-run.js";
+import { runBenchmarkInvariantsCommand } from "./benchmark-invariants.js";
+import { runBenchmarkReportCommand } from "./benchmark-report.js";
+import {
+  BENCHMARK_AGENT_MODEL,
+  LEAD_MODEL,
+} from "@forwardimpact/libutil/models";
+
+export const definition = {
+  name: "fit-benchmark",
+  description:
+    "Run coding-agent task families, grade hidden tests, and aggregate pass@k across runs.",
+  commands: [
+    {
+      name: "run",
+      args: [],
+      handler: runBenchmarkRunCommand,
+      description:
+        "Run every task in a family for N runs and emit one result record per (task, runIndex).",
+      options: {
+        family: {
+          type: "string",
+          description: "Path or git URL to a task family",
+        },
+        task: {
+          type: "string",
+          description:
+            "Run only this task id (directory name under tasks/, default: every task)",
+        },
+        "skills-from": {
+          type: "string",
+          description:
+            "Stage .claude/ from this directory (a root containing .claude/) instead of running apm install — exercise local, unpublished skills",
+        },
+        output: {
+          type: "string",
+          description:
+            "Run-output directory (created if missing, default: benchmark-runs)",
+        },
+        runs: {
+          type: "string",
+          description: "Runs per task (integer ≥ 1, default: 5)",
+        },
+        "agent-model": {
+          type: "string",
+          description: `Claude model for the agent-under-test (default: ${BENCHMARK_AGENT_MODEL})`,
+        },
+        "lead-model": {
+          type: "string",
+          description: `Claude model for the lead role (default: ${LEAD_MODEL})`,
+        },
+        "judge-model": {
+          type: "string",
+          description: `Claude model for the judge (default: ${LEAD_MODEL})`,
+        },
+        "agent-profile": {
+          type: "string",
+          description: "Agent-under-test profile name",
+        },
+        "judge-profile": {
+          type: "string",
+          description: "Judge profile name",
+        },
+        "work-tracker": {
+          type: "string",
+          description:
+            "Active work-item tracker (github|filesystem, default: github)",
+        },
+        "max-turns": {
+          type: "string",
+          description:
+            "Agent-under-test turn budget (default: 50, 0 = unlimited)",
+        },
+        concurrency: {
+          type: "string",
+          description:
+            "Max cells run concurrently (positive integer; default: CPU-aware min(4, max(2, cores/2)); env: LIBHARNESS_BENCHMARK_CONCURRENCY)",
+        },
+        shard: {
+          type: "string",
+          description:
+            "Run only shard i of N as i/N (1-based; default: the whole family). Each shard writes a partial results.jsonl; report --input merges them.",
+        },
+        "allowed-tools": {
+          type: "string",
+          description:
+            "Comma-separated tool allowlist for the agent-under-test (default: Bash,Read,Glob,Grep,Write,Edit,Agent,TodoWrite)",
+        },
+      },
+    },
+    {
+      name: "invariants",
+      args: [],
+      handler: runBenchmarkInvariantsCommand,
+      description:
+        "Check a single task's invariants against a post-run workdir without invoking an agent.",
+      options: {
+        family: {
+          type: "string",
+          description: "Path or git URL to a task family",
+        },
+        task: {
+          type: "string",
+          description: "Task id (directory name under tasks/)",
+        },
+        "run-dir": {
+          type: "string",
+          description:
+            "Post-run directory whose cwd/ subdir is the agent CWD; invariants run against that cwd — the path hooks receive as $AGENT_CWD",
+        },
+        output: {
+          type: "string",
+          description: "Output file (defaults to stdout; one JSONL line)",
+        },
+      },
+    },
+    {
+      name: "report",
+      args: [],
+      handler: runBenchmarkReportCommand,
+      description:
+        "Aggregate result records into pass@k via the OpenAI HumanEval estimator.",
+      options: {
+        input: {
+          type: "string",
+          description:
+            "Run-output directory containing results.jsonl (default: benchmark-runs)",
+        },
+        k: {
+          type: "string",
+          description: "Comma-separated k values (default: 1,3,5)",
+        },
+        format: {
+          type: "string",
+          description: "Output format (json|text, default: json)",
+        },
+      },
+    },
+  ],
+  globalOptions: {
+    help: { type: "boolean", short: "h", description: "Show this help" },
+    version: { type: "boolean", description: "Show version" },
+    json: { type: "boolean", description: "Output help as JSON" },
+  },
+  examples: [
+    "fit-benchmark run --family=./families/coding",
+    "fit-benchmark run --family=./families/coding --task=todo-api --runs=1",
+    "fit-benchmark run --family=./families/coding --work-tracker=filesystem",
+    "fit-benchmark run --family=./families/coding --skills-from=. --task=todo-api",
+    `fit-benchmark run --family=./families/coding --runs=10 --agent-model=${BENCHMARK_AGENT_MODEL}`,
+    "fit-benchmark invariants --family=./families/coding --task=todo-api --run-dir=./benchmark-runs/runs/todo-api/0",
+    "fit-benchmark report --format=text",
+    "fit-benchmark report --input=./runs/today --k=1,3,5 --format=text",
+  ],
+  documentation: [
+    {
+      title: "Run a Benchmark",
+      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/index.md",
+      description:
+        "Author a coding-task family, run a benchmark across multiple runs, and read the pass@k report.",
+    },
+    {
+      title: "Automate with GitHub Actions",
+      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/ci-workflow/index.md",
+      description:
+        "Run benchmarks in CI with the forwardimpact/fit-benchmark action.",
+    },
+  ],
+};

package/src/commands/benchmark-invariants.js

@@ -0,0 +1,73 @@
+/**
+ * `fit-benchmark invariants` — check a single task's invariants against a
+ * post-run workdir directory without invoking an agent. Useful for
+ * re-checking an agent's output against revised grading material.
+ */
+
+import { join, resolve } from "node:path";
+import { createServer } from "node:net";
+
+import { validateInvariantsRecord } from "../benchmark/result.js";
+import { runInvariants } from "../benchmark/invariants.js";
+import { loadTaskFamily } from "../benchmark/task-family.js";
+
+/**
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
+ */
+export async function runBenchmarkInvariantsCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
+  const familyInput = values.family;
+  if (!familyInput)
+    return { ok: false, code: 1, error: "--family is required" };
+  const taskId = values.task;
+  if (!taskId) return { ok: false, code: 1, error: "--task is required" };
+  const runDirArg = values["run-dir"];
+  if (!runDirArg) return { ok: false, code: 1, error: "--run-dir is required" };
+
+  const family = await loadTaskFamily(familyInput, runtime);
+  const task = family.tasks().find((t) => t.id === taskId);
+  if (!task)
+    return { ok: false, code: 1, error: `task not found in family: ${taskId}` };
+
+  const runDir = resolve(runDirArg);
+  const cwd = join(runDir, "cwd");
+  const port = await allocatePort();
+
+  const invariants = await runInvariants(task, { cwd, port, runDir }, runtime);
+  const record = {
+    taskId: task.id,
+    invariants,
+    exitCode: invariants.exitCode,
+  };
+  validateInvariantsRecord(record);
+
+  const line = JSON.stringify(record) + "\n";
+  if (values.output) {
+    runtime.fsSync.writeFileSync(resolve(values.output), line);
+  } else {
+    runtime.proc.stdout.write(line);
+  }
+  return invariants.verdict === "pass"
+    ? { ok: true }
+    : { ok: false, code: 1, error: "" };
+}
+
+function allocatePort() {
+  return new Promise((res, rej) => {
+    const server = createServer();
+    server.unref();
+    server.on("error", rej);
+    server.listen(0, "127.0.0.1", () => {
+      const addr = server.address();
+      if (!addr || typeof addr === "string") {
+        server.close();
+        rej(new Error("failed to allocate port"));
+        return;
+      }
+      const port = addr.port;
+      server.close(() => res(port));
+    });
+  });
+}

package/src/commands/benchmark-report.js

@@ -0,0 +1,51 @@
+/**
+ * `fit-benchmark report` — aggregate `results.jsonl` into pass@k via the
+ * OpenAI HumanEval estimator. Output is JSON by default; pass --format=text
+ * to render a markdown table.
+ */
+
+import { resolve } from "node:path";
+
+import { aggregate, renderTextReport } from "../benchmark/report.js";
+
+/**
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
+ */
+export async function runBenchmarkReportCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
+  const inputDir = values.input ?? "benchmark-runs";
+  const kRaw = values.k ?? "1,3,5";
+  let kValues;
+  try {
+    kValues = kRaw.split(",").map((t) => {
+      const n = Number.parseInt(t.trim(), 10);
+      if (!Number.isFinite(n) || n < 1) {
+        throw new Error(
+          "--k must be a comma-separated list of positive integers",
+        );
+      }
+      return n;
+    });
+  } catch (err) {
+    return { ok: false, code: 1, error: err.message };
+  }
+  const format = values.format ?? "json";
+  if (format !== "json" && format !== "text") {
+    return { ok: false, code: 1, error: "--format must be 'json' or 'text'" };
+  }
+
+  const report = await aggregate({
+    inputDir: resolve(inputDir),
+    kValues,
+    includeRuns: format === "text",
+    runtime,
+  });
+  if (format === "text") {
+    runtime.proc.stdout.write(renderTextReport(report, kValues) + "\n");
+  } else {
+    runtime.proc.stdout.write(JSON.stringify(report, null, 2) + "\n");
+  }
+  return { ok: true };
+}

package/src/commands/benchmark-run.js

@@ -0,0 +1,175 @@
+/**
+ * `fit-benchmark run` — run every task in a family for N runs, stream each
+ * ResultRecord to stdout (one JSON line per record), and append to the
+ * canonical `<output>/results.jsonl` for the report subcommand.
+ */
+
+import { resolve } from "node:path";
+import { availableParallelism } from "node:os";
+
+import { createConfig } from "@forwardimpact/libconfig";
+import { createBenchmarkRunner } from "../benchmark/runner.js";
+import { resolveWorkTracker } from "./work-tracker.js";
+import {
+  BENCHMARK_AGENT_MODEL,
+  LEAD_MODEL,
+} from "@forwardimpact/libutil/models";
+
+/**
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
+ */
+export async function runBenchmarkRunCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
+  let opts;
+  try {
+    opts = parseRunOptions(values, runtime.proc.env);
+  } catch (err) {
+    return { ok: false, code: 1, error: err.message };
+  }
+  const config = await createConfig("script", "benchmark");
+  runtime.proc.env.ANTHROPIC_API_KEY = await config.anthropicToken();
+  // The benchmark agent runs via createBenchmarkRunner, not the supervise
+  // command, so the active-tracker env must land here before the runner
+  // spawns the subprocess that inherits process.env.
+  runtime.proc.env.LIBHARNESS_WORK_TRACKER = opts.workTracker;
+
+  // The Claude Agent SDK spawns a `claude` subprocess that inherits
+  // process.env. NODE_EXTRA_CA_CERTS causes undici (the HTTP client
+  // inside that subprocess) to fail with UND_ERR_INVALID_ARG on
+  // Node 22+, aborting every API call after 10 retries. Strip it
+  // before the SDK loads so the subprocess gets a clean environment.
+  delete runtime.proc.env.NODE_EXTRA_CA_CERTS;
+
+  const { query } = await import("@anthropic-ai/claude-agent-sdk");
+  const runner = createBenchmarkRunner({ ...opts, query, runtime });
+
+  let anyFail = false;
+  let count = 0;
+  for await (const record of runner.run()) {
+    count++;
+    runtime.proc.stdout.write(JSON.stringify(record) + "\n");
+    if (record.verdict !== "pass") anyFail = true;
+  }
+  if (count === 0) return resolveZeroRecordOutcome(opts, runtime);
+  return anyFail ? { ok: false, code: 1, error: "" } : { ok: true };
+}
+
+/**
+ * Decide the exit outcome when a run streamed zero records. A run that emits no
+ * records normally did nothing (no tasks discovered, or the agent never
+ * produced output) — a failure, surfaced loudly so CI does not go green on an
+ * empty benchmark. The one exception is a deliberately-empty shard: a
+ * high-index `--shard=i/N` with `N > cell count` legitimately selects zero
+ * cells, so it exits 0 with a stderr note. Exported so the relaxed-guard branch
+ * is testable without the full handler's config/SDK setup.
+ * @param {{shard: {index: number, total: number} | null}} opts
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @returns {{ok: true} | {ok: false, code: number, error: string}}
+ */
+export function resolveZeroRecordOutcome(opts, runtime) {
+  if (opts.shard) {
+    runtime.proc.stderr.write(
+      `shard ${opts.shard.index}/${opts.shard.total} selected no cells\n`,
+    );
+    return { ok: true };
+  }
+  return {
+    ok: false,
+    code: 1,
+    error:
+      "benchmark produced no result records — no task ran to completion; check the family's tasks/, apm install, and agent availability (ANTHROPIC_API_KEY / claude CLI / IS_SANDBOX)",
+  };
+}
+
+/**
+ * Parse and validate benchmark run options. Exported so tests can verify
+ * defaults, including the resolved work tracker.
+ * @param {Record<string, string|undefined>} values - Parsed option values
+ * @param {Record<string, string|undefined>} [env] - Process environment, read
+ *   for the `LIBHARNESS_WORK_TRACKER` fallback when `--work-tracker` is absent.
+ * @returns {object}
+ */
+export function parseRunOptions(values, env = {}) {
+  const family = values.family;
+  if (!family) throw new Error("--family is required");
+  const output = values.output ?? "benchmark-runs";
+  const runs = Number.parseInt(values.runs ?? "5", 10);
+  if (!Number.isFinite(runs) || runs < 1)
+    throw new Error("--runs must be a positive integer");
+  return {
+    family,
+    runs,
+    task: values.task ?? null,
+    skillsFrom: values["skills-from"] ?? null,
+    output: resolve(output),
+    agentModel: values["agent-model"] || BENCHMARK_AGENT_MODEL,
+    supervisorModel: values["lead-model"] || LEAD_MODEL,
+    judgeModel: values["judge-model"] || LEAD_MODEL,
+    workTracker: resolveWorkTracker(values, env),
+    profiles: {
+      agent: values["agent-profile"] ?? null,
+      judge: values["judge-profile"] ?? null,
+    },
+    maxTurns: parseMaxTurns(values["max-turns"]),
+    concurrency: resolveConcurrency(values, env),
+    shard: parseShard(values.shard),
+    allowedTools: values["allowed-tools"]
+      ? values["allowed-tools"]
+          .split(",")
+          .map((s) => s.trim())
+          .filter(Boolean)
+      : undefined,
+  };
+}
+
+function parseMaxTurns(raw) {
+  if (raw === undefined) return undefined;
+  if (raw === "0") return 0;
+  return Number.parseInt(raw, 10);
+}
+
+/**
+ * Parse a `--shard=<i>/<N>` selector into `{index, total}` (1-based), or `null`
+ * for an unsharded run. Validates `1 ≤ index ≤ total` with integer parts.
+ * @param {string|undefined} raw
+ * @returns {{index: number, total: number} | null}
+ */
+export function parseShard(raw) {
+  if (raw == null || raw === "") return null;
+  const m = /^(\d+)\/(\d+)$/.exec(raw.trim());
+  if (!m) throw new Error("--shard must be in the form i/N (e.g. 1/4)");
+  const index = Number.parseInt(m[1], 10);
+  const total = Number.parseInt(m[2], 10);
+  if (total < 1 || index < 1 || index > total)
+    throw new Error("--shard requires 1 ≤ i ≤ N");
+  return { index, total };
+}
+
+// Conservative because each cell spawns ~3 agent subprocesses (lead +
+// agent-under-test + judge); a low ceiling keeps a single runner from
+// thrashing. The bulk of the CI speedup comes from Layer-2 sharding across
+// machines, not from raising this in-job default.
+const CONCURRENCY_CEILING = 4;
+
+/**
+ * Resolve the cell concurrency: `--concurrency` flag > the
+ * `LIBHARNESS_BENCHMARK_CONCURRENCY` env var > a CPU-aware default of
+ * `min(CONCURRENCY_CEILING, max(2, ⌊cores/2⌋))`. The default is `> 1` so
+ * concurrency is on transparently without any consumer opting in.
+ * @param {Record<string, string|undefined>} values
+ * @param {Record<string, string|undefined>} [env]
+ * @returns {number}
+ */
+export function resolveConcurrency(values, env = {}) {
+  const raw = values.concurrency ?? env.LIBHARNESS_BENCHMARK_CONCURRENCY;
+  if (raw != null && raw !== "") {
+    const n = Number.parseInt(raw, 10);
+    if (!Number.isFinite(n) || n < 1)
+      throw new Error("--concurrency must be a positive integer");
+    return n;
+  }
+  const cores = availableParallelism();
+  return Math.min(CONCURRENCY_CEILING, Math.max(2, Math.floor(cores / 2)));
+}