@forwardimpact/libharness 0.1.20 → 1.0.0

package/src/commands/benchmark-definition.js

@@ -0,0 +1,165 @@
+/**
+ * `fit-benchmark` CLI definition. Lives in `src/` so the bin stays an
+ * execute-on-import entry point — launcher packages import the bin to run
+ * it — while tests import the definition without running the CLI.
+ */
+
+import { runBenchmarkRunCommand } from "./benchmark-run.js";
+import { runBenchmarkInvariantsCommand } from "./benchmark-invariants.js";
+import { runBenchmarkReportCommand } from "./benchmark-report.js";
+import {
+  BENCHMARK_AGENT_MODEL,
+  LEAD_MODEL,
+} from "@forwardimpact/libutil/models";
+
+export const definition = {
+  name: "fit-benchmark",
+  description:
+    "Run coding-agent task families, grade hidden tests, and aggregate pass@k across runs.",
+  commands: [
+    {
+      name: "run",
+      args: [],
+      handler: runBenchmarkRunCommand,
+      description:
+        "Run every task in a family for N runs and emit one result record per (task, runIndex).",
+      options: {
+        family: {
+          type: "string",
+          description: "Path or git URL to a task family",
+        },
+        task: {
+          type: "string",
+          description:
+            "Run only this task id (directory name under tasks/, default: every task)",
+        },
+        "skills-from": {
+          type: "string",
+          description:
+            "Stage .claude/ from this directory (a root containing .claude/) instead of running apm install — exercise local, unpublished skills",
+        },
+        output: {
+          type: "string",
+          description:
+            "Run-output directory (created if missing, default: benchmark-runs)",
+        },
+        runs: {
+          type: "string",
+          description: "Runs per task (integer ≥ 1, default: 5)",
+        },
+        "agent-model": {
+          type: "string",
+          description: `Claude model for the agent-under-test (default: ${BENCHMARK_AGENT_MODEL})`,
+        },
+        "lead-model": {
+          type: "string",
+          description: `Claude model for the lead role (default: ${LEAD_MODEL})`,
+        },
+        "judge-model": {
+          type: "string",
+          description: `Claude model for the judge (default: ${LEAD_MODEL})`,
+        },
+        "agent-profile": {
+          type: "string",
+          description: "Agent-under-test profile name",
+        },
+        "judge-profile": {
+          type: "string",
+          description: "Judge profile name",
+        },
+        "work-tracker": {
+          type: "string",
+          description:
+            "Active work-item tracker (github|filesystem, default: github)",
+        },
+        "max-turns": {
+          type: "string",
+          description:
+            "Agent-under-test turn budget (default: 50, 0 = unlimited)",
+        },
+        "allowed-tools": {
+          type: "string",
+          description:
+            "Comma-separated tool allowlist for the agent-under-test (default: Bash,Read,Glob,Grep,Write,Edit,Agent,TodoWrite)",
+        },
+      },
+    },
+    {
+      name: "invariants",
+      args: [],
+      handler: runBenchmarkInvariantsCommand,
+      description:
+        "Check a single task's invariants against a post-run workdir without invoking an agent.",
+      options: {
+        family: {
+          type: "string",
+          description: "Path or git URL to a task family",
+        },
+        task: {
+          type: "string",
+          description: "Task id (directory name under tasks/)",
+        },
+        "run-dir": {
+          type: "string",
+          description:
+            "Post-run directory whose cwd/ subdir is the agent CWD; invariants run against that cwd — the path hooks receive as $AGENT_CWD",
+        },
+        output: {
+          type: "string",
+          description: "Output file (defaults to stdout; one JSONL line)",
+        },
+      },
+    },
+    {
+      name: "report",
+      args: [],
+      handler: runBenchmarkReportCommand,
+      description:
+        "Aggregate result records into pass@k via the OpenAI HumanEval estimator.",
+      options: {
+        input: {
+          type: "string",
+          description:
+            "Run-output directory containing results.jsonl (default: benchmark-runs)",
+        },
+        k: {
+          type: "string",
+          description: "Comma-separated k values (default: 1,3,5)",
+        },
+        format: {
+          type: "string",
+          description: "Output format (json|text, default: json)",
+        },
+      },
+    },
+  ],
+  globalOptions: {
+    help: { type: "boolean", short: "h", description: "Show this help" },
+    version: { type: "boolean", description: "Show version" },
+    json: { type: "boolean", description: "Output help as JSON" },
+  },
+  examples: [
+    "fit-benchmark run --family=./families/coding",
+    "fit-benchmark run --family=./families/coding --task=todo-api --runs=1",
+    "fit-benchmark run --family=./families/coding --work-tracker=filesystem",
+    "fit-benchmark run --family=./families/coding --skills-from=. --task=todo-api",
+    `fit-benchmark run --family=./families/coding --runs=10 --agent-model=${BENCHMARK_AGENT_MODEL}`,
+    "fit-benchmark invariants --family=./families/coding --task=todo-api --run-dir=./benchmark-runs/runs/todo-api/0",
+    "fit-benchmark report --format=text",
+    "fit-benchmark report --input=./runs/today --k=1,3,5 --format=text",
+  ],
+  documentation: [
+    {
+      title: "Run a Benchmark",
+      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/index.md",
+      description:
+        "Author a coding-task family, run a benchmark across multiple runs, and read the pass@k report.",
+    },
+    {
+      title: "Automate with GitHub Actions",
+      url: "https://www.forwardimpact.team/docs/libraries/prove-changes/run-benchmark/ci-workflow/index.md",
+      description:
+        "Run benchmarks in CI with the forwardimpact/fit-benchmark action.",
+    },
+  ],
+};

package/src/commands/benchmark-invariants.js

@@ -0,0 +1,73 @@
+/**
+ * `fit-benchmark invariants` — check a single task's invariants against a
+ * post-run workdir directory without invoking an agent. Useful for
+ * re-checking an agent's output against revised grading material.
+ */
+
+import { join, resolve } from "node:path";
+import { createServer } from "node:net";
+
+import { validateInvariantsRecord } from "../benchmark/result.js";
+import { runInvariants } from "../benchmark/invariants.js";
+import { loadTaskFamily } from "../benchmark/task-family.js";
+
+/**
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
+ */
+export async function runBenchmarkInvariantsCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
+  const familyInput = values.family;
+  if (!familyInput)
+    return { ok: false, code: 1, error: "--family is required" };
+  const taskId = values.task;
+  if (!taskId) return { ok: false, code: 1, error: "--task is required" };
+  const runDirArg = values["run-dir"];
+  if (!runDirArg) return { ok: false, code: 1, error: "--run-dir is required" };
+
+  const family = await loadTaskFamily(familyInput, runtime);
+  const task = family.tasks().find((t) => t.id === taskId);
+  if (!task)
+    return { ok: false, code: 1, error: `task not found in family: ${taskId}` };
+
+  const runDir = resolve(runDirArg);
+  const cwd = join(runDir, "cwd");
+  const port = await allocatePort();
+
+  const invariants = await runInvariants(task, { cwd, port, runDir }, runtime);
+  const record = {
+    taskId: task.id,
+    invariants,
+    exitCode: invariants.exitCode,
+  };
+  validateInvariantsRecord(record);
+
+  const line = JSON.stringify(record) + "\n";
+  if (values.output) {
+    runtime.fsSync.writeFileSync(resolve(values.output), line);
+  } else {
+    runtime.proc.stdout.write(line);
+  }
+  return invariants.verdict === "pass"
+    ? { ok: true }
+    : { ok: false, code: 1, error: "" };
+}
+
+function allocatePort() {
+  return new Promise((res, rej) => {
+    const server = createServer();
+    server.unref();
+    server.on("error", rej);
+    server.listen(0, "127.0.0.1", () => {
+      const addr = server.address();
+      if (!addr || typeof addr === "string") {
+        server.close();
+        rej(new Error("failed to allocate port"));
+        return;
+      }
+      const port = addr.port;
+      server.close(() => res(port));
+    });
+  });
+}

package/src/commands/benchmark-report.js

@@ -0,0 +1,51 @@
+/**
+ * `fit-benchmark report` — aggregate `results.jsonl` into pass@k via the
+ * OpenAI HumanEval estimator. Output is JSON by default; pass --format=text
+ * to render a markdown table.
+ */
+
+import { resolve } from "node:path";
+
+import { aggregate, renderTextReport } from "../benchmark/report.js";
+
+/**
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
+ */
+export async function runBenchmarkReportCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
+  const inputDir = values.input ?? "benchmark-runs";
+  const kRaw = values.k ?? "1,3,5";
+  let kValues;
+  try {
+    kValues = kRaw.split(",").map((t) => {
+      const n = Number.parseInt(t.trim(), 10);
+      if (!Number.isFinite(n) || n < 1) {
+        throw new Error(
+          "--k must be a comma-separated list of positive integers",
+        );
+      }
+      return n;
+    });
+  } catch (err) {
+    return { ok: false, code: 1, error: err.message };
+  }
+  const format = values.format ?? "json";
+  if (format !== "json" && format !== "text") {
+    return { ok: false, code: 1, error: "--format must be 'json' or 'text'" };
+  }
+
+  const report = await aggregate({
+    inputDir: resolve(inputDir),
+    kValues,
+    includeRuns: format === "text",
+    runtime,
+  });
+  if (format === "text") {
+    runtime.proc.stdout.write(renderTextReport(report, kValues) + "\n");
+  } else {
+    runtime.proc.stdout.write(JSON.stringify(report, null, 2) + "\n");
+  }
+  return { ok: true };
+}

package/src/commands/benchmark-run.js

@@ -0,0 +1,111 @@
+/**
+ * `fit-benchmark run` — run every task in a family for N runs, stream each
+ * ResultRecord to stdout (one JSON line per record), and append to the
+ * canonical `<output>/results.jsonl` for the report subcommand.
+ */
+
+import { resolve } from "node:path";
+
+import { createConfig } from "@forwardimpact/libconfig";
+import { createBenchmarkRunner } from "../benchmark/runner.js";
+import { resolveWorkTracker } from "./work-tracker.js";
+import {
+  BENCHMARK_AGENT_MODEL,
+  LEAD_MODEL,
+} from "@forwardimpact/libutil/models";
+
+/**
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
+ */
+export async function runBenchmarkRunCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
+  let opts;
+  try {
+    opts = parseRunOptions(values, runtime.proc.env);
+  } catch (err) {
+    return { ok: false, code: 1, error: err.message };
+  }
+  const config = await createConfig("script", "benchmark");
+  runtime.proc.env.ANTHROPIC_API_KEY = await config.anthropicToken();
+  // The benchmark agent runs via createBenchmarkRunner, not the supervise
+  // command, so the active-tracker env must land here before the runner
+  // spawns the subprocess that inherits process.env.
+  runtime.proc.env.LIBHARNESS_WORK_TRACKER = opts.workTracker;
+
+  // The Claude Agent SDK spawns a `claude` subprocess that inherits
+  // process.env. NODE_EXTRA_CA_CERTS causes undici (the HTTP client
+  // inside that subprocess) to fail with UND_ERR_INVALID_ARG on
+  // Node 22+, aborting every API call after 10 retries. Strip it
+  // before the SDK loads so the subprocess gets a clean environment.
+  delete runtime.proc.env.NODE_EXTRA_CA_CERTS;
+
+  const { query } = await import("@anthropic-ai/claude-agent-sdk");
+  const runner = createBenchmarkRunner({ ...opts, query, runtime });
+
+  let anyFail = false;
+  let count = 0;
+  for await (const record of runner.run()) {
+    count++;
+    runtime.proc.stdout.write(JSON.stringify(record) + "\n");
+    if (record.verdict !== "pass") anyFail = true;
+  }
+  // A run that emits zero records did nothing (no tasks discovered, or the
+  // agent never produced output). That is a failure, not a silent success —
+  // surface it loudly so CI does not go green on an empty benchmark.
+  if (count === 0) {
+    return {
+      ok: false,
+      code: 1,
+      error:
+        "benchmark produced no result records — no task ran to completion; check the family's tasks/, apm install, and agent availability (ANTHROPIC_API_KEY / claude CLI / IS_SANDBOX)",
+    };
+  }
+  return anyFail ? { ok: false, code: 1, error: "" } : { ok: true };
+}
+
+/**
+ * Parse and validate benchmark run options. Exported so tests can verify
+ * defaults, including the resolved work tracker.
+ * @param {Record<string, string|undefined>} values - Parsed option values
+ * @param {Record<string, string|undefined>} [env] - Process environment, read
+ *   for the `LIBHARNESS_WORK_TRACKER` fallback when `--work-tracker` is absent.
+ * @returns {object}
+ */
+export function parseRunOptions(values, env = {}) {
+  const family = values.family;
+  if (!family) throw new Error("--family is required");
+  const output = values.output ?? "benchmark-runs";
+  const runs = Number.parseInt(values.runs ?? "5", 10);
+  if (!Number.isFinite(runs) || runs < 1)
+    throw new Error("--runs must be a positive integer");
+  return {
+    family,
+    runs,
+    task: values.task ?? null,
+    skillsFrom: values["skills-from"] ?? null,
+    output: resolve(output),
+    agentModel: values["agent-model"] || BENCHMARK_AGENT_MODEL,
+    supervisorModel: values["lead-model"] || LEAD_MODEL,
+    judgeModel: values["judge-model"] || LEAD_MODEL,
+    workTracker: resolveWorkTracker(values, env),
+    profiles: {
+      agent: values["agent-profile"] ?? null,
+      judge: values["judge-profile"] ?? null,
+    },
+    maxTurns: parseMaxTurns(values["max-turns"]),
+    allowedTools: values["allowed-tools"]
+      ? values["allowed-tools"]
+          .split(",")
+          .map((s) => s.trim())
+          .filter(Boolean)
+      : undefined,
+  };
+}
+
+function parseMaxTurns(raw) {
+  if (raw === undefined) return undefined;
+  if (raw === "0") return 0;
+  return Number.parseInt(raw, 10);
+}

package/src/commands/by-discussion.js

@@ -0,0 +1,94 @@
+import { join } from "node:path";
+
+const FIRST_LINE_CAP = 64 * 1024;
+
+/**
+ * Read the first newline-terminated line of a file, bounded to the first
+ * {@link FIRST_LINE_CAP} bytes. Trace `.ndjson` files can be many MB; the
+ * Step 2.6 meta header is always small, so a bounded positional read avoids
+ * loading whole files into memory just to inspect the header. The positional
+ * `openSync`/`readSync`/`closeSync` trio is read off the injected
+ * `runtime.fsSync` surface.
+ *
+ * @param {object} fsSync - Sync filesystem surface (`runtime.fsSync`).
+ * @param {string} path
+ * @returns {string}
+ */
+function readFirstLine(fsSync, path) {
+  const fd = fsSync.openSync(path, "r");
+  try {
+    const buf = Buffer.alloc(FIRST_LINE_CAP);
+    const bytes = fsSync.readSync(fd, buf, 0, buf.length, 0);
+    const text = buf.toString("utf8", 0, bytes);
+    const nl = text.indexOf("\n");
+    return nl === -1 ? text : text.slice(0, nl);
+  } finally {
+    fsSync.closeSync(fd);
+  }
+}
+
+/**
+ * Scan a directory for `.ndjson` files whose meta header carries the
+ * given discussion_id. The Step 2.6 first-line guarantee makes the
+ * lookup cheap: we read only the first line per file. Files without a
+ * meta header (e.g. legacy supervise/facilitate traces) are skipped
+ * silently — not erroneous.
+ *
+ * @param {string} dir
+ * @param {string} discussionId
+ * @param {object} fsSync - Sync filesystem surface (`runtime.fsSync`).
+ * @returns {Array<{path: string, mtimeMs: number}>}
+ */
+export function findTracesByDiscussion(dir, discussionId, fsSync) {
+  const matches = [];
+  let entries;
+  try {
+    entries = fsSync.readdirSync(dir);
+  } catch {
+    return [];
+  }
+  for (const entry of entries) {
+    if (!entry.endsWith(".ndjson")) continue;
+    const path = join(dir, entry);
+    let firstLine;
+    try {
+      firstLine = readFirstLine(fsSync, path);
+    } catch {
+      continue;
+    }
+    let parsed;
+    try {
+      parsed = JSON.parse(firstLine);
+    } catch {
+      continue;
+    }
+    const event = parsed.event ?? parsed;
+    if (event?.type !== "meta") continue;
+    if (event.discussion_id !== discussionId) continue;
+    matches.push({ path, mtimeMs: fsSync.statSync(path).mtimeMs });
+  }
+  matches.sort((a, b) => a.mtimeMs - b.mtimeMs);
+  return matches;
+}
+
+/**
+ * `fit-trace by-discussion <discussion-id> [trace-dir]` — list trace
+ * files whose meta header carries the given discussion_id, one per
+ * line, ordered by first-event timestamp (file mtime ascending). The
+ * result is usable with `xargs cat` for a chronological merge.
+ *
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
+ */
+export async function runByDiscussionCommand(ctx) {
+  const runtime = ctx.deps.runtime;
+  const discussionId = ctx.args["discussion-id"];
+  if (!discussionId)
+    return { ok: false, code: 1, error: "<discussion-id> is required" };
+  const dir = ctx.args["trace-dir"] ?? ctx.options["trace-dir"] ?? "traces";
+  const matches = findTracesByDiscussion(dir, discussionId, runtime.fsSync);
+  for (const { path } of matches) {
+    runtime.proc.stdout.write(`${path}\n`);
+  }
+  return { ok: true };
+}

package/src/commands/callback.js

@@ -0,0 +1,119 @@
+import { sumTraceCost } from "../cost.js";
+
+/**
+ * Scan an NDJSON trace and return the last orchestrator summary event,
+ * the first `meta` event's `discussion_id`, and any structured replies
+ * collected by the discusser. Skips malformed lines.
+ *
+ * The runner is verdict-agnostic — verbatim passthrough of whatever the
+ * trace carries ("success"/"failure" from supervise/facilitate; canonical
+ * "adjourned"/"recessed"/"failed" from discuss). The bridge layer maps to
+ * its channel semantics.
+ *
+ * @param {string} content - Raw NDJSON trace content.
+ * @returns {{verdict: string, summary: string, replies: object[], trigger?: object, discussionId?: string} | null}
+ */
+// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: NDJSON scan with malformed-line tolerance + meta/summary dual extraction
+function readTraceSummary(content) {
+  let summary = null;
+  let metaDiscussionId = null;
+  for (const line of content.split("\n")) {
+    if (!line.trim()) continue;
+    let record;
+    try {
+      record = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (record.source !== "orchestrator") continue;
+    if (record.event?.type === "meta" && !metaDiscussionId) {
+      metaDiscussionId = record.event.discussion_id ?? null;
+    }
+    if (record.event?.type === "summary") {
+      summary = {
+        verdict: record.event.verdict ?? "failed",
+        summary: record.event.summary ?? "",
+        replies: Array.isArray(record.event.replies)
+          ? record.event.replies
+          : [],
+        ...(record.event.trigger && { trigger: record.event.trigger }),
+        ...(record.event.discussion_id && {
+          discussionId: record.event.discussion_id,
+        }),
+        ...(typeof record.event.lastActedSeq === "number" && {
+          lastActedSeq: record.event.lastActedSeq,
+        }),
+      };
+    }
+  }
+  if (summary && !summary.discussionId && metaDiscussionId) {
+    summary.discussionId = metaDiscussionId;
+  }
+  return summary;
+}
+
+/**
+ * Callback command — read an NDJSON trace, extract the terminal
+ * orchestrator summary, and POST a canonical callback body to the
+ * configured URL. Used by `kata-dispatch.yml` to deliver the lead's
+ * conclusion to the bridge that dispatched the run.
+ *
+ * Wire shape (single shape across modes):
+ *
+ * ```
+ * {
+ *   correlation_id, verdict, summary, run_url,
+ *   discussion_id?, replies: [], trigger?
+ * }
+ * ```
+ *
+ * @param {import("@forwardimpact/libcli").InvocationContext} ctx
+ * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
+ */
+export async function runCallbackCommand(ctx) {
+  const values = ctx.options;
+  const runtime = ctx.deps.runtime;
+  const traceFile = values["trace-file"];
+  const callbackUrl = values["callback-url"];
+  const correlationId = values["correlation-id"];
+  const runUrl = values["run-url"] ?? "";
+  const discussionIdOverride = values["discussion-id"] ?? null;
+
+  if (!traceFile)
+    return { ok: false, code: 1, error: "--trace-file is required" };
+  if (!callbackUrl)
+    return { ok: false, code: 1, error: "--callback-url is required" };
+
+  const content = runtime.fsSync.readFileSync(traceFile, "utf8");
+  const found = readTraceSummary(content) ?? {
+    verdict: "failed",
+    summary: "Run ended without producing a summary.",
+    replies: [],
+  };
+  // Total spend across every participant in the trace — the bridge surfaces
+  // it alongside the verdict so a dispatched run reports what it cost.
+  const { totalCostUsd } = sumTraceCost(content.split("\n"));
+
+  const discussionId = found.discussionId ?? discussionIdOverride ?? null;
+  const payload = {
+    correlation_id: correlationId,
+    kind: "terminal",
+    verdict: found.verdict,
+    summary: found.summary,
+    run_url: runUrl,
+    cost_usd: totalCostUsd,
+    replies: found.replies,
+    last_acted_seq: found.lastActedSeq ?? -1,
+    ...(discussionId && { discussion_id: discussionId }),
+    ...(found.trigger && { trigger: found.trigger }),
+  };
+  const res = await fetch(callbackUrl, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(payload),
+  });
+  if (!res.ok) {
+    return { ok: false, code: 1, error: `Callback POST failed: ${res.status}` };
+  }
+  return { ok: true };
+}