@forwardimpact/libharness 0.1.22 → 1.0.0

package/src/benchmark/report.js

@@ -0,0 +1,522 @@
+/**
+ * ReportAggregator — read a run-output directory's `results.jsonl`, group
+ * records by `taskId`, and compute pass@k via the OpenAI HumanEval
+ * unbiased estimator: `1 - C(n-c, k) / C(n, k)`.
+ *
+ * When `includeRuns` is true, each task carries per-run detail (invariant
+ * checks, judge commentary, cost, duration) and the text renderer produces
+ * a full markdown report instead of just the pass@k table.
+ *
+ * Records that fail schema validation are skipped with a stderr warning
+ * (counted under `totals.skipped`) so a corrupt line cannot abort the
+ * whole report.
+ */
+
+import { join } from "node:path";
+
+import { validateResultRecord } from "./result.js";
+
+/**
+ * @typedef {object} RunDetail
+ * @property {number} runIndex
+ * @property {"pass"|"fail"} verdict
+ * @property {{verdict: string, details: unknown[], exitCode: number}} [invariants]
+ * @property {{verdict: string, summary: string}} [judgeVerdict]
+ * @property {number} costUsd
+ * @property {number} turns
+ * @property {number} durationMs
+ * @property {{message: string, aborted: boolean}} [agentError]
+ * @property {{phase: string, message: string, exitCode: number}} [preflightError]
+ */
+
+/**
+ * @typedef {object} TaskReport
+ * @property {string} taskId
+ * @property {number} n - Total runs.
+ * @property {number} c - Passing runs.
+ * @property {Record<string|number, number|null>} passAtK
+ * @property {RunDetail[]} [runs] - Per-run detail (only when includeRuns).
+ */
+
+/**
+ * @param {{inputDir: string, kValues: number[], includeRuns?: boolean, runtime: import("@forwardimpact/libutil/runtime").Runtime}} opts
+ * @returns {Promise<{tasks: TaskReport[], totals: object}>}
+ */
+export async function aggregate({
+  inputDir,
+  kValues,
+  includeRuns = false,
+  runtime,
+}) {
+  if (!runtime) throw new Error("runtime is required");
+  const records = await loadRecords(inputDir, runtime);
+  const grouped = groupByTask(records.records);
+  const tasks = [];
+  let totalRuns = 0;
+  let totalCost = 0;
+  const allDurations = [];
+  const allTurns = [];
+  let firstRecord = null;
+
+  for (const [taskId, group] of grouped) {
+    const n = group.length;
+    const c = group.filter((r) => r.verdict === "pass").length;
+    totalRuns += n;
+    const passAtK = {};
+    for (const k of kValues) passAtK[k] = passAtKValue(n, c, k);
+
+    const task = { taskId, n, c, passAtK };
+
+    if (includeRuns) {
+      if (!firstRecord) firstRecord = group[0];
+      const accumulators = { allDurations, allTurns };
+      task.runs = group
+        .map((r) => {
+          totalCost += r.costUsd ?? 0;
+          return buildRunDetail(r, accumulators);
+        })
+        .sort((a, b) => a.runIndex - b.runIndex);
+    }
+
+    tasks.push(task);
+  }
+  tasks.sort((a, b) =>
+    a.taskId < b.taskId ? -1 : a.taskId > b.taskId ? 1 : 0,
+  );
+
+  const totals = {
+    tasks: tasks.length,
+    runs: totalRuns,
+    skipped: records.skipped,
+  };
+
+  if (includeRuns) {
+    totals.costUsd = totalCost;
+    totals.medianDurationMs = median(allDurations);
+    totals.medianTurns = median(allTurns);
+    totals.model = firstRecord?.model ?? "";
+    totals.skillSetHash = firstRecord?.skillSetHash ?? "";
+    totals.familyRevision = firstRecord?.familyRevision ?? "";
+  }
+
+  return { tasks, totals };
+}
+
+/**
+ * Build a normalized per-run detail object and accumulate duration/turn
+ * samples for median calculation. Extracted from `aggregate` to keep its
+ * cognitive complexity below the lint ceiling.
+ * @param {object} r - Raw record.
+ * @param {{allDurations: number[], allTurns: number[]}} acc
+ * @returns {RunDetail}
+ */
+function buildRunDetail(r, acc) {
+  if (r.durationMs != null) acc.allDurations.push(r.durationMs);
+  if (r.turns != null) acc.allTurns.push(r.turns);
+  return {
+    runIndex: r.runIndex,
+    verdict: r.verdict,
+    ...(r.invariants && { invariants: r.invariants }),
+    ...(r.judgeVerdict && { judgeVerdict: r.judgeVerdict }),
+    costUsd: r.costUsd ?? 0,
+    turns: r.turns ?? 0,
+    durationMs: r.durationMs ?? 0,
+    ...(r.agentError && { agentError: r.agentError }),
+    ...(r.preflightError && { preflightError: r.preflightError }),
+  };
+}
+
+/**
+ * Render an aggregate report as markdown. When the report contains per-run
+ * detail (from `includeRuns: true`), renders a full report with summary,
+ * pass@k table, and per-task detail sections. Otherwise falls back to the
+ * compact pass@k table.
+ * @param {Awaited<ReturnType<typeof aggregate>>} report
+ * @param {number[]} kValues
+ * @returns {string}
+ */
+export function renderTextReport(report, kValues) {
+  if (report.tasks[0]?.runs) {
+    return renderFullReport(report, kValues);
+  }
+  return renderCompactReport(report, kValues);
+}
+
+// ---------------------------------------------------------------------------
+// Compact report (legacy path)
+// ---------------------------------------------------------------------------
+
+function renderCompactReport(report, kValues) {
+  const lines = [
+    renderPassAtKTable(report, kValues),
+    "",
+    renderTotalsLine(report),
+  ];
+  return lines.join("\n");
+}
+
+// ---------------------------------------------------------------------------
+// Full report
+// ---------------------------------------------------------------------------
+
+function renderFullReport(report, kValues) {
+  const sections = [
+    renderSummary(report),
+    "## Pass@k",
+    "",
+    renderPassAtKTable(report, kValues),
+    "",
+    renderTotalsLine(report),
+    "",
+    "## Task Details",
+  ];
+
+  for (const task of report.tasks) {
+    sections.push("");
+    sections.push(renderTaskDetail(task));
+  }
+
+  return sections.join("\n");
+}
+
+function renderSummary(report) {
+  const { totals } = report;
+  const passing = report.tasks.filter((t) => t.c > 0 && t.c === t.n).length;
+  const icon = statusIcon(passing === totals.tasks);
+  const lines = [
+    "# Benchmark Report",
+    "",
+    `${icon} **${passing}/${totals.tasks} tasks passing** | ${totals.runs} runs${totals.skipped ? ` | ${totals.skipped} skipped` : ""}`,
+  ];
+
+  const headers = [];
+  const values = [];
+  if (totals.costUsd != null) {
+    headers.push("Cost");
+    values.push(formatCost(totals.costUsd));
+  }
+  if (totals.medianDurationMs != null) {
+    headers.push("Median Duration");
+    values.push(formatDuration(totals.medianDurationMs));
+  }
+  if (totals.medianTurns != null) {
+    headers.push("Median Turns");
+    values.push(String(totals.medianTurns));
+  }
+  if (headers.length) {
+    lines.push("");
+    lines.push(`| ${headers.join(" | ")} |`);
+    lines.push(`| ${headers.map(() => "---").join(" | ")} |`);
+    lines.push(`| ${values.join(" | ")} |`);
+  }
+
+  const meta = [];
+  if (totals.model) {
+    meta.push(`Agent: \`${totals.model.agent}\``);
+    meta.push(`Supervisor: \`${totals.model.supervisor}\``);
+    meta.push(`Judge: \`${totals.model.judge}\``);
+  }
+  if (totals.skillSetHash) meta.push(`Skill set: \`${totals.skillSetHash}\``);
+  if (totals.familyRevision) meta.push(`Family: \`${totals.familyRevision}\``);
+  if (meta.length) {
+    lines.push("");
+    lines.push(meta.join(" | "));
+  }
+
+  lines.push("");
+  return lines.join("\n");
+}
+
+// ---------------------------------------------------------------------------
+// Pass@k table (shared between compact and full)
+// ---------------------------------------------------------------------------
+
+function renderPassAtKTable(report, kValues) {
+  const header = ["taskId", "n", "c", ...kValues.map((k) => `pass@${k}`)];
+  const rows = [header, header.map(() => "---")];
+  for (const t of report.tasks) {
+    rows.push([
+      t.taskId,
+      String(t.n),
+      String(t.c),
+      ...kValues.map((k) => formatPassAt(t.passAtK[k])),
+    ]);
+  }
+  return rows.map((r) => `| ${r.join(" | ")} |`).join("\n");
+}
+
+function renderTotalsLine(report) {
+  return `Totals — tasks: ${report.totals.tasks}, runs: ${report.totals.runs}, skipped: ${report.totals.skipped}`;
+}
+
+// ---------------------------------------------------------------------------
+// Per-task detail
+// ---------------------------------------------------------------------------
+
+function renderTaskDetail(task) {
+  const runs = task.runs ?? [];
+  const icon = statusIcon(task.c === task.n);
+  const singleRun = runs.length === 1;
+
+  const lines = [
+    `### ${task.taskId}`,
+    "",
+    `${icon} **${task.c}/${task.n} runs passed**`,
+  ];
+
+  lines.push("", renderRunsTable(runs));
+
+  const checks = renderInvariantChecks(runs, singleRun);
+  if (checks) lines.push("", checks);
+
+  const commentary = renderJudgeCommentary(runs, singleRun);
+  if (commentary) lines.push("", commentary);
+
+  const errors = renderErrors(runs);
+  if (errors) lines.push("", errors);
+
+  return lines.join("\n");
+}
+
+function renderRunsTable(runs) {
+  const header = [
+    "Run",
+    "Verdict",
+    "Invariants",
+    "Judge",
+    "Cost",
+    "Turns",
+    "Duration",
+  ];
+  const rows = [header, header.map(() => "---")];
+  for (const r of runs) {
+    const invariantsCell = r.preflightError
+      ? "preflight error"
+      : r.invariants
+        ? statusIcon(r.invariants.verdict === "pass")
+        : "—";
+    const judgeCell = r.preflightError
+      ? "—"
+      : r.judgeVerdict
+        ? statusIcon(r.judgeVerdict.verdict === "pass")
+        : "—";
+    rows.push([
+      String(r.runIndex),
+      statusIcon(r.verdict === "pass"),
+      invariantsCell,
+      judgeCell,
+      formatCost(r.costUsd),
+      String(r.turns),
+      formatDuration(r.durationMs),
+    ]);
+  }
+  return rows.map((r) => `| ${r.join(" | ")} |`).join("\n");
+}
+
+function renderInvariantChecks(runs, singleRun) {
+  const rows = collectInvariantRows(runs);
+  if (!rows.length) return null;
+
+  const header = singleRun
+    ? ["Check", "Result", "Message"]
+    : ["Run", "Check", "Result", "Message"];
+  const lines = [
+    "#### Invariant Checks",
+    "",
+    `| ${header.join(" | ")} |`,
+    `| ${header.map(() => "---").join(" | ")} |`,
+  ];
+  for (const row of rows) {
+    const cells = singleRun
+      ? [row.check, row.result, row.message]
+      : [String(row.run), row.check, row.result, row.message];
+    lines.push(`| ${cells.join(" | ")} |`);
+  }
+  return lines.join("\n");
+}
+
+function collectInvariantRows(runs) {
+  const rows = [];
+  for (const r of runs) {
+    if (!r.invariants?.details?.length) continue;
+    for (const d of r.invariants.details) {
+      rows.push({
+        run: r.runIndex,
+        check: escapeCell(String(d.test ?? "(unnamed)")),
+        result: statusIcon(d.pass),
+        message: escapeCell(String(d.message ?? "")),
+      });
+    }
+  }
+  return rows;
+}
+
+function renderJudgeCommentary(runs, singleRun) {
+  const entries = runs.filter((r) => r.judgeVerdict?.summary);
+  if (!entries.length) return null;
+
+  const lines = ["#### Judge Commentary", ""];
+  for (let i = 0; i < entries.length; i++) {
+    const r = entries[i];
+    const summary = r.judgeVerdict.summary.replace(/\n/g, "\n> ");
+    if (singleRun) {
+      lines.push(`> ${summary}`);
+    } else {
+      lines.push(`> **Run ${r.runIndex}:** ${summary}`);
+    }
+    if (i < entries.length - 1) lines.push(">");
+  }
+  return lines.join("\n");
+}
+
+function renderErrors(runs) {
+  const lines = [];
+  for (const r of runs) {
+    if (r.agentError) {
+      lines.push(
+        `- **Run ${r.runIndex}:** Agent error — "${escapeCell(r.agentError.message)}" (aborted: ${r.agentError.aborted})`,
+      );
+    }
+    if (r.preflightError) {
+      lines.push(
+        `- **Run ${r.runIndex}:** Preflight error — "${escapeCell(r.preflightError.message)}" (exit ${r.preflightError.exitCode})`,
+      );
+    }
+  }
+  if (!lines.length) return null;
+  return ["#### Errors", "", ...lines].join("\n");
+}
+
+// ---------------------------------------------------------------------------
+// Formatting helpers
+// ---------------------------------------------------------------------------
+
+function statusIcon(pass) {
+  return pass ? "✅" : "❌";
+}
+
+function formatPassAt(v) {
+  if (v == null) return "—";
+  if (typeof v === "object" && "error" in v) return v.error;
+  return Number(v).toFixed(4);
+}
+
+function formatDuration(ms) {
+  if (ms == null || ms === 0) return "0s";
+  const totalSeconds = Math.round(ms / 1000);
+  if (totalSeconds < 60) return `${totalSeconds}s`;
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = totalSeconds % 60;
+  return seconds > 0 ? `${minutes}m ${seconds}s` : `${minutes}m`;
+}
+
+function formatCost(usd) {
+  if (usd == null) return "$0.00";
+  return `$${usd.toFixed(2)}`;
+}
+
+function escapeCell(str) {
+  return str.replace(/\|/g, "\\|");
+}
+
+function median(arr) {
+  if (!arr.length) return 0;
+  const sorted = [...arr].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  if (sorted.length % 2 === 0) {
+    return Math.round((sorted[mid - 1] + sorted[mid]) / 2);
+  }
+  return sorted[mid];
+}
+
+// ---------------------------------------------------------------------------
+// Record loading
+// ---------------------------------------------------------------------------
+
+async function loadRecords(inputDir, runtime) {
+  const path = join(inputDir, "results.jsonl");
+  let content;
+  try {
+    content = await runtime.fs.readFile(path, "utf8");
+  } catch (e) {
+    // Re-throw with the stack collapsed to the message line so the CLI's
+    // error rendering stays free of node-internal async `readFile` frames
+    // (matching the pre-1370 stream-error shape the golden captured).
+    const err = new Error(e.message);
+    if (e.code) err.code = e.code;
+    err.stack = `Error: ${e.message}`;
+    throw err;
+  }
+  const records = [];
+  let skipped = 0;
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    let record;
+    try {
+      record = JSON.parse(trimmed);
+    } catch (e) {
+      runtime.proc.stderr.write(
+        `benchmark report: skipped malformed JSON line — ${e.message}\n`,
+      );
+      skipped++;
+      continue;
+    }
+    try {
+      validateResultRecord(record);
+    } catch (e) {
+      runtime.proc.stderr.write(
+        `benchmark report: skipped record failing schema — ${describeError(e)}\n`,
+      );
+      skipped++;
+      continue;
+    }
+    records.push(record);
+  }
+  return { records, skipped };
+}
+
+function describeError(e) {
+  if (e && Array.isArray(e.issues)) {
+    return e.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ");
+  }
+  return e.message ?? String(e);
+}
+
+function groupByTask(records) {
+  const out = new Map();
+  for (const r of records) {
+    if (!out.has(r.taskId)) out.set(r.taskId, []);
+    out.get(r.taskId).push(r);
+  }
+  return out;
+}
+
+/**
+ * pass@k = 1 - C(n - c, k) / C(n, k). Compute with BigInt to avoid
+ * floating-point loss on large n.
+ * @param {number} n
+ * @param {number} c
+ * @param {number} k
+ * @returns {number | {error: string}}
+ */
+function passAtKValue(n, c, k) {
+  if (k > n) return { error: "k > n" };
+  if (n - c < k) return 1;
+  const total = binomial(BigInt(n), BigInt(k));
+  const fail = binomial(BigInt(n - c), BigInt(k));
+  const passing = total - fail;
+  return Number(passing) / Number(total);
+}
+
+function binomial(n, k) {
+  if (k < 0n || k > n) return 0n;
+  if (k === 0n || k === n) return 1n;
+  let kk = k;
+  if (kk > n - kk) kk = n - kk;
+  let result = 1n;
+  for (let i = 0n; i < kk; i++) {
+    result = (result * (n - i)) / (i + 1n);
+  }
+  return result;
+}

package/src/benchmark/result.js

@@ -0,0 +1,127 @@
+/**
+ * Result-record schemas and runtime validators.
+ *
+ * Two schemas live here:
+ *   - RESULT_RECORD_SCHEMA — one record per (task, runIndex) from a full
+ *     benchmark run. Has a happy branch (invariants + judge present) and a
+ *     pre-flight-failure branch (invariants/judgeVerdict/submission absent).
+ *   - INVARIANTS_RECORD_SCHEMA — narrower output of `benchmark-invariants`:
+ *     ad-hoc grading without a full lifecycle.
+ *
+ * Validation is throw-on-mismatch so the runner can wrap every JSONL append
+ * in a guard and reject schema drift at write time.
+ */
+
+import { z } from "zod";
+
+const VERDICT_ENUM = z.enum(["pass", "fail"]);
+
+const INVARIANTS_SHAPE = z.object({
+  verdict: VERDICT_ENUM,
+  details: z.array(z.unknown()),
+  exitCode: z.number().int(),
+  stderr: z.string().optional(),
+});
+
+const JUDGE_VERDICT_SHAPE = z.object({
+  verdict: VERDICT_ENUM,
+  summary: z.string(),
+});
+
+/**
+ * Per-participant cost attribution. `costUsd` is the sum of these; the
+ * breakdown lets reports show where the spend went. The judge runs as its
+ * own SDK session, so its cost is tracked separately from agent/supervisor.
+ */
+const COST_BREAKDOWN_SHAPE = z.object({
+  agent: z.number(),
+  supervisor: z.number(),
+  judge: z.number(),
+});
+
+const PROFILES_SHAPE = z.object({
+  agent: z.union([z.string(), z.null()]),
+  supervisor: z.union([z.string(), z.null()]),
+  judge: z.union([z.string(), z.null()]),
+});
+
+const PREFLIGHT_ERROR_SHAPE = z.object({
+  phase: z.string(),
+  message: z.string(),
+  exitCode: z.number().int(),
+});
+
+const COMMON_FIELDS = {
+  taskId: z.string().min(1),
+  runIndex: z.number().int().min(0),
+  verdict: VERDICT_ENUM,
+  costUsd: z.number(),
+  costBreakdown: COST_BREAKDOWN_SHAPE.optional(),
+  turns: z.number().int().min(0),
+  profiles: PROFILES_SHAPE,
+  model: z.object({
+    agent: z.string(),
+    supervisor: z.string().optional(),
+    judge: z.string().optional(),
+  }),
+  skillSetHash: z.string(),
+  familyRevision: z.string(),
+  durationMs: z.number().int().min(0),
+};
+
+const AGENT_ERROR_SHAPE = z.object({
+  message: z.string(),
+  aborted: z.boolean(),
+});
+
+const HAPPY_RECORD = z.object({
+  ...COMMON_FIELDS,
+  invariants: INVARIANTS_SHAPE,
+  submission: z.string(),
+  judgeVerdict: JUDGE_VERDICT_SHAPE.optional(),
+  agentTracePath: z.string(),
+  supervisorTracePath: z.string(),
+  judgeTracePath: z.string(),
+  agentError: AGENT_ERROR_SHAPE.optional(),
+  preflightError: z.undefined().optional(),
+});
+
+const PREFLIGHT_RECORD = z.object({
+  ...COMMON_FIELDS,
+  costUsd: z.literal(0),
+  preflightError: PREFLIGHT_ERROR_SHAPE,
+  // Trace paths are populated even on preflight failure (the runner allocates
+  // them in WorkdirManager.start) so the record is uniform across branches
+  // and downstream consumers can reference them without conditional fields.
+  agentTracePath: z.string(),
+  supervisorTracePath: z.string(),
+  judgeTracePath: z.string(),
+  invariants: z.undefined().optional(),
+  submission: z.undefined().optional(),
+  judgeVerdict: z.undefined().optional(),
+  agentError: z.undefined().optional(),
+});
+
+export const RESULT_RECORD_SCHEMA = z.union([HAPPY_RECORD, PREFLIGHT_RECORD]);
+
+export const INVARIANTS_RECORD_SCHEMA = z.object({
+  taskId: z.string().min(1),
+  invariants: INVARIANTS_SHAPE,
+  exitCode: z.number().int(),
+});
+
+/**
+ * Throw on schema mismatch.
+ * @param {object} record
+ */
+export function validateResultRecord(record) {
+  RESULT_RECORD_SCHEMA.parse(record);
+}
+
+/**
+ * Throw on schema mismatch.
+ * @param {object} record
+ */
+export function validateInvariantsRecord(record) {
+  INVARIANTS_RECORD_SCHEMA.parse(record);
+}