@forwardimpact/libharness 1.4.0 → 2.0.0

package/src/benchmark/report.js

@@ -15,12 +15,16 @@
 import { join } from "node:path";
 
 import { validateResultRecord } from "./result.js";
+import { mergeRows } from "./grade.js";
 
 /**
  * @typedef {object} RunDetail
  * @property {number} runIndex
  * @property {"pass"|"fail"} verdict
- * @property {{verdict: string, details: unknown[], exitCode: number}} [invariants]
+ * @property {{details: unknown[], exitCode: number}} [invariants]
+ * @property {{verdict: string, gatesPass: boolean, score?: number, malformed?: number}} [grade]
+ * @property {{details: unknown[], error?: string}} [hiddenTests]
+ * @property {number} [score] - Effective judge-zeroed score (scored tasks).
  * @property {{verdict: string, summary: string}} [judgeVerdict]
  * @property {number} costUsd
  * @property {number} turns
@@ -66,6 +70,7 @@ export async function aggregate({
     for (const k of kValues) passAtK[k] = passAtKValue(n, c, k);
 
     const task = { taskId, n, c, passAtK };
+    applyScoreFields(task, group, kValues);
 
     if (includeRuns) {
       if (!firstRecord) firstRecord = group[0];
@@ -102,6 +107,25 @@ export async function aggregate({
   return { tasks, totals };
 }
 
+/**
+ * Attach `meanScore` and `scoreAtK` to a scored task group. A group is
+ * scored iff any record carries an effective score; a score-less record in
+ * a scored group (a preflight failure never reached grading, or a binary
+ * run) contributes its verdict as the degenerate score — skipping it would
+ * inflate the mean exactly when the agent fails hardest. Binary groups gain
+ * neither field.
+ * @param {object} task - Mutated.
+ * @param {object[]} group
+ * @param {number[]} kValues
+ */
+function applyScoreFields(task, group, kValues) {
+  if (!group.some((r) => r.score !== undefined)) return;
+  const scores = group.map((r) => r.score ?? (r.verdict === "pass" ? 1 : 0));
+  task.meanScore = scores.reduce((a, b) => a + b, 0) / scores.length;
+  task.scoreAtK = {};
+  for (const k of kValues) task.scoreAtK[k] = scoreAtKValue(scores, k);
+}
+
 /**
  * Build a normalized per-run detail object and accumulate duration/turn
  * samples for median calculation. Extracted from `aggregate` to keep its
@@ -117,6 +141,9 @@ function buildRunDetail(r, acc) {
     runIndex: r.runIndex,
     verdict: r.verdict,
     ...(r.invariants && { invariants: r.invariants }),
+    ...(r.grade && { grade: r.grade }),
+    ...(r.hiddenTests && { hiddenTests: r.hiddenTests }),
+    ...(r.score !== undefined && { score: r.score }),
     ...(r.judgeVerdict && { judgeVerdict: r.judgeVerdict }),
     costUsd: r.costUsd ?? 0,
     turns: r.turns ?? 0,
@@ -156,7 +183,7 @@ function renderCompactReport(report, kValues) {
   const lines = [
     `${icon} **${passing}/${totals.tasks} tasks passing** | ${totals.runs} runs${totals.skipped ? ` | ${totals.skipped} skipped` : ""}`,
     "",
-    renderPassAtKTable(report, kValues),
+    renderPassAtKTable(report, kValues, hasScoredTask(report)),
     "",
     renderTotalsLine(report),
   ];
@@ -167,12 +194,18 @@ function renderCompactReport(report, kValues) {
 // Full report
 // ---------------------------------------------------------------------------
 
+/** Score columns render only when the report has at least one scored task. */
+function hasScoredTask(report) {
+  return report.tasks.some((t) => t.meanScore !== undefined);
+}
+
 function renderFullReport(report, kValues) {
+  const scored = hasScoredTask(report);
   const sections = [
     renderSummary(report),
     "## Pass@k",
     "",
-    renderPassAtKTable(report, kValues),
+    renderPassAtKTable(report, kValues, scored),
     "",
     renderTotalsLine(report),
     "",
@@ -181,7 +214,7 @@ function renderFullReport(report, kValues) {
 
   for (const task of report.tasks) {
     sections.push("");
-    sections.push(renderTaskDetail(task));
+    sections.push(renderTaskDetail(task, scored));
   }
 
   return sections.join("\n");
@@ -239,16 +272,27 @@ function renderSummary(report) {
 // Pass@k table (shared between compact and full)
 // ---------------------------------------------------------------------------
 
-function renderPassAtKTable(report, kValues) {
+function renderPassAtKTable(report, kValues, scored) {
   const header = ["taskId", "n", "c", ...kValues.map((k) => `pass@${k}`)];
+  if (scored) {
+    header.push("score", ...kValues.map((k) => `score@${k}`));
+  }
   const rows = [header, header.map(() => "---")];
   for (const t of report.tasks) {
-    rows.push([
+    const row = [
       t.taskId,
       String(t.n),
       String(t.c),
       ...kValues.map((k) => formatPassAt(t.passAtK[k])),
-    ]);
+    ];
+    if (scored) {
+      // Binary tasks render "—" in every score column.
+      row.push(
+        formatPassAt(t.meanScore ?? null),
+        ...kValues.map((k) => formatPassAt(t.scoreAtK?.[k] ?? null)),
+      );
+    }
+    rows.push(row);
   }
   return rows.map((r) => `| ${r.join(" | ")} |`).join("\n");
 }
@@ -261,7 +305,7 @@ function renderTotalsLine(report) {
 // Per-task detail
 // ---------------------------------------------------------------------------
 
-function renderTaskDetail(task) {
+function renderTaskDetail(task, scored) {
   const runs = task.runs ?? [];
   const icon = statusIcon(task.c === task.n);
   const singleRun = runs.length === 1;
@@ -272,9 +316,9 @@ function renderTaskDetail(task) {
     `${icon} **${task.c}/${task.n} runs passed**`,
   ];
 
-  lines.push("", renderRunsTable(runs));
+  lines.push("", renderRunsTable(runs, scored));
 
-  const checks = renderInvariantChecks(runs, singleRun);
+  const checks = renderChecks(runs, singleRun);
   if (checks) lines.push("", checks);
 
   const commentary = renderJudgeCommentary(runs, singleRun);
@@ -286,33 +330,32 @@ function renderTaskDetail(task) {
   return lines.join("\n");
 }
 
-function renderRunsTable(runs) {
+function renderRunsTable(runs, scored) {
   const header = [
     "Run",
     "Verdict",
-    "Invariants",
+    "Checks",
     "Judge",
+    ...(scored ? ["Score"] : []),
     "Cost",
     "Turns",
     "Duration",
   ];
   const rows = [header, header.map(() => "---")];
   for (const r of runs) {
-    const invariantsCell = r.preflightError
-      ? "preflight error"
-      : r.invariants
-        ? statusIcon(r.invariants.verdict === "pass")
-        : "—";
-    const judgeCell = r.preflightError
-      ? "—"
-      : r.judgeVerdict
-        ? statusIcon(r.judgeVerdict.verdict === "pass")
-        : "—";
+    // A preflight-failure record is the one grade-less branch in the schema.
+    const checksCell = r.grade ? statusIcon(r.grade.verdict === "pass") : "—";
+    const judgeCell = r.judgeVerdict
+      ? statusIcon(r.judgeVerdict.verdict === "pass")
+      : "—";
     rows.push([
       String(r.runIndex),
       statusIcon(r.verdict === "pass"),
-      invariantsCell,
+      checksCell,
       judgeCell,
+      ...(scored
+        ? [r.score !== undefined ? Number(r.score).toFixed(4) : "—"]
+        : []),
       formatCost(r.costUsd),
       String(r.turns),
       formatDuration(r.durationMs),
@@ -321,42 +364,45 @@ function renderRunsTable(runs) {
   return rows.map((r) => `| ${r.join(" | ")} |`).join("\n");
 }
 
-function renderInvariantChecks(runs, singleRun) {
-  const rows = collectInvariantRows(runs);
+function renderChecks(runs, singleRun) {
+  const { rows, hasHidden } = collectCheckRows(runs);
   if (!rows.length) return null;
 
-  const header = singleRun
-    ? ["Check", "Result", "Message"]
-    : ["Run", "Check", "Result", "Message"];
+  const header = singleRun ? ["Check"] : ["Run", "Check"];
+  if (hasHidden) header.push("Source");
+  header.push("Result", "Message");
   const lines = [
-    "#### Invariant Checks",
+    "#### Checks",
     "",
     `| ${header.join(" | ")} |`,
     `| ${header.map(() => "---").join(" | ")} |`,
   ];
   for (const row of rows) {
-    const cells = singleRun
-      ? [row.check, row.result, row.message]
-      : [String(row.run), row.check, row.result, row.message];
+    const cells = singleRun ? [row.check] : [String(row.run), row.check];
+    if (hasHidden) cells.push(row.source);
+    cells.push(row.result, row.message);
     lines.push(`| ${cells.join(" | ")} |`);
   }
   return lines.join("\n");
 }
 
-function collectInvariantRows(runs) {
-  const rows = [];
-  for (const r of runs) {
-    if (!r.invariants?.details?.length) continue;
-    for (const d of r.invariants.details) {
-      rows.push({
+/**
+ * Merge both producers' check rows per run. The Source column appears only
+ * when at least one row came from a hidden test suite.
+ */
+function collectCheckRows(runs) {
+  const rows = runs.flatMap((r) =>
+    mergeRows(r.invariants?.details ?? [], r.hiddenTests?.details ?? [])
+      .filter((d) => d !== null && typeof d === "object")
+      .map((d) => ({
         run: r.runIndex,
         check: escapeCell(String(d.test ?? "(unnamed)")),
+        source: d.source ?? "",
         result: statusIcon(d.pass),
         message: escapeCell(String(d.message ?? "")),
-      });
-    }
-  }
-  return rows;
+      })),
+  );
+  return { rows, hasHidden: rows.some((row) => row.source === "tests") };
 }
 
 function renderJudgeCommentary(runs, singleRun) {
@@ -378,23 +424,36 @@ function renderJudgeCommentary(runs, singleRun) {
 }
 
 function renderErrors(runs) {
-  const lines = [];
-  for (const r of runs) {
-    if (r.agentError) {
-      lines.push(
-        `- **Run ${r.runIndex}:** Agent error — "${escapeCell(r.agentError.message)}" (aborted: ${r.agentError.aborted})`,
-      );
-    }
-    if (r.preflightError) {
-      lines.push(
-        `- **Run ${r.runIndex}:** Preflight error — "${escapeCell(r.preflightError.message)}" (exit ${r.preflightError.exitCode})`,
-      );
-    }
-  }
+  const lines = runs.flatMap(runErrorLines);
   if (!lines.length) return null;
   return ["#### Errors", "", ...lines].join("\n");
 }
 
+function runErrorLines(r) {
+  const lines = [];
+  if (r.grade?.malformed) {
+    lines.push(
+      `- **Run ${r.runIndex}:** ⚠️ ${r.grade.malformed} malformed check row(s) — counted as failing`,
+    );
+  }
+  if (r.hiddenTests?.error) {
+    lines.push(
+      `- **Run ${r.runIndex}:** Hidden-test engine error — "${escapeCell(r.hiddenTests.error)}"`,
+    );
+  }
+  if (r.agentError) {
+    lines.push(
+      `- **Run ${r.runIndex}:** Agent error — "${escapeCell(r.agentError.message)}" (aborted: ${r.agentError.aborted})`,
+    );
+  }
+  if (r.preflightError) {
+    lines.push(
+      `- **Run ${r.runIndex}:** Preflight error — "${escapeCell(r.preflightError.message)}" (exit ${r.preflightError.exitCode})`,
+    );
+  }
+  return lines;
+}
+
 // ---------------------------------------------------------------------------
 // Formatting helpers
 // ---------------------------------------------------------------------------
@@ -599,6 +658,33 @@ function passAtKValue(n, c, k) {
   return Number(passing) / Number(total);
 }
 
+/**
+ * score@k — the expected **maximum** score over k runs drawn without
+ * replacement from the n recorded scores; the continuous analog of pass@k.
+ * With scores sorted ascending s₍₁₎…s₍ₙ₎:
+ *
+ *   score@k = Σ_{i=k..n} s₍ᵢ₎ · C(i−1, k−1) / C(n, k)
+ *
+ * Each term weights s₍ᵢ₎ by the probability it is the k-subset's maximum.
+ * Binary scores reduce exactly to the pass@k estimator (same BigInt binomial
+ * helper); `k > n` yields the same `{error}` value — one idiom.
+ * @param {number[]} scores - Effective per-record scores.
+ * @param {number} k
+ * @returns {number | {error: string}}
+ */
+function scoreAtKValue(scores, k) {
+  const n = scores.length;
+  if (k > n) return { error: "k > n" };
+  const sorted = [...scores].sort((a, b) => a - b);
+  const total = Number(binomial(BigInt(n), BigInt(k)));
+  let sum = 0;
+  for (let i = k; i <= n; i++) {
+    const weight = Number(binomial(BigInt(i - 1), BigInt(k - 1))) / total;
+    sum += sorted[i - 1] * weight;
+  }
+  return sum;
+}
+
 function binomial(n, k) {
   if (k < 0n || k > n) return 0n;
   if (k === 0n || k === n) return 1n;

package/src/benchmark/result.js

@@ -3,10 +3,14 @@
  *
  * Two schemas live here:
  *   - RESULT_RECORD_SCHEMA — one record per (task, runIndex) from a full
- *     benchmark run. Has a happy branch (invariants + judge present) and a
- *     pre-flight-failure branch (invariants/judgeVerdict/submission absent).
- *   - INVARIANTS_RECORD_SCHEMA — narrower output of `benchmark-invariants`:
- *     ad-hoc grading without a full lifecycle.
+ *     benchmark run. Has a happy branch (grade + collectors + judge present)
+ *     and a pre-flight-failure branch (grade/judgeVerdict/submission absent).
+ *   - GRADE_RECORD_SCHEMA — narrower output of `benchmark-grade`: ad-hoc
+ *     grading without a full lifecycle.
+ *
+ * The check rows are the authoritative grading channel: the happy branch
+ * requires a `grade` object, so a pre-break record fails validation rather
+ * than rendering under semantics it never carried.
  *
  * Validation is throw-on-mismatch so the runner can wrap every JSONL append
  * in a guard and reject schema drift at write time.
@@ -17,12 +21,27 @@ import { z } from "zod";
 const VERDICT_ENUM = z.enum(["pass", "fail"]);
 
 const INVARIANTS_SHAPE = z.object({
-  verdict: VERDICT_ENUM,
   details: z.array(z.unknown()),
   exitCode: z.number().int(),
   stderr: z.string().optional(),
 });
 
+/**
+ * The normalized grading projection: `score` appears only on scored tasks,
+ * `malformed` only when at least one row was malformed.
+ */
+const GRADE_SHAPE = z.object({
+  verdict: VERDICT_ENUM,
+  gatesPass: z.boolean(),
+  score: z.number().min(0).max(1).optional(),
+  malformed: z.number().int().min(1).optional(),
+});
+
+const HIDDEN_TESTS_SHAPE = z.object({
+  details: z.array(z.unknown()),
+  error: z.string().optional(),
+});
+
 const JUDGE_VERDICT_SHAPE = z.object({
   verdict: VERDICT_ENUM,
   summary: z.string(),
@@ -77,6 +96,11 @@ const AGENT_ERROR_SHAPE = z.object({
 const HAPPY_RECORD = z.object({
   ...COMMON_FIELDS,
   invariants: INVARIANTS_SHAPE,
+  grade: GRADE_SHAPE,
+  hiddenTests: HIDDEN_TESTS_SHAPE.optional(),
+  // The effective, judge-zeroed score `report` aggregates — present only on
+  // scored tasks.
+  score: z.number().min(0).max(1).optional(),
   submission: z.string(),
   judgeVerdict: JUDGE_VERDICT_SHAPE.optional(),
   agentTracePath: z.string(),
@@ -97,6 +121,9 @@ const PREFLIGHT_RECORD = z.object({
   supervisorTracePath: z.string(),
   judgeTracePath: z.string(),
   invariants: z.undefined().optional(),
+  grade: z.undefined().optional(),
+  hiddenTests: z.undefined().optional(),
+  score: z.undefined().optional(),
   submission: z.undefined().optional(),
   judgeVerdict: z.undefined().optional(),
   agentError: z.undefined().optional(),
@@ -104,9 +131,17 @@ const PREFLIGHT_RECORD = z.object({
 
 export const RESULT_RECORD_SCHEMA = z.union([HAPPY_RECORD, PREFLIGHT_RECORD]);
 
-export const INVARIANTS_RECORD_SCHEMA = z.object({
+export const GRADE_RECORD_SCHEMA = z.object({
   taskId: z.string().min(1),
+  // Unlike the happy result record — where `grade.score` is the raw
+  // weighted fraction and the effective (zeroed) value lives on the
+  // top-level `score` — this record has no second score field, so its
+  // `grade.score` carries the effective health/gate-zeroed value.
+  grade: GRADE_SHAPE,
   invariants: INVARIANTS_SHAPE,
+  hiddenTests: HIDDEN_TESTS_SHAPE.optional(),
+  // Mirrors the invariants script's exit for diagnosis; the graded verdict
+  // is what drives the command's process exit.
   exitCode: z.number().int(),
 });
 
@@ -122,6 +157,6 @@ export function validateResultRecord(record) {
  * Throw on schema mismatch.
  * @param {object} record
  */
-export function validateInvariantsRecord(record) {
-  INVARIANTS_RECORD_SCHEMA.parse(record);
+export function validateGradeRecord(record) {
+  GRADE_RECORD_SCHEMA.parse(record);
 }