@forwardimpact/libharness 1.4.0 → 2.0.0

package/src/benchmark/runner.js

@@ -4,8 +4,10 @@
  * Phases per (task, runIndex):
  *   1. WorkdirManager.start → seed CWD + run pre-flight probe
  *   2. Supervisor session (agent + supervisor) → produce traces + submission
- *   3. Invariants.runInvariants → exit-code-driven verdict via fd-3 NDJSON
- *   4. Judge.runJudge → Conclude-driven verdict mapped to pass/fail
+ *   3. Invariants collector + hidden-test engine → merged check rows,
+ *      graded by `gradeChecks` (rows are authoritative; script exit is
+ *      grader health only)
+ *   4. Judge.runJudge → Conclude-driven binary gate mapped to pass/fail
  *   5. WorkdirManager.teardown → process-group cleanup
  *
  * Cells run with bounded in-process concurrency (`CellScheduler`); `run()`
@@ -17,7 +19,6 @@
  * stream.
  */
 
-import { createInterface } from "node:readline";
 import { join, resolve as resolvePath } from "node:path";
 
 import { DEFAULT_ENV_ALLOWLIST, createRedactor } from "../redaction.js";
@@ -28,7 +29,10 @@ import { installNpm as defaultInstallNpm } from "./npm-installer.js";
 import { runJudge } from "./judge.js";
 import { validateResultRecord } from "./result.js";
 import { runInvariants } from "./invariants.js";
+import { runHiddenTests } from "./hidden-tests.js";
+import { runProducersAndGrade } from "./grade.js";
 import { assertJudgeProfileStaged, loadTaskFamily } from "./task-family.js";
+import { splitAndSummarize } from "./trace-split.js";
 import { createWorkdirManager } from "./workdir.js";
 import { CellScheduler } from "./scheduler.js";
 
@@ -82,9 +86,13 @@ export class BenchmarkRunner {
    *   threaded into the installers, workdir manager, invariants, and judge.
    * @param {Function} [opts.runInvariants] - Test seam: replaces `runInvariants`.
    *   Same contract as `runInvariants(task, ctx, runtime)`. Internal testing only.
+   * @param {Function} [opts.runHiddenTests] - Test seam: replaces
+   *   `runHiddenTests`. Same contract as `runHiddenTests(task, ctx, runtime)`.
+   *   Internal testing only.
    * @param {Function} [opts.runJudge] - Test seam: replaces `runJudge`. Same
-   *   contract as `runJudge(task, workdir, invariants, deps)` (deps carries
-   *   `runtime`). Internal testing only.
+   *   contract as `runJudge(task, workdir, gradeResult, deps)` where
+   *   `gradeResult` is the normalized grade plus the merged, source-stamped
+   *   check `rows` (deps carries `runtime`). Internal testing only.
    * @param {Function} [opts.installApm] - Test seam: replaces `installApm`.
    *   Same contract as `installApm(family, outputDir, runtime)`. Lets tests
    *   inject a fake subprocess (or skip the install entirely) so the suite
@@ -114,6 +122,7 @@ export class BenchmarkRunner {
     // Test seams — default to the real implementations.
     runAgent,
     runInvariants: runInvariantsHook,
+    runHiddenTests: runHiddenTestsHook,
     runJudge: runJudgeHook,
     installApm: installApmHook,
     installNpm: installNpmHook,
@@ -141,6 +150,7 @@ export class BenchmarkRunner {
     this.termGraceMs = termGraceMs;
     this._runAgentHook = runAgent ?? null;
     this._runInvariantsHook = runInvariantsHook ?? runInvariants;
+    this._runHiddenTestsHook = runHiddenTestsHook ?? runHiddenTests;
     this._runJudgeHook = runJudgeHook ?? runJudge;
     this._installApmHook = installApmHook ?? defaultInstallApm;
     this._installNpmHook = installNpmHook ?? defaultInstallNpm;
@@ -291,55 +301,37 @@ export class BenchmarkRunner {
       const agentRun = await this.#runAgentSafe(task, workdir);
       const { costUsd, turns, submission, agentError } = agentRun;
       const breakdown = agentRun.costBreakdown ?? { agent: 0, supervisor: 0 };
-      const invariants = await this._runInvariantsHook(
+      const graded = await this.#gradeCell(family, task, workdir);
+      const { invariants, hiddenRows, engineError, rows, grade } = graded;
+      const { judgeVerdict, judgeCost } = await this.#judgeCell({
         task,
-        {
-          cwd: workdir.cwd,
-          port: workdir.port,
-          runDir: workdir.runDir,
-          familyDir: family.rootPath,
-        },
-        this.runtime,
-      );
-      let judgeVerdict = null;
-      let judgeCost = 0;
-      if (task.paths.judge) {
-        const judgeContext = await this.#buildJudgeContext(
-          task,
-          workdir,
-          skillSetHash,
-        );
-        const judgeResult = await this._runJudgeHook(
-          task,
-          workdir,
-          invariants,
-          {
-            query: this.query,
-            model: this.judgeModel,
-            judgeProfile: this.profiles.judge ?? undefined,
-            profilesDir: judgeProfilesDir,
-            runtime: this.runtime,
-          },
-          judgeContext,
-        );
-        judgeCost = judgeResult.costUsd ?? 0;
-        // The record's judgeVerdict carries only the verdict + summary; the
-        // judge's cost is folded into costUsd / costBreakdown instead.
-        judgeVerdict = {
-          verdict: judgeResult.verdict,
-          summary: judgeResult.summary,
-        };
-      }
-      const verdict =
-        invariants.verdict === "pass" &&
-        (judgeVerdict === null || judgeVerdict.verdict === "pass")
-          ? "pass"
-          : "fail";
+        workdir,
+        gradeResult: { ...grade, rows },
+        skillSetHash,
+        judgeProfilesDir,
+      });
+      const judgePass =
+        judgeVerdict === null || judgeVerdict.verdict === "pass";
+      const verdict = grade.verdict === "pass" && judgePass ? "pass" : "fail";
+      // Gates protect the score: an unhealthy grader, a failing gate row, or
+      // a failing judge zeroes the effective score. Full marks does not — a
+      // fractional score with verdict fail is the point.
+      const scoreValid = graded.healthy && grade.gatesPass && judgePass;
       const record = {
         taskId: task.id,
         runIndex,
         verdict,
         invariants,
+        grade,
+        ...(task.tests && {
+          hiddenTests: {
+            details: hiddenRows,
+            ...(engineError && { error: engineError.message }),
+          },
+        }),
+        ...(grade.score !== undefined && {
+          score: scoreValid ? grade.score : 0,
+        }),
         submission,
         ...(judgeVerdict && { judgeVerdict }),
         costUsd: costUsd + judgeCost,
@@ -371,6 +363,65 @@ export class BenchmarkRunner {
     }
   }
 
+  /**
+   * Run the judge (when the task ships a template) over the grade result.
+   * The record's judgeVerdict carries only the verdict + summary; the
+   * judge's cost is folded into costUsd / costBreakdown instead.
+   */
+  async #judgeCell({
+    task,
+    workdir,
+    gradeResult,
+    skillSetHash,
+    judgeProfilesDir,
+  }) {
+    if (!task.paths.judge) return { judgeVerdict: null, judgeCost: 0 };
+    const judgeContext = await this.#buildJudgeContext(
+      task,
+      workdir,
+      skillSetHash,
+    );
+    const judgeResult = await this._runJudgeHook(
+      task,
+      workdir,
+      gradeResult,
+      {
+        query: this.query,
+        model: this.judgeModel,
+        judgeProfile: this.profiles.judge ?? undefined,
+        profilesDir: judgeProfilesDir,
+        runtime: this.runtime,
+      },
+      judgeContext,
+    );
+    return {
+      judgeVerdict: {
+        verdict: judgeResult.verdict,
+        summary: judgeResult.summary,
+      },
+      judgeCost: judgeResult.costUsd ?? 0,
+    };
+  }
+
+  /**
+   * Run both check-row producers against the post-run CWD and grade the
+   * merged rows via the shared derivation. Restoration happens inside the
+   * engine, so the judge (which runs after) sees the workdir exactly as the
+   * agent left it.
+   */
+  #gradeCell(family, task, workdir) {
+    const ctx = {
+      cwd: workdir.cwd,
+      port: workdir.port,
+      runDir: workdir.runDir,
+      familyDir: family.rootPath,
+    };
+    return runProducersAndGrade(task, ctx, this.runtime, {
+      runInvariants: this._runInvariantsHook,
+      runHiddenTests: this._runHiddenTestsHook,
+    });
+  }
+
   /**
    * Dispatch to either the injected hook or the default `#runAgent`. Either
    * path can throw; catch here so a thrown error becomes an `agentError` on
@@ -614,67 +665,6 @@ async function writeRecord(stream, record) {
   });
 }
 
-/**
- * Split the combined supervisor trace into agent and supervisor files and
- * extract turn count and submission in a single pass. Agent-source events go
- * to `agentPath`; supervisor and orchestrator events go to `supervisorPath`.
- *
- * Cost is deliberately not summed here — the caller derives it from the same
- * combined trace via `sumTraceCost`, so there is one cost path across the
- * benchmark, callback, and `fit-trace cost` consumers.
- */
-// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: stream-splitting state machine
-async function splitAndSummarize(
-  runtime,
-  combinedPath,
-  agentPath,
-  supervisorPath,
-) {
-  const fs = runtime.fs;
-  const agentStream = fs.createWriteStream(agentPath);
-  const supStream = fs.createWriteStream(supervisorPath);
-  const rl = createInterface({
-    input: fs.createReadStream(combinedPath),
-    crlfDelay: Infinity,
-  });
-  let turns = 0;
-  let submission = "";
-  for await (const line of rl) {
-    if (!line.trim()) continue;
-    let event;
-    try {
-      event = JSON.parse(line);
-    } catch {
-      continue;
-    }
-    const target = event.source === "agent" ? agentStream : supStream;
-    target.write(line + "\n");
-    const inner = event.event;
-    if (!inner) continue;
-    if (event.source === "agent" && inner.type === "assistant") {
-      const text = extractText(inner);
-      if (text) submission = text;
-    }
-    if (event.source === "orchestrator" && inner.type === "summary") {
-      turns = inner.turns ?? 0;
-    }
-  }
-  await Promise.all([
-    new Promise((r) => agentStream.end(r)),
-    new Promise((r) => supStream.end(r)),
-  ]);
-  return { turns, submission };
-}
-
-function extractText(inner) {
-  const content = inner.message?.content ?? inner.content;
-  if (!Array.isArray(content)) return null;
-  for (let i = content.length - 1; i >= 0; i--) {
-    if (content[i].type === "text" && content[i].text) return content[i].text;
-  }
-  return null;
-}
-
 /**
  * Factory function — wires real dependencies.
  * @param {ConstructorParameters<typeof BenchmarkRunner>[0]} opts

package/src/benchmark/task-family.js

@@ -10,9 +10,16 @@
  *       hooks/                # harness-only; never copied to agent CWD
  *         preflight.sh
  *         invariants.sh
+ *       tests/                # optional hidden test suite; harness-only overlay
  *       specs/                # copied into agent CWD
  *       workdir/              # copied into agent CWD
  *
+ * `tests/` is an overlay mirror of the agent CWD: a file's path under
+ * `tests/` is its staging path. Every `*.test.js` file is one check —
+ * `*.gate.test.js` marks a gate, any other `*.test.js` is scored — and every
+ * other file is support material, staged but never graded. The layout is
+ * validated eagerly here so authoring errors fail before any agent spend.
+ *
  * Local paths or git URLs are both accepted; git URLs are shallow-cloned into
  * a temp dir and `familyRevision` becomes `git:<sha>` of HEAD at clone time.
  * Local paths use the canonical-tree algorithm from design § Family revision
@@ -113,36 +120,124 @@ async function discoverTasks(runtime, rootPath) {
   }
   for (const entry of entries) {
     if (!entry.isDirectory()) continue;
-    const taskDir = join(tasksRoot, entry.name);
-    const supervisorPath = join(taskDir, "supervisor.task.md");
-    const judgePath = join(taskDir, "judge.task.md");
-    const preflightPath = join(taskDir, "hooks", "preflight.sh");
-    const invariantsPath = join(taskDir, "hooks", "invariants.sh");
-    tasks.push({
-      id: entry.name,
-      paths: {
-        taskDir,
-        instructions: join(taskDir, "agent.task.md"),
-        supervisor: (await fileExists(fs, supervisorPath))
-          ? supervisorPath
-          : null,
-        judge: (await fileExists(fs, judgePath)) ? judgePath : null,
-        hooks: join(taskDir, "hooks"),
-        preflight: (await fileExecutable(fs, preflightPath))
-          ? preflightPath
-          : null,
-        invariants: (await fileExecutable(fs, invariantsPath))
-          ? invariantsPath
-          : null,
-        specs: join(taskDir, "specs"),
-        workdir: join(taskDir, "workdir"),
-      },
-    });
+    tasks.push(await loadTask(fs, join(tasksRoot, entry.name), entry.name));
   }
   tasks.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
   return tasks;
 }
 
+async function loadTask(fs, taskDir, id) {
+  const supervisorPath = join(taskDir, "supervisor.task.md");
+  const judgePath = join(taskDir, "judge.task.md");
+  const preflightPath = join(taskDir, "hooks", "preflight.sh");
+  const invariantsPath = join(taskDir, "hooks", "invariants.sh");
+  const suite = await discoverSuite(fs, taskDir);
+  return {
+    id,
+    paths: {
+      taskDir,
+      instructions: join(taskDir, "agent.task.md"),
+      supervisor: (await fileExists(fs, supervisorPath))
+        ? supervisorPath
+        : null,
+      judge: (await fileExists(fs, judgePath)) ? judgePath : null,
+      hooks: join(taskDir, "hooks"),
+      preflight: (await fileExecutable(fs, preflightPath))
+        ? preflightPath
+        : null,
+      invariants: (await fileExecutable(fs, invariantsPath))
+        ? invariantsPath
+        : null,
+      tests: suite ? join(taskDir, "tests") : null,
+      specs: join(taskDir, "specs"),
+      workdir: join(taskDir, "workdir"),
+    },
+    tests: suite,
+  };
+}
+
+const CHECK_SUFFIX = ".test.js";
+const GATE_SUFFIX = ".gate.test.js";
+
+/**
+ * Discover and validate a task's hidden test suite under `<taskDir>/tests/`.
+ * Returns null when the directory is absent; throws on an invalid layout
+ * (no check files, a dangling symlink, duplicate check names) so
+ * `loadTaskFamily` rejects broken suites before any agent spend.
+ * @param {object} fs - Async filesystem surface (`runtime.fs`).
+ * @param {string} taskDir
+ * @returns {Promise<HiddenSuite | null>}
+ */
+async function discoverSuite(fs, taskDir) {
+  const testsRoot = join(taskDir, "tests");
+  try {
+    const st = await fs.lstat(testsRoot);
+    if (!st.isDirectory()) return null;
+  } catch {
+    return null;
+  }
+  const files = [];
+  await walkSuiteFiles(fs, testsRoot, testsRoot, files);
+  files.sort((a, b) =>
+    a.stagePath < b.stagePath ? -1 : a.stagePath > b.stagePath ? 1 : 0,
+  );
+
+  const checks = [];
+  const support = [];
+  for (const file of files) {
+    const base = file.stagePath.split(sep).at(-1);
+    if (!base.endsWith(CHECK_SUFFIX)) {
+      support.push(file);
+      continue;
+    }
+    const gate = base.endsWith(GATE_SUFFIX);
+    const name = base.slice(0, -(gate ? GATE_SUFFIX : CHECK_SUFFIX).length);
+    checks.push({ name, gate, ...file });
+  }
+
+  if (checks.length === 0) {
+    throw new Error(`hidden test suite has no check files: ${testsRoot}`);
+  }
+  const seen = new Set();
+  for (const check of checks) {
+    if (seen.has(check.name)) {
+      throw new Error(
+        `hidden test suite has duplicate check name '${check.name}': ${check.sourcePath}`,
+      );
+    }
+    seen.add(check.name);
+  }
+  return { checks, support };
+}
+
+/**
+ * Walk a suite tree collecting `{sourcePath, stagePath}` entries. Unlike
+ * `walkFiles` (which silently skips dangling symlinks for hashing), every
+ * entry here must be a regular file after symlink resolution — a dangling
+ * symlink or a link to a non-file target is an authoring error.
+ */
+async function walkSuiteFiles(fs, root, dir, out) {
+  const entries = await fs.readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      await walkSuiteFiles(fs, root, full, out);
+    } else if (entry.isFile()) {
+      out.push({ sourcePath: full, stagePath: relative(root, full) });
+    } else if (entry.isSymbolicLink()) {
+      const resolved = await resolveSymlinkToFile(fs, full);
+      if (!resolved) {
+        throw new Error(
+          `hidden test suite entry is not a regular file: ${full}`,
+        );
+      }
+      out.push({ sourcePath: full, stagePath: relative(root, full) });
+    } else {
+      throw new Error(`hidden test suite entry is not a regular file: ${full}`);
+    }
+  }
+}
+
 async function fileExists(fs, path) {
   try {
     await fs.access(path);
@@ -246,10 +341,28 @@ async function git(runtime, args) {
   return stdout;
 }
 
+/**
+ * @typedef {object} HiddenCheck
+ * @property {string} name - Basename stem with the check suffix stripped.
+ * @property {boolean} gate - True iff the filename ends `.gate.test.js`.
+ * @property {string} sourcePath - Absolute path under `tests/`.
+ * @property {string} stagePath - Path relative to `tests/` — the overlay
+ *   mirror of the staging path under the agent CWD.
+ */
+
+/**
+ * @typedef {object} HiddenSuite
+ * @property {HiddenCheck[]} checks - In sorted stage-path order.
+ * @property {{sourcePath: string, stagePath: string}[]} support - Non-check
+ *   files, staged for the whole pass but never graded.
+ */
+
 /**
  * @typedef {object} Task
  * @property {string} id - Task name (directory name under tasks/)
- * @property {{taskDir: string, instructions: string, supervisor: string|null, judge: string|null, hooks: string, preflight: string|null, invariants: string|null, specs: string, workdir: string}} paths
+ * @property {{taskDir: string, instructions: string, supervisor: string|null, judge: string|null, hooks: string, preflight: string|null, invariants: string|null, tests: string|null, specs: string, workdir: string}} paths
+ * @property {HiddenSuite | null} tests - Hidden test suite (null when the
+ *   task ships no `tests/` directory)
  */
 
 /**

package/src/benchmark/trace-split.js

@@ -0,0 +1,73 @@
+/**
+ * Combined-supervisor-trace splitting for the benchmark runner: one pass
+ * over the tagged NDJSON envelope stream separates agent events from
+ * supervisor/orchestrator events and extracts the run summary.
+ */
+
+import { createInterface } from "node:readline";
+
+/**
+ * Split the combined supervisor trace into agent and supervisor files and
+ * extract turn count and submission in a single pass. Agent-source events go
+ * to `agentPath`; supervisor and orchestrator events go to `supervisorPath`.
+ *
+ * Cost is deliberately not summed here — the caller derives it from the same
+ * combined trace via `sumTraceCost`, so there is one cost path across the
+ * benchmark, callback, and `fit-trace cost` consumers.
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @param {string} combinedPath
+ * @param {string} agentPath
+ * @param {string} supervisorPath
+ * @returns {Promise<{turns: number, submission: string}>}
+ */
+// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: stream-splitting state machine
+export async function splitAndSummarize(
+  runtime,
+  combinedPath,
+  agentPath,
+  supervisorPath,
+) {
+  const fs = runtime.fs;
+  const agentStream = fs.createWriteStream(agentPath);
+  const supStream = fs.createWriteStream(supervisorPath);
+  const rl = createInterface({
+    input: fs.createReadStream(combinedPath),
+    crlfDelay: Infinity,
+  });
+  let turns = 0;
+  let submission = "";
+  for await (const line of rl) {
+    if (!line.trim()) continue;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    const target = event.source === "agent" ? agentStream : supStream;
+    target.write(line + "\n");
+    const inner = event.event;
+    if (!inner) continue;
+    if (event.source === "agent" && inner.type === "assistant") {
+      const text = extractText(inner);
+      if (text) submission = text;
+    }
+    if (event.source === "orchestrator" && inner.type === "summary") {
+      turns = inner.turns ?? 0;
+    }
+  }
+  await Promise.all([
+    new Promise((r) => agentStream.end(r)),
+    new Promise((r) => supStream.end(r)),
+  ]);
+  return { turns, submission };
+}
+
+function extractText(inner) {
+  const content = inner.message?.content ?? inner.content;
+  if (!Array.isArray(content)) return null;
+  for (let i = content.length - 1; i >= 0; i--) {
+    if (content[i].type === "text" && content[i].text) return content[i].text;
+  }
+  return null;
+}

package/src/benchmark/workdir.js

@@ -268,7 +268,12 @@ async function runPreflight(runtime, script, cwd, port, vars) {
   };
 }
 
-function probeFreePort() {
+/**
+ * Allocate a free TCP port by binding to 0 and releasing it. Shared with the
+ * `grade` subcommand, which needs a plausible `$PORT` for the hook env.
+ * @returns {Promise<number>}
+ */
+export function probeFreePort() {
   return new Promise((res, rej) => {
     const server = createServer();
     server.unref();

package/src/commands/assert.js

@@ -62,12 +62,74 @@ export function evaluateAssertion(values, args, fsSync) {
 
   const output = { test: testName, pass: result.pass };
   if (result.message) output.message = result.message;
+  applyGradingFlags(values, output);
   return output;
 }
 
+/**
+ * Attach the check-row grading role: `--gate` marks a gate check, `--weight`
+ * attaches a numeric weight (0 marks the row diagnostic). `--gate` with any
+ * `--weight` — 0 included — is invalid: a stray weight must never silently
+ * disarm a gate.
+ * @param {object} values
+ * @param {{test: string, pass: boolean, message?: string}} output - Mutated.
+ */
+function applyGradingFlags(values, output) {
+  const hasWeight = values.weight !== undefined;
+  if (values.gate && hasWeight) {
+    throw new Error("assert: --gate cannot be combined with --weight");
+  }
+  if (values.gate) output.gate = true;
+  if (hasWeight) {
+    const weight = parseWeight(values.weight);
+    if (weight === null) {
+      throw new Error(
+        `assert: invalid --weight '${values.weight}' (expected a finite number ≥ 0)`,
+      );
+    }
+    output.weight = weight;
+  }
+}
+
+/**
+ * Parse a `--weight` value; null when invalid. A blank string is invalid —
+ * `Number("")` is 0, which would silently demote the check to a diagnostic.
+ * @param {string} raw
+ * @returns {number | null}
+ */
+function parseWeight(raw) {
+  if (typeof raw === "string" && raw.trim() === "") return null;
+  const weight = Number(raw);
+  return Number.isFinite(weight) && weight >= 0 ? weight : null;
+}
+
+/**
+ * The grading role an emit-then-fail row keeps: a failing check must not
+ * lose its authored role — an errored gate that demoted to a scored row
+ * would let a broken scaffold earn partial credit instead of zeroing the
+ * score. Invalid or conflicting flags yield no role (the row fails as a
+ * unit-weight scored check).
+ * @param {object} values
+ * @returns {{gate?: true, weight?: number}}
+ */
+function errorRowRole(values) {
+  const hasWeight = values.weight !== undefined;
+  if (values.gate && !hasWeight) return { gate: true };
+  if (!values.gate && hasWeight) {
+    const weight = parseWeight(values.weight);
+    if (weight !== null) return { weight };
+  }
+  return {};
+}
+
 /**
  * Run an assertion, write JSON to stdout, and return a failure envelope when
  * the assertion does not pass.
+ *
+ * Emit-then-fail on every failure path: an invalid grading flag or an
+ * errored evaluation (e.g. `--grep` against a file the agent deleted) writes
+ * a failing row before the nonzero exit, so a typo or a vanished target
+ * shrinks the score, never the denominator.
  * @param {import("@forwardimpact/libcli").InvocationContext} ctx
  * @returns {Promise<{ok: true} | {ok: false, code: number, error: string}>}
  */
@@ -78,6 +140,16 @@ export async function runAssertCommand(ctx) {
   try {
     result = evaluateAssertion(ctx.options, args, runtime.fsSync);
   } catch (err) {
+    const reason = err.message.startsWith("assert: ")
+      ? err.message
+      : `assert: ${err.message}`;
+    const row = {
+      test: ctx.args["test-name"] ?? "(missing test name)",
+      pass: false,
+      ...errorRowRole(ctx.options),
+      message: reason,
+    };
+    runtime.proc.stdout.write(JSON.stringify(row) + "\n");
     return { ok: false, code: 1, error: err.message };
   }
   runtime.proc.stdout.write(JSON.stringify(result) + "\n");

package/src/commands/benchmark-definition.js

@@ -5,7 +5,7 @@
  */
 
 import { runBenchmarkRunCommand } from "./benchmark-run.js";
-import { runBenchmarkInvariantsCommand } from "./benchmark-invariants.js";
+import { runBenchmarkGradeCommand } from "./benchmark-grade.js";
 import { runBenchmarkReportCommand } from "./benchmark-report.js";
 import {
   BENCHMARK_AGENT_MODEL,
@@ -95,11 +95,11 @@ export const definition = {
       },
     },
     {
-      name: "invariants",
+      name: "grade",
       args: [],
-      handler: runBenchmarkInvariantsCommand,
+      handler: runBenchmarkGradeCommand,
       description:
-        "Check a single task's invariants against a post-run workdir without invoking an agent.",
+        "Grade a single task against a post-run workdir without invoking an agent: run the hidden test suite and the invariants script, then derive the verdict from the check rows (the exit mirrors it).",
       options: {
         family: {
           type: "string",
@@ -112,7 +112,7 @@ export const definition = {
         "run-dir": {
           type: "string",
           description:
-            "Post-run directory whose cwd/ subdir is the agent CWD; invariants run against that cwd — the path hooks receive as $AGENT_CWD",
+            "Post-run directory whose cwd/ subdir is the agent CWD; both producers run against that cwd — the path hooks receive as $AGENT_CWD",
         },
         output: {
           type: "string",
@@ -159,7 +159,7 @@ export const definition = {
     "fit-benchmark run --family=./families/coding --work-tracker=filesystem",
     "fit-benchmark run --family=./families/coding --skills-from=. --task=todo-api",
     `fit-benchmark run --family=./families/coding --runs=10 --agent-model=${BENCHMARK_AGENT_MODEL}`,
-    "fit-benchmark invariants --family=./families/coding --task=todo-api --run-dir=./benchmark-runs/runs/todo-api/0",
+    "fit-benchmark grade --family=./families/coding --task=todo-api --run-dir=./benchmark-runs/runs/todo-api/0",
     "fit-benchmark report --format=text",
     "fit-benchmark report --input=./runs/today --k=1,3,5 --format=text",
   ],