@forwardimpact/libharness 1.4.0 → 3.0.0

package/README.md

@@ -17,12 +17,12 @@ traces they produce, and edits skill files under controlled conditions.
 
 | CLI             | Purpose                                                                |
 | --------------- | ---------------------------------------------------------------------- |
-| `fit-harness`      | Run agents in `run`/`supervise`/`facilitate`/`discuss` subcommands.    |
-| `fit-trace`     | Download, query, and analyze NDJSON traces produced by `fit-harness`.     |
-| `fit-benchmark` | Run task families for N runs each and aggregate pass@k.                |
-| `fit-selfedit`  | Write stdin to `.claude/**` paths, gated by settings.json + branch.    |
+| `gemba-harness`      | Run agents in `run`/`supervise`/`facilitate`/`discuss` subcommands.    |
+| `gemba-trace`     | Download, query, and analyze NDJSON traces produced by `gemba-harness`.     |
+| `gemba-benchmark` | Run task families for N runs each and aggregate pass@k.                |
+| `gemba-selfedit`  | Write stdin to `.claude/**` paths, gated by settings.json + branch.    |
 
-`fit-harness`'s subcommands share one orchestration loop and one async tool
+`gemba-harness`'s subcommands share one orchestration loop and one async tool
 surface, below. The `judge` role is a profile passed to `supervise`.
 
 ## Modes
@@ -147,9 +147,9 @@ Each line is `{ "source": "<participant|orchestrator>", "seq": N, "event":
 {…} }`. `seq` is monotonic across the whole trace; `orchestrator` emits
 `session_start`, `agent_start`, `protocol_violation`, `lead_turn_limit`,
 and `summary`. `event` is the SDK event verbatim or the orchestrator
-payload. `fit-trace` consumes this format.
+payload. `gemba-trace` consumes this format.
 
-Redaction is on by default for `fit-harness run`/`supervise`/`facilitate`
+Redaction is on by default for `gemba-harness run`/`supervise`/`facilitate`
 and composes two layers:
 
 - **Env-var allowlist** — `ANTHROPIC_API_KEY`, `GH_TOKEN`, `GITHUB_TOKEN`
@@ -178,7 +178,7 @@ downloadable through retention.
 | `trace-collector.js` / `trace-query.js` / `trace-github.js` | Trace ingestion / querying / GitHub-attachment helpers.              |
 | `redaction.js`                                              | Env-var allowlist + credential-shape pattern redaction.              |
 
-## fit-selfedit
+## gemba-selfedit
 
 A narrow, audited bypass for sessions where `Edit`/`Write` (and bash
 writes) are blocked against paths the project's own allowlist permits.
@@ -186,7 +186,7 @@ Reads stdin, writes the target, exits 0 / 2 (safeguard violation) / 1
 (I/O error).
 
 ```sh
-echo "<content>" | bunx fit-selfedit <path>
+echo "<content>" | bunx gemba-selfedit <path>
 ```
 
 Two safeguards, checked in order:
@@ -221,7 +221,7 @@ lists the `Edit()` rules that were tried.
   — end-to-end workflow from dataset generation through evaluation to trace
   analysis, including multi-agent collaboration sessions.
 - [Analyze Traces](https://www.forwardimpact.team/docs/libraries/prove-changes/trace-analysis/index.md)
-  — read the NDJSON traces produced by `fit-harness` with `fit-trace`.
+  — read the NDJSON traces produced by `gemba-harness` with `gemba-trace`.
 - [Agent Teams](https://www.forwardimpact.team/docs/products/agent-teams/index.md)
   — author the profiles consumed by `--agent-profile`, `--lead-profile`, and
   `--agent-profiles`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libharness",
-  "version": "1.4.0",
+  "version": "3.0.0",
   "description": "Autonomous agent team harness — coordinate a lead and participant agents in one async session, with eval, benchmark, and trace tooling to prove the changes worked.",
   "keywords": [
     "orchestration",
@@ -40,20 +40,22 @@
   "main": "./src/index.js",
   "exports": {
     ".": "./src/index.js",
-    "./bin/fit-harness.js": "./bin/fit-harness.js",
-    "./bin/fit-trace.js": "./bin/fit-trace.js",
-    "./bin/fit-benchmark.js": "./bin/fit-benchmark.js",
-    "./bin/fit-selfedit.js": "./bin/fit-selfedit.js"
-  },
-  "bin": {
-    "fit-harness": "./bin/fit-harness.js",
-    "fit-trace": "./bin/fit-trace.js",
-    "fit-benchmark": "./bin/fit-benchmark.js",
-    "fit-selfedit": "./bin/fit-selfedit.js"
+    "./commands/output.js": "./src/commands/output.js",
+    "./commands/tee.js": "./src/commands/tee.js",
+    "./commands/run.js": "./src/commands/run.js",
+    "./commands/supervise.js": "./src/commands/supervise.js",
+    "./commands/facilitate.js": "./src/commands/facilitate.js",
+    "./commands/discuss.js": "./src/commands/discuss.js",
+    "./commands/callback.js": "./src/commands/callback.js",
+    "./commands/scan-logs.js": "./src/commands/scan-logs.js",
+    "./commands/trace.js": "./src/commands/trace.js",
+    "./commands/assert.js": "./src/commands/assert.js",
+    "./commands/by-discussion.js": "./src/commands/by-discussion.js",
+    "./commands/benchmark-definition.js": "./src/commands/benchmark-definition.js",
+    "./commands/selfedit.js": "./src/commands/selfedit.js"
   },
   "files": [
     "src/**/*.js",
-    "bin/**/*.js",
     "README.md"
   ],
   "scripts": {

package/src/agent-runner.js

@@ -1,7 +1,7 @@
 /**
  * AgentRunner — runs a single Claude Agent SDK session and emits raw
- * NDJSON events to an output stream. Building block for `fit-harness run`,
- * `fit-harness supervise`, `fit-harness facilitate`, and `fit-harness discuss`.
+ * NDJSON events to an output stream. Building block for `gemba-harness run`,
+ * `gemba-harness supervise`, `gemba-harness facilitate`, and `gemba-harness discuss`.
  *
  * Follows OO+DI: constructor injection, factory function, tests bypass factory.
  */
@@ -37,7 +37,7 @@ function modelDidWork(result) {
   return tokens > 0 || (cost ?? 0) > 0;
 }
 
-// fit-harness and kata-action run headless in CI/CD with no human to answer
+// gemba-harness and kata-action run headless in CI/CD with no human to answer
 // permission prompts. The SDK is always launched in bypass mode — not
 // overridable — so a future caller can't accidentally reduce permissions.
 const PERMISSION_MODE = "bypassPermissions";

package/src/benchmark/grade.js

@@ -0,0 +1,222 @@
+/**
+ * Grading derivation — the sole home of the check-row arithmetic.
+ *
+ * Check rows are the single authoritative grading channel. Every row is a
+ * check by default; a row declares its role with its own fields, checked in
+ * order:
+ *
+ *   1. Gate       — `gate` is exactly `true`, `pass` is boolean, and no
+ *                   `weight` key is present. Any failing gate → `gatesPass`
+ *                   false.
+ *   2. Diagnostic — no `gate` key and `weight` is exactly `0`. Free-form;
+ *                   never graded.
+ *   3. Scored     — no `gate` key, boolean `pass`, `weight` absent (defaults
+ *                   to 1) or finite > 0.
+ *   4. Malformed  — everything else: any `gate`+`weight` co-occurrence (a
+ *                   stray weight must never silently disarm a gate), a
+ *                   non-boolean `gate`, a missing or non-boolean `pass` on a
+ *                   graded row, an invalid `weight`, an fd-3 line that failed
+ *                   to parse, a non-object row. Counts as a **failing scored
+ *                   check** — dropping a defect could mint full marks;
+ *                   failing the whole run would zero completed work.
+ *
+ * The producers' `source` stamp is display metadata, never a grading input.
+ */
+
+/**
+ * @typedef {object} GradeResult
+ * @property {"pass" | "fail"} verdict - `healthy ∧ gatesPass ∧ fullMarks`.
+ * @property {boolean} gatesPass - Every gate row passes (vacuously true).
+ * @property {number | null} score - Weighted fraction of passing scored
+ *   checks; `null` when the cell has zero scored checks (binary task).
+ * @property {boolean} fullMarks - Integer count predicate: no malformed rows
+ *   and every scored check passes. Never a float comparison, so fractional
+ *   weights carry no equality hazard. Vacuously true with zero scored checks.
+ * @property {number} malformed - Malformed row count.
+ */
+
+/**
+ * Grade the merged check rows against grader health.
+ *
+ * `healthy` is the completion signal a crashed grader cannot fake: when it is
+ * false the verdict is `fail` whatever the rows say, so a hook that dies
+ * after emitting passing rows can never mint marks.
+ * @param {unknown[]} details - Merged check rows from both producers.
+ * @param {boolean} healthy - Invariants exited 0 AND the hidden-test engine
+ *   did not throw.
+ * @returns {GradeResult}
+ */
+export function gradeChecks(details, healthy) {
+  const tally = {
+    gatesPass: true,
+    malformed: 0,
+    scored: 0,
+    passing: 0,
+    weightAll: 0,
+    weightPassing: 0,
+  };
+  for (const row of details) tallyRow(tally, row);
+
+  const score =
+    tally.scored + tally.malformed === 0
+      ? null
+      : tally.weightPassing / tally.weightAll;
+  const fullMarks = tally.malformed === 0 && tally.passing === tally.scored;
+  const verdict = healthy && tally.gatesPass && fullMarks ? "pass" : "fail";
+  return {
+    verdict,
+    gatesPass: tally.gatesPass,
+    score,
+    fullMarks,
+    malformed: tally.malformed,
+  };
+}
+
+/**
+ * Fold one row into the running tally per its classified role.
+ * @param {{gatesPass: boolean, malformed: number, scored: number, passing: number, weightAll: number, weightPassing: number}} tally
+ * @param {unknown} row
+ */
+function tallyRow(tally, row) {
+  const role = classifyRow(row);
+  if (role === "gate") {
+    if (!row.pass) tally.gatesPass = false;
+  } else if (role === "scored") {
+    const weight = row.weight ?? 1;
+    tally.scored++;
+    tally.weightAll += weight;
+    if (row.pass) {
+      tally.passing++;
+      tally.weightPassing += weight;
+    }
+  } else if (role === "malformed") {
+    tally.malformed++;
+    tally.weightAll += malformedWeight(row);
+  }
+}
+
+/**
+ * Run both check-row producers and grade the merged rows — the one
+ * composition shared by the runner and the `grade` subcommand. An engine
+ * throw is grader fault: its message lands on the returned `engineError`
+ * and health fails, so a crashed grader can never mint marks from rows it
+ * happened to emit first.
+ * @param {import("./task-family.js").Task} task
+ * @param {{cwd: string, port: number, runDir: string, familyDir?: string|null}} ctx
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @param {{runInvariants: Function, runHiddenTests: Function}} producers -
+ *   The two producer functions (real implementations or test seams).
+ * @returns {Promise<{invariants: object, hiddenRows: object[], engineError: Error|null, rows: unknown[], healthy: boolean, grade: object}>}
+ */
+export async function runProducersAndGrade(task, ctx, runtime, producers) {
+  const invariants = await producers.runInvariants(task, ctx, runtime);
+  let hiddenRows = [];
+  let engineError = null;
+  try {
+    const hidden = await producers.runHiddenTests(task, ctx, runtime);
+    hiddenRows = hidden.details;
+  } catch (e) {
+    engineError = e;
+  }
+  const rows = mergeRows(invariants.details, hiddenRows);
+  const healthy = invariants.exitCode === 0 && !engineError;
+  const grade = normalizeGrade(gradeChecks(rows, healthy));
+  return { invariants, hiddenRows, engineError, rows, healthy, grade };
+}
+
+/**
+ * Merge the two producers' rows (invariants first) and stamp each row's
+ * provenance. The stamp is display metadata, never a grading input, and
+ * non-object rows (malformed by contract) pass through verbatim.
+ * @param {unknown[]} invariantsDetails
+ * @param {unknown[]} hiddenDetails
+ * @returns {unknown[]}
+ */
+export function mergeRows(invariantsDetails, hiddenDetails) {
+  return [
+    ...invariantsDetails.map((row) => stampSource(row, "invariants")),
+    ...hiddenDetails.map((row) => stampSource(row, "tests")),
+  ];
+}
+
+function stampSource(row, source) {
+  if (row === null || typeof row !== "object" || Array.isArray(row)) {
+    return row;
+  }
+  return { ...row, source };
+}
+
+/**
+ * Project the raw `gradeChecks` return onto the record schema: `fullMarks`
+ * is derivable and dropped, `score` is omitted on binary tasks (`null`),
+ * `malformed` is omitted when clean.
+ * @param {GradeResult} raw
+ * @returns {{verdict: "pass"|"fail", gatesPass: boolean, score?: number, malformed?: number}}
+ */
+export function normalizeGrade({ verdict, gatesPass, score, malformed }) {
+  return {
+    verdict,
+    gatesPass,
+    ...(score !== null && { score }),
+    ...(malformed > 0 && { malformed }),
+  };
+}
+
+/**
+ * Classify one row per the role order in the module contract.
+ * @param {unknown} row
+ * @returns {"gate" | "diagnostic" | "scored" | "malformed"}
+ */
+function classifyRow(row) {
+  if (row === null || typeof row !== "object" || Array.isArray(row)) {
+    return "malformed";
+  }
+  if ("gate" in row) return classifyGateRow(row);
+  if ("weight" in row) return classifyWeightedRow(row);
+  return typeof row.pass === "boolean" ? "scored" : "malformed";
+}
+
+/**
+ * A row carrying a `gate` key: valid only as `gate: true` with a boolean
+ * `pass` and no `weight` key — any co-occurring weight is malformed so a
+ * stray weight can never silently disarm a gate.
+ * @param {object} row
+ * @returns {"gate" | "malformed"}
+ */
+function classifyGateRow(row) {
+  if ("weight" in row) return "malformed";
+  return row.gate === true && typeof row.pass === "boolean"
+    ? "gate"
+    : "malformed";
+}
+
+/**
+ * A gate-less row carrying a `weight` key: exactly 0 is a diagnostic, a
+ * finite positive weight with a boolean `pass` is scored, anything else is
+ * malformed.
+ * @param {object} row
+ * @returns {"diagnostic" | "scored" | "malformed"}
+ */
+function classifyWeightedRow(row) {
+  if (row.weight === 0) return "diagnostic";
+  return isValidWeight(row.weight) && typeof row.pass === "boolean"
+    ? "scored"
+    : "malformed";
+}
+
+/**
+ * A malformed row fails at its own weight when it carries a valid positive
+ * one, else at unit weight 1.
+ * @param {unknown} row
+ * @returns {number}
+ */
+function malformedWeight(row) {
+  if (row !== null && typeof row === "object" && isValidWeight(row.weight)) {
+    return row.weight;
+  }
+  return 1;
+}
+
+function isValidWeight(w) {
+  return typeof w === "number" && Number.isFinite(w) && w > 0;
+}

package/src/benchmark/hidden-tests.js

@@ -0,0 +1,180 @@
+/**
+ * Hidden-test engine — executes a task's `tests/` overlay against the
+ * post-run agent CWD: stage each file at its mirrored path, run each check
+ * with `node --test`, convert the exit status into one check row, and
+ * restore the tree so the judge sees the workdir exactly as the agent left
+ * it.
+ *
+ * Fault attribution is the engine's contract: a stage or spawn failure (the
+ * agent deleted the scaffold) is a *failing row* — agent fault; the engine
+ * itself throwing is grader fault, which the caller records as unhealthy so
+ * a crashed grader can never mint marks.
+ */
+
+import { dirname, join } from "node:path";
+
+import { buildHookEnv } from "./hook-env.js";
+
+// Fixed per-check budget. A wedged test process runs outside the agent
+// watchdog, so this bound is what keeps a hung hidden test from stalling the
+// cell; the timeout row keeps the failure visible.
+const CHECK_TIMEOUT_MS = 120_000;
+const STDERR_TAIL_CHARS = 500;
+
+/**
+ * Run the task's hidden test suite.
+ * @param {import("./task-family.js").Task} task
+ * @param {{cwd: string, port: number, runDir: string, familyDir?: string|null}} ctx
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @param {{timeoutMs?: number}} [opts] - Test seam for the per-check timeout.
+ * @returns {Promise<{details: object[]}>}
+ */
+export async function runHiddenTests(task, ctx, runtime, opts = {}) {
+  if (!runtime) throw new Error("runtime is required");
+  if (!task.tests) return { details: [] };
+  const timeoutMs = opts.timeoutMs ?? CHECK_TIMEOUT_MS;
+  const fs = runtime.fs;
+  const details = [];
+  const supportStager = newStager();
+  try {
+    for (const file of task.tests.support) {
+      await stageFile(fs, ctx.cwd, supportStager, file);
+    }
+    for (const check of task.tests.checks) {
+      details.push(await runOneCheck(task, ctx, runtime, timeoutMs, check));
+    }
+  } finally {
+    await unstage(fs, supportStager);
+  }
+  return { details };
+}
+
+/**
+ * Stage one check, run it, and restore its staging — the check's own row is
+ * the only trace it leaves. A stage failure is the agent's fault (a deleted
+ * scaffold), so it becomes a failing row rather than a throw.
+ */
+async function runOneCheck(task, ctx, runtime, timeoutMs, check) {
+  const stager = newStager();
+  try {
+    try {
+      await stageFile(runtime.fs, ctx.cwd, stager, check);
+    } catch (e) {
+      return checkRow(check, false, `stage failed: ${e.message}`);
+    }
+    return await spawnCheck(task, ctx, runtime, timeoutMs, check);
+  } finally {
+    await unstage(runtime.fs, stager);
+  }
+}
+
+/**
+ * Spawn `node --test <staged path>` from the agent CWD under the hook env
+ * and map the exit status onto one row. The clock timer SIGKILLs a child
+ * that outlives the per-check budget; the row fails with a timeout message.
+ */
+async function spawnCheck(task, ctx, runtime, timeoutMs, check) {
+  const env = buildHookEnv(runtime.proc.env, {
+    cwd: ctx.cwd,
+    port: ctx.port,
+    taskId: task.id,
+    taskDir: task.paths.taskDir,
+    hooksDir: task.paths.hooks,
+    familyDir: ctx.familyDir,
+  });
+  // An inherited test-runner context makes the child `node --test` report
+  // exit 0 even when its tests fail — a failing check would mint a passing
+  // row whenever the harness itself runs under `node --test`.
+  delete env.NODE_TEST_CONTEXT;
+  const child = runtime.subprocess.spawn("node", ["--test", check.stagePath], {
+    cwd: ctx.cwd,
+    env,
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+  let timedOut = false;
+  const timer = runtime.clock.setTimeout(() => {
+    timedOut = true;
+    child.kill("SIGKILL");
+  }, timeoutMs);
+  const drainStdout = (async () => {
+    for await (const _chunk of child.stdout) {
+      // discard
+    }
+  })();
+  let stderr = "";
+  for await (const chunk of child.stderr) stderr += chunk.toString();
+  await drainStdout;
+  const exit = await child.exitCode;
+  runtime.clock.clearTimeout(timer);
+
+  if (timedOut) {
+    return checkRow(check, false, `timed out after ${timeoutMs}ms`);
+  }
+  if (exit === 0) return checkRow(check, true);
+  const tail = stderr.trim().slice(-STDERR_TAIL_CHARS);
+  return checkRow(check, false, `exit ${exit}${tail ? `: ${tail}` : ""}`);
+}
+
+function checkRow(check, pass, message) {
+  return {
+    test: check.name,
+    pass,
+    ...(check.gate && { gate: true }),
+    ...(message && { message }),
+  };
+}
+
+function newStager() {
+  return { staged: [], backups: [], createdDirs: [] };
+}
+
+/**
+ * Copy the symlink-resolved source to its mirrored path under the agent CWD,
+ * backing up a collided file's bytes and tracking every directory created so
+ * `unstage` can put the tree back exactly.
+ */
+async function stageFile(fs, cwd, stager, { sourcePath, stagePath }) {
+  const target = join(cwd, stagePath);
+  let collided = null;
+  try {
+    collided = await fs.readFile(target);
+  } catch {
+    // no collision
+  }
+  if (collided !== null) stager.backups.push({ target, bytes: collided });
+  await ensureParents(fs, cwd, stager, dirname(target));
+  const resolved = await fs.realpath(sourcePath);
+  await fs.copyFile(resolved, target);
+  stager.staged.push(target);
+}
+
+async function ensureParents(fs, cwd, stager, dir) {
+  if (dir === cwd) return;
+  try {
+    await fs.access(dir);
+    return;
+  } catch {
+    // missing — create below
+  }
+  await ensureParents(fs, cwd, stager, dirname(dir));
+  await fs.mkdir(dir);
+  stager.createdDirs.push(dir);
+}
+
+/**
+ * Reverse the staging: staged copies out, collided bytes back, created
+ * directories removed (deepest first — a check's own artifacts inside a
+ * created directory go with it, since that directory did not exist when the
+ * agent finished).
+ */
+async function unstage(fs, stager) {
+  for (const target of stager.staged) {
+    await fs.rm(target, { force: true });
+  }
+  for (const backup of stager.backups) {
+    await fs.writeFile(backup.target, backup.bytes);
+  }
+  for (const dir of [...stager.createdDirs].reverse()) {
+    await fs.rm(dir, { recursive: true, force: true });
+  }
+}

package/src/benchmark/invariants.js

@@ -1,7 +1,10 @@
 /**
  * Invariants — runs `<task.paths.hooks>/invariants.sh` from the template path
- * against the post-run agent CWD. The exit code is authoritative for the
- * verdict; structured per-check rows arrive on fd 3 (`$RESULTS_FD=3`) as NDJSON.
+ * against the post-run agent CWD. A pure collector with no verdict of its
+ * own: structured per-check rows arrive on fd 3 (`$RESULTS_FD=3`) as NDJSON
+ * and grading happens downstream over the merged rows. The exit code is
+ * script health only — nonzero means the grader itself failed, never that a
+ * check failed.
  *
  * Subprocess access flows through `runtime.subprocess.spawn`; the fd-3 backing
  * store and the stderr log use the sync filesystem surface (`runtime.fsSync`) —
@@ -14,9 +17,9 @@ import { buildHookEnv } from "./hook-env.js";
 
 /**
  * @typedef {object} InvariantsResult
- * @property {"pass" | "fail"} verdict
  * @property {Array<object>} details
- * @property {number} exitCode
+ * @property {number} exitCode - Script health: nonzero means the hook itself
+ *   failed, never that a check failed.
  * @property {string} [stderr] - Trimmed script stderr, present only when the
  *   script wrote to stderr. Surfaces hook failures (e.g. a missing tool) that
  *   leave `details` empty, so they read distinctly from a real invariant miss.
@@ -32,7 +35,7 @@ import { buildHookEnv } from "./hook-env.js";
 export async function runInvariants(task, ctx, runtime) {
   if (!runtime) throw new Error("runtime is required");
   if (!task.paths.invariants) {
-    return { verdict: "pass", details: [], exitCode: 0 };
+    return { details: [], exitCode: 0 };
   }
   const fsSync = runtime.fsSync;
   const script = task.paths.invariants;
@@ -84,11 +87,7 @@ export async function runInvariants(task, ctx, runtime) {
   const details = [];
   parseFd3Buffer(raw, details);
   const exitCode = typeof code === "number" ? code : -1;
-  const result = {
-    verdict: exitCode === 0 ? "pass" : "fail",
-    details,
-    exitCode,
-  };
+  const result = { details, exitCode };
   const trimmedStderr = stderr.trim();
   if (trimmedStderr) result.stderr = trimmedStderr;
   return result;

package/src/benchmark/judge.js

@@ -9,7 +9,7 @@
  *   {{AGENT_INSTRUCTIONS}}  — contents of agent.task.md
  *   {{AGENT_PROFILE}}       — agent profile body (empty string if none)
  *   {{AGENT_TRACE_PATH}}    — path to agent.ndjson
- *   {{INVARIANTS_RESULT}}   — JSON invariants object
+ *   {{GRADE_RESULT}}        — JSON grade object plus the merged check rows
  *   {{SKILL_SET_HASH}}      — SHA-256 from apm.lock.yaml
  *   {{TASK_ID}}             — task name (directory under tasks/)
  *   {{TASK_DIR}}            — agent working directory path
@@ -40,22 +40,25 @@ import { sumTraceCost } from "../cost.js";
  */
 
 /**
- * Run the judge over a completed task run.
+ * Run the judge over a completed task run. The judge is a binary gate over
+ * the grade's validity, never a grade itself: `gradeResult` reaches the
+ * template as evidence, and the verdict stays pass/fail.
  * @param {import("./task-family.js").Task} task
  * @param {import("./workdir.js").Workdir} workdir
- * @param {import("./invariants.js").InvariantsResult} invariants
+ * @param {{verdict: string, gatesPass: boolean, score?: number, malformed?: number, rows: unknown[]}} gradeResult -
+ *   The normalized grade plus the merged, source-stamped check rows.
  * @param {{query: Function, model: string, judgeProfile?: string, profilesDir?: string, runtime: import("@forwardimpact/libutil/runtime").Runtime}} deps
  * @param {JudgeContext} [context]
  * @returns {Promise<JudgeVerdict>}
  */
-export async function runJudge(task, workdir, invariants, deps, context) {
+export async function runJudge(task, workdir, gradeResult, deps, context) {
   const runtime = deps.runtime;
   if (!runtime) throw new Error("runtime is required");
   const fs = runtime.fs;
   const template = await fs.readFile(task.paths.judge, "utf8");
-  const invariantsJson = JSON.stringify(invariants, null, 2);
+  const gradeJson = JSON.stringify(gradeResult, null, 2);
   const taskText = template
-    .replaceAll("{{INVARIANTS_RESULT}}", invariantsJson)
+    .replaceAll("{{GRADE_RESULT}}", gradeJson)
     .replaceAll("{{AGENT_TRACE_PATH}}", workdir.agentTracePath)
     .replaceAll("{{AGENT_INSTRUCTIONS}}", context?.agentInstructions ?? "")
     .replaceAll("{{AGENT_PROFILE}}", context?.agentProfile ?? "")