@forwardimpact/libharness 1.3.0 → 2.0.0

package/src/benchmark/result.js

@@ -3,10 +3,14 @@
  *
  * Two schemas live here:
  *   - RESULT_RECORD_SCHEMA — one record per (task, runIndex) from a full
- *     benchmark run. Has a happy branch (invariants + judge present) and a
- *     pre-flight-failure branch (invariants/judgeVerdict/submission absent).
- *   - INVARIANTS_RECORD_SCHEMA — narrower output of `benchmark-invariants`:
- *     ad-hoc grading without a full lifecycle.
+ *     benchmark run. Has a happy branch (grade + collectors + judge present)
+ *     and a pre-flight-failure branch (grade/judgeVerdict/submission absent).
+ *   - GRADE_RECORD_SCHEMA — narrower output of `benchmark-grade`: ad-hoc
+ *     grading without a full lifecycle.
+ *
+ * The check rows are the authoritative grading channel: the happy branch
+ * requires a `grade` object, so a pre-break record fails validation rather
+ * than rendering under semantics it never carried.
  *
  * Validation is throw-on-mismatch so the runner can wrap every JSONL append
  * in a guard and reject schema drift at write time.
@@ -17,12 +21,27 @@ import { z } from "zod";
 const VERDICT_ENUM = z.enum(["pass", "fail"]);
 
 const INVARIANTS_SHAPE = z.object({
-  verdict: VERDICT_ENUM,
   details: z.array(z.unknown()),
   exitCode: z.number().int(),
   stderr: z.string().optional(),
 });
 
+/**
+ * The normalized grading projection: `score` appears only on scored tasks,
+ * `malformed` only when at least one row was malformed.
+ */
+const GRADE_SHAPE = z.object({
+  verdict: VERDICT_ENUM,
+  gatesPass: z.boolean(),
+  score: z.number().min(0).max(1).optional(),
+  malformed: z.number().int().min(1).optional(),
+});
+
+const HIDDEN_TESTS_SHAPE = z.object({
+  details: z.array(z.unknown()),
+  error: z.string().optional(),
+});
+
 const JUDGE_VERDICT_SHAPE = z.object({
   verdict: VERDICT_ENUM,
   summary: z.string(),
@@ -77,6 +96,11 @@ const AGENT_ERROR_SHAPE = z.object({
 const HAPPY_RECORD = z.object({
   ...COMMON_FIELDS,
   invariants: INVARIANTS_SHAPE,
+  grade: GRADE_SHAPE,
+  hiddenTests: HIDDEN_TESTS_SHAPE.optional(),
+  // The effective, judge-zeroed score `report` aggregates — present only on
+  // scored tasks.
+  score: z.number().min(0).max(1).optional(),
   submission: z.string(),
   judgeVerdict: JUDGE_VERDICT_SHAPE.optional(),
   agentTracePath: z.string(),
@@ -97,6 +121,9 @@ const PREFLIGHT_RECORD = z.object({
   supervisorTracePath: z.string(),
   judgeTracePath: z.string(),
   invariants: z.undefined().optional(),
+  grade: z.undefined().optional(),
+  hiddenTests: z.undefined().optional(),
+  score: z.undefined().optional(),
   submission: z.undefined().optional(),
   judgeVerdict: z.undefined().optional(),
   agentError: z.undefined().optional(),
@@ -104,9 +131,17 @@ const PREFLIGHT_RECORD = z.object({
 
 export const RESULT_RECORD_SCHEMA = z.union([HAPPY_RECORD, PREFLIGHT_RECORD]);
 
-export const INVARIANTS_RECORD_SCHEMA = z.object({
+export const GRADE_RECORD_SCHEMA = z.object({
   taskId: z.string().min(1),
+  // Unlike the happy result record — where `grade.score` is the raw
+  // weighted fraction and the effective (zeroed) value lives on the
+  // top-level `score` — this record has no second score field, so its
+  // `grade.score` carries the effective health/gate-zeroed value.
+  grade: GRADE_SHAPE,
   invariants: INVARIANTS_SHAPE,
+  hiddenTests: HIDDEN_TESTS_SHAPE.optional(),
+  // Mirrors the invariants script's exit for diagnosis; the graded verdict
+  // is what drives the command's process exit.
   exitCode: z.number().int(),
 });
 
@@ -122,6 +157,6 @@ export function validateResultRecord(record) {
  * Throw on schema mismatch.
  * @param {object} record
  */
-export function validateInvariantsRecord(record) {
-  INVARIANTS_RECORD_SCHEMA.parse(record);
+export function validateGradeRecord(record) {
+  GRADE_RECORD_SCHEMA.parse(record);
 }

package/src/benchmark/runner.js

@@ -4,8 +4,10 @@
  * Phases per (task, runIndex):
  *   1. WorkdirManager.start → seed CWD + run pre-flight probe
  *   2. Supervisor session (agent + supervisor) → produce traces + submission
- *   3. Invariants.runInvariants → exit-code-driven verdict via fd-3 NDJSON
- *   4. Judge.runJudge → Conclude-driven verdict mapped to pass/fail
+ *   3. Invariants collector + hidden-test engine → merged check rows,
+ *      graded by `gradeChecks` (rows are authoritative; script exit is
+ *      grader health only)
+ *   4. Judge.runJudge → Conclude-driven binary gate mapped to pass/fail
  *   5. WorkdirManager.teardown → process-group cleanup
  *
  * Cells run with bounded in-process concurrency (`CellScheduler`); `run()`
@@ -17,7 +19,6 @@
  * stream.
  */
 
-import { createInterface } from "node:readline";
 import { join, resolve as resolvePath } from "node:path";
 
 import { DEFAULT_ENV_ALLOWLIST, createRedactor } from "../redaction.js";
@@ -28,7 +29,10 @@ import { installNpm as defaultInstallNpm } from "./npm-installer.js";
 import { runJudge } from "./judge.js";
 import { validateResultRecord } from "./result.js";
 import { runInvariants } from "./invariants.js";
+import { runHiddenTests } from "./hidden-tests.js";
+import { runProducersAndGrade } from "./grade.js";
 import { assertJudgeProfileStaged, loadTaskFamily } from "./task-family.js";
+import { splitAndSummarize } from "./trace-split.js";
 import { createWorkdirManager } from "./workdir.js";
 import { CellScheduler } from "./scheduler.js";
 
@@ -82,9 +86,13 @@ export class BenchmarkRunner {
    *   threaded into the installers, workdir manager, invariants, and judge.
    * @param {Function} [opts.runInvariants] - Test seam: replaces `runInvariants`.
    *   Same contract as `runInvariants(task, ctx, runtime)`. Internal testing only.
+   * @param {Function} [opts.runHiddenTests] - Test seam: replaces
+   *   `runHiddenTests`. Same contract as `runHiddenTests(task, ctx, runtime)`.
+   *   Internal testing only.
    * @param {Function} [opts.runJudge] - Test seam: replaces `runJudge`. Same
-   *   contract as `runJudge(task, workdir, invariants, deps)` (deps carries
-   *   `runtime`). Internal testing only.
+   *   contract as `runJudge(task, workdir, gradeResult, deps)` where
+   *   `gradeResult` is the normalized grade plus the merged, source-stamped
+   *   check `rows` (deps carries `runtime`). Internal testing only.
    * @param {Function} [opts.installApm] - Test seam: replaces `installApm`.
    *   Same contract as `installApm(family, outputDir, runtime)`. Lets tests
    *   inject a fake subprocess (or skip the install entirely) so the suite
@@ -114,6 +122,7 @@ export class BenchmarkRunner {
     // Test seams — default to the real implementations.
     runAgent,
     runInvariants: runInvariantsHook,
+    runHiddenTests: runHiddenTestsHook,
     runJudge: runJudgeHook,
     installApm: installApmHook,
     installNpm: installNpmHook,
@@ -141,6 +150,7 @@ export class BenchmarkRunner {
     this.termGraceMs = termGraceMs;
     this._runAgentHook = runAgent ?? null;
     this._runInvariantsHook = runInvariantsHook ?? runInvariants;
+    this._runHiddenTestsHook = runHiddenTestsHook ?? runHiddenTests;
     this._runJudgeHook = runJudgeHook ?? runJudge;
     this._installApmHook = installApmHook ?? defaultInstallApm;
     this._installNpmHook = installNpmHook ?? defaultInstallNpm;
@@ -291,55 +301,37 @@ export class BenchmarkRunner {
       const agentRun = await this.#runAgentSafe(task, workdir);
       const { costUsd, turns, submission, agentError } = agentRun;
       const breakdown = agentRun.costBreakdown ?? { agent: 0, supervisor: 0 };
-      const invariants = await this._runInvariantsHook(
+      const graded = await this.#gradeCell(family, task, workdir);
+      const { invariants, hiddenRows, engineError, rows, grade } = graded;
+      const { judgeVerdict, judgeCost } = await this.#judgeCell({
         task,
-        {
-          cwd: workdir.cwd,
-          port: workdir.port,
-          runDir: workdir.runDir,
-          familyDir: family.rootPath,
-        },
-        this.runtime,
-      );
-      let judgeVerdict = null;
-      let judgeCost = 0;
-      if (task.paths.judge) {
-        const judgeContext = await this.#buildJudgeContext(
-          task,
-          workdir,
-          skillSetHash,
-        );
-        const judgeResult = await this._runJudgeHook(
-          task,
-          workdir,
-          invariants,
-          {
-            query: this.query,
-            model: this.judgeModel,
-            judgeProfile: this.profiles.judge ?? undefined,
-            profilesDir: judgeProfilesDir,
-            runtime: this.runtime,
-          },
-          judgeContext,
-        );
-        judgeCost = judgeResult.costUsd ?? 0;
-        // The record's judgeVerdict carries only the verdict + summary; the
-        // judge's cost is folded into costUsd / costBreakdown instead.
-        judgeVerdict = {
-          verdict: judgeResult.verdict,
-          summary: judgeResult.summary,
-        };
-      }
-      const verdict =
-        invariants.verdict === "pass" &&
-        (judgeVerdict === null || judgeVerdict.verdict === "pass")
-          ? "pass"
-          : "fail";
+        workdir,
+        gradeResult: { ...grade, rows },
+        skillSetHash,
+        judgeProfilesDir,
+      });
+      const judgePass =
+        judgeVerdict === null || judgeVerdict.verdict === "pass";
+      const verdict = grade.verdict === "pass" && judgePass ? "pass" : "fail";
+      // Gates protect the score: an unhealthy grader, a failing gate row, or
+      // a failing judge zeroes the effective score. Full marks does not — a
+      // fractional score with verdict fail is the point.
+      const scoreValid = graded.healthy && grade.gatesPass && judgePass;
       const record = {
         taskId: task.id,
         runIndex,
         verdict,
         invariants,
+        grade,
+        ...(task.tests && {
+          hiddenTests: {
+            details: hiddenRows,
+            ...(engineError && { error: engineError.message }),
+          },
+        }),
+        ...(grade.score !== undefined && {
+          score: scoreValid ? grade.score : 0,
+        }),
         submission,
         ...(judgeVerdict && { judgeVerdict }),
         costUsd: costUsd + judgeCost,
@@ -371,6 +363,65 @@ export class BenchmarkRunner {
     }
   }
 
+  /**
+   * Run the judge (when the task ships a template) over the grade result.
+   * The record's judgeVerdict carries only the verdict + summary; the
+   * judge's cost is folded into costUsd / costBreakdown instead.
+   */
+  async #judgeCell({
+    task,
+    workdir,
+    gradeResult,
+    skillSetHash,
+    judgeProfilesDir,
+  }) {
+    if (!task.paths.judge) return { judgeVerdict: null, judgeCost: 0 };
+    const judgeContext = await this.#buildJudgeContext(
+      task,
+      workdir,
+      skillSetHash,
+    );
+    const judgeResult = await this._runJudgeHook(
+      task,
+      workdir,
+      gradeResult,
+      {
+        query: this.query,
+        model: this.judgeModel,
+        judgeProfile: this.profiles.judge ?? undefined,
+        profilesDir: judgeProfilesDir,
+        runtime: this.runtime,
+      },
+      judgeContext,
+    );
+    return {
+      judgeVerdict: {
+        verdict: judgeResult.verdict,
+        summary: judgeResult.summary,
+      },
+      judgeCost: judgeResult.costUsd ?? 0,
+    };
+  }
+
+  /**
+   * Run both check-row producers against the post-run CWD and grade the
+   * merged rows via the shared derivation. Restoration happens inside the
+   * engine, so the judge (which runs after) sees the workdir exactly as the
+   * agent left it.
+   */
+  #gradeCell(family, task, workdir) {
+    const ctx = {
+      cwd: workdir.cwd,
+      port: workdir.port,
+      runDir: workdir.runDir,
+      familyDir: family.rootPath,
+    };
+    return runProducersAndGrade(task, ctx, this.runtime, {
+      runInvariants: this._runInvariantsHook,
+      runHiddenTests: this._runHiddenTestsHook,
+    });
+  }
+
   /**
    * Dispatch to either the injected hook or the default `#runAgent`. Either
    * path can throw; catch here so a thrown error becomes an `agentError` on
@@ -614,67 +665,6 @@ async function writeRecord(stream, record) {
   });
 }
 
-/**
- * Split the combined supervisor trace into agent and supervisor files and
- * extract turn count and submission in a single pass. Agent-source events go
- * to `agentPath`; supervisor and orchestrator events go to `supervisorPath`.
- *
- * Cost is deliberately not summed here — the caller derives it from the same
- * combined trace via `sumTraceCost`, so there is one cost path across the
- * benchmark, callback, and `fit-trace cost` consumers.
- */
-// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: stream-splitting state machine
-async function splitAndSummarize(
-  runtime,
-  combinedPath,
-  agentPath,
-  supervisorPath,
-) {
-  const fs = runtime.fs;
-  const agentStream = fs.createWriteStream(agentPath);
-  const supStream = fs.createWriteStream(supervisorPath);
-  const rl = createInterface({
-    input: fs.createReadStream(combinedPath),
-    crlfDelay: Infinity,
-  });
-  let turns = 0;
-  let submission = "";
-  for await (const line of rl) {
-    if (!line.trim()) continue;
-    let event;
-    try {
-      event = JSON.parse(line);
-    } catch {
-      continue;
-    }
-    const target = event.source === "agent" ? agentStream : supStream;
-    target.write(line + "\n");
-    const inner = event.event;
-    if (!inner) continue;
-    if (event.source === "agent" && inner.type === "assistant") {
-      const text = extractText(inner);
-      if (text) submission = text;
-    }
-    if (event.source === "orchestrator" && inner.type === "summary") {
-      turns = inner.turns ?? 0;
-    }
-  }
-  await Promise.all([
-    new Promise((r) => agentStream.end(r)),
-    new Promise((r) => supStream.end(r)),
-  ]);
-  return { turns, submission };
-}
-
-function extractText(inner) {
-  const content = inner.message?.content ?? inner.content;
-  if (!Array.isArray(content)) return null;
-  for (let i = content.length - 1; i >= 0; i--) {
-    if (content[i].type === "text" && content[i].text) return content[i].text;
-  }
-  return null;
-}
-
 /**
  * Factory function — wires real dependencies.
  * @param {ConstructorParameters<typeof BenchmarkRunner>[0]} opts

package/src/benchmark/task-family.js

@@ -10,9 +10,16 @@
  *       hooks/                # harness-only; never copied to agent CWD
  *         preflight.sh
  *         invariants.sh
+ *       tests/                # optional hidden test suite; harness-only overlay
  *       specs/                # copied into agent CWD
  *       workdir/              # copied into agent CWD
  *
+ * `tests/` is an overlay mirror of the agent CWD: a file's path under
+ * `tests/` is its staging path. Every `*.test.js` file is one check —
+ * `*.gate.test.js` marks a gate, any other `*.test.js` is scored — and every
+ * other file is support material, staged but never graded. The layout is
+ * validated eagerly here so authoring errors fail before any agent spend.
+ *
  * Local paths or git URLs are both accepted; git URLs are shallow-cloned into
  * a temp dir and `familyRevision` becomes `git:<sha>` of HEAD at clone time.
  * Local paths use the canonical-tree algorithm from design § Family revision
@@ -113,36 +120,124 @@ async function discoverTasks(runtime, rootPath) {
   }
   for (const entry of entries) {
     if (!entry.isDirectory()) continue;
-    const taskDir = join(tasksRoot, entry.name);
-    const supervisorPath = join(taskDir, "supervisor.task.md");
-    const judgePath = join(taskDir, "judge.task.md");
-    const preflightPath = join(taskDir, "hooks", "preflight.sh");
-    const invariantsPath = join(taskDir, "hooks", "invariants.sh");
-    tasks.push({
-      id: entry.name,
-      paths: {
-        taskDir,
-        instructions: join(taskDir, "agent.task.md"),
-        supervisor: (await fileExists(fs, supervisorPath))
-          ? supervisorPath
-          : null,
-        judge: (await fileExists(fs, judgePath)) ? judgePath : null,
-        hooks: join(taskDir, "hooks"),
-        preflight: (await fileExecutable(fs, preflightPath))
-          ? preflightPath
-          : null,
-        invariants: (await fileExecutable(fs, invariantsPath))
-          ? invariantsPath
-          : null,
-        specs: join(taskDir, "specs"),
-        workdir: join(taskDir, "workdir"),
-      },
-    });
+    tasks.push(await loadTask(fs, join(tasksRoot, entry.name), entry.name));
   }
   tasks.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
   return tasks;
 }
 
+async function loadTask(fs, taskDir, id) {
+  const supervisorPath = join(taskDir, "supervisor.task.md");
+  const judgePath = join(taskDir, "judge.task.md");
+  const preflightPath = join(taskDir, "hooks", "preflight.sh");
+  const invariantsPath = join(taskDir, "hooks", "invariants.sh");
+  const suite = await discoverSuite(fs, taskDir);
+  return {
+    id,
+    paths: {
+      taskDir,
+      instructions: join(taskDir, "agent.task.md"),
+      supervisor: (await fileExists(fs, supervisorPath))
+        ? supervisorPath
+        : null,
+      judge: (await fileExists(fs, judgePath)) ? judgePath : null,
+      hooks: join(taskDir, "hooks"),
+      preflight: (await fileExecutable(fs, preflightPath))
+        ? preflightPath
+        : null,
+      invariants: (await fileExecutable(fs, invariantsPath))
+        ? invariantsPath
+        : null,
+      tests: suite ? join(taskDir, "tests") : null,
+      specs: join(taskDir, "specs"),
+      workdir: join(taskDir, "workdir"),
+    },
+    tests: suite,
+  };
+}
+
+const CHECK_SUFFIX = ".test.js";
+const GATE_SUFFIX = ".gate.test.js";
+
+/**
+ * Discover and validate a task's hidden test suite under `<taskDir>/tests/`.
+ * Returns null when the directory is absent; throws on an invalid layout
+ * (no check files, a dangling symlink, duplicate check names) so
+ * `loadTaskFamily` rejects broken suites before any agent spend.
+ * @param {object} fs - Async filesystem surface (`runtime.fs`).
+ * @param {string} taskDir
+ * @returns {Promise<HiddenSuite | null>}
+ */
+async function discoverSuite(fs, taskDir) {
+  const testsRoot = join(taskDir, "tests");
+  try {
+    const st = await fs.lstat(testsRoot);
+    if (!st.isDirectory()) return null;
+  } catch {
+    return null;
+  }
+  const files = [];
+  await walkSuiteFiles(fs, testsRoot, testsRoot, files);
+  files.sort((a, b) =>
+    a.stagePath < b.stagePath ? -1 : a.stagePath > b.stagePath ? 1 : 0,
+  );
+
+  const checks = [];
+  const support = [];
+  for (const file of files) {
+    const base = file.stagePath.split(sep).at(-1);
+    if (!base.endsWith(CHECK_SUFFIX)) {
+      support.push(file);
+      continue;
+    }
+    const gate = base.endsWith(GATE_SUFFIX);
+    const name = base.slice(0, -(gate ? GATE_SUFFIX : CHECK_SUFFIX).length);
+    checks.push({ name, gate, ...file });
+  }
+
+  if (checks.length === 0) {
+    throw new Error(`hidden test suite has no check files: ${testsRoot}`);
+  }
+  const seen = new Set();
+  for (const check of checks) {
+    if (seen.has(check.name)) {
+      throw new Error(
+        `hidden test suite has duplicate check name '${check.name}': ${check.sourcePath}`,
+      );
+    }
+    seen.add(check.name);
+  }
+  return { checks, support };
+}
+
+/**
+ * Walk a suite tree collecting `{sourcePath, stagePath}` entries. Unlike
+ * `walkFiles` (which silently skips dangling symlinks for hashing), every
+ * entry here must be a regular file after symlink resolution — a dangling
+ * symlink or a link to a non-file target is an authoring error.
+ */
+async function walkSuiteFiles(fs, root, dir, out) {
+  const entries = await fs.readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      await walkSuiteFiles(fs, root, full, out);
+    } else if (entry.isFile()) {
+      out.push({ sourcePath: full, stagePath: relative(root, full) });
+    } else if (entry.isSymbolicLink()) {
+      const resolved = await resolveSymlinkToFile(fs, full);
+      if (!resolved) {
+        throw new Error(
+          `hidden test suite entry is not a regular file: ${full}`,
+        );
+      }
+      out.push({ sourcePath: full, stagePath: relative(root, full) });
+    } else {
+      throw new Error(`hidden test suite entry is not a regular file: ${full}`);
+    }
+  }
+}
+
 async function fileExists(fs, path) {
   try {
     await fs.access(path);
@@ -246,10 +341,28 @@ async function git(runtime, args) {
   return stdout;
 }
 
+/**
+ * @typedef {object} HiddenCheck
+ * @property {string} name - Basename stem with the check suffix stripped.
+ * @property {boolean} gate - True iff the filename ends `.gate.test.js`.
+ * @property {string} sourcePath - Absolute path under `tests/`.
+ * @property {string} stagePath - Path relative to `tests/` — the overlay
+ *   mirror of the staging path under the agent CWD.
+ */
+
+/**
+ * @typedef {object} HiddenSuite
+ * @property {HiddenCheck[]} checks - In sorted stage-path order.
+ * @property {{sourcePath: string, stagePath: string}[]} support - Non-check
+ *   files, staged for the whole pass but never graded.
+ */
+
 /**
  * @typedef {object} Task
  * @property {string} id - Task name (directory name under tasks/)
- * @property {{taskDir: string, instructions: string, supervisor: string|null, judge: string|null, hooks: string, preflight: string|null, invariants: string|null, specs: string, workdir: string}} paths
+ * @property {{taskDir: string, instructions: string, supervisor: string|null, judge: string|null, hooks: string, preflight: string|null, invariants: string|null, tests: string|null, specs: string, workdir: string}} paths
+ * @property {HiddenSuite | null} tests - Hidden test suite (null when the
+ *   task ships no `tests/` directory)
  */
 
 /**

package/src/benchmark/trace-split.js

@@ -0,0 +1,73 @@
+/**
+ * Combined-supervisor-trace splitting for the benchmark runner: one pass
+ * over the tagged NDJSON envelope stream separates agent events from
+ * supervisor/orchestrator events and extracts the run summary.
+ */
+
+import { createInterface } from "node:readline";
+
+/**
+ * Split the combined supervisor trace into agent and supervisor files and
+ * extract turn count and submission in a single pass. Agent-source events go
+ * to `agentPath`; supervisor and orchestrator events go to `supervisorPath`.
+ *
+ * Cost is deliberately not summed here — the caller derives it from the same
+ * combined trace via `sumTraceCost`, so there is one cost path across the
+ * benchmark, callback, and `fit-trace cost` consumers.
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @param {string} combinedPath
+ * @param {string} agentPath
+ * @param {string} supervisorPath
+ * @returns {Promise<{turns: number, submission: string}>}
+ */
+// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: stream-splitting state machine
+export async function splitAndSummarize(
+  runtime,
+  combinedPath,
+  agentPath,
+  supervisorPath,
+) {
+  const fs = runtime.fs;
+  const agentStream = fs.createWriteStream(agentPath);
+  const supStream = fs.createWriteStream(supervisorPath);
+  const rl = createInterface({
+    input: fs.createReadStream(combinedPath),
+    crlfDelay: Infinity,
+  });
+  let turns = 0;
+  let submission = "";
+  for await (const line of rl) {
+    if (!line.trim()) continue;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    const target = event.source === "agent" ? agentStream : supStream;
+    target.write(line + "\n");
+    const inner = event.event;
+    if (!inner) continue;
+    if (event.source === "agent" && inner.type === "assistant") {
+      const text = extractText(inner);
+      if (text) submission = text;
+    }
+    if (event.source === "orchestrator" && inner.type === "summary") {
+      turns = inner.turns ?? 0;
+    }
+  }
+  await Promise.all([
+    new Promise((r) => agentStream.end(r)),
+    new Promise((r) => supStream.end(r)),
+  ]);
+  return { turns, submission };
+}
+
+function extractText(inner) {
+  const content = inner.message?.content ?? inner.content;
+  if (!Array.isArray(content)) return null;
+  for (let i = content.length - 1; i >= 0; i--) {
+    if (content[i].type === "text" && content[i].text) return content[i].text;
+  }
+  return null;
+}

package/src/benchmark/workdir.js

@@ -268,7 +268,12 @@ async function runPreflight(runtime, script, cwd, port, vars) {
   };
 }
 
-function probeFreePort() {
+/**
+ * Allocate a free TCP port by binding to 0 and releasing it. Shared with the
+ * `grade` subcommand, which needs a plausible `$PORT` for the hook env.
+ * @returns {Promise<number>}
+ */
+export function probeFreePort() {
   return new Promise((res, rej) => {
     const server = createServer();
     server.unref();

package/src/commands/advisor-flags.js

@@ -0,0 +1,28 @@
+/**
+ * Shared advisor-flag parsing for the four session-mode commands. The two
+ * flags are identical everywhere: `--advisor-model` (no default — absent
+ * means the Advisor tool is not offered) and `--advisor-max-uses`
+ * (default 3), which is a usage error without the model flag.
+ */
+
+/**
+ * Parse `--advisor-model` / `--advisor-max-uses` from parsed option values.
+ * A malformed max-uses is a usage error, not a silent fallback: NaN would
+ * make the budget check (`used >= maxUses`) permanently false and disable
+ * the code-enforced cap the flag exists to guarantee.
+ * @param {object} values - Parsed option values from cli.parse()
+ * @returns {{advisorModel: string|undefined, advisorMaxUses: number}}
+ */
+export function parseAdvisorOptions(values) {
+  if (values["advisor-max-uses"] && !values["advisor-model"]) {
+    throw new Error("--advisor-max-uses requires --advisor-model");
+  }
+  const advisorMaxUses = parseInt(values["advisor-max-uses"] || "3", 10);
+  if (Number.isNaN(advisorMaxUses) || advisorMaxUses < 1) {
+    throw new Error("--advisor-max-uses must be a positive integer");
+  }
+  return {
+    advisorModel: values["advisor-model"] || undefined,
+    advisorMaxUses,
+  };
+}