@gethmy/harness 1.2.0 → 1.3.0

package/src/oracle.ts

@@ -64,11 +64,25 @@
  * Failing the gate rather than writing is the point.
  */
 import type { ChildProcess } from "node:child_process";
-import { lstat, mkdir, realpath, rm, writeFile } from "node:fs/promises";
-import { dirname, isAbsolute, resolve, sep } from "node:path";
+import { lstatSync, readFileSync, statSync } from "node:fs";
+import {
+  chmod,
+  lstat,
+  mkdir,
+  mkdtemp,
+  realpath,
+  rm,
+  writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, isAbsolute, join, resolve, sep } from "node:path";
+import { StringDecoder } from "node:string_decoder";
 import { DEFAULT_METRIC_TIMEOUT_MS } from "./exec-types.js";
+import { log } from "./log.js";
 import { reapGroup, spawnInGroup, terminateGroup } from "./process-group.js";
 
+const TAG = "oracle";
+
 export interface HeldOracle {
   /**
    * The `stage_oracles` row id, when the API returns one (card #927 — older
@@ -90,6 +104,24 @@ export interface OracleDeps {
    * into `fetchOracle`'s third argument.
    */
   sessionId: string;
+  /**
+   * The stage the held oracle is KEYED TO — the downstream stage whose gate is
+   * `oracle_passed`, as resolved by `findOracleTargetStage`. Needed only by the
+   * `oracle_red` gate (#921) and `null` when there is no such stage.
+   *
+   * A green gate does not use this: the stage that runs `oracle_passed` IS the
+   * target, so `context.stageId` already addresses the row. A RED gate is the
+   * author's own stage, one stage EARLIER, and the row is keyed to the target
+   * rather than to the writer (see the `/stage-oracle` route: "The row is keyed
+   * to the TARGET"). So the red collector must ask for the target's id, or it
+   * would look up an oracle under its own stage id and find nothing.
+   *
+   * Keying stays single: ONE held test, written once by the author, read by both
+   * gates. That is also why no edge-side authorization change was needed — both
+   * reads name a target stage that declares `oracle_passed`, which is exactly
+   * what `mayReadOracle` requires.
+   */
+  targetStageId?: string | null;
   /** Reads via POST /stage-oracle/fetch with purpose "gate_evaluation". */
   fetchOracle(
     cardId: string,
@@ -107,7 +139,23 @@ export interface OracleDeps {
   run(
     repoPath: string,
     oracle: HeldOracle,
-  ): Promise<{ exitCode: number; output: string }>;
+  ): Promise<{
+    exitCode: number;
+    /** stdout + stderr merged, for the operator's log. Never the verdict. */
+    output: string;
+    /**
+     * The runner's OWN structured report of what it executed — the only thing
+     * the `oracle_red` gate grades (#1019). `null` ⇒ no report this recognizes,
+     * which the red gate must treat as `blocked`, never as a pass.
+     *
+     * Read from a file the runner writes OUTSIDE the worktree, never from the
+     * child's stdout or stderr. See {@link OracleReportSpec} for why no
+     * in-band channel can carry this: the held test shares stdout and stderr
+     * with the runner, and the drain window after the leader exits is exactly
+     * where a detached grandchild gets to write the last word.
+     */
+    report: OracleRunSummary | null;
+  }>;
 }
 
 /**
@@ -215,14 +263,287 @@ export async function remove(
  * already have: an absent runner must fail the gate, not fetch a package onto
  * the motor's host mid-stage.
  */
-const ORACLE_RUNNERS: Record<string, (path: string) => OracleRunnerArgv> = {
-  vitest: (path) => ({
-    command: "npx",
-    args: ["--no-install", "vitest", "run", path],
-  }),
-  bun: (path) => ({ command: "bun", args: ["test", path] }),
+const ORACLE_RUNNERS: Record<string, OracleRunnerSpec> = {
+  vitest: {
+    argv: (path) => ({
+      command: "npx",
+      args: ["--no-install", "vitest", "run", path],
+    }),
+    // Measured: vitest prints its human summary block on STDOUT. Diagnostic
+    // only since #1019 — nothing grades this stream.
+    verdictStream: "stdout",
+    report: {
+      file: "report.json",
+      // `--reporter=default` is listed FIRST and is not decoration: measured
+      // against vitest 4.1.11, `--reporter=json` ALONE replaces the default
+      // reporter, so stdout loses the `Tests  N failed (N)` block entirely and
+      // the operator's log is left with one line saying a file was written.
+      // Naming both reporters keeps the human log AND gets the machine report.
+      flags: (reportPath) => [
+        "--reporter=default",
+        "--reporter=json",
+        `--outputFile=${reportPath}`,
+      ],
+      // Measured against vitest 4.1.11:
+      //   genuine mixed failure  → numTotalTests 3, numFailedTests 2
+      //   clean pass             → numTotalTests N, numFailedTests 0
+      //   syntax/collect error   → numTotalTests 0  (`Tests  no tests`)
+      //   file with no tests     → numTotalTests 0
+      //   path matched nothing   → numTotalTests 0
+      //   runner absent          → NO FILE AT ALL, so `null` before this runs
+      // Every one of the zero-total cases exits non-zero and is a "could not
+      // run", which `gradeOracleRed` maps to `no_verdict` on `total === 0`.
+      parse: (content) => {
+        const parsed: unknown = JSON.parse(content);
+        if (typeof parsed !== "object" || parsed === null) return null;
+        const { numTotalTests, numFailedTests } = parsed as Record<
+          string,
+          unknown
+        >;
+        // Both counts must be present and finite. A report missing either is a
+        // shape we do not know, and `null` (⇒ blocked) is the safe reading —
+        // never "assume it ran".
+        if (
+          typeof numTotalTests !== "number" ||
+          typeof numFailedTests !== "number" ||
+          !Number.isFinite(numTotalTests) ||
+          !Number.isFinite(numFailedTests)
+        ) {
+          return null;
+        }
+        return { total: numTotalTests, failed: numFailedTests };
+      },
+    },
+  },
+  bun: {
+    argv: (path) => ({ command: "bun", args: ["test", path] }),
+    // Measured: bun test prints ` N fail` and `Ran N tests…` on STDERR, not
+    // stdout (stdout carries only its 28-byte version banner). Diagnostic only
+    // since #1019 — nothing grades this stream.
+    verdictStream: "stderr",
+    report: {
+      file: "report.xml",
+      // #921 rejected `--reporter=junit` because bun refuses it without
+      // `--reporter-outfile`, and a report file INSIDE the worktree is one the
+      // completion path could commit — handing the implementer the held test's
+      // shape. #1019 answers that objection rather than living with it: the
+      // outfile goes to a temp dir outside the worktree (see
+      // `runHeldOracle`), so there is nothing in the tree to commit. Measured:
+      // bun keeps printing its human stderr summary with this flag on.
+      flags: (reportPath) => [
+        "--reporter=junit",
+        `--reporter-outfile=${reportPath}`,
+      ],
+      // Measured against bun 1.3.13. The root element carries the run-wide
+      // counts:
+      //   `<testsuites name="bun test" tests="2" assertions="2" failures="1" …>`
+      // A thrown non-assertion error counts under `failures` too (measured),
+      // and the XML carries NO assertion text — only `type="AssertionError"`.
+      // An unmatched path and a file with no tests both write NO FILE at all
+      // (measured), so those reach `gradeOracleRed` as `null` ⇒ `no_verdict`
+      // without this parser running.
+      parse: (content) => {
+        // The ROOT `<testsuites>` element only — never a per-file
+        // `<testsuite>`. Anchoring on the run-wide totals means a multi-file
+        // report cannot be read as just its first file's counts.
+        const root = /<testsuites\b[^>]*>/.exec(content);
+        if (!root) return null;
+        const total = /\btests="(\d+)"/.exec(root[0]);
+        const failed = /\bfailures="(\d+)"/.exec(root[0]);
+        if (!total || !failed) return null;
+        return { total: Number(total[1]), failed: Number(failed[1]) };
+      },
+    },
+  },
 };
 
+/**
+ * WHY THE VERDICT IS NOT READ FROM stdout/stderr AT ALL (#1019).
+ *
+ * Until #1019 the red gate parsed the runner's human summary out of the child's
+ * verdict stream, taking the LAST match on the reasoning that both runners print
+ * their summary last. Three forgeries were closed in that order, and the third
+ * could not be:
+ *
+ *   1. **First-match forgery** (#921). A held test that PASSES while printing
+ *      `Tests  7 failed (7)` was measured reading as 7 failures. Closed by
+ *      reading the last match, since anything the test prints precedes the
+ *      runner's own summary. This was the accidental case too — an honest test
+ *      that merely logs.
+ *   2. **Cross-pipe forgery** (PR #721's security review, MEDIUM
+ *      `authorization_bypass`). stdout and stderr are appended by two
+ *      independent `data` handlers with no ordering guarantee BETWEEN the pipes,
+ *      so a forged line on the stream the runner does not use could land after
+ *      the real summary. Closed by reading one measured stream per runner.
+ *   3. **The detached grandchild** — not closable in-band, which is why the
+ *      channel moved. `runHeldOracle` waits {@link ORACLE_DRAIN_GRACE_MS} after
+ *      the leader exits so a slow pipe can still deliver legitimate trailing
+ *      output, and `reapGroup` addresses the group by number, so it cannot
+ *      reach a grandchild that called `setsid` to leave it. A held test that
+ *      spawns a detached writer inheriting the runner's verdict fd therefore
+ *      gets to emit into that same stream AFTER the runner's real summary:
+ *
+ *          spawn('sh', ['-c', 'sleep 0.1; echo "Ran 9 tests across 9 files."'],
+ *                { detached: true, stdio: 'inherit' });
+ *
+ *      Read as `{ total: 9, failed: 9 }` ⇒ `reproduced`, on a run where nothing
+ *      failed. The same test passes the green gate on exit 0, so red-green is
+ *      satisfied end to end by a contract that proves nothing.
+ *
+ * Every in-band tightening trades the hole for a worse property, which is what
+ * makes this a channel problem rather than a parser problem:
+ *
+ *   - **Snapshot the buffer at leader exit.** The drain window exists BECAUSE
+ *     the summary may not have been delivered yet, so this trades an
+ *     adversarial hole for a flaky gate. It is also defeatable: a sprayer that
+ *     writes continuously lands a line in the window between the runner's own
+ *     summary and its exit, which is inside any such snapshot.
+ *   - **Kill the group harder first.** `setsid` is precisely what puts the
+ *     grandchild out of reach — there is nothing left to signal.
+ *   - **Require the summary to be the final line, or preceded by the runner's
+ *     banner.** Legitimate trailing output exists (coverage tables, teardown
+ *     warnings), so ordinary runs would report `blocked` — and a sprayer can
+ *     emit a banner too.
+ *
+ * So the graded channel is a FILE the runner writes, in a temp dir outside the
+ * worktree (see {@link OracleReportSpec} and `runHeldOracle`). The streams are
+ * still captured for the operator's log, and nothing parses them.
+ *
+ * WHAT THAT DOES AND DOES NOT BUY, measured rather than assumed. It closes the
+ * three stream forgeries above outright: nothing written to stdout or stderr
+ * changes a verdict any more, so the accidental case, the first-match case and
+ * the detached-grandchild case are all gone.
+ *
+ * It does NOT make the verdict unforgeable, and nothing here should be read as
+ * saying so. The held test is arbitrary code running as the motor's own user, on
+ * the same host, so it can write the report file itself:
+ *
+ *   - The directory is NOT secret. `mkdtemp` randomizes the suffix, but the
+ *     prefix is a fixed string and `readdirSync(os.tmpdir())` enumerates the
+ *     directory in one call — measured from inside a real held test, under both
+ *     runners. 0700 does not hide it either; the held test is the owner.
+ *   - The forgery does NOT need to win a race. A held test can write its own
+ *     report and then drop write permission (on the file, or on the directory)
+ *     so the runner's later write cannot land. Measured: a trivially GREEN bun
+ *     held test doing this produced `{ total: 1, failed: 1 }` and exit 0 — a
+ *     `reproduced` red gate AND a passing green gate from one file.
+ *
+ * {@link assertUntampered} refuses that one shape, because both ways of making
+ * it stick change a mode the motor set. It does NOT close the class, and the
+ * cheapest bypass needs no mode change and no race at all: the held test
+ * pre-writes the report and then stops the runner from writing its own.
+ * `process.exit(0)` in the test body is enough — measured, 3 runs of 3 under
+ * bun, giving `{ total: 1, failed: 1 }` with exit 0, which is `reproduced` from
+ * the red gate AND a pass from the green one. Under vitest the same trick yields
+ * the same forged counts. `oracle-run.test.ts` pins that this is still open, so
+ * the description cannot quietly drift back to "closed".
+ *
+ * Two other reasons not to read the check as a boundary: `statSync` follows
+ * symlinks, so REPLACING the directory (rmdir + mkdir 0700, or a symlink) leaves
+ * every mode the motor set intact; and a mode change is restorable by a
+ * grandchild that waits for the runner to exit and chmods back.
+ *
+ * So the accurate summary, and what the docs say: this gate is evidence against
+ * a CARELESS author, not against a hostile one, and the mode check raises the
+ * cost of one attack rather than establishing a property. Anything short of not
+ * running the held test as the motor's own user is a mitigation — the sandbox
+ * `repair-sandbox.ts` (#1015) already uses for untrusted code, with the report
+ * read from outside it, is the actual fix. Tracked at card #1021; see
+ * `docs/agent-daemon.md`.
+ */
+
+/**
+ * What a runner reported about the tests it actually executed. This exists for
+ * the `oracle_red` gate (#921) and for nothing else: `oracle_passed` needs only
+ * the exit code, because a green gate that misreads "could not run" as `failed`
+ * merely under-grants, which is safe.
+ *
+ * A red gate has no such luxury. It passes on FAILURE, so reading "could not
+ * run" as a failure would over-grant — and the three cases are indistinguishable
+ * by exit code alone. All of these exit 1, measured against the real runners:
+ *
+ *   - a genuine assertion failure          (the signal the gate wants)
+ *   - `npx --no-install vitest`, absent    (the runner never started)
+ *   - `bun test ./unmatched-path`          (nothing was collected)
+ *
+ * So the red gate requires the runner's own count of executed and failed tests,
+ * and reports `blocked` whenever it cannot read one.
+ */
+export interface OracleRunSummary {
+  /** Tests the runner reported having EXECUTED. */
+  total: number;
+  /** Of those, how many the runner reported as FAILING. */
+  failed: number;
+}
+
+/**
+ * How one allow-listed runner writes a MACHINE report, and how to read it.
+ *
+ * This is the red gate's only graded channel (#1019). The point is not that the
+ * format is structured — it is that the held test does not share the channel.
+ * A file the runner writes outside the worktree is not something an inherited
+ * stdio fd reaches, so the detached-grandchild forgery
+ * that no in-band parser could refuse has nowhere to land. The long comment
+ * above `OracleRunnerSpec` has the full account.
+ *
+ * `flags` must keep the runner's HUMAN output intact as well: the operator's log
+ * is how a person understands a blocked gate, and a reporter flag that replaces
+ * the default reporter (measured: vitest's `--reporter=json` does exactly that)
+ * would silently empty it.
+ */
+interface OracleReportSpec {
+  /**
+   * The report's filename inside the dir `runHeldOracle` creates. A plain fixed
+   * name on purpose: the directory's random suffix is not a secret either (the
+   * held test can enumerate it — see `runHeldOracle`), so a random filename
+   * would buy nothing and only make the operator's log harder to read.
+   */
+  file: string;
+  /** Extra argv elements that make the runner write its report to `reportPath`. */
+  flags(reportPath: string): string[];
+  /**
+   * Read `{ total, failed }` out of the runner's own report, or `null` when the
+   * content carries no report this recognizes.
+   *
+   * `null` is load-bearing and must stay the DEFAULT for anything unfamiliar:
+   * the red gate maps it to `blocked`, so a runner that changes its report
+   * format degrades to "we could not tell" rather than to a false red. Never
+   * widen one of these to "assume it ran" — that inverts the safe direction.
+   *
+   * May throw (a JSON parse error is the obvious one); `runHeldOracle` catches
+   * and treats a throw exactly as `null`.
+   *
+   * The formats are pinned by tests in `oracle-red.test.ts` against captured
+   * real-runner reports, so drift fails a test instead of going quiet — and by
+   * an end-to-end test against the real runners in `oracle-run.test.ts`, so a
+   * runner that stops honoring the flag fails there rather than leaving the gate
+   * permanently `blocked` in production.
+   */
+  parse(content: string): OracleRunSummary | null;
+}
+
+/** One allow-listed runner: the argv to run it, and how to read its report. */
+interface OracleRunnerSpec {
+  argv(path: string): OracleRunnerArgv;
+  /**
+   * WHICH stream this runner prints its human summary on.
+   *
+   * DIAGNOSTIC ONLY since #1019 — nothing grades it. `runHeldOracle` keeps this
+   * stream in a buffer of its own so the operator's log can show the runner's
+   * own summary apart from whatever the held test printed, and so the
+   * end-to-end tests can pin which fd each runner uses. A verdict is never
+   * parsed out of it: see the comment above `OracleRunnerSpec` for why no
+   * in-band channel can carry one.
+   *
+   * Measured per runner, because they differ and the difference is not
+   * guessable: **vitest** prints `Tests  N failed (N)` on **stdout**, while
+   * **bun** prints ` N fail` / `Ran N tests…` on **stderr**.
+   */
+  verdictStream: "stdout" | "stderr";
+  /** The machine report: the gate's ONLY graded channel. */
+  report: OracleReportSpec;
+}
+
 /** The hints {@link resolveOracleRunner} accepts, for the refusal message. */
 export const ORACLE_RUNNER_HINTS: readonly string[] =
   Object.keys(ORACLE_RUNNERS).sort();
@@ -240,6 +561,14 @@ export interface OracleRunnerArgv {
  * way `runMetricCommand` must (there, stdout IS the measurement, so a partial
  * read would corrupt the number). Memory stays bounded either way, and the
  * timeout still bounds the process.
+ *
+ * Truncation keeps the TAIL, and that outlived its original reason. #921 made
+ * it security-critical (the red gate parsed the runner's summary block, printed
+ * LAST, out of this very buffer, so trimming the end destroyed the gate's only
+ * signal on a verbose held test). #1019 moved the verdict to a report file, so
+ * the tail is no longer load-bearing for GRADING — but it stays, because the
+ * runner's summary is still the most useful thing in an operator's log and the
+ * head of a verbose run is the least. See `append` in {@link runHeldOracle}.
  */
 const ORACLE_OUTPUT_LIMIT = 64 * 1024;
 
@@ -252,7 +581,13 @@ const ORACLE_SIGTERM_GRACE_MS = 3_000;
 /** How long to wait after the leader exits for its stdio pipes to deliver the
  * tail of the output. Bounded for the same reason `command-metric.ts` bounds
  * its own: a grandchild that left the group with `setsid()` holds the pipe open
- * forever, and a run that already produced its exit code must not wait on it. */
+ * forever, and a run that already produced its exit code must not wait on it.
+ *
+ * This window used to be the red gate's exposure (#1019): a detached grandchild
+ * could write a forged summary into the verdict stream during it and win the
+ * parse. Since the verdict comes from a report file read AT the `exit` event,
+ * anything appended during this window reaches the operator's log and nothing
+ * else — so the window can stay as generous as legitimate late output needs. */
 const ORACLE_DRAIN_GRACE_MS = 500;
 
 /**
@@ -276,20 +611,128 @@ const ORACLE_DRAIN_GRACE_MS = 500;
  * `failed` instead, so this follows the outcome it named, not the mechanism.
  */
 export function resolveOracleRunner(oracle: HeldOracle): OracleRunnerArgv {
+  return resolveOracleRunnerSpec(oracle).argv(argvPath(oracle.path));
+}
+
+/**
+ * The whole allow-list entry for a held test's `runnerHint` — argv AND summary
+ * reader. Throws on an unknown or absent hint, exactly as
+ * {@link resolveOracleRunner} does (they share this function), so the refusal
+ * message and the `blocked` outcome are identical for both gate polarities.
+ */
+function resolveOracleRunnerSpec(oracle: HeldOracle): OracleRunnerSpec {
   const hint = oracle.runnerHint?.trim().toLowerCase() ?? "";
   // `Object.hasOwn`, never a bare lookup: an inherited key ("constructor",
   // "toString") would otherwise resolve to a function and be called.
-  const build = Object.hasOwn(ORACLE_RUNNERS, hint)
+  const spec = Object.hasOwn(ORACLE_RUNNERS, hint)
     ? ORACLE_RUNNERS[hint]
     : undefined;
-  if (!build) {
+  if (!spec) {
     throw new Error(
       `refusing to run the held test: runner_hint ${JSON.stringify(
         oracle.runnerHint,
       )} is not in the motor's allow-list (${ORACLE_RUNNER_HINTS.join(", ")})`,
     );
   }
-  return build(argvPath(oracle.path));
+  return spec;
+}
+
+/**
+ * Read the runner's own test counts out of a machine report's CONTENT, through
+ * the same allow-list that chose the argv. `null` ⇒ no report this recognizes,
+ * which the red gate must treat as `blocked` (never as a pass). A parser that
+ * throws (malformed JSON, most obviously) is `null` too — the safe reading of
+ * "we could not tell".
+ *
+ * EXPORTED AS A TEST SEAM, and has no production caller by design: production
+ * reads the report through `runHeldOracle`, which owns the file's lifetime.
+ * The export exists so `oracle-red.test.ts` can pin each runner's report format
+ * against captured real output without going near the filesystem, which is the
+ * only thing that makes a format drift fail loudly instead of degrading the gate
+ * to a silent `blocked`. Nothing else should be handed report content: it is
+ * secret-bearing for one of the two runners (vitest's JSON carries
+ * `failureMessages`, measured — see `captureReport`).
+ *
+ * The green gate never calls this: `oracle_passed` decides on the exit code
+ * alone, and widening it to consult a report would be a behavior change to a
+ * security-reviewed gate for no gain — a green gate that misreads "could not
+ * run" as `failed` under-grants, which is already the safe direction.
+ */
+export function summarizeOracleReport(
+  oracle: HeldOracle,
+  content: string,
+): OracleRunSummary | null {
+  // Resolved OUTSIDE the try, so an unknown `runner_hint` still THROWS. The two
+  // failures are not the same thing and must not collapse: an unrecognized
+  // report is "we could not tell" (`null` ⇒ blocked), while an unknown hint is a
+  // defect in the oracle's own declaration that `OracleCollector` reports as
+  // `blocked` with the refusal message attached. Swallowing the throw here
+  // would lose the message that names the hint and the allow-list.
+  const spec = resolveOracleRunnerSpec(oracle);
+  try {
+    return spec.report.parse(content) ?? null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * The three outcomes a RED reading of a held-test run can have. Pure, and
+ * separated from the collector so the decision table is testable on its own.
+ *
+ * `no_verdict` is the outcome that makes the gate honest: it is the answer
+ * whenever the run cannot be shown to have executed the held test and found it
+ * failing. It maps to `blocked`, which can never satisfy a gate.
+ */
+export type OracleRedOutcome =
+  /** The held test ran and reported at least one failing test — the bug is reproduced. */
+  | { outcome: "reproduced"; summary: OracleRunSummary }
+  /** The held test ran and nothing failed — this does not reproduce the bug. */
+  | { outcome: "not_reproduced"; summary: OracleRunSummary }
+  /** The run produced no trustworthy red/green answer at all. */
+  | { outcome: "no_verdict"; reason: string };
+
+/**
+ * Grade a completed held-test run RED.
+ *
+ * The order of these checks is the safety property, not a style choice. Every
+ * "could not run" case exits non-zero exactly like a genuine assertion failure
+ * (see {@link OracleRunSummary}), so the exit code is consulted LAST and only
+ * ever to reject — never to grant. A pass requires a positive count of failing
+ * tests from the runner's own summary.
+ */
+export function gradeOracleRed(
+  summary: OracleRunSummary | null,
+  exitCode: number,
+): OracleRedOutcome {
+  if (!summary) {
+    return {
+      outcome: "no_verdict",
+      reason:
+        `the runner exited ${exitCode} but the motor has no report it could read — absent, unreadable, or refused, ` +
+        "so it cannot be shown to have run the held test (an absent runner and a failing test share this exit code). " +
+        "The report is a file the runner writes outside the worktree; the motor's local log says which of the three it was",
+    };
+  }
+  if (summary.total === 0) {
+    return {
+      outcome: "no_verdict",
+      reason: `the runner started but executed no tests (exit ${exitCode}), so the held test produced no verdict`,
+    };
+  }
+  if (summary.failed > 0) {
+    return { outcome: "reproduced", summary };
+  }
+  if (exitCode === 0) {
+    return { outcome: "not_reproduced", summary };
+  }
+  // Every test the runner ran passed, yet it still failed overall — a coverage
+  // threshold, a post-run hook, an unhandled rejection. That is not a
+  // reproduction of the card's bug, and it is not a clean green either.
+  return {
+    outcome: "no_verdict",
+    reason: `all ${summary.total} test(s) passed but the runner exited ${exitCode}, so the failure is not the held test's`,
+  };
 }
 
 /**
@@ -334,108 +777,413 @@ export async function runHeldOracle(
   repoPath: string,
   oracle: HeldOracle,
   timeoutMs: number = DEFAULT_METRIC_TIMEOUT_MS,
-): Promise<{ exitCode: number; output: string }> {
-  const { command, args } = resolveOracleRunner(oracle);
+): Promise<{
+  exitCode: number;
+  output: string;
+  verdict: string;
+  report: OracleRunSummary | null;
+}> {
+  // Resolve the runner BEFORE creating anything: an unknown hint must reject
+  // without leaving a temp directory behind (and without spawning, which the
+  // "rejects an unknown hint before it spawns anything" test pins).
+  const spec = resolveOracleRunnerSpec(oracle);
+
+  // The graded channel (#1019). OUTSIDE the worktree, because a report file
+  // inside the tree is one the completion path could commit — handing the
+  // implementer the held test's shape, the exact hazard `removeBestEffort`
+  // exists to prevent, and the objection that made #921 avoid a report file at
+  // all.
+  //
+  // `mkdtemp` randomizes the SUFFIX and creates the directory 0700. Neither
+  // hides the path from the held test: the prefix is a fixed string and
+  // `readdirSync(tmpdir())` enumerates the directory in one call, 0700 included
+  // (the held test runs as the motor's own user, who owns it). This was measured,
+  // not assumed. So the randomness is not a secret and nothing here should be
+  // read as one — see `assertUntampered` for what actually defends the file, and
+  // the comment above `OracleRunnerSpec` for what remains open.
+  const reportDir = await mkdtemp(join(tmpdir(), reportDirPrefix()));
+  const reportPath = join(reportDir, spec.report.file);
+  try {
+    return await spawnHeldOracle(spec, repoPath, oracle, reportPath, timeoutMs);
+  } finally {
+    await removeReportDir(reportDir);
+  }
+}
+
+/**
+ * The report directory's fixed name prefix. Deliberately a constant and not a
+ * secret (see `runHeldOracle`); it exists so an operator can recognize a
+ * stranded directory as this module's, and so {@link removeReportDir} logs
+ * something identifiable.
+ */
+function reportDirPrefix(): string {
+  return "harmony-oracle-report-";
+}
+
+/**
+ * Refuse a report whose file or directory shows that something other than the
+ * runner has been at it. THROWS on any doubt; `captureReport` turns a throw into
+ * `null` ⇒ `blocked`, which can never satisfy a gate.
+ *
+ * What this catches, and it is worth being exact because the rest is open: a
+ * held test can find this directory (the prefix is fixed and `readdir` lists
+ * it), so the cheapest forgery is to write the report ITSELF and then remove
+ * write permission — from the file, or from the directory — so the runner's own
+ * write cannot land on top of it. That variant is deterministic and needs no
+ * race. Both halves of it change a mode the motor set, so both are visible here:
+ *
+ *   - the directory must still be exactly 0700, as `mkdtemp` created it
+ *   - the report must be a REGULAR file (never a symlink, which would redirect
+ *     the read somewhere the motor did not create)
+ *   - the report must still be writable by its owner — a 0444 report is one the
+ *     runner could not have overwritten, so it is not necessarily the runner's
+ *
+ * What this does NOT catch, stated so nobody reads it as more than it is:
+ *
+ *   - **Pre-write, then stop the runner writing.** No mode change, no race:
+ *     `process.exit(0)` in the held test skips the reporter's write entirely and
+ *     leaves the forgery as the only file on disk, at ordinary modes inside a
+ *     still-0700 directory. Measured under both runners. This is CHEAPER than
+ *     the attack above, so the check below is one shape of an open class.
+ *   - **Replacing the directory.** `statSync` follows symlinks, so an rmdir +
+ *     mkdir 0700, or a symlink to the attacker's own directory, changes no mode
+ *     the motor set.
+ *   - **Restoring the mode.** A grandchild that waits for the runner to exit and
+ *     chmods back defeats the file-mode arm.
+ *
+ * Mode checks raise the cost of one attack; they are not a boundary. The only
+ * boundary is not running the held test as the motor's user at all — see the
+ * comment above `OracleRunnerSpec` and card #1021.
+ *
+ * The exact-0700 requirement is deliberate but is a false-positive risk worth
+ * knowing about: `mkdtemp` asks for 0700 through `mkdir`, so an exotic umask
+ * that clears an owner bit, or a `TMPDIR` on a filesystem that does not carry
+ * POSIX modes, would refuse every report and block the gate permanently. Both
+ * runners were measured writing the report 0644 inside a 0700 directory under a
+ * normal umask. `captureReport` logs the refusal reason so that case is
+ * diagnosable rather than silent.
+ */
+function assertUntampered(reportPath: string): void {
+  const dir = statSync(dirname(reportPath));
+  // `& 0o777` drops the file-type bits; only the permission bits are the claim.
+  if ((dir.mode & 0o777) !== 0o700) {
+    throw new Error(
+      `the oracle report directory's mode changed to ${(dir.mode & 0o777).toString(8)} — refusing the report`,
+    );
+  }
+  // `lstat`, never `stat`: a symlink is exactly what a redirect would look like,
+  // and `stat` would follow it and report the target as a fine regular file.
+  const file = lstatSync(reportPath);
+  if (!file.isFile()) {
+    throw new Error("the oracle report is not a regular file — refusing it");
+  }
+  if ((file.mode & 0o200) === 0) {
+    throw new Error(
+      `the oracle report is not owner-writable (mode ${(file.mode & 0o777).toString(8)}), so the runner could not have written it last — refusing it`,
+    );
+  }
+}
 
-  return await new Promise<{ exitCode: number; output: string }>(
-    (settleOk, settleErr) => {
-      let child: ChildProcess;
+/**
+ * Delete the report directory. Attempted on EVERY path — completion, rejection,
+ * timeout — because the report is secret-bearing for one of the two runners:
+ * vitest's JSON carries `failureMessages` with the assertion text verbatim
+ * (measured), which is itself a partial oracle.
+ *
+ * BEST EFFORT, and said so out loud rather than in a silent `catch`. A held test
+ * that chmods the directory to remove write permission makes the removal fail,
+ * and a stranded directory then accumulates on a long-lived daemon host with the
+ * assertion text still in it. So this restores the mode first — the motor owns
+ * the directory, so it may — and WARNS when the removal still fails, giving the
+ * operator the path to clean up. It never throws: a directory we could not
+ * delete must not turn a completed gate into a blocked one.
+ */
+async function removeReportDir(reportDir: string): Promise<void> {
+  try {
+    // A held test may have chmod'ed this to make its forged report stick. We
+    // own it, so take the permission back before removing.
+    await chmod(reportDir, 0o700).catch(() => {});
+    await rm(reportDir, { recursive: true, force: true });
+  } catch (err) {
+    log.warn(
+      TAG,
+      `Could not remove the oracle report directory ${reportDir} (${
+        err instanceof Error ? err.message : String(err)
+      }) — it may still hold the runner's report, which carries assertion text for vitest. Remove it by hand.`,
+    );
+  }
+}
+
+/**
+ * The supervised spawn half of {@link runHeldOracle}, split out so the report
+ * directory's lifetime is a plain `try`/`finally` in one place rather than a
+ * cleanup duplicated down every settle path.
+ */
+async function spawnHeldOracle(
+  spec: OracleRunnerSpec,
+  repoPath: string,
+  oracle: HeldOracle,
+  reportPath: string,
+  timeoutMs: number,
+): Promise<{
+  exitCode: number;
+  output: string;
+  verdict: string;
+  report: OracleRunSummary | null;
+}> {
+  const { command, args: baseArgs } = spec.argv(argvPath(oracle.path));
+  // The report flags ride on BOTH gate polarities, not just the red one. #921
+  // kept the two polarities on one argv on purpose, so the security-reviewed
+  // allow-list stays a single shape per runner; making the report conditional
+  // would have split it. The green gate simply ignores the report it produces.
+  //
+  // The cost of that choice, so it is not a surprise: the green gate grades on
+  // the EXIT CODE alone, and a runner that cannot write its report may exit
+  // non-zero for that reason (measured: vitest does). `oracle_passed` would then
+  // report `failed` — blaming the implementer and spending an attempt — for a
+  // problem with the motor's temp dir rather than with the diff. That has not
+  // been seen outside a deliberately broken temp dir, and the alternative (two
+  // argv shapes) was judged worse, but it is the trade being made.
+  const args = [...baseArgs, ...spec.report.flags(reportPath)];
+
+  return await new Promise<{
+    exitCode: number;
+    output: string;
+    verdict: string;
+    report: OracleRunSummary | null;
+  }>((settleOk, settleErr) => {
+    let child: ChildProcess;
+    try {
+      child = spawnInGroup(command, args, {
+        cwd: repoPath,
+        stdio: ["ignore", "pipe", "pipe"],
+      });
+    } catch (err) {
+      // A synchronous spawn throw (unusable cwd, bad argv) — no group exists.
+      settleErr(err);
+      return;
+    }
+
+    // Record the pgid AT SPAWN: `reapGroup` addresses the group by number, so
+    // it still sweeps stragglers once the leader handle is spent.
+    const pgid = child.pid;
+    let output = "";
+    let truncated = false;
+    // ONE DECODER PER PIPE, not one shared: each pipe delivers its own byte
+    // stream, so a shared decoder would hand a partial character from stdout
+    // to the next stderr chunk and corrupt both.
+    const outDecoder = new StringDecoder("utf8");
+    const errDecoder = new StringDecoder("utf8");
+    /** Only the runner's own stream. Diagnostic since #1019; nothing parses it. */
+    let verdict = "";
+    /**
+     * The buffer as the caller sees it: the kept tail, labelled if trimmed.
+     *
+     * Flushes the decoder first, so bytes it is still holding for an
+     * incomplete character at end-of-stream are not silently dropped. Called
+     * once — `settle` is guarded by `settled` — and guarded here too, because
+     * `decoder.end()` is not idempotent.
+     */
+    let flushed = false;
+    const flush = (): void => {
+      if (flushed) return;
+      flushed = true;
+      const outTail = outDecoder.end();
+      const errTail = errDecoder.end();
+      output += outTail + errTail;
+      verdict += spec.verdictStream === "stdout" ? outTail : errTail;
+    };
+    const finalOutput = (): string => {
+      flush();
+      return truncated
+        ? `… earlier output dropped; kept the last ${ORACLE_OUTPUT_LIMIT} characters\n${output}`
+        : output;
+    };
+    /**
+     * The verdict stream, flushed. The operator's per-runner view of what the
+     * runner itself said; NOT parsed for a verdict (#1019).
+     */
+    const finalVerdict = (): string => {
+      flush();
+      return verdict;
+    };
+
+    /**
+     * The runner's machine report, captured AT THE `exit` EVENT.
+     *
+     * `readFileSync`, deliberately, in the `exit` handler — not an awaited
+     * async read. Both runners write this file and then exit, so at `exit` it is
+     * complete; reading it synchronously right there leaves no gap in which
+     * anything else could rewrite it. An async read would hand the held test's
+     * detached grandchild a scheduling window to overwrite the file, which is
+     * the same shape of race the drain window used to be. The file is a few KB
+     * and the run is already over, so blocking the loop here costs nothing.
+     *
+     * Every failure is `null`, which `gradeOracleRed` maps to `no_verdict` ⇒
+     * `blocked`: the runner never wrote a report (measured — bun writes none at
+     * all for an unmatched path or a file with no tests), the content is
+     * malformed, the parser does not recognize the shape, or the file shows
+     * signs of tampering. Never a pass.
+     */
+    let report: OracleRunSummary | null = null;
+    const captureReport = (): void => {
       try {
-        child = spawnInGroup(command, args, {
-          cwd: repoPath,
-          stdio: ["ignore", "pipe", "pipe"],
-        });
+        assertUntampered(reportPath);
+        report = spec.report.parse(readFileSync(reportPath, "utf8"));
       } catch (err) {
-        // A synchronous spawn throw (unusable cwd, bad argv) — no group exists.
-        settleErr(err);
-        return;
+        report = null;
+        // Never silent. This is the module's only tamper DETECTOR, and a
+        // detection that logs nothing is not a detection an operator can act
+        // on. It is also the one place a false positive can produce a
+        // permanently blocked gate (see `assertUntampered` on the exact-0700
+        // requirement), and the gate's own reason cannot name which case it
+        // was — so the message that distinguishes "no report at all" from
+        // "a report we refused" has to land here, in the motor's local log.
+        //
+        // ENOENT is the ordinary "runner wrote nothing" path (measured: bun
+        // writes no report for an unmatched path or a file with no tests), so
+        // it is info, not a warning — it is not evidence of anything wrong.
+        const message = err instanceof Error ? err.message : String(err);
+        const absent =
+          err instanceof Error &&
+          (err as NodeJS.ErrnoException).code === "ENOENT";
+        if (absent) {
+          log.info(TAG, `No oracle report at ${reportPath} — no verdict.`);
+        } else {
+          log.warn(
+            TAG,
+            `Refusing the oracle report at ${reportPath}: ${message} — the gate will report no verdict.`,
+          );
+        }
       }
+    };
+    let settled = false;
+    let killing = false;
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    let drainTimer: ReturnType<typeof setTimeout> | undefined;
 
-      // Record the pgid AT SPAWN: `reapGroup` addresses the group by number, so
-      // it still sweeps stragglers once the leader handle is spent.
-      const pgid = child.pid;
-      let output = "";
-      let settled = false;
-      let killing = false;
-      let timer: ReturnType<typeof setTimeout> | undefined;
-      let drainTimer: ReturnType<typeof setTimeout> | undefined;
-
-      /** Settle once, and never without sweeping the group. */
-      const settle = (
-        failure: Error | null,
-        result?: { exitCode: number; output: string },
-      ): void => {
-        if (settled) return;
-        settled = true;
-        if (timer) clearTimeout(timer);
-        if (drainTimer) clearTimeout(drainTimer);
-        reapGroup(pgid);
-        if (failure) settleErr(failure);
-        else settleOk(result!);
-      };
-
-      // stdout and stderr both land in one buffer: a test runner splits its
-      // report across them, and the collector only ever hands this to a human.
-      const append = (chunk: Buffer): void => {
-        if (output.length >= ORACLE_OUTPUT_LIMIT) return;
-        output += chunk.toString("utf8");
-        if (output.length > ORACLE_OUTPUT_LIMIT) {
-          output = `${output.slice(0, ORACLE_OUTPUT_LIMIT)}\n… output truncated at ${ORACLE_OUTPUT_LIMIT} characters`;
-        }
-      };
-      child.stdout?.on("data", append);
-      child.stderr?.on("data", append);
-
-      // A missing binary surfaces as an async `error` event (an async spawn
-      // never throws it) — no verdict, so it rejects.
-      child.once("error", (err) => settle(err));
-
-      const settleFromExit = (
-        code: number | null,
-        signal: NodeJS.Signals | null,
-      ): void => {
-        if (drainTimer) clearTimeout(drainTimer);
-        if (code === null) {
-          // A signal nobody here sent (an operator's `kill`, the OOM killer).
-          // The held test produced no verdict, so this is not a `failed` gate.
-          settle(new Error(`the held test was terminated by signal ${signal}`));
-          return;
+    /** Settle once, and never without sweeping the group. */
+    const settle = (
+      failure: Error | null,
+      result?: {
+        exitCode: number;
+        output: string;
+        verdict: string;
+        report: OracleRunSummary | null;
+      },
+    ): void => {
+      if (settled) return;
+      settled = true;
+      if (timer) clearTimeout(timer);
+      if (drainTimer) clearTimeout(drainTimer);
+      reapGroup(pgid);
+      if (failure) settleErr(failure);
+      else settleOk(result!);
+    };
+
+    // stdout and stderr both land in one buffer: a test runner splits its
+    // report across them.
+    //
+    // Truncation keeps the TAIL, not the head. Both runners print their
+    // summary block LAST, so the tail is the useful end of an operator's log
+    // and the head is the least useful. This used to be load-bearing for
+    // GRADING — the red gate parsed that block out of this buffer, and keeping
+    // the head instead left the gate permanently `blocked` on any verbose held
+    // test — but since #1019 the verdict comes from the report file, so a
+    // truncated buffer costs a reader context and costs the gate nothing.
+    //
+    // Re-slicing on every append keeps memory bounded to ~the cap without
+    // holding the whole stream, and the notice is composed at settle time
+    // rather than stored, so repeated trims cannot mangle it.
+    // Decoded through a StringDecoder, NOT `chunk.toString("utf8")` per
+    // chunk: a multi-byte sequence split across a chunk boundary decodes to
+    // U+FFFD that way. Now that nothing parses these buffers this is a
+    // legibility property rather than a correctness one — a replacement
+    // character in the operator's log is merely ugly — but it is still what
+    // keeps a UTF-8 test name readable. The decoder holds the partial bytes
+    // until the next chunk completes them.
+    const append = (stream: "stdout" | "stderr", chunk: Buffer): void => {
+      const text = (stream === "stdout" ? outDecoder : errDecoder).write(chunk);
+      output += text;
+      if (output.length > ORACLE_OUTPUT_LIMIT) {
+        output = output.slice(-ORACLE_OUTPUT_LIMIT);
+        truncated = true;
+      }
+      // The runner's own stream is ALSO kept unmerged, so an operator can read
+      // what the RUNNER said apart from what the held test printed. Nothing
+      // grades it (see `verdictStream`); it exists for the log and for the
+      // end-to-end tests that pin which fd each runner uses.
+      if (stream === spec.verdictStream) {
+        verdict += text;
+        if (verdict.length > ORACLE_OUTPUT_LIMIT) {
+          verdict = verdict.slice(-ORACLE_OUTPUT_LIMIT);
         }
-        settle(null, { exitCode: code, output });
-      };
-
-      child.once("exit", (code, signal) => {
-        if (killing) return; // the timeout path owns this outcome
-        // The timeout governs the RUN, which is over. Disarm it so a slow pipe
-        // drain cannot turn a finished run into a reported timeout.
-        if (timer) clearTimeout(timer);
-        // Reap before waiting on the pipes: a backgrounded grandchild inherits
-        // the stdout pipe, so `close` cannot fire while it lives.
-        reapGroup(pgid);
-        drainTimer = setTimeout(
-          () => settleFromExit(code, signal),
-          ORACLE_DRAIN_GRACE_MS,
-        );
-        child.once("close", () => settleFromExit(code, signal));
+      }
+    };
+    child.stdout?.on("data", (chunk: Buffer) => append("stdout", chunk));
+    child.stderr?.on("data", (chunk: Buffer) => append("stderr", chunk));
+
+    // A missing binary surfaces as an async `error` event (an async spawn
+    // never throws it) — no verdict, so it rejects.
+    child.once("error", (err) => settle(err));
+
+    const settleFromExit = (
+      code: number | null,
+      signal: NodeJS.Signals | null,
+    ): void => {
+      if (drainTimer) clearTimeout(drainTimer);
+      if (code === null) {
+        // A signal nobody here sent (an operator's `kill`, the OOM killer).
+        // The held test produced no verdict, so this is not a `failed` gate.
+        settle(new Error(`the held test was terminated by signal ${signal}`));
+        return;
+      }
+      settle(null, {
+        exitCode: code,
+        output: finalOutput(),
+        verdict: finalVerdict(),
+        // Captured at `exit`, never re-read here: `settleFromExit` runs once
+        // the pipes drain, up to ORACLE_DRAIN_GRACE_MS later, and re-reading
+        // then would reopen the very window the at-exit capture closes.
+        report,
       });
+    };
 
-      timer = setTimeout(() => {
-        if (settled) return;
-        killing = true;
-        terminateGroup(child, {
-          sigintTimeoutMs: ORACLE_SIGINT_GRACE_MS,
-          sigtermTimeoutMs: ORACLE_SIGTERM_GRACE_MS,
+    child.once("exit", (code, signal) => {
+      if (killing) return; // the timeout path owns this outcome
+      // FIRST, before anything that yields: the runner has written its report
+      // and exited, so this is the moment the file is both complete and not yet
+      // reachable by a straggler. See `captureReport`.
+      captureReport();
+      // The timeout governs the RUN, which is over. Disarm it so a slow pipe
+      // drain cannot turn a finished run into a reported timeout.
+      if (timer) clearTimeout(timer);
+      // Reap before waiting on the pipes: a backgrounded grandchild inherits
+      // the stdout pipe, so `close` cannot fire while it lives.
+      reapGroup(pgid);
+      drainTimer = setTimeout(
+        () => settleFromExit(code, signal),
+        ORACLE_DRAIN_GRACE_MS,
+      );
+      child.once("close", () => settleFromExit(code, signal));
+    });
+
+    timer = setTimeout(() => {
+      if (settled) return;
+      killing = true;
+      terminateGroup(child, {
+        sigintTimeoutMs: ORACLE_SIGINT_GRACE_MS,
+        sigtermTimeoutMs: ORACLE_SIGTERM_GRACE_MS,
+      })
+        .catch(() => {
+          // terminateGroup swallows its own signal errors; guard anyway so a
+          // rejection can never strand the promise unsettled.
         })
-          .catch(() => {
-            // terminateGroup swallows its own signal errors; guard anyway so a
-            // rejection can never strand the promise unsettled.
-          })
-          .then(() => {
-            settle(
-              new Error(`the held test did not finish within ${timeoutMs}ms`),
-            );
-          });
-      }, timeoutMs);
-    },
-  );
+        .then(() => {
+          settle(
+            new Error(`the held test did not finish within ${timeoutMs}ms`),
+          );
+        });
+    }, timeoutMs);
+  });
 }