@gethmy/harness 1.2.1 → 1.3.0

package/src/oracle-collector.ts

@@ -1,7 +1,17 @@
 /**
- * Evidence collector for the `oracle_passed` gate. It runs only AFTER the stage's
- * subagent has exited (see stage-run.ts): the held test and a model must never
- * share a filesystem window.
+ * Evidence collectors for the two held-test gates — `oracle_passed` (green: the
+ * held test must pass) and `oracle_red` (#921, red: it must run and fail). Both
+ * run only AFTER the stage's subagent has exited (see stage-run.ts): the held
+ * test and a model must never share a filesystem window.
+ *
+ * ONE held test serves both. The author stage writes it keyed to the downstream
+ * stage whose gate is `oracle_passed`; the red gate reads that same row one stage
+ * earlier (`OracleDeps.targetStageId`). So a `Fix a Bug` chain is red-green over a
+ * single contract: the author's test must fail before the fix and pass after it,
+ * and the implementer never sees it in either direction.
+ *
+ * The two polarities are NOT mirror images, and `HeldOracleCollector`'s doc
+ * comment explains where they diverge and why the divergence is one-directional.
  *
  * It produces evidence and nothing else — `gateEvaluate` in @harmony/shared turns
  * it into a verdict. A missing oracle is `blocked`, never `passed`: a gate with no
@@ -27,28 +37,89 @@ import type {
   GateEvidenceCollector,
   GateEvidenceContext,
 } from "@harmony/shared";
+import { GATE_CONFIG_ERROR_MARK } from "./gate-config-error.js";
 import { log } from "./log.js";
-import type { HeldOracle, OracleDeps } from "./oracle.js";
+import {
+  gradeOracleRed,
+  type HeldOracle,
+  type OracleDeps,
+  type OracleRunSummary,
+} from "./oracle.js";
 
 const TAG = "oracle-collector";
 
-export class OracleCollector implements GateEvidenceCollector {
-  readonly kind = "oracle_passed" as const;
+/** The digest identifying the exact contract a verdict graded (card #927). */
+interface OracleIdentity {
+  oracleId: string | null;
+  contentHash: string;
+}
+
+/**
+ * The shared half of both held-test gates: fetch the oracle, place it, run it,
+ * remove it — always. Only the two things that genuinely differ per polarity are
+ * abstract:
+ *
+ *   - {@link oracleStageId} — WHICH stage the oracle row is keyed to. The green
+ *     gate's own stage IS the target; the red gate's is one stage earlier.
+ *   - {@link verdict} — how a completed run is graded.
+ *
+ * Everything else is deliberately NOT duplicated. `removeBestEffort` and the
+ * `finally` that calls it are the security-critical part of this module (a held
+ * test left in the worktree gets auto-committed and hands the implementer the
+ * one file it must never see), so there is exactly one copy of them and both
+ * gates inherit it. A second polarity that quietly forgot the `finally` is the
+ * failure this shape exists to make impossible.
+ */
+abstract class HeldOracleCollector {
+  constructor(protected readonly deps: OracleDeps) {}
 
-  constructor(private readonly deps: OracleDeps) {}
+  /** Which stage's held oracle this gate grades. `null` ⇒ nothing to grade. */
+  protected abstract oracleStageId(context: GateEvidenceContext): string | null;
+
+  /** Grade a run that produced an exit code. Never sees a run that threw. */
+  protected abstract verdict(args: {
+    oracle: HeldOracle;
+    identity: OracleIdentity;
+    exitCode: number;
+    /** stdout + stderr merged — the operator's log. NOT the verdict channel. */
+    output: string;
+    /**
+     * The runner's own machine report, read from a file outside the worktree
+     * (#1019) — the red gate's only graded input. `null` ⇒ no trustworthy
+     * count, which that gate turns into `blocked`.
+     */
+    report: OracleRunSummary | null;
+  }): GateEvidence;
 
   async collect(context: GateEvidenceContext): Promise<GateEvidence> {
+    const stageId = this.oracleStageId(context);
+    if (!stageId) {
+      // Reached only by a red gate on a stage with no downstream
+      // `oracle_passed` stage — a playbook shape defect, not a fact about the
+      // card's code. So it carries the #823 config-error mark: re-running the
+      // stage cannot change the answer, and the engine should hold for a human
+      // instead of spending the card's whole attempt budget rediscovering it.
+      const reason =
+        `No stage downstream of ${context.stageId} declares an \`oracle_passed\` gate, ` +
+        "so there is no held test for this gate to grade. A red gate needs a later stage that runs the same test green.";
+      log.warn(TAG, `${reason} — blocked (config)`);
+      return {
+        result: "blocked",
+        structured: { reason, ...GATE_CONFIG_ERROR_MARK },
+      };
+    }
+
     const oracle = await this.deps.fetchOracle(
       context.cardId,
-      context.stageId,
+      stageId,
       this.deps.sessionId,
     );
     if (!oracle) {
-      log.info(TAG, `No oracle held for stage ${context.stageId} — blocked`);
+      log.info(TAG, `No oracle held for stage ${stageId} — blocked`);
       return {
         result: "blocked",
         structured: {
-          reason: `No oracle is held for stage ${context.stageId}.`,
+          reason: `No oracle is held for stage ${stageId}.`,
         },
       };
     }
@@ -68,7 +139,7 @@ export class OracleCollector implements GateEvidenceCollector {
     };
     await this.deps.place(this.deps.repoPath, oracle);
     try {
-      const { exitCode, output } = await this.deps.run(
+      const { exitCode, output, report } = await this.deps.run(
         this.deps.repoPath,
         oracle,
       );
@@ -81,18 +152,13 @@ export class OracleCollector implements GateEvidenceCollector {
       } else {
         log.warn(TAG, logLine);
       }
-      return {
-        result: exitCode === 0 ? "passed" : "failed",
-        structured: {
-          oracle: {
-            exitCode,
-            path: oracle.path,
-            ...identity,
-            output:
-              "withheld — oracle_passed is a secrecy gate; see the motor's local log",
-          },
-        },
-      };
+      return this.verdict({
+        oracle,
+        identity,
+        exitCode,
+        output,
+        report: report ?? null,
+      });
     } catch (err) {
       const message = errText(err);
       log.warn(TAG, `Oracle run threw: ${message} — blocked`);
@@ -157,6 +223,161 @@ export class OracleCollector implements GateEvidenceCollector {
   }
 }
 
+/**
+ * The GREEN gate: `oracle_passed`. The held test must pass against the
+ * implementer's worktree.
+ *
+ * Grades on the exit code alone, unchanged since Task 12. That is safe here
+ * precisely because it is the conservative direction: a run that could not
+ * start exits non-zero and reads as `failed`, which under-grants. Do not
+ * "improve" this to consult the runner's summary — the red gate needs that
+ * because its safe direction is the opposite one, and giving both gates the
+ * same machinery would trade a proven behavior for a symmetry nobody needs.
+ */
+export class OracleCollector
+  extends HeldOracleCollector
+  implements GateEvidenceCollector
+{
+  readonly kind = "oracle_passed" as const;
+
+  /** The stage running this gate IS the stage the oracle is keyed to. */
+  protected oracleStageId(context: GateEvidenceContext): string | null {
+    return context.stageId;
+  }
+
+  protected verdict({
+    oracle,
+    identity,
+    exitCode,
+  }: {
+    oracle: HeldOracle;
+    identity: OracleIdentity;
+    exitCode: number;
+    output: string;
+    report: OracleRunSummary | null;
+  }): GateEvidence {
+    return {
+      result: exitCode === 0 ? "passed" : "failed",
+      structured: {
+        oracle: {
+          exitCode,
+          path: oracle.path,
+          ...identity,
+          output:
+            "withheld — oracle_passed is a secrecy gate; see the motor's local log",
+        },
+      },
+    };
+  }
+}
+
+/**
+ * The RED gate: `oracle_red` (#921). The same held test must RUN and FAIL
+ * against the current tree — the reproduction half of red-green.
+ *
+ * Why this is not `oracle_passed` with the verdict flipped: a red gate passes on
+ * failure, so anything it misreads as a failure it OVER-grants. And the exit code
+ * cannot tell a real failure from a runner that never started — measured against
+ * the real allow-listed runners, `npx --no-install vitest` with vitest absent,
+ * `bun test` on an unmatched path, and a genuine assertion failure all exit 1.
+ *
+ * So a pass here requires a positive count of failing tests from the runner's own
+ * machine report, and every case where that cannot be read is `blocked` with a
+ * reason naming which case it was. `blocked` can never satisfy a gate, so the
+ * gate is unsatisfiable exactly when it cannot be trusted.
+ *
+ * That report is read from a FILE the runner writes outside the worktree (#1019),
+ * never from the child's stdout or stderr. The held test shares those streams
+ * with the runner, and the drain window after the leader exits is precisely where
+ * a detached grandchild gets the last word — so no in-band parser could refuse
+ * the forgery.
+ *
+ * HOW FAR THAT GOES: it closes the stream forgeries outright, and it does NOT
+ * make the verdict unforgeable. The held test runs as the motor's own user and
+ * can write the report file itself — the directory is findable by a fixed name
+ * prefix, and pre-writing the report then dropping write permission needs no
+ * race at all. `runHeldOracle`'s mode check refuses that shape; a writer that
+ * leaves the modes alone still wins. This gate is evidence against a careless
+ * author, not a hostile one. `oracle.ts` has the full account, measurements
+ * included, above `OracleRunnerSpec`.
+ *
+ * Secrecy is inherited whole. The report is parsed in memory and only its two
+ * counts reach the persisted evidence — never the assertion text, which is itself
+ * a partial oracle. `runHeldOracle` removes the report directory on every path
+ * (best effort, and logged when it fails), which matters because vitest's JSON
+ * report carries `failureMessages` verbatim. The full output still goes to the
+ * motor's local log for a human operator, exactly as the green gate does.
+ */
+export class OracleRedCollector
+  extends HeldOracleCollector
+  implements GateEvidenceCollector
+{
+  readonly kind = "oracle_red" as const;
+
+  /**
+   * The DOWNSTREAM stage's id, not this stage's: the author writes one held test
+   * keyed to the stage that must later pass it green, and this gate reads that
+   * same row early. See `OracleDeps.targetStageId`.
+   */
+  protected oracleStageId(): string | null {
+    return this.deps.targetStageId ?? null;
+  }
+
+  protected verdict({
+    oracle,
+    identity,
+    exitCode,
+    report,
+  }: {
+    oracle: HeldOracle;
+    identity: OracleIdentity;
+    exitCode: number;
+    output: string;
+    report: OracleRunSummary | null;
+  }): GateEvidence {
+    // The runner's own MACHINE REPORT, and nothing else (#1019). There is
+    // deliberately no fallback to the merged buffer or to the runner's verdict
+    // stream: the held test shares both of those with the runner, and a
+    // fallback is all an attacker needs — make the report unreadable, and the
+    // gate drops back to a channel it can write. An unreadable report is
+    // `blocked`, which can never satisfy a gate.
+    //
+    // `output` stays in the signature because the base class logs it for the
+    // operator; this gate does not read it.
+    const graded = gradeOracleRed(report, exitCode);
+    const base = { exitCode, path: oracle.path, ...identity };
+
+    if (graded.outcome === "no_verdict") {
+      log.warn(TAG, `Red gate for ${oracle.path}: ${graded.reason} — blocked`);
+      return {
+        result: "blocked",
+        // NOT marked as a config error (#823 rule 1: opt in, and only for
+        // static config). A summary the motor could not read may well be a
+        // runner the next attempt resolves, or a transient tool failure — the
+        // retryable default is the honest one. The one genuinely static case,
+        // a red gate with no downstream green stage, is marked in `collect`.
+        structured: { oracle: base, reason: graded.reason },
+      };
+    }
+
+    // Counts only — never the assertion text. `total`/`failed` say THAT the
+    // held test failed, which is the verdict; they say nothing about WHAT it
+    // asserts, which is the secret.
+    return {
+      result: graded.outcome === "reproduced" ? "passed" : "failed",
+      structured: {
+        oracle: {
+          ...base,
+          testsRun: graded.summary.total,
+          testsFailed: graded.summary.failed,
+          output:
+            "withheld — oracle_red is a secrecy gate; see the motor's local log",
+        },
+      },
+    };
+  }
+}
+
 function errText(err: unknown): string {
   return err instanceof Error ? err.message : String(err);
 }