omp-conductor 0.16.2 → 0.17.1

package/src/tracker/github.ts

@@ -385,7 +385,13 @@ function checkVerdict(check: GhCheck): CheckVerdict {
     return "failed";
   }
   if (check.status !== "COMPLETED") return "pending";
-  if (check.conclusion === "SUCCESS" || check.conclusion === "SKIPPED") return "green";
+  if (
+    check.conclusion === "SUCCESS" ||
+    check.conclusion === "SKIPPED" ||
+    check.conclusion === "NEUTRAL"
+  ) {
+    return "green";
+  }
   return "failed";
 }
 
@@ -461,6 +467,311 @@ function failedCheck(raw: string): GhCheck | undefined {
   return checks.find((check) => checkVerdict(check) === "failed");
 }
 
+/** REST projection of a pull request read by the verifyPr fallback (#653). */
+interface RestPrProjection {
+  draft?: unknown;
+  head_sha?: unknown;
+}
+
+/** One REST check, normalized onto the rollup verdict vocabulary. */
+interface RestCheckConclusion {
+  name: string;
+  /** The terminal `conclusion`/`status` (run) or `state` (commit status), as
+   *  GitHub spelled it — surfaced in refusals exactly like the GraphQL one. */
+  state: string;
+  verdict: CheckVerdict;
+}
+
+/** One REST check plane, parsed strictly. */
+type RestCheckEvidence =
+  | { readonly ok: true; readonly checks: RestCheckConclusion[] }
+  | { readonly ok: false };
+
+/** The live head SHA from a REST pull-request projection, or undefined. */
+function restPrHeadFrom(raw: string): string | undefined {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw) as unknown;
+  } catch {
+    return undefined;
+  }
+  if (parsed === null || typeof parsed !== "object") return undefined;
+  const sha = (parsed as RestPrProjection).head_sha;
+  return typeof sha === "string" && /^[0-9a-f]{40,64}$/i.test(sha) ? sha : undefined;
+}
+
+/** Whether a REST pull-request projection is a draft. Absent or malformed is
+ *  "draft state unknown", the same fail-closed answer {@link prVerificationFrom}
+ *  gives for a missing rollup — it must never be read as a clean non-draft. */
+function restPrDraftFrom(raw: string): boolean | undefined {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw) as unknown;
+  } catch {
+    return undefined;
+  }
+  if (parsed === null || typeof parsed !== "object") return undefined;
+  const draft = (parsed as RestPrProjection).draft;
+  return typeof draft === "boolean" ? draft : undefined;
+}
+
+/** One REST check-run member onto the rollup vocabulary, or undefined when the
+ *  member's identity or lifecycle fields cannot be read. */
+function restRunConclusionFrom(row: { readonly [key: string]: unknown }): RestCheckConclusion | undefined {
+  const name = row["name"];
+  const status = row["status"];
+  const conclusion = row["conclusion"];
+  if (typeof name !== "string" || name === "") return undefined;
+  if (typeof status !== "string") return undefined;
+  if (conclusion !== undefined && conclusion !== null && typeof conclusion !== "string") return undefined;
+  // statusCheckRollup semantics: an unfinished run is pending,
+  // `success`/`skipped`/`neutral` are green — GitHub's rollup counts a
+  // completed `neutral` check as successful — anything else is failed.
+  const verdict: CheckVerdict =
+    status !== "completed"
+      ? "pending"
+      : conclusion === "success" || conclusion === "skipped" || conclusion === "neutral"
+        ? "green"
+        : "failed";
+  return { name, state: typeof conclusion === "string" ? conclusion : status, verdict };
+}
+
+/** One REST commit-status member onto the rollup vocabulary, or undefined when
+ *  the identity or state cannot be read. */
+function restStatusConclusionFrom(row: { readonly [key: string]: unknown }): RestCheckConclusion | undefined {
+  const context = row["context"];
+  const state = row["state"];
+  if (typeof context !== "string" || context === "") return undefined;
+  if (typeof state !== "string") return undefined;
+  // StatusContext semantics: `success` is green, `pending` pending, anything
+  // else (`failure`/`error`) is failed.
+  const verdict: CheckVerdict = state === "success" ? "green" : state === "pending" ? "pending" : "failed";
+  return { name: context, state, verdict };
+}
+
+/**
+ * Parse one raw REST page body of a check plane (`check_runs` or `statuses`)
+ * strictly. Any malformed view answers `{ ok: false }`, never a partial list:
+ * unparseable JSON, a top-level body that is not an object, a plane that is
+ * missing or not an array, or any member whose identity cannot be read. A
+ * garbage entry beside a green one must not let the plane read as green.
+ */
+function restPlaneFrom(
+  raw: string,
+  plane: "check_runs" | "statuses",
+  member: (row: { readonly [key: string]: unknown }) => RestCheckConclusion | undefined,
+): RestCheckEvidence {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw) as unknown;
+  } catch {
+    return { ok: false };
+  }
+  if (parsed === null || typeof parsed !== "object") return { ok: false };
+  const list = (parsed as { readonly [key: string]: unknown })[plane];
+  if (!Array.isArray(list)) return { ok: false };
+  const checks: RestCheckConclusion[] = [];
+  for (const entry of list) {
+    if (entry === null || typeof entry !== "object") return { ok: false };
+    const conclusion = member(entry as { readonly [key: string]: unknown });
+    if (conclusion === undefined) return { ok: false };
+    checks.push(conclusion);
+  }
+  return { ok: true, checks };
+}
+
+/**
+ * The projection-only verdict of a REST pull-request read (#653): state, draft
+ * and exact head, decided strictly from the one raw projection. These facts do
+ * not need the check endpoints, so a caller proves them as soon as the
+ * projection arrives — a stale head or a non-open PR answered here stays that
+ * refusal even when the check planes are independently degraded, and is never
+ * masked as unresolvable. A projection that cannot prove the head throws, so
+ * the caller maps it to `head-unresolvable` exactly as it does for the GraphQL
+ * parser's failures.
+ */
+function restProjectionVerdict(
+  prRaw: string,
+  expectedHead?: string,
+  opts?: { read?: boolean },
+): { readonly ok: true; readonly headSha: string } | { readonly ok: false; readonly verdict: PrVerification } {
+  const headSha = restPrHeadFrom(prRaw);
+  if (headSha === undefined) throw new Error("PR head is unavailable");
+  const state = prStateFromRest(prRaw);
+  if (state !== "open") {
+    const stateName = state ?? "unknown";
+    if (opts?.read === true) {
+      return {
+        ok: false,
+        verdict: {
+          status: "failed",
+          reason: `PR is ${stateName}; checks are unavailable for a non-open PR`,
+          headSha,
+        },
+      };
+    }
+    return {
+      ok: false,
+      verdict: { status: "failed", reason: `PR is ${stateName}, expected OPEN`, headSha },
+    };
+  }
+  if (restPrDraftFrom(prRaw) !== false) {
+    return {
+      ok: false,
+      verdict: { status: "failed", reason: "PR is draft or draft state is unknown", headSha },
+    };
+  }
+  if (expectedHead !== undefined && headSha.toLowerCase() !== expectedHead.toLowerCase()) {
+    return {
+      ok: false,
+      verdict: {
+        status: "failed",
+        reason: `PR head changed: expected ${expectedHead}, found ${headSha}`,
+        headSha,
+      },
+    };
+  }
+  return { ok: true, headSha };
+}
+
+/**
+ * REST fallback for {@link prVerificationFrom} (#653): prove the same
+ * execution-time contract from the REST PR projection plus the check-runs and
+ * commit-status endpoints, when the primary GraphQL read cannot answer.
+ *
+ * The head is the REST projection's own `head.sha` — the same read that
+ * supplied state and draft — and the check evidence is pinned to that head via
+ * the commit endpoints, never to the caller's observation. Every page of both
+ * planes is flattened before the verdict is calculated: GitHub defaults each
+ * endpoint to 30 per page, and a non-green context past page one is exactly as
+ * decisive as one on page one. Missing check evidence is pending, never green.
+ * Throws on a payload that cannot prove the head or either check plane, so the
+ * caller maps it to `head-unresolvable` exactly as it does for the GraphQL
+ * parser.
+ */
+export function restVerificationFrom(
+  prRaw: string,
+  checkRunPages: string[],
+  statusPages: string[],
+  expectedHead?: string,
+  opts?: { read?: boolean },
+): PrVerification {
+  const projection = restProjectionVerdict(prRaw, expectedHead, opts);
+  if (!projection.ok) return projection.verdict;
+  const headSha = projection.headSha;
+
+  const checks: RestCheckConclusion[] = [];
+  for (const page of checkRunPages) {
+    const evidence = restPlaneFrom(page, "check_runs", restRunConclusionFrom);
+    if (!evidence.ok) throw new Error("check-run evidence is unresolvable");
+    checks.push(...evidence.checks);
+  }
+  for (const page of statusPages) {
+    const evidence = restPlaneFrom(page, "statuses", restStatusConclusionFrom);
+    if (!evidence.ok) throw new Error("commit-status evidence is unresolvable");
+    checks.push(...evidence.checks);
+  }
+  if (checks.length === 0) {
+    return { status: "pending", reason: "GitHub has not reported any checks yet", headSha };
+  }
+  const failed = checks.filter((c) => c.verdict === "failed");
+  if (failed.length > 0) {
+    return {
+      status: "failed",
+      reason: `Checks failed: ${failed.map((c) => `${c.name} (${c.state})`).join(", ")}`,
+      headSha,
+    };
+  }
+  const pending = checks.filter((c) => c.verdict === "pending");
+  if (pending.length > 0) {
+    return {
+      status: "pending",
+      reason: `Checks pending: ${pending.map((c) => c.name).join(", ")}`,
+      headSha,
+    };
+  }
+  return { status: "green", reason: `${checks.length} checks succeeded or were skipped`, headSha };
+}
+
+/**
+ * The REST half of {@link Tracker.verifyPr} (#653): re-prove the execution-time
+ * exact-head and check verdict when the primary GraphQL read could not answer.
+ * State, draft and exact head are decided from the PR projection the moment it
+ * arrives, before any check-plane fetch — a refusal the projection already
+ * proves (a stale head, a non-open or draft PR) is returned as that refusal,
+ * never downgraded to unresolvable by an independently degraded check
+ * endpoint. Returns `undefined` only when the live head or the complete check
+ * evidence cannot be proven from REST, so a degraded verification port never
+ * weakens the merge gate — it only ever turns a would-be refusal into the same
+ * refusal reported as unresolvable.
+ */
+async function verifyPrRest(
+  runGh: typeof gh,
+  cache: RestListCache,
+  url: string,
+  expectedHead?: string,
+  opts?: { read?: boolean },
+  onNotModified?: () => void,
+): Promise<PrVerification | undefined> {
+  const parts = prUrlParts(url);
+  if (parts === undefined) return undefined;
+  const base = `repos/${parts.owner}/${parts.repo}`;
+
+  let prRaw: string;
+  let headSha: string;
+  try {
+    prRaw = await runGh([
+      "api",
+      `${base}/pulls/${parts.number}`,
+      "--jq",
+      "{state, draft, head_sha: .head.sha, merged_at}",
+    ]);
+    // The projection alone proves state, draft and exact head, so those
+    // verdicts are returned before any check evidence is fetched: a stale
+    // head proven here is reported as such even when the check endpoints are
+    // independently degraded — the check planes must not be able to mask an
+    // already-proven stale head as unresolvable.
+    const projection = restProjectionVerdict(prRaw, expectedHead, opts);
+    if (!projection.ok) return projection.verdict;
+    headSha = projection.headSha;
+  } catch {
+    return undefined;
+  }
+
+  let checkRunPages: string[];
+  let statusPages: string[];
+  try {
+    // Every page of both planes is fetched through the tracked REST funnel
+    // (breakers, rate-limit ledger, conditional revalidation) and flattened
+    // before the verdict runs: each endpoint defaults to 30 per page, so a PR
+    // with more checks than that would otherwise let a non-green context hide
+    // past page one. The evidence is pinned to the head proven by the PR
+    // projection — never to the caller's observation, which could be stale.
+    checkRunPages = await conditionalListPages(
+      runGh,
+      cache,
+      `${base}/commits/${headSha}/check-runs?per_page=100`,
+      /* paginate */ true,
+      onNotModified,
+    );
+    statusPages = await conditionalListPages(
+      runGh,
+      cache,
+      `${base}/commits/${headSha}/status?per_page=100`,
+      /* paginate */ true,
+      onNotModified,
+    );
+  } catch {
+    return undefined;
+  }
+
+  try {
+    return restVerificationFrom(prRaw, checkRunPages, statusPages, expectedHead, opts);
+  } catch {
+    return undefined;
+  }
+}
+
 function runLogArgs(detailsUrl: string): string[] | undefined {
   const match =
     /^https:\/\/github\.com\/([^/]+\/[^/]+)\/actions\/runs\/(\d+)(?:\/job\/(\d+))?/.exec(
@@ -972,6 +1283,103 @@ export function isRateLimitRefusal(err: unknown): boolean {
   );
 }
 
+/**
+ * True when a `gh` stderr text carries a transient GitHub server-side 5xx —
+ * the API answering "try later" rather than judging the request. gh's stderr
+ * carries the status in two spellings: the REST `HTTP 500: Internal Server
+ * Error` style, and the GraphQL `No server is currently available to service
+ * your request. (HTTP 503)` recorded in #642. Both mean "stop asking for a
+ * while", and both open the tracker's bounded GraphQL breaker so admission
+ * does not multiply one provider outage into O(queue size) spawns per pass.
+ *
+ * Deliberately narrower than "anything nonzero": a 4xx answers the request
+ * (bad credentials, not found, an exhausted-but-named budget), and a GraphQL
+ * product error carries no HTTP status at all — neither is a provider outage,
+ * and neither may masquerade as the breaker event that holds every candidate.
+ * The status must be a complete three-digit 5xx token (`HTTP 503)`, `HTTP 500:`):
+ * a malformed suffix like `HTTP 5030` is an invalid response, not a provider
+ * outage, so it stays a plain failure instead of opening the breaker.
+ */
+export function isTransientServer5xx(stderr: string): boolean {
+  return /HTTP 5\d\d\b/i.test(stderr);
+}
+
+export function isTransientServerError(err: unknown): boolean {
+  return err instanceof GhError && isTransientServer5xx(err.stderr);
+}
+
+/**
+ * True when a `gh` argv rides the GraphQL API surface rather than core REST.
+ *
+ * `gh api graphql` is the raw surface. The named subcommands ride GraphQL
+ * mutations in the gh version this fleet pins (2.86): `issue edit`
+ * (`addLabelsToLabelable` / `removeLabelsFromLabelable` / `updateIssue`),
+ * `issue comment` (`addComment`), `issue close` (`closeIssue`), `pr create`
+ * (`createPullRequest`), `pr edit` (`updatePullRequest`), `pr merge`
+ * (`mergePullRequest`) and `pr update-branch` (`updatePullRequestBranch`).
+ * gh's transports move between versions, which is why this list is pinned and
+ * tested rather than guessed: a subcommand that stops riding GraphQL only
+ * stops being gated — it never stops working.
+ *
+ * Everything else the tracker and the verb actions shell — `gh api
+ * repos/...`, `gh pr diff`, `gh release create` — is core REST and must keep
+ * running while the GraphQL breaker is open: #642 measured REST healthy
+ * through the whole outage, and the ready-queue read and settle sweeps run on
+ * it.
+ */
+export function isGraphqlSurface(argv: readonly string[]): boolean {
+  if (argv[0] === "api") return argv[1] === "graphql";
+  if (argv[0] === "issue") {
+    return argv[1] === "edit" || argv[1] === "comment" || argv[1] === "close";
+  }
+  if (argv[0] === "pr") {
+    return argv[1] === "create" || argv[1] === "edit" || argv[1] === "merge" || argv[1] === "update-branch";
+  }
+  return false;
+}
+
+/**
+ * The process-local transient-server-error breaker for the GraphQL surface
+ * (#642). One instance is shared by the tracker and the verb actions of a
+ * project, so a 503 observed by either side gates both: admission's
+ * `parentOf` / `openCloserFor` checks AND the orchestrator mutation commands
+ * that ride GraphQL in gh 2.86 (`gh issue edit`, `gh pr merge`,
+ * `gh pr update-branch`, ...) fast-fail instead of each spawning `gh` into
+ * the same unavailable surface.
+ *
+ * Deliberately separate from the rate-limit breaker: a 5xx is a provider
+ * outage, not budget exhaustion, and must not fire `onRefusal` or touch the
+ * `gh_refusals` ledger. The cooldown is the same fixed window, because gh's
+ * stderr carries no `Retry-After` for either class.
+ */
+export class GraphqlBreaker {
+  private refusedUntil = 0;
+
+  constructor(private readonly cooldownMs: number = RATE_LIMIT_COOLDOWN_MS) {}
+
+  /** Record a transient 5xx: every GraphQL-surface call fast-fails until the
+   *  bounded cooldown elapses. */
+  open(at = Date.now()): void {
+    this.refusedUntil = at + this.cooldownMs;
+  }
+
+  /** Whether a GraphQL-surface call must fast-fail right now. */
+  refused(at = Date.now()): boolean {
+    return at < this.refusedUntil;
+  }
+
+  /** Wall-clock ms after which a real call may be attempted again; 0 = closed. */
+  get until(): number {
+    return this.refusedUntil;
+  }
+}
+
+/** The fast-fail wording both the tracker's throw and the verb actions'
+ *  failed result carry, so a refusal reads the same from either side. */
+export function graphqlBreakerRefusal(untilMs: number): string {
+  return `circuit breaker open (a prior GitHub server error holds the GraphQL cooldown until ${new Date(untilMs).toISOString()})`;
+}
+
 /**
  * A rate-limit refusal rethrown by {@link makeTracker}. This is the one
  * failure class the tracker names instead of swallowing: the circuit breaker
@@ -990,7 +1398,9 @@ export class GhRateLimitError extends GhError {
   }
 }
 
-/** How long a rate-limit refusal holds the circuit breaker open. */
+/** How long a rate-limit refusal or a transient server 5xx holds the tracker's
+ *  circuit breaker open. One window bounds both breakers: the all-call gate a
+ *  rate-limit refusal opens and the GraphQL-only gate a 5xx opens. */
 export const RATE_LIMIT_COOLDOWN_MS = 60_000;
 
 /** Instrumentation hooks the daemon binds the tracker to (#198). */
@@ -998,7 +1408,10 @@ export interface TrackerHooks {
   /** Fired immediately before each `gh` spawn, so the daemon can count its own
    *  API spend. Not fired for a breaker fast-fail, which spawns nothing. */
   onCall?: () => void;
-  /** Fired once per observed refusal, with the wall-clock moment. */
+  /** Fired once per observed rate-limit refusal, with the wall-clock moment.
+   *  A transient server 5xx opens the GraphQL breaker but is not a rate-limit
+   *  refusal and does not fire this hook (#642): the `gh_refusals` ledger and
+   *  the rate-limit decisions that read it stay about rate limits. */
   onRefusal?: (at: number) => void;
   /** Fired once per list page GitHub answered 304 for — a spawn that cost no
    *  primary-rate-limit budget. Lets the daemon's call counter tell billed
@@ -1007,8 +1420,13 @@ export interface TrackerHooks {
 }
 
 interface TrackerOpts {
-  /** Override {@link RATE_LIMIT_COOLDOWN_MS} (tests use a 0ms window). */
+  /** Override {@link RATE_LIMIT_COOLDOWN_MS}, the shared window for both
+   *  breakers (tests use a 0ms window). */
   rateLimitCooldownMs?: number;
+  /** The project's shared transient-server-error breaker (#642). Absent, the
+   *  tracker owns a fresh one; the daemon passes the same instance to the
+   *  verb actions so a 503 observed by either side gates both. */
+  graphqlBreaker?: GraphqlBreaker;
   /** Conditional-request cache for REST list reads; defaults to the shared
    *  module-level cache. Tests pass a fresh Map. */
   listCache?: RestListCache;
@@ -1024,15 +1442,39 @@ export function makeTracker(
   const refuseForMs = opts.rateLimitCooldownMs ?? RATE_LIMIT_COOLDOWN_MS;
   const cache = opts.listCache ?? sharedRestListCache;
 
-  // The single funnel around the injected `gh`. A caught refusal records the
-  // user-observed moment, opens the 60s breaker and rethrows named; while the
-  // breaker is open every call fails fast WITHOUT spawning gh — a burst under a
-  // persistent limit degrades the tick into holds (`open-pr-lookup-error` /
-  // `parent-lookup-error` fail closed per candidate and retry next tick)
-  // instead of an error storm or a stalled loop. No in-wrapper sleeps.
+  // The single funnel around the injected `gh`, with two bounded breakers:
+  //
+  // - A rate-limit refusal is account-wide — every budget the token can spend
+  //   (REST and GraphQL) is exhausted or throttled — so it opens a breaker
+  //   that fast-fails EVERY call.
+  // - A transient server 5xx is surface-specific. #642 measured a GraphQL 503
+  //   that admission then re-asked once per candidate (`parentOf` /
+  //   `openCloserFor`), multiplying one outage into O(queue size) spawns,
+  //   TLS handshakes, failures and latency per pass while REST stayed
+  //   healthy. It opens a breaker that fast-fails GraphQL calls only — every
+  //   GraphQL-surface call, tracker reads AND the mutation commands the
+  //   orchestrator verbs shell (`gh issue edit`, `gh pr merge`, ...), because
+  //   both ride the same broken surface — so the ready-queue read and the
+  //   settle sweeps keep running and the pass still degrades into holds
+  //   instead of dying.
+  //
+  // The GraphQL breaker is shared with the verb actions through the daemon
+  // (#642 comment 4): a 503 observed while serving `conductor_pr_update_branch`
+  // must gate the next admission pass too, and vice versa — one provider
+  // outage must not be re-asked by every candidate AND every mutation.
+  //
+  // While a breaker is open every gated call fails fast WITHOUT spawning gh —
+  // a burst under a persistent outage degrades the tick into holds
+  // (`open-pr-lookup-error` / `parent-lookup-error` fail closed per candidate
+  // and retry next tick) instead of an error storm or a stalled loop. No
+  // in-wrapper sleeps. A 5xx is not a rate-limit refusal: it never fires
+  // `onRefusal`, so the `gh_refusals` ledger and the rate-limit decisions
+  // that read it are untouched by a provider outage.
+  const graphqlBreaker = opts.graphqlBreaker ?? new GraphqlBreaker(refuseForMs);
   let refusedUntil = 0;
   const runGh = async (argv: string[], stdin?: string): Promise<string> => {
     const now = Date.now();
+    const graphql = isGraphqlSurface(argv);
     if (now < refusedUntil) {
       throw new GhRateLimitError(
         argv,
@@ -1041,6 +1483,9 @@ export function makeTracker(
         refusedUntil,
       );
     }
+    if (graphql && graphqlBreaker.refused(now)) {
+      throw new GhError(argv, 0, graphqlBreakerRefusal(graphqlBreaker.until));
+    }
     hooks.onCall?.();
     try {
       return await injectedRunGh(argv, stdin);
@@ -1058,6 +1503,14 @@ export function makeTracker(
           cause?.stdout ?? "",
         );
       }
+      // A transient 5xx on the GraphQL surface proves THAT surface unavailable
+      // for this pass: the next candidate's parent/closer checks AND the next
+      // GraphQL-backed mutation must fail fast instead of re-asking it. The
+      // original error is rethrown unchanged, so the failing candidate holds
+      // on the true provider message.
+      if (graphql && isTransientServerError(err)) {
+        graphqlBreaker.open();
+      }
       throw err;
     }
   };
@@ -1307,6 +1760,98 @@ export function makeTracker(
       }
     },
 
+    async runFailedAttemptLogs(
+      repo: string,
+      runUrl: string,
+      maxAttempts: number,
+    ): Promise<string[] | undefined> {
+      // The attempt-blind Tracker surface cannot reach a failed job that a
+      // rerun-to-green left behind; the attempt-scoped jobs register still
+      // names it. Routed through this adapter's guarded `runGh`, so call/
+      // refusal accounting and the rate-limit breaker apply to the historical
+      // reconciliation's reads exactly as they do to a live tick's (#638).
+      const match =
+        /^https:\/\/github\.com\/([^/\s]+\/[^/\s]+)\/actions\/runs\/(\d+)(?:\/job\/\d+)?/.exec(
+          runUrl,
+        );
+      const runId = match?.[2];
+      if (runId === undefined || !/^[^/\s]+\/[^/\s]+$/.test(repo)) return [];
+      let latest: number;
+      try {
+        const raw = await runGh([
+          "api",
+          `repos/${repo}/actions/runs/${runId}`,
+          "--jq",
+          ".run_attempt",
+        ]);
+        latest = Number.parseInt(raw.trim(), 10);
+      } catch {
+        // The run register could not be read: a no-mutation refusal the next
+        // pass asks again, not a determinately-clean answer.
+        return undefined;
+      }
+      if (!Number.isInteger(latest) || latest <= 0) return undefined;
+      // Refuse rather than classify a prefix as the complete attempt history:
+      // infra evidence in attempts 1..N-1 must not waive a row whose later
+      // attempt carries the real compile/test failure (review #654).
+      if (latest > maxAttempts) return undefined;
+      const chunks: string[] = [];
+      for (let attempt = 1; attempt <= latest; attempt++) {
+        // One chunk per failed job, not one aggregate per attempt: a setup
+        // job's codeload 429 in the same attempt as a sibling matrix job's
+        // compile error must surface as two chunks so the caller's all-infra
+        // guard sees the mixed failure (review #654). Each chunk is the
+        // failed steps of that one job at that attempt — the exact surface
+        // the forward classifier's `checkLog` reads (`--log-failed`), so a
+        // repaired row and a fresh row share one definition of "the log
+        // proves infra". Full and untruncated: a truncated last-400-lines
+        // view can discard contrary failure evidence.
+        let failedJobIds: number[];
+        try {
+          const raw = await runGh([
+            "api",
+            `repos/${repo}/actions/runs/${runId}/attempts/${attempt}/jobs`,
+            "--jq",
+            '[.jobs[] | select(.conclusion == "action_required" or .conclusion == "failure" or .conclusion == "startup_failure" or .conclusion == "timed_out") | .id]',
+          ]);
+          failedJobIds = JSON.parse(raw) as number[];
+        } catch {
+          // The attempt's job register could not be read: undecided, never a
+          // clean answer assembled from whatever earlier attempts yielded.
+          return undefined;
+        }
+        for (const jobId of failedJobIds) {
+          try {
+            const raw = await runGh([
+              "run",
+              "view",
+              runId,
+              "--repo",
+              repo,
+              "--job",
+              String(jobId),
+              "--attempt",
+              String(attempt),
+              "--log-failed",
+            ]);
+            const cleaned = raw.replaceAll(/\u001b\[[0-9;]*m/g, "");
+            // Push even an empty chunk: a failed job whose failed steps carry
+            // no signature is determinately not infrastructure, and omitting
+            // it would let a sibling 429 chunk waive the row on partial
+            // evidence.
+            chunks.push(cleaned);
+          } catch {
+            // Any per-attempt log read failure is a no-mutation refusal: the
+            // earlier chunks are not the complete evidence set, and returning
+            // them would let a later compile/test failure go unseen (review
+            // #654).
+            return undefined;
+          }
+        }
+      }
+      return chunks;
+    },
+
     async previousWorkflowRun(
       repo: string,
       workflowId: number,
@@ -1370,9 +1915,13 @@ export function makeTracker(
           return verification;
         }
       } catch {
-        // Lookup or payload failure is not permission to accept success. The
-        // daemon records a pending row and asks again on a later tick.
-        return undefined;
+        // The primary read is GraphQL (`gh pr view ... statusCheckRollup`).
+        // When that plane is degraded (#653), the verification port is not
+        // single-transport: re-prove the exact head and the complete check
+        // verdict from REST, or report the head unresolvable. A REST read that
+        // cannot prove green never weakens the contract — it only replaces one
+        // refusal with the same refusal reported as unresolvable.
+        return verifyPrRest(runGh, cache, url, expectedHead, opts, hooks.onNotModified);
       }
     },