forgedock 1.18.0 → 1.20.0

package/README.md

@@ -303,6 +303,8 @@ If you're carrying an older `--global` habit in scripts or CI, it's harmless to
 
 </details>
 
+**After upgrading**: run `npx forgedock labels setup` once per repo. Repos created before the `workflow:remediating` state (forge#3541) lack that label; the pipeline now creates it on demand (best-effort), but the setup command is the supported way to sync all labels.
+
 ---
 
 ## For companies

package/bin/engine/outcome-codes.mjs

@@ -21,6 +21,7 @@ export const OUTCOME_CODES = Object.freeze({
   REVIEW_REREVIEW_SKIPPED: "review.rereview_skipped",
   REVIEW_BLOCKING_ISSUE: "review.blocking_issue",
   REVIEW_PR_OPEN: "review.pr_open",
+  REVIEW_CI_PENDING_TIMEOUT: "review.ci_pending_timeout",
   BUILD_ACCEPTANCE_FAILED: "build.acceptance_failed",
   BUILD_ACCEPTANCE_MALFORMED: "build.acceptance_malformed",
   BUILD_SIZE_OVER_GATE: "build.size_over_gate", // retired (forge#3862): never emitted; kept so historic runs parse
@@ -29,12 +30,15 @@ export const OUTCOME_CODES = Object.freeze({
   REMEDIATE_DEFERRED_LIVE_RUN: "remediate.deferred_live_run",
   REMEDIATE_RE_ESCALATED: "remediate.re_escalated",
   REMEDIATE_UNFIXABLE: "remediate.unfixable",
+  REMEDIATE_INPR_REQUESTED: "remediate.inpr_requested",
   CI_RED: "ci.red",
   CI_TIMEOUT: "ci.timeout",
   BASE_CONFLICT: "base.conflict",
   RATE_LIMITED: "rate_limited",
   ENGINE_ERROR: "engine.error",
   INFRA_DISK_QUOTA: "infra.disk_quota",
+  INFRA_CLI_SPAWN_TRANSIENT: "infra.cli_spawn_transient",
+  CLI_TIMEOUT: "cli.timeout",
   LEASE_HELD: "lease.held",
   DEPENDENCY_OPEN: "dependency.open",
   DEPENDENCY_UNREADABLE: "dependency.unreadable",
@@ -74,6 +78,11 @@ export function isDiskQuotaError(err) {
   return isDiskQuotaText(err.message) || isDiskQuotaText(err.stderr);
 }
 
+/** True when a thrown runner error is a transient CLI spawn failure (ETIMEDOUT/EAGAIN/ENOMEM/EMFILE at spawn), forge#4021. */
+export function isTransientSpawnError(err) {
+  return !!err && typeof err === "object" && err.code === "CLI_SPAWN_TRANSIENT";
+}
+
 /** Code for a thrown runner error, from `err.code` / session-limit evidence only. */
 export function codeForRunnerError(err) {
   if (!err || typeof err !== "object") return UNCLASSIFIED;
@@ -81,6 +90,10 @@ export function codeForRunnerError(err) {
   if (err.usageLimit === true || (typeof err.resetAtEpochMs === "number" && Number.isFinite(err.resetAtEpochMs)))
     return OUTCOME_CODES.RATE_LIMITED;
   switch (err.code) {
+    case "CLI_SPAWN_TRANSIENT":
+      return OUTCOME_CODES.INFRA_CLI_SPAWN_TRANSIENT;
+    case "CLI_TIMEOUT":
+      return OUTCOME_CODES.CLI_TIMEOUT;
     case "NO_API_KEY":
     case "NO_SDK":
     case "CLI_BACKEND_FAILED":

package/bin/engine/phases.mjs

@@ -220,7 +220,8 @@ function latestGateMarker(comments) {
     const b = comments[i];
     if (startsWithHeader(b, "FORGE:BUILD_BLOCKED")) return { state: "blocked", index: i };
     if (!startsWithHeader(b, gate)) continue;
-    if (hasSentinelLine(b, `${gate}:PASSED`)) return { state: "pass", index: i };
+    // :DEGRADED is pass-class (forge#3628): the build completed, and review gates the merge on the degraded paths.
+    if (hasSentinelLine(b, `${gate}:PASSED`) || hasSentinelLine(b, `${gate}:DEGRADED`)) return { state: "pass", index: i };
     if (hasSentinelLine(b, `${gate}:FAILED`) || hasSentinelLine(b, `${gate}:BLOCKED`)) return { state: "blocked", index: i };
   }
   return { state: "none", index: -1 };
@@ -553,6 +554,9 @@ function isRemediationKind(k) {
   return typeof k === "string" && Object.hasOwn(REMEDIATION_BOUND_MARKERS, k);
 }
 
+// forge#4021: the newest remediation trail reported an in-PR fix request (remediate.md M8).
+const INPR_REQUESTED_RE = /\*\*Re-gate outcome\*\*:\s*INPR-FIX-REQUESTED\b/;
+
 const REPO_RE = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/;
 const BRANCH_RE = /^[A-Za-z0-9._][A-Za-z0-9._\/-]*$/;
 const LANE_RE = BRANCH_RE;
@@ -842,6 +846,11 @@ export const PHASES = [
       // An explicit REVIEW_RESULT status is authoritative: BLOCKED/COMPLETE exits
       // (phase trail, ci gate, base conflict, merge refusal) also label the issue
       // needs-human but are NOT remediation handoffs, so skip the label fallback.
+      // CI still running when the merge gate's wait window ended: nothing failed, so this is a wait the engine
+      // polls (runIssue), not a block. Structured `blocker` field only, never PR/comment prose; "ci gate"
+      // (red CI) is a disjoint phrase and keeps the REVIEW_PR_OPEN path below.
+      if (rr && rr.status === "BLOCKED" && isCiPendingBlocker(rr.blocker))
+        return { status: "failed", detail: "checks still pending", retryable: false, handoff: false, wait: "ci-pending", outputs, code: OC.REVIEW_PR_OPEN };
       if (rr && rr.status) return { status: "failed", detail: "PR open, not merged", retryable: false, handoff: false, outputs, code: OC.REVIEW_PR_OPEN };
       let snap = null;
       try { snap = await issueSnapshot(state.issue, io); } catch { snap = null; }
@@ -902,7 +911,15 @@ export const PHASES = [
         const { comments } = await issueMarkers(state.issue, io);
         // Resume idempotency: a handoff interrupted after the marker post must not double-count.
         // forge#3542: only a full-line marker in a trusted comment counts.
-        if (!comments.some((b) => hasSentinelLine(b, `FORGE:${boundMarker}: pr=${pr}`))) {
+        // forge#4021: an in-PR fix re-armed by the re-gate (the newest trail after the newest marker reports
+        // INPR-FIX-REQUESTED) needs a FRESH marker, or M0's per-kind guard sees the first trail as this dispatch's
+        // and returns ALREADY_DONE. The marker count is also the bound (detectOutcome allows one re-arm).
+        const sentinel = `FORGE:${boundMarker}: pr=${pr}`;
+        let markerAt = -1;
+        comments.forEach((b, i) => { if (hasSentinelLine(b, sentinel)) markerAt = i; });
+        const trailAfter = markerAt < 0 ? null : latestMarked(comments.slice(markerAt + 1), PHASE_MARKERS.remediate.header, PHASE_MARKERS.remediate.completionMarker);
+        const rearmed = kind === "inpr-fix" && trailAfter !== null && INPR_REQUESTED_RE.test(trailAfter);
+        if (markerAt < 0 || rearmed) {
           await io.gh(["issue", "comment", String(state.issue), "--body",
             `${marker}\nReview handed PR #${pr} to remediation (${kind}); the engine is dispatching it once.`]);
         }
@@ -912,7 +929,8 @@ export const PHASES = [
       await postDispatchMarker();
       return legacyArgs();
     },
-    entryCondition: (s) => s.committed.includes("review") && s.terminalReason === "needs-human",
+    // forge#4021: also re-entered (with remediationKind "inpr-fix") after a re-gate that requested one in-PR fix round.
+    entryCondition: (s) => s.committed.includes("review") && (s.terminalReason === "needs-human" || s.remediationKind === "inpr-fix"),
     async detectOutcome(state, io) {
       // forge#3530: read the NEWEST completed trail only. A concatenated oldest-first blob
       // returned a stale outcome from an earlier remediation when a later one posted nothing.
@@ -966,6 +984,12 @@ export const PHASES = [
       if (heldAt >= 0 && heldAt > completeAt) {
         // Merged PR hands to close; otherwise awaiting-merge (terminal for the engine, re-entered by an operator), never a retry.
         if (await prMerged()) return mergedResult("AUTO-LANDED");
+        // CI still pending at M7 is a wait, not a park: the engine polls CI (no model) and re-dispatches this phase, which
+        // resumes at M7. Only the structured `Hold reason` line of the newest trusted held record counts; the stale-head
+        // and deploy-gate holds keep the terminal awaiting-merge result below.
+        if (/^\*\*Hold reason\*\*:[ \t]*CI pending\b/m.test(scoped[heldAt]))
+          return { status: "failed", detail: "checks still pending", retryable: false, handoff: false, wait: "ci-pending",
+                   outputs: { reGateOutcome: "HELD-AWAITING-MERGE" }, code: OC.REVIEW_PR_OPEN };
         return { status: "committed", terminalReason: "awaiting-merge", outputs: { reGateOutcome: "HELD-AWAITING-MERGE" } };
       }
       if (latest === null) {
@@ -1024,11 +1048,30 @@ export const PHASES = [
           return { status: "committed", terminalReason: "needs-human",
                    code: reGateOutcome === "UNFIXABLE" ? OC.REMEDIATE_UNFIXABLE : OC.REMEDIATE_RE_ESCALATED,
                    outputs: { reGateOutcome } };
+        case "INPR-FIX-REQUESTED": {
+          // forge#4021: the re-review approved but posted a FORGE:INPR_FIX work order. One further in-PR fix
+          // remediation is allowed (never needs-human on the first request); the bound marker count caps it.
+          if (await remediationHeadStale(state, io))
+            return { status: "failed", code: OC.REMEDIATE_NO_MARKER, detail: `stale ${completion} trail: PR head moved since it was recorded` };
+          if (await prMerged()) return mergedResult(reGateOutcome);
+          const round = /\*\*INPR round\*\*:\s*(\d+)/.exec(latest)?.[1] ?? null;
+          const outputs = { reGateOutcome, ...(round ? { inprRound: Number(round) } : {}) };
+          const boundSentinel = `FORGE:${REMEDIATION_BOUND_MARKERS["inpr-fix"]}: pr=${state.pr}`;
+          const priorMarkers = state.pr == null ? 2 : comments.filter((b) => hasSentinelLine(b, boundSentinel)).length;
+          if (priorMarkers >= 2)
+            return { status: "committed", terminalReason: "needs-human", code: OC.REMEDIATE_RE_ESCALATED, outputs };
+          return { status: "committed", code: OC.REMEDIATE_INPR_REQUESTED, outputs: { ...outputs, remediation: "inpr-fix" } };
+        }
+        case "REREVIEW-REQUIRED":
+          if (await remediationHeadStale(state, io))
+            return { status: "failed", code: OC.REMEDIATE_NO_MARKER, detail: `stale ${completion} trail: PR head moved since it was recorded` };
+          return { status: "committed", terminalReason: "needs-human", code: OC.REMEDIATE_RE_ESCALATED, outputs: { reGateOutcome } };
         default:
           return { status: "failed", detail: `FORGE:REMEDIATION:COMPLETE present but Re-gate outcome unrecognized/missing: ${reGateOutcome || "none"}` };
       }
     },
-    isTerminalAfter: () => true,
+    // forge#4021: a committed in-PR fix request continues the run (remediate re-arms) instead of ending it.
+    isTerminalAfter: (s) => s.remediationKind !== "inpr-fix",
   },
   {
     id: "close",
@@ -1164,6 +1207,10 @@ export async function detectTrustedOutcome(phaseId, state, io) {
  * Untrusted model output: only the fixed keys below are read, via anchored
  * line matches; `pr_number` must be digits-only. Returns null when absent.
  */
+export function isCiPendingBlocker(blocker) {
+  return typeof blocker === "string" && /checks still pending/i.test(blocker) && !/ci gate/i.test(blocker);
+}
+
 export function parseReviewResult(text) {
   if (typeof text !== "string" || !text) return null;
   const lines = text.split(/\r?\n/);
@@ -1174,6 +1221,9 @@ export function parseReviewResult(text) {
   for (let i = start + 1; i < lines.length; i++) {
     const m = /^\s+(status|next|remediation|pr_number):[ \t]*(\S*)[ \t]*$/.exec(lines[i]);
     if (m) { if (!(m[1] in out)) out[m[1]] = m[2]; continue; }
+    // `blocker` is free text (spaces): rest of the anchored line, first wins, only ever substring-tested.
+    const b = /^\s+blocker:[ \t]*(.*?)[ \t]*$/.exec(lines[i]);
+    if (b) { if (!("blocker" in out)) out.blocker = b[1].slice(0, 500); continue; }
     if (/^\s+[A-Za-z_]+:/.test(lines[i]) || !lines[i].trim()) continue;
     break;
   }

package/bin/engine/runlog.mjs

@@ -91,7 +91,13 @@ export function deriveState(events) {
         // forge#3530: the review's remediation kind must survive resume. A later review
         // commit replaces it; a committed remediate consumes it, so it is never reused.
         if (e.phase === "review") s.remediationKind = e.outputs?.remediation ?? null;
-        else if (e.phase === "remediate") s.remediationKind = null;
+        else if (e.phase === "remediate") {
+          // forge#4021: a re-gate that requested an in-PR fix re-arms remediate once (bounded in
+          // phases.mjs by the INPR_REMEDIATION marker count); every other remediate commit consumes the kind.
+          const rearm = e.outputs?.remediation === "inpr-fix";
+          s.remediationKind = rearm ? "inpr-fix" : null;
+          if (rearm) s.committed = s.committed.filter((ph) => ph !== "remediate");
+        }
         break;
       case "RUN_TERMINAL":
         s.terminal = true; s.terminalReason = e.reason ?? "done"; s.v = e.seq;
@@ -112,6 +118,8 @@ export function deriveState(events) {
         if (Array.isArray(e.uncommit) && e.uncommit.length) {
           s.committed = s.committed.filter((p) => !e.uncommit.includes(p));
           s.terminalReason = null; s.outcomeCode = null;
+          // the revoked handoff reason must not be restored by a later RUN_REOPEN (e.g. after an engine-error)
+          lastCommitReason = null; lastCommitCode = null;
         }
         // forge#3754: a remediate that ended needs-human is re-run after a human clears the
         // label. Un-commit it (new array, no in-place mutation) so the forge#3506 resume guard
@@ -121,7 +129,7 @@ export function deriveState(events) {
         // keeps remediate committed so the resume guard is not bypassed.
         if (lastCommitPhase === "remediate" && lastCommitReason === "needs-human"
             && lastTerminalReason === "needs-human") {
-          s.committed = s.committed.filter((p) => p !== "remediate");
+          s.committed = s.committed.filter((ph) => ph !== "remediate");
         }
         lastTerminalReason = null;
         s.lastRateLimit = null; // forge#3791: an explicit reopen never re-enters a stale pause

package/bin/engine/telemetry-seam.mjs

@@ -37,6 +37,9 @@ export function mapRecord(rec, wallMs) {
   if (rec.event === "RUN_TERMINAL") {
     return { event: "run_terminal", phase: null, payload: compact({ reason: rec.reason, code: rec.code, wall_ms: wallMs }) };
   }
+  if (rec.event === "CI_PENDING_WAIT") {
+    return { event: "ci_pending_wait", phase: rec.phase ?? "review", payload: compact({ pr: rec.pr, minutes_waited: rec.minutes_waited, result: rec.result, polls: rec.polls, rate_limit_sleeps: rec.rate_limit_sleeps }) };
+  }
   return null;
 }
 

package/bin/engine/telemetry.test.mjs

@@ -1,8 +1,9 @@
 // SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
 // SPDX-License-Identifier: AGPL-3.0-or-later
+import { tmpDir } from "../tests/_tmp.mjs";
 import { describe, it, beforeEach, afterEach } from "node:test";
 import assert from "node:assert/strict";
-import { chmodSync, existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { chmodSync, existsSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { spawnSync } from "node:child_process";
@@ -31,7 +32,7 @@ const setComments = (bodies) => writeFileSync(join(td, "comments.json"),
 
 beforeEach(() => {
   saved = Object.fromEntries(KEYS.map((k) => [k, process.env[k]]));
-  td = mkdtempSync(join(tmpdir(), "tel-"));
+  td = tmpDir("tel-");
   Object.assign(process.env, { TD: td, FORGE_TELEMETRY_BUFFER_DIR: join(td, "buf"), FORGE_REPO: "o/r", FORGE_ISSUE: "5" });
   for (const k of ["FORGE_TELEMETRY", "FORGE_PR", "FORGE_RUN", "FORGE_PHASE", "FORGE_TELEMETRY_MAX_CHARS"]) delete process.env[k];
 });

package/bin/engine-cli.mjs

@@ -6,13 +6,15 @@
  */
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
-import { readdirSync, statSync } from "node:fs";
+import { readdirSync, statSync, readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
 import { homedir, hostname } from "node:os";
 import { join } from "node:path";
 import { runIssue, DEFAULT_MAX_ATTEMPTS, sanitizeLeaseHost } from "./engine.mjs";
 import { makeProjector } from "./engine/projector.mjs";
 import { readLog, deriveState } from "./engine/runlog.mjs";
 import { assertOpenCodeNativeRuntime } from "./runner.mjs";
+import { parseForgeYaml } from "./forge-utils.mjs";
 import { makeEmitTelemetry } from "./engine/telemetry-seam.mjs";
 
 /**
@@ -53,7 +55,34 @@ export function makeIo() {
     const { stdout } = await pexec(bin, args, { maxBuffer: 100 * 1024 * 1024, timeout: 10000 });
     return stdout;
   };
-  return { gh: run("gh"), git: run("git") };
+  // CI wait seam (scripts/wait-ci-green.sh, argv only, no shell). Resolves { rc, text } for any exit code.
+  const waitCiGreen = async (pr, repo, timeoutSec) => {
+    const script = fileURLToPath(new URL("../scripts/wait-ci-green.sh", import.meta.url));
+    try {
+      // The engine owns the backoff and the budget: each call is ONE read (no in-script polling, no settle hold;
+      // the re-dispatched phase re-runs the full gate before any merge).
+      const { stdout } = await pexec("bash", [script, String(pr), "-R", String(repo), "--timeout", String(timeoutSec)],
+        { maxBuffer: 10 * 1024 * 1024, timeout: (timeoutSec + 120) * 1000, env: { ...process.env, FORGE_CI_SETTLE: "0" } });
+      return { rc: 0, text: stdout };
+    } catch (e) { return { rc: Number.isInteger(e?.code) ? e.code : 2, text: String(e?.stdout ?? "") }; }
+  };
+  // REST budget seam (`gh api rate_limit`, argv only; the call itself is not rate-limited). Resolves { remaining, limit, reset } or null.
+  const rateLimit = async () => {
+    try {
+      const { stdout } = await pexec("gh", ["api", "rate_limit", "--jq", ".resources.core"], { maxBuffer: 1024 * 1024, timeout: 10000 });
+      const j = JSON.parse(stdout);
+      return { remaining: j.remaining, limit: j.limit, reset: j.reset };
+    } catch { return null; }
+  };
+  return { gh: run("gh"), git: run("git"), waitCiGreen, rateLimit };
+}
+
+/** forge.yaml `engine.ci_pending_max_minutes` from the cwd checkout; undefined when absent or invalid. */
+export function readCiPendingMaxMinutes(cwd = process.cwd()) {
+  try {
+    const n = Number(parseForgeYaml(readFileSync(join(cwd, "forge.yaml"), "utf8"))?.engine?.ci_pending_max_minutes);
+    return Number.isInteger(n) && n > 0 ? n : undefined;
+  } catch { return undefined; }
 }
 
 export function runDir() { return join(homedir(), ".forge", "runs"); }
@@ -408,6 +437,7 @@ export async function runFromCli(argv, deps = {}) {
     // runIssue's/runner.mjs's existing defaults (forge#2028).
     ...(backend ? { backend } : {}),
     ...(model ? { model } : {}),
+    ...(readCiPendingMaxMinutes() ? { ciPendingMaxMinutes: readCiPendingMaxMinutes() } : {}),
     ...(retry ? { retry: true } : {}),
     ...(waitLease ? { waitLease: true } : {}) });
   console.log(`issue #${issue} → ${res.terminalReason}`);

package/bin/engine.mjs

@@ -4,7 +4,8 @@
  * GitHub state (phase.detectOutcome). All effects are injected → fully testable.
  */
 import { fileURLToPath } from "node:url";
-import { readFileSync, rmSync } from "node:fs";
+import { readFileSync, rmSync, readdirSync } from "node:fs";
+import { execFileSync } from "node:child_process";
 import { createHash, scryptSync } from "node:crypto";
 import { hostname as osHostname, homedir } from "node:os";
 import { appendEvent, readLog, deriveState, rewriteLog } from "./engine/runlog.mjs";
@@ -12,7 +13,7 @@ import { pickPhase, TERMINAL_REASONS, issueSnapshot, hasInvestigationExit, postC
 import { reconcileState } from "./engine/reconcile.mjs";
 import { makeProjector } from "./engine/projector.mjs";
 import { VALID_BACKENDS, resolveRunTmpRoot, runTmpDir } from "./runner.mjs";
-import { OUTCOME_CODES, UNCLASSIFIED, normalizeCode, codeForRunnerError, codeForTerminalReason, isDiskQuotaError, isDiskQuotaText } from "./engine/outcome-codes.mjs";
+import { OUTCOME_CODES, UNCLASSIFIED, normalizeCode, codeForRunnerError, codeForTerminalReason, isDiskQuotaError, isDiskQuotaText, isTransientSpawnError } from "./engine/outcome-codes.mjs";
 
 // Exported (forge#2175) so bin/engine-cli.mjs can render "failed N/M attempts"
 // diagnostics without duplicating the retry budget constant.
@@ -38,6 +39,39 @@ export const DEFAULT_MAX_SESSION_LIMIT_PAUSES = 5;
 // reset epoch. The runner never fabricates a reset time; this default lives in
 // the engine and is still capped by `maxSessionLimitPauses`.
 export const DEFAULT_SESSION_LIMIT_WAIT_MS = 30 * 60 * 1000;
+// CI still pending at the merge gate: total wait budget (engine.ci_pending_max_minutes) and the poll scheduler bounds.
+export const DEFAULT_CI_PENDING_MAX_MINUTES = 90;
+export const CI_BACKOFF_START_MS = 60 * 1000;
+export const CI_BACKOFF_MAX_MS = 5 * 60 * 1000;
+export const CI_JITTER_FRACTION = 0.2;
+/** Below this fraction of the GitHub REST budget the scheduler sleeps until the window resets instead of polling. */
+export const CI_RATE_RESERVE_FRACTION = 0.1;
+const CI_RESET_SPREAD_MS = 30 * 1000;
+const MAX_CI_PENDING_WAITS = 3;
+
+/**
+ * Pure CI-poll scheduler: delay before poll number `pollIndex` (0-based). 60 s, x2 per poll, capped at 5 min,
+ * +/-20% jitter so lanes waiting on the same CI do not poll in lockstep. `rand` is injectable (returns [0,1)).
+ */
+export function ciPollDelayMs(pollIndex, rand = Math.random) {
+  const base = Math.min(CI_BACKOFF_START_MS * 2 ** Math.max(0, pollIndex), CI_BACKOFF_MAX_MS);
+  const r = Math.min(Math.max(Number(rand()) || 0, 0), 1);
+  return Math.round(base * (1 + (r * 2 - 1) * CI_JITTER_FRACTION));
+}
+
+/**
+ * Pure budget check: ms to sleep before the next poll so the shared REST budget keeps its reserve.
+ * `rl` is `{ remaining, limit, reset }` (reset in epoch seconds, as `gh api rate_limit` reports it). Returns 0 when the
+ * budget is healthy or unreadable (a failed read never blocks polling); otherwise the time until reset plus a random
+ * spread so lanes released by the same reset do not stampede.
+ */
+export function ciRateLimitSleepMs(rl, nowMs, rand = Math.random) {
+  const remaining = Number(rl?.remaining), limit = Number(rl?.limit), reset = Number(rl?.reset);
+  if (![remaining, limit, reset].every(Number.isFinite) || limit <= 0) return 0;
+  if (remaining > limit * CI_RATE_RESERVE_FRACTION) return 0;
+  const untilReset = Math.max(0, reset * 1000 - nowMs);
+  return untilReset + Math.round(Math.min(Math.max(Number(rand()) || 0, 0), 1) * CI_RESET_SPREAD_MS);
+}
 
 /**
  * forge#3791: non-secret identity of the backend/account a usage-limit pause belongs to.
@@ -235,6 +269,46 @@ export function isLeaseHolderDead({ by, host = osHostname(), killFn = process.ki
   }
 }
 
+/**
+ * forge#3724: a dead lease-holder pid proves the engine is dead, not that its phase child is. The runner
+ * stamps every phase child with `FORGE_LEASE_HOLDER=<by>` (inherited by grandchildren), so a surviving
+ * orphan is found by scanning process environments. Returns FALSE only when the probe affirmatively finds
+ * no live process carrying the marker; unsupported platform, unreadable /proc, a failed `ps`, or any thrown
+ * error returns TRUE (fail-safe: the caller keeps waiting for `lease.until`). Per-process EACCES/ENOENT is
+ * skipped (exited, or another uid). Residual: a child that scrubs its env (`env -i`) is not detected.
+ */
+export function leaseChildAlive({ by, platform = process.platform, procRoot = "/proc", ownPid = process.pid,
+  readdirFn = readdirSync, readFileFn = readFileSync, psFn } = {}) {
+  try {
+    const marker = `FORGE_LEASE_HOLDER=${by}`;
+    if (!parseLeaseHolder(by)) return true;
+    if (platform === "linux") {
+      for (const name of readdirFn(procRoot)) {
+        if (!/^[0-9]+$/.test(name) || Number(name) === ownPid) continue;
+        let env;
+        try { env = readFileFn(`${procRoot}/${name}/environ`, "latin1"); }
+        catch (e) { if (e?.code === "EACCES" || e?.code === "ENOENT" || e?.code === "ESRCH" || e?.code === "EPERM") continue; return true; }
+        if (String(env).split("\0").includes(marker)) return true;
+      }
+      return false;
+    }
+    if (platform === "darwin") {
+      const run = psFn || (() => execFileSync("ps", ["eww", "-Ao", "pid,command"], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: 10000 }));
+      const out = run();
+      if (typeof out !== "string" || !out.trim()) return true;
+      for (const line of out.split("\n")) {
+        const m = /^\s*(\d+)\s/.exec(line);
+        if (!m || Number(m[1]) === ownPid) continue;
+        if (line.split(/\s+/).includes(marker)) return true;
+      }
+      return false;
+    }
+    return true;
+  } catch {
+    return true;
+  }
+}
+
 /** Max startup wait for a persisted rate-limit reset (guards a corrupt far-future resetAt). */
 export const MAX_RESUME_RATE_LIMIT_WAIT_MS = 6 * 60 * 60 * 1000;
 
@@ -297,7 +371,8 @@ export const MAX_RESUME_RATE_LIMIT_WAIT_MS = 6 * 60 * 60 * 1000;
  *   the lease, poll (via `sleep`, bounded by `leaseTtlMs`) until it expires or
  *   its holder is reclaimable, then continue; `deferred` on timeout.
  * @param {string} [opts.hostname] - injectable host name (tests).
- * @param {Function} [opts.killFn] - injectable `process.kill` (tests).
+  * @param {Function} [opts.killFn] - injectable `process.kill` (tests).
+ * @param {Function} [opts.childAliveFn] - forge#3724: injectable `leaseChildAlive` (orphaned phase-child probe; tests).
  */
 // forge#3824: stamped on RUN_START for per-release cost attribution; null when unreadable.
 export const ENGINE_VERSION = (() => {
@@ -326,10 +401,17 @@ export async function runIssue(opts) {
           sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
           maxSessionLimitPauses = DEFAULT_MAX_SESSION_LIMIT_PAUSES,
           sessionLimitDefaultWaitMs = DEFAULT_SESSION_LIMIT_WAIT_MS,
+          // CI still pending at the merge gate: engine-side wait budget in minutes (forge.yaml
+          // `engine.ci_pending_max_minutes`). Invalid values fall back to the default.
+          ciPendingMaxMinutes: ciPendingMaxMinutesOpt,
+          // CI-poll jitter source (tests inject a deterministic value).
+          random = Math.random,
           retry = false,
           waitLease = false,
           hostname = osHostname(),
           killFn = process.kill.bind(process),
+          // forge#3724: injectable orphaned-phase-child probe (tests).
+          childAliveFn = leaseChildAlive,
           onProgress = () => {},
           // forge#3696: per-attempt telemetry seam (FORGE:TELEMETRY emitter, #3834). No-op by
           // default; failures are swallowed so telemetry can never fail a phase.
@@ -338,6 +420,8 @@ export async function runIssue(opts) {
   // forge#2452: extracted into makeProgressEmitter() — see its docstring for
   // the forge#2240 rationale this wrapper implements.
   const emitProgress = makeProgressEmitter(onProgress);
+  const ciPendingMaxMinutes = Number.isFinite(ciPendingMaxMinutesOpt) && ciPendingMaxMinutesOpt > 0
+    ? ciPendingMaxMinutesOpt : DEFAULT_CI_PENDING_MAX_MINUTES;
 
   // forge#3833: one seam for coded outcome records that are not run-log phase events. Appends the
   // event and forwards it to the telemetry hook (#3834 owns the FORGE:TELEMETRY writer), which can
@@ -360,7 +444,7 @@ export async function runIssue(opts) {
   let remote = await projector.readState(issue);
   let { state, action } = reconcileState(local, remote);
   const leaseHeld = (r) => !!(r?.lease && r.lease.until > now() && r.lease.by !== agentId &&
-    !isLeaseHolderDead({ by: r.lease.by, host: hostname, killFn }));
+    !(isLeaseHolderDead({ by: r.lease.by, host: hostname, killFn }) && !childAliveFn({ by: r.lease.by })));
   const deferredResult = (r) => ({
     terminalReason: "deferred",
     code: OUTCOME_CODES.LEASE_HELD,
@@ -770,9 +854,13 @@ export async function runIssue(opts) {
         // the entire duration of this loop, including any `sleep()` below —
         // no separate heartbeat mechanism is needed for the wait itself.
         let pauseCount = 0;
+        let ciWaits = 0; // bounded: a poll that reads green while the phase still sees pending cannot loop forever
+        // CI-pending wait: the review phase may return BLOCKED "checks still pending" (nothing failed). The outer
+        // loop re-dispatches the phase after each deterministic wait below (at most MAX_CI_PENDING_WAITS times) (no model calls while waiting).
+        for (;;) {
         for (;;) {
           try {
-            outcome = await runPhaseWithRetry(phase, state, { io, runner, dir, issue, commandsDir, maxAttempts, backend, model, resolveRepo, agentId, emitTelemetry });
+            outcome = await runPhaseWithRetry(phase, state, { io, runner, dir, issue, commandsDir, maxAttempts, backend, model, resolveRepo, agentId, emitTelemetry, sleep });
             break;
           } catch (e) {
             const resetAtMs = e.resetAtEpochMs;
@@ -815,6 +903,14 @@ export async function runIssue(opts) {
             // the exhausted one) — AC2.
           }
         }
+        if (ciWaits < MAX_CI_PENDING_WAITS && (phase.id === "review" || phase.id === "remediate") && outcome?.status === "blocked" && outcome.wait === "ci-pending") {
+          ciWaits++;
+          const w = await waitForCiGreen(outcome, phase.id);
+          if (w.redispatch) continue;
+          if (w.outcome) outcome = w.outcome;
+        }
+        break;
+        }
       } catch (e) {
         // forge#2261: NO_API_KEY/NO_SDK/CLI_BACKEND_FAILED are fail-fast
         // rethrown by runPhaseWithRetry() (see its own catch below) instead
@@ -996,6 +1092,68 @@ export async function runIssue(opts) {
     return true;
   }
 
+  // CI still pending at the merge gate (review AND remediate's M7 re-gate). Polls `io.waitCiGreen`
+  // (scripts/wait-ci-green.sh one-shot, no model) on the ciPollDelayMs schedule inside `ciPendingMaxMinutes`; before
+  // each poll `io.rateLimit` (when present) is read and, below the reserve, the wait sleeps until the budget resets.
+  // The in-flight `renewTimer` keeps the lease alive. Resolves to
+  // `{ redispatch: true }` (green, red or a genuine error: the phase runs once more and merges or reproduces the
+  // block), `{ outcome }` (budget exhausted: needs-human `review.ci_pending_timeout`), or `{}` (no seam/PR/repo:
+  // the original outcome stands). Rate-limit and transient results (rc 4, a failed read) are waits: they retry the poll
+  // and NEVER redispatch. The start time is a run-log event so a crash resumes the remaining budget.
+  async function waitForCiGreen(outcome, phaseId = "review") {
+    const pr = outcome.outputs?.pr ?? state.pr;
+    const repoName = await resolveRepo();
+    if (typeof io.waitCiGreen !== "function" || pr == null || !repoName) return {};
+    const budgetMs = ciPendingMaxMinutes * 60000;
+    const open = readLog(dir, issue).filter((ev) => ev.event === "CI_PENDING_WAIT" && ev.pr === pr).pop();
+    const startedAt = open && open.state === "start" && Number.isFinite(open.startedAt) ? open.startedAt : now();
+    if (!(open && open.state === "start")) appendEvent(dir, issue, { event: "CI_PENDING_WAIT", state: "start", pr, startedAt });
+    let slept = 0, polls = 0, rateLimitSleeps = 0, pollIndex = 0;
+    const spentMs = () => Math.max(now() - startedAt, slept);
+    const report = async (result) => {
+      const minutes = Math.round(spentMs() / 600) / 100;
+      appendEvent(dir, issue, { event: "CI_PENDING_WAIT", state: "end", pr, startedAt, minutes_waited: minutes, result, polls, rate_limit_sleeps: rateLimitSleeps });
+      try { await emitTelemetry?.({ event: "CI_PENDING_WAIT", phase: phaseId, issue, pr, minutes_waited: minutes, result, polls, rate_limit_sleeps: rateLimitSleeps }); } catch { /* telemetry never fails a run */ }
+      return minutes;
+    };
+    const nap = async (ms, reason, detail) => {
+      emitProgress({ event: "phase_paused", phase: phaseId, reason, detail });
+      if (ms > 0) { await sleep(ms); slept += ms; }
+    };
+    for (;;) {
+      const remainingMs = budgetMs - spentMs();
+      if (remainingMs <= 0) break;
+      // Budget first: a read failure or absent seam never blocks the poll.
+      if (typeof io.rateLimit === "function") {
+        let holdMs = 0;
+        try { holdMs = ciRateLimitSleepMs(await io.rateLimit(), now(), random); } catch { holdMs = 0; }
+        if (holdMs > 0) {
+          rateLimitSleeps++;
+          await nap(Math.min(holdMs, remainingMs), "ci-rate-limit",
+            `GitHub API budget below reserve — sleeping ${Math.round(Math.min(holdMs, remainingMs) / 1000)}s until reset before polling PR #${pr}`);
+          continue;
+        }
+      }
+      let rc = 2;
+      polls++;
+      try { rc = (await io.waitCiGreen(pr, repoName, 0))?.rc ?? 2; } catch { rc = 2; }
+      if (rc !== 3 && rc !== 4) {
+        // 0 green, 1 red, 2 genuine error or head moved: one phase dispatch resolves each (merge, ci-gate handoff, re-review).
+        await report(rc === 0 ? "green" : rc === 1 ? "red" : "error");
+        return { redispatch: true };
+      }
+      if (rc === 4) rateLimitSleeps++;
+      const waitMs = Math.min(ciPollDelayMs(pollIndex++, random), Math.max(0, budgetMs - spentMs()));
+      // ci_pending_wait: the progress observer sees each wait cycle (distinct from a failure).
+      await nap(waitMs, rc === 4 ? "ci-rate-limit" : "ci-pending",
+        `${rc === 4 ? "GitHub rate-limited or unavailable; retrying" : "CI still pending"} on PR #${pr} — waited ${Math.round(spentMs() / 60000)} of ${ciPendingMaxMinutes} min`);
+    }
+    const minutes = await report("timeout");
+    return { outcome: { status: "blocked", reason: "needs-human", handoff: false, outputs: outcome.outputs,
+      detail: `checks still pending after ${minutes} min (engine.ci_pending_max_minutes=${ciPendingMaxMinutes}); re-run with --retry once CI finishes`,
+      code: OUTCOME_CODES.REVIEW_CI_PENDING_TIMEOUT, usage: outcome.usage ?? null, last_attempt_usage: outcome.last_attempt_usage ?? null } };
+  }
+
   async function terminate(s, reason, detail, code) {
     // forge#3833: absent -> the reason's own code; unknown -> `unclassified` with the raw text kept in `detail`.
     const outcomeCode = code == null ? codeForTerminalReason(reason) : normalizeCode(code);
@@ -1044,15 +1202,18 @@ export async function runIssue(opts) {
 export function cleanupRunTmpdir(issue, { cwd = process.cwd(), env = process.env, rm = rmSync } = {}) {
   try {
     const root = resolveRunTmpRoot(cwd, env);
-    const dir = runTmpDir(root, issue);
+    const dir = runTmpDir(root, issue, cwd);
     if (!dir || dir === root) return false;
     rm(dir, { recursive: true, force: true });
     return true;
   } catch { return false; }
 }
 
+/** forge#4021: backoff before each transient-spawn retry; its length is the cap (3). */
+export const INFRA_SPAWN_BACKOFF_MS = Object.freeze([10_000, 30_000, 60_000]);
+
 async function runPhaseWithRetry(phase, state, ctx) {
-  const { io, runner, dir, issue, commandsDir, maxAttempts, backend, model, resolveRepo, agentId, emitTelemetry } = ctx;
+  const { io, runner, dir, issue, commandsDir, maxAttempts, backend, model, resolveRepo, agentId, emitTelemetry, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) } = ctx;
   // forge#3499: build the phase's full argument set ONCE, up front. Every
   // work-on phase sub-skill stops with BLOCKED when a required flag is absent,
   // so an incomplete/invalid set is a deterministic engine failure — fail
@@ -1105,6 +1266,10 @@ async function runPhaseWithRetry(phase, state, ctx) {
     appendEvent(dir, issue, rec);
     try { await emitTelemetry?.({ ...rec, issue }); } catch { /* telemetry never fails a phase */ }
   };
+  // forge#4021: transient CLI spawn failures (ETIMEDOUT/EAGAIN/ENOMEM at spawn) retry on their own bounded
+  // counter and never consume a phase attempt; once spent they fall through to normal attempt accounting,
+  // so a CLI that merely runs long cannot loop forever.
+  let infraRetries = 0;
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
     appendEvent(dir, issue, { event: "PHASE_START", phase: phase.id, attempt });
     let result;
@@ -1112,6 +1277,7 @@ async function runPhaseWithRetry(phase, state, ctx) {
     try {
       result = await runner({
         commandsDir, commandName: phase.command, args: phaseArgs,
+        leaseHolder: agentId,
         // Only forwarded when explicitly provided — omitting them preserves
         // runner.mjs's existing default ("auto" backend / DEFAULT_MODEL).
         ...(backend ? { backend } : {}),
@@ -1154,14 +1320,25 @@ async function runPhaseWithRetry(phase, state, ctx) {
         appendEvent(dir, issue, { event: "PHASE_FAILED", phase: phase.id, attempt, reason: e.message, code: codeForRunnerError(e), maxAttempts });
         throw e;
       }
+      // Transient spawn class: e.code === "CLI_SPAWN_TRANSIENT" (see isTransientSpawnError); retried with infra backoff.
+      if (isTransientSpawnError(e) && infraRetries < INFRA_SPAWN_BACKOFF_MS.length) {
+        await recordAttempt(attempt, null, t0, null);
+        appendEvent(dir, issue, { event: "PHASE_FAILED", phase: phase.id, attempt, reason: e.message, code: codeForRunnerError(e), maxAttempts, infraRetry: infraRetries + 1 });
+        await sleep(INFRA_SPAWN_BACKOFF_MS[infraRetries++]);
+        attempt--;
+        continue;
+      }
       if (e.code === "NO_API_KEY" || e.code === "NO_SDK" || e.code === "CLI_BACKEND_FAILED") throw e;
       // forge#2377: no `usage` field here — the runner threw, so no result
       // (and therefore no usage data) was ever produced for this attempt.
       // Do not fabricate a value; omitting the field (rather than a stale
       // `lastUsage` from a prior attempt) keeps this event's usage
       // trustworthy as "usage actually observed on this attempt".
-      await recordAttempt(attempt, null, t0, null);
-      appendEvent(dir, issue, { event: "PHASE_FAILED", phase: phase.id, attempt, reason: e.message, code: codeForRunnerError(e), maxAttempts });
+      // forge#4037: a wall-clock kill (CLI_TIMEOUT) consumes this attempt; its usage is recorded when the CLI emitted any.
+      const killUsage = e.code === "CLI_TIMEOUT" ? e.usage ?? null : null;
+      if (killUsage) addUsage(killUsage);
+      await recordAttempt(attempt, killUsage, t0, null);
+      appendEvent(dir, issue, { event: "PHASE_FAILED", phase: phase.id, attempt, reason: e.message, code: codeForRunnerError(e), maxAttempts, ...(killUsage ? { usage: killUsage } : {}) });
       lastCode = OUTCOME_CODES.ENGINE_ERROR;
       continue;
     }
@@ -1190,7 +1367,7 @@ async function runPhaseWithRetry(phase, state, ctx) {
       // forge#3521: keep outcome.outputs (e.g. the PR number) for persistence and
       // forward `handoff` so only an explicit opt-in escalation is honoured; a
       // no-signal failure carries no handoff and stops at needs-human.
-      return { status: "blocked", detail: outcome.detail, outputs: outcome.outputs, handoff: outcome.handoff, code: lastCode, usage: cumulativeUsage, last_attempt_usage: lastUsage };
+      return { status: "blocked", detail: outcome.detail, outputs: outcome.outputs, handoff: outcome.handoff, ...(outcome.wait ? { wait: outcome.wait } : {}), code: lastCode, usage: cumulativeUsage, last_attempt_usage: lastUsage };
     }
   }
   // Exhausted transient retries → escalate (spec §7).