forgedock 1.17.0 → 1.19.0

package/bin/engine/telemetry.test.mjs

@@ -1,8 +1,9 @@
 // SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
 // SPDX-License-Identifier: AGPL-3.0-or-later
+import { tmpDir } from "../tests/_tmp.mjs";
 import { describe, it, beforeEach, afterEach } from "node:test";
 import assert from "node:assert/strict";
-import { chmodSync, existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { chmodSync, existsSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { spawnSync } from "node:child_process";
@@ -31,7 +32,7 @@ const setComments = (bodies) => writeFileSync(join(td, "comments.json"),
 
 beforeEach(() => {
   saved = Object.fromEntries(KEYS.map((k) => [k, process.env[k]]));
-  td = mkdtempSync(join(tmpdir(), "tel-"));
+  td = tmpDir("tel-");
   Object.assign(process.env, { TD: td, FORGE_TELEMETRY_BUFFER_DIR: join(td, "buf"), FORGE_REPO: "o/r", FORGE_ISSUE: "5" });
   for (const k of ["FORGE_TELEMETRY", "FORGE_PR", "FORGE_RUN", "FORGE_PHASE", "FORGE_TELEMETRY_MAX_CHARS"]) delete process.env[k];
 });

package/bin/engine-cli.mjs

@@ -6,13 +6,15 @@
  */
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
-import { readdirSync, statSync } from "node:fs";
+import { readdirSync, statSync, readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
 import { homedir, hostname } from "node:os";
 import { join } from "node:path";
 import { runIssue, DEFAULT_MAX_ATTEMPTS, sanitizeLeaseHost } from "./engine.mjs";
 import { makeProjector } from "./engine/projector.mjs";
 import { readLog, deriveState } from "./engine/runlog.mjs";
 import { assertOpenCodeNativeRuntime } from "./runner.mjs";
+import { parseForgeYaml } from "./forge-utils.mjs";
 import { makeEmitTelemetry } from "./engine/telemetry-seam.mjs";
 
 /**
@@ -53,7 +55,34 @@ export function makeIo() {
     const { stdout } = await pexec(bin, args, { maxBuffer: 100 * 1024 * 1024, timeout: 10000 });
     return stdout;
   };
-  return { gh: run("gh"), git: run("git") };
+  // CI wait seam (scripts/wait-ci-green.sh, argv only, no shell). Resolves { rc, text } for any exit code.
+  const waitCiGreen = async (pr, repo, timeoutSec) => {
+    const script = fileURLToPath(new URL("../scripts/wait-ci-green.sh", import.meta.url));
+    try {
+      // The engine owns the backoff and the budget: each call is ONE read (no in-script polling, no settle hold;
+      // the re-dispatched phase re-runs the full gate before any merge).
+      const { stdout } = await pexec("bash", [script, String(pr), "-R", String(repo), "--timeout", String(timeoutSec)],
+        { maxBuffer: 10 * 1024 * 1024, timeout: (timeoutSec + 120) * 1000, env: { ...process.env, FORGE_CI_SETTLE: "0" } });
+      return { rc: 0, text: stdout };
+    } catch (e) { return { rc: Number.isInteger(e?.code) ? e.code : 2, text: String(e?.stdout ?? "") }; }
+  };
+  // REST budget seam (`gh api rate_limit`, argv only; the call itself is not rate-limited). Resolves { remaining, limit, reset } or null.
+  const rateLimit = async () => {
+    try {
+      const { stdout } = await pexec("gh", ["api", "rate_limit", "--jq", ".resources.core"], { maxBuffer: 1024 * 1024, timeout: 10000 });
+      const j = JSON.parse(stdout);
+      return { remaining: j.remaining, limit: j.limit, reset: j.reset };
+    } catch { return null; }
+  };
+  return { gh: run("gh"), git: run("git"), waitCiGreen, rateLimit };
+}
+
+/** forge.yaml `engine.ci_pending_max_minutes` from the cwd checkout; undefined when absent or invalid. */
+export function readCiPendingMaxMinutes(cwd = process.cwd()) {
+  try {
+    const n = Number(parseForgeYaml(readFileSync(join(cwd, "forge.yaml"), "utf8"))?.engine?.ci_pending_max_minutes);
+    return Number.isInteger(n) && n > 0 ? n : undefined;
+  } catch { return undefined; }
 }
 
 export function runDir() { return join(homedir(), ".forge", "runs"); }
@@ -408,6 +437,7 @@ export async function runFromCli(argv, deps = {}) {
     // runIssue's/runner.mjs's existing defaults (forge#2028).
     ...(backend ? { backend } : {}),
     ...(model ? { model } : {}),
+    ...(readCiPendingMaxMinutes() ? { ciPendingMaxMinutes: readCiPendingMaxMinutes() } : {}),
     ...(retry ? { retry: true } : {}),
     ...(waitLease ? { waitLease: true } : {}) });
   console.log(`issue #${issue} → ${res.terminalReason}`);
@@ -420,6 +450,11 @@ export async function runFromCli(argv, deps = {}) {
     console.log(res.detail ?? "run is not retryable");
     return res;
   }
+  // forge#3932: refused before any state write (open or unreadable **Depends on**); nothing ran.
+  if (res.terminalReason === "dependency-open" || res.terminalReason === "dependency-unreadable") {
+    console.log(res.detail ?? "run refused: dependency check");
+    return res;
+  }
   // forge#2175: a non-success termination previously printed nothing beyond
   // the bare reason above — the actual failing phase/attempt/reason was only
   // recoverable by manually reading ~/.forge/runs/{issue}.jsonl and then the

package/bin/engine.mjs

@@ -4,15 +4,15 @@
  * GitHub state (phase.detectOutcome). All effects are injected → fully testable.
  */
 import { fileURLToPath } from "node:url";
-import { readFileSync } from "node:fs";
+import { readFileSync, rmSync } from "node:fs";
 import { createHash, scryptSync } from "node:crypto";
 import { hostname as osHostname, homedir } from "node:os";
 import { appendEvent, readLog, deriveState, rewriteLog } from "./engine/runlog.mjs";
-import { pickPhase, TERMINAL_REASONS, issueSnapshot, hasInvestigationExit, postCycleMarker, nextCycleNumber } from "./engine/phases.mjs";
+import { pickPhase, TERMINAL_REASONS, issueSnapshot, hasInvestigationExit, postCycleMarker, nextCycleNumber, newestCycle, cycleResetProvable, remediationHeadStale, decompositionRevoked, openDependency } from "./engine/phases.mjs";
 import { reconcileState } from "./engine/reconcile.mjs";
 import { makeProjector } from "./engine/projector.mjs";
-import { VALID_BACKENDS } from "./runner.mjs";
-import { OUTCOME_CODES, UNCLASSIFIED, normalizeCode, codeForRunnerError, codeForTerminalReason } from "./engine/outcome-codes.mjs";
+import { VALID_BACKENDS, resolveRunTmpRoot, runTmpDir } from "./runner.mjs";
+import { OUTCOME_CODES, UNCLASSIFIED, normalizeCode, codeForRunnerError, codeForTerminalReason, isDiskQuotaError, isDiskQuotaText, isTransientSpawnError } from "./engine/outcome-codes.mjs";
 
 // Exported (forge#2175) so bin/engine-cli.mjs can render "failed N/M attempts"
 // diagnostics without duplicating the retry budget constant.
@@ -38,6 +38,39 @@ export const DEFAULT_MAX_SESSION_LIMIT_PAUSES = 5;
 // reset epoch. The runner never fabricates a reset time; this default lives in
 // the engine and is still capped by `maxSessionLimitPauses`.
 export const DEFAULT_SESSION_LIMIT_WAIT_MS = 30 * 60 * 1000;
+// CI still pending at the merge gate: total wait budget (engine.ci_pending_max_minutes) and the poll scheduler bounds.
+export const DEFAULT_CI_PENDING_MAX_MINUTES = 90;
+export const CI_BACKOFF_START_MS = 60 * 1000;
+export const CI_BACKOFF_MAX_MS = 5 * 60 * 1000;
+export const CI_JITTER_FRACTION = 0.2;
+/** Below this fraction of the GitHub REST budget the scheduler sleeps until the window resets instead of polling. */
+export const CI_RATE_RESERVE_FRACTION = 0.1;
+const CI_RESET_SPREAD_MS = 30 * 1000;
+const MAX_CI_PENDING_WAITS = 3;
+
+/**
+ * Pure CI-poll scheduler: delay before poll number `pollIndex` (0-based). 60 s, x2 per poll, capped at 5 min,
+ * +/-20% jitter so lanes waiting on the same CI do not poll in lockstep. `rand` is injectable (returns [0,1)).
+ */
+export function ciPollDelayMs(pollIndex, rand = Math.random) {
+  const base = Math.min(CI_BACKOFF_START_MS * 2 ** Math.max(0, pollIndex), CI_BACKOFF_MAX_MS);
+  const r = Math.min(Math.max(Number(rand()) || 0, 0), 1);
+  return Math.round(base * (1 + (r * 2 - 1) * CI_JITTER_FRACTION));
+}
+
+/**
+ * Pure budget check: ms to sleep before the next poll so the shared REST budget keeps its reserve.
+ * `rl` is `{ remaining, limit, reset }` (reset in epoch seconds, as `gh api rate_limit` reports it). Returns 0 when the
+ * budget is healthy or unreadable (a failed read never blocks polling); otherwise the time until reset plus a random
+ * spread so lanes released by the same reset do not stampede.
+ */
+export function ciRateLimitSleepMs(rl, nowMs, rand = Math.random) {
+  const remaining = Number(rl?.remaining), limit = Number(rl?.limit), reset = Number(rl?.reset);
+  if (![remaining, limit, reset].every(Number.isFinite) || limit <= 0) return 0;
+  if (remaining > limit * CI_RATE_RESERVE_FRACTION) return 0;
+  const untilReset = Math.max(0, reset * 1000 - nowMs);
+  return untilReset + Math.round(Math.min(Math.max(Number(rand()) || 0, 0), 1) * CI_RESET_SPREAD_MS);
+}
 
 /**
  * forge#3791: non-secret identity of the backend/account a usage-limit pause belongs to.
@@ -326,6 +359,11 @@ export async function runIssue(opts) {
           sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
           maxSessionLimitPauses = DEFAULT_MAX_SESSION_LIMIT_PAUSES,
           sessionLimitDefaultWaitMs = DEFAULT_SESSION_LIMIT_WAIT_MS,
+          // CI still pending at the merge gate: engine-side wait budget in minutes (forge.yaml
+          // `engine.ci_pending_max_minutes`). Invalid values fall back to the default.
+          ciPendingMaxMinutes: ciPendingMaxMinutesOpt,
+          // CI-poll jitter source (tests inject a deterministic value).
+          random = Math.random,
           retry = false,
           waitLease = false,
           hostname = osHostname(),
@@ -338,6 +376,8 @@ export async function runIssue(opts) {
   // forge#2452: extracted into makeProgressEmitter() — see its docstring for
   // the forge#2240 rationale this wrapper implements.
   const emitProgress = makeProgressEmitter(onProgress);
+  const ciPendingMaxMinutes = Number.isFinite(ciPendingMaxMinutesOpt) && ciPendingMaxMinutesOpt > 0
+    ? ciPendingMaxMinutesOpt : DEFAULT_CI_PENDING_MAX_MINUTES;
 
   // forge#3833: one seam for coded outcome records that are not run-log phase events. Appends the
   // event and forwards it to the telemetry hook (#3834 owns the FORGE:TELEMETRY writer), which can
@@ -390,6 +430,27 @@ export async function runIssue(opts) {
     remote = cur;
     ({ state, action } = reconcileState(local, remote));
   }
+  // forge#3932: `Depends on` admission. A trusted `**Depends on**: #N` line whose dependency is still OPEN
+  // refuses the run BEFORE any state/lease write (mirrors deferredResult), so building on top of an
+  // unmerged dependency cannot conflict with it. Checked on every start; a closed dependency, no
+  // declaration or an untrusted body proceeds. An unreadable body/dependency fails closed.
+  {
+    let dep = null;
+    try { dep = await openDependency(issue, io); } catch (e) {
+      dep = { unreadable: true, detail: `issue ${issue} **Depends on** check failed: ${e?.message || e}` };
+    }
+    if (dep?.unreadable) {
+      return { terminalReason: "dependency-unreadable", code: OUTCOME_CODES.DEPENDENCY_UNREADABLE, detail: dep.detail };
+    }
+    if (dep?.open?.length) {
+      return {
+        terminalReason: "dependency-open",
+        code: OUTCOME_CODES.DEPENDENCY_OPEN,
+        dependencies: dep.open,
+        detail: `issue ${issue} **Depends on** ${dep.open.map((n) => `#${n}`).join(", ")} which is still open; re-run once it is closed`,
+      };
+    }
+  }
   // forge#3824: hoisted so RUN_START can carry the repo (lazy, memoised; null when unresolved).
   let repoPromise = null;
   const resolveRepo = () => (repoPromise ??= (async () => {
@@ -412,6 +473,20 @@ export async function runIssue(opts) {
     // and regress committed/issue/v (C2).
     rewriteLog(dir, issue, eventsFromIndex(state, await runMeta()));
     state = deriveState(readLog(dir, issue));
+    // forge#3931: the index does not carry the cycle, so a hydrated log always starts at cycle 0. When a
+    // trusted FORGE:CYCLE exists, cycle-scoped progress (or a needs-human/phase-complete/engine-error terminal)
+    // belongs to an older cycle ONLY when provably so (cycleResetProvable: terminal index, issue OPEN, no needs-human/remediating label, no open PR): reset via the existing RUN_NEXT_CYCLE event so build runs again instead of
+    // terminating. An unreadable trail resets nothing (fail closed). merged/decomposed/invalid terminals stay.
+    if (state.committed.some((p) => ["build", "review", "remediate", "close"].includes(p)) &&
+        state.terminal && ["needs-human", "phase-complete", "engine-error"].includes(state.terminalReason) &&
+        await cycleResetProvable(state, io)) {
+      let cyc = 0;
+      try { cyc = await newestCycle(issue, io); } catch { cyc = 0; }
+      if (cyc > 0) {
+        appendEvent(dir, issue, { event: "RUN_NEXT_CYCLE", issue });
+        state = deriveState(readLog(dir, issue));
+      }
+    }
   }
 
   // forge#3511: explicit `--retry` reopens a transient `engine-error` terminal
@@ -424,12 +499,19 @@ export async function runIssue(opts) {
   // forge#3562: `phase-complete` is also retryable, but as a NEW cycle
   // (RUN_NEXT_CYCLE clears committed) rather than a reopen of the same one.
   let reopened = false;
+  let reopenUncommit = null;
   if (retry && state?.terminal) {
-    if (state.terminalReason !== "engine-error" && state.terminalReason !== "phase-complete" &&
+    // forge#3862 / forge#3932: a `decomposed` terminal is reopened at build when trusted GitHub state shows
+    // the decomposition revoked (a trusted FORGE:DECOMPOSED:REVOKED newer than FORGE:DECOMPOSED); the
+    // revocation alone suffices now that the size gate is gone. Anything else keeps the terminal; an
+    // unreadable comment trail fails closed.
+    if (state.terminalReason === "decomposed" && await decompositionRevoked(issue, io)) {
+      reopenUncommit = ["build", "decompose"];
+    } else if (state.terminalReason !== "engine-error" && state.terminalReason !== "phase-complete" &&
         state.terminalReason !== "needs-human") {
       return {
         terminalReason: "not-retryable",
-        detail: `issue ${issue} ended ${state.terminalReason}; --retry only resumes an engine-error, phase-complete or needs-human run`,
+        detail: `issue ${issue} ended ${state.terminalReason}; --retry only resumes an engine-error, phase-complete or needs-human run (or a decomposed run whose FORGE:DECOMPOSED was revoked by a trusted FORGE:DECOMPOSED:REVOKED)`,
       };
     }
     // forge#3571: a human clearing `needs-human` is the hand-back signal. Reopen
@@ -457,6 +539,13 @@ export async function runIssue(opts) {
         };
       }
     }
+    // forge#3931: a remediate that ended needs-human was recorded against an older PR head. When the
+    // live head differs from the head in its trusted trail, un-commit remediate AND review so review
+    // re-runs at the new head instead of the old RE-ESCALATED result replaying. Unreadable => no change.
+    if (state.terminalReason === "needs-human" && !reopenUncommit &&
+        state.committed[state.committed.length - 1] === "remediate" && await remediationHeadStale(state, io)) {
+      reopenUncommit = ["remediate", "review"];
+    }
     // forge#3562 (SEC-3): a new cycle discards `committed`, so check the live
     // issue first. A CLOSED issue has nothing left to run; an unreadable
     // snapshot fails closed rather than resetting a possibly-finished run.
@@ -486,6 +575,7 @@ export async function runIssue(opts) {
     appendEvent(dir, issue, {
       event: state.terminalReason === "phase-complete" ? "RUN_NEXT_CYCLE" : "RUN_REOPEN",
       issue,
+      ...(reopenUncommit ? { uncommit: reopenUncommit } : {}),
     });
     state = deriveState(readLog(dir, issue));
     reopened = true;
@@ -720,9 +810,13 @@ export async function runIssue(opts) {
         // the entire duration of this loop, including any `sleep()` below —
         // no separate heartbeat mechanism is needed for the wait itself.
         let pauseCount = 0;
+        let ciWaits = 0; // bounded: a poll that reads green while the phase still sees pending cannot loop forever
+        // CI-pending wait: the review phase may return BLOCKED "checks still pending" (nothing failed). The outer
+        // loop re-dispatches the phase after each deterministic wait below (at most MAX_CI_PENDING_WAITS times) (no model calls while waiting).
+        for (;;) {
         for (;;) {
           try {
-            outcome = await runPhaseWithRetry(phase, state, { io, runner, dir, issue, commandsDir, maxAttempts, backend, model, resolveRepo, agentId, emitTelemetry });
+            outcome = await runPhaseWithRetry(phase, state, { io, runner, dir, issue, commandsDir, maxAttempts, backend, model, resolveRepo, agentId, emitTelemetry, sleep });
             break;
           } catch (e) {
             const resetAtMs = e.resetAtEpochMs;
@@ -765,6 +859,14 @@ export async function runIssue(opts) {
             // the exhausted one) — AC2.
           }
         }
+        if (ciWaits < MAX_CI_PENDING_WAITS && (phase.id === "review" || phase.id === "remediate") && outcome?.status === "blocked" && outcome.wait === "ci-pending") {
+          ciWaits++;
+          const w = await waitForCiGreen(outcome, phase.id);
+          if (w.redispatch) continue;
+          if (w.outcome) outcome = w.outcome;
+        }
+        break;
+        }
       } catch (e) {
         // forge#2261: NO_API_KEY/NO_SDK/CLI_BACKEND_FAILED are fail-fast
         // rethrown by runPhaseWithRetry() (see its own catch below) instead
@@ -779,7 +881,7 @@ export async function runIssue(opts) {
         // "needs-human" — this is the engine/tool breaking, not a genuine
         // human-judgment block (see #2244/#2261). Any other thrown error is
         // a true unexpected crash and keeps propagating unchanged.
-        if (e.code === "NO_API_KEY" || e.code === "NO_SDK" || e.code === "CLI_BACKEND_FAILED" || e.code === "CLI_MAX_TURNS") {
+        if (e.code === "NO_API_KEY" || e.code === "NO_SDK" || e.code === "CLI_BACKEND_FAILED" || e.code === "CLI_MAX_TURNS" || isDiskQuotaError(e)) {
           // forge#2241: when the runner attached a session-limit reset time
           // (bin/runner.mjs's extractSessionLimitResetTime(), only ever set
           // for a genuine session-limit CLI_BACKEND_FAILED — never
@@ -789,7 +891,7 @@ export async function runIssue(opts) {
           const resetSuffix = e.resetAt ? ` (resets: ${e.resetAt})` : "";
           const detail = e.code === "CLI_MAX_TURNS"
             ? `phase ${phase.id}: CLI_MAX_TURNS - max-turns exhausted${e.numTurns != null ? ` after ${e.numTurns} turns` : ""}: ${e.message}`
-            : `phase ${phase.id}: ${e.code} - ${e.message}${resetSuffix}`;
+            : `phase ${phase.id}: ${e.code ?? "DISK_QUOTA"} - ${e.message}${resetSuffix}`;
           // forge#2240 (review finding): this fail-fast path previously left
           // phase_exit unreported — a caller tailing progress output would see
           // "→ phase X started" and then nothing, dangling exactly on the
@@ -845,6 +947,9 @@ export async function runIssue(opts) {
         await emitOutcome({ event: "PHASE_DEFERRED", phase: phase.id, code: normalizeCode(outcome.code), reason: outcome.detail });
         return { terminalReason: "deferred", code: normalizeCode(outcome.code), detail: outcome.detail };
       }
+      // forge#3572: a phase that returned blocked because the disk/quota ran out is a retryable infra failure.
+      if (isDiskQuotaText(outcome.detail))
+        return await terminate(state, "engine-error", outcome.detail, OUTCOME_CODES.INFRA_DISK_QUOTA);
       if (!isRemediationHandoff)
         return await terminate(state, outcome.reason || "needs-human", outcome.detail, outcome.code);
     } else {
@@ -910,8 +1015,8 @@ export async function runIssue(opts) {
     // is seen), phase.id is "decompose" — not "investigate" — so this
     // exemption does not apply and the normal terminate() path below fires,
     // ending the run for real.
-    // build also hands off: the B5.5 size gate (NEEDS_DECOMPOSE) reports "decomposed".
-    const isDecomposeHandoff = (phase.id === "investigate" || phase.id === "build") && terminalReason === "decomposed";
+    // forge#3862: only investigate hands off (decomposition is a planning decision); build never reports "decomposed".
+    const isDecomposeHandoff = phase.id === "investigate" && terminalReason === "decomposed";
     // forge#3624: remediate AUTO-LANDED whose issue is not yet CLOSED+workflow:merged hands off
     // to `close` instead of ending the run (close is idempotent). Exempts BOTH terminate paths.
     const isCloseHandoff = phase.id === "remediate" && outcome.status === "committed" &&
@@ -943,6 +1048,68 @@ export async function runIssue(opts) {
     return true;
   }
 
+  // CI still pending at the merge gate (review AND remediate's M7 re-gate). Polls `io.waitCiGreen`
+  // (scripts/wait-ci-green.sh one-shot, no model) on the ciPollDelayMs schedule inside `ciPendingMaxMinutes`; before
+  // each poll `io.rateLimit` (when present) is read and, below the reserve, the wait sleeps until the budget resets.
+  // The in-flight `renewTimer` keeps the lease alive. Resolves to
+  // `{ redispatch: true }` (green, red or a genuine error: the phase runs once more and merges or reproduces the
+  // block), `{ outcome }` (budget exhausted: needs-human `review.ci_pending_timeout`), or `{}` (no seam/PR/repo:
+  // the original outcome stands). Rate-limit and transient results (rc 4, a failed read) are waits: they retry the poll
+  // and NEVER redispatch. The start time is a run-log event so a crash resumes the remaining budget.
+  async function waitForCiGreen(outcome, phaseId = "review") {
+    const pr = outcome.outputs?.pr ?? state.pr;
+    const repoName = await resolveRepo();
+    if (typeof io.waitCiGreen !== "function" || pr == null || !repoName) return {};
+    const budgetMs = ciPendingMaxMinutes * 60000;
+    const open = readLog(dir, issue).filter((ev) => ev.event === "CI_PENDING_WAIT" && ev.pr === pr).pop();
+    const startedAt = open && open.state === "start" && Number.isFinite(open.startedAt) ? open.startedAt : now();
+    if (!(open && open.state === "start")) appendEvent(dir, issue, { event: "CI_PENDING_WAIT", state: "start", pr, startedAt });
+    let slept = 0, polls = 0, rateLimitSleeps = 0, pollIndex = 0;
+    const spentMs = () => Math.max(now() - startedAt, slept);
+    const report = async (result) => {
+      const minutes = Math.round(spentMs() / 600) / 100;
+      appendEvent(dir, issue, { event: "CI_PENDING_WAIT", state: "end", pr, startedAt, minutes_waited: minutes, result, polls, rate_limit_sleeps: rateLimitSleeps });
+      try { await emitTelemetry?.({ event: "CI_PENDING_WAIT", phase: phaseId, issue, pr, minutes_waited: minutes, result, polls, rate_limit_sleeps: rateLimitSleeps }); } catch { /* telemetry never fails a run */ }
+      return minutes;
+    };
+    const nap = async (ms, reason, detail) => {
+      emitProgress({ event: "phase_paused", phase: phaseId, reason, detail });
+      if (ms > 0) { await sleep(ms); slept += ms; }
+    };
+    for (;;) {
+      const remainingMs = budgetMs - spentMs();
+      if (remainingMs <= 0) break;
+      // Budget first: a read failure or absent seam never blocks the poll.
+      if (typeof io.rateLimit === "function") {
+        let holdMs = 0;
+        try { holdMs = ciRateLimitSleepMs(await io.rateLimit(), now(), random); } catch { holdMs = 0; }
+        if (holdMs > 0) {
+          rateLimitSleeps++;
+          await nap(Math.min(holdMs, remainingMs), "ci-rate-limit",
+            `GitHub API budget below reserve — sleeping ${Math.round(Math.min(holdMs, remainingMs) / 1000)}s until reset before polling PR #${pr}`);
+          continue;
+        }
+      }
+      let rc = 2;
+      polls++;
+      try { rc = (await io.waitCiGreen(pr, repoName, 0))?.rc ?? 2; } catch { rc = 2; }
+      if (rc !== 3 && rc !== 4) {
+        // 0 green, 1 red, 2 genuine error or head moved: one phase dispatch resolves each (merge, ci-gate handoff, re-review).
+        await report(rc === 0 ? "green" : rc === 1 ? "red" : "error");
+        return { redispatch: true };
+      }
+      if (rc === 4) rateLimitSleeps++;
+      const waitMs = Math.min(ciPollDelayMs(pollIndex++, random), Math.max(0, budgetMs - spentMs()));
+      // ci_pending_wait: the progress observer sees each wait cycle (distinct from a failure).
+      await nap(waitMs, rc === 4 ? "ci-rate-limit" : "ci-pending",
+        `${rc === 4 ? "GitHub rate-limited or unavailable; retrying" : "CI still pending"} on PR #${pr} — waited ${Math.round(spentMs() / 60000)} of ${ciPendingMaxMinutes} min`);
+    }
+    const minutes = await report("timeout");
+    return { outcome: { status: "blocked", reason: "needs-human", handoff: false, outputs: outcome.outputs,
+      detail: `checks still pending after ${minutes} min (engine.ci_pending_max_minutes=${ciPendingMaxMinutes}); re-run with --retry once CI finishes`,
+      code: OUTCOME_CODES.REVIEW_CI_PENDING_TIMEOUT, usage: outcome.usage ?? null, last_attempt_usage: outcome.last_attempt_usage ?? null } };
+  }
+
   async function terminate(s, reason, detail, code) {
     // forge#3833: absent -> the reason's own code; unknown -> `unclassified` with the raw text kept in `detail`.
     const outcomeCode = code == null ? codeForTerminalReason(reason) : normalizeCode(code);
@@ -980,12 +1147,29 @@ export async function runIssue(opts) {
         try { await projector.setLabel(issue, "needs-human"); } catch { /* best-effort */ }
       }
     }
+    // forge#3572: drop the per-run scratch dir (venvs, session data) on a terminal outcome. Kept for
+    // `engine-error` so `--retry` resumes with it, and when another live engine owns the issue.
+    if (reason !== "engine-error" && foreign === undefined) cleanupRunTmpdir(issue);
     return { terminalReason: reason, code: outcomeCode, detail };
   }
 }
 
+/** Best-effort removal of the issue's per-run scratch dir. Confined to the resolved tmp root; never throws. */
+export function cleanupRunTmpdir(issue, { cwd = process.cwd(), env = process.env, rm = rmSync } = {}) {
+  try {
+    const root = resolveRunTmpRoot(cwd, env);
+    const dir = runTmpDir(root, issue, cwd);
+    if (!dir || dir === root) return false;
+    rm(dir, { recursive: true, force: true });
+    return true;
+  } catch { return false; }
+}
+
+/** forge#4021: backoff before each transient-spawn retry; its length is the cap (3). */
+export const INFRA_SPAWN_BACKOFF_MS = Object.freeze([10_000, 30_000, 60_000]);
+
 async function runPhaseWithRetry(phase, state, ctx) {
-  const { io, runner, dir, issue, commandsDir, maxAttempts, backend, model, resolveRepo, agentId, emitTelemetry } = ctx;
+  const { io, runner, dir, issue, commandsDir, maxAttempts, backend, model, resolveRepo, agentId, emitTelemetry, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) } = ctx;
   // forge#3499: build the phase's full argument set ONCE, up front. Every
   // work-on phase sub-skill stops with BLOCKED when a required flag is absent,
   // so an incomplete/invalid set is a deterministic engine failure — fail
@@ -1038,6 +1222,10 @@ async function runPhaseWithRetry(phase, state, ctx) {
     appendEvent(dir, issue, rec);
     try { await emitTelemetry?.({ ...rec, issue }); } catch { /* telemetry never fails a phase */ }
   };
+  // forge#4021: transient CLI spawn failures (ETIMEDOUT/EAGAIN/ENOMEM at spawn) retry on their own bounded
+  // counter and never consume a phase attempt; once spent they fall through to normal attempt accounting,
+  // so a CLI that merely runs long cannot loop forever.
+  let infraRetries = 0;
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
     appendEvent(dir, issue, { event: "PHASE_START", phase: phase.id, attempt });
     let result;
@@ -1080,14 +1268,32 @@ async function runPhaseWithRetry(phase, state, ctx) {
         });
         throw e;
       }
+      // forge#3572: a full disk / exhausted quota (EDQUOT/ENOSPC) is infra, not content: retrying burns the budget on the same
+      // failure, so rethrow fail-fast and let runIssue end it as a retryable `engine-error` (never needs-human).
+      if (isDiskQuotaError(e)) {
+        await recordAttempt(attempt, null, t0, null);
+        appendEvent(dir, issue, { event: "PHASE_FAILED", phase: phase.id, attempt, reason: e.message, code: codeForRunnerError(e), maxAttempts });
+        throw e;
+      }
+      // Transient spawn class: e.code === "CLI_SPAWN_TRANSIENT" (see isTransientSpawnError); retried with infra backoff.
+      if (isTransientSpawnError(e) && infraRetries < INFRA_SPAWN_BACKOFF_MS.length) {
+        await recordAttempt(attempt, null, t0, null);
+        appendEvent(dir, issue, { event: "PHASE_FAILED", phase: phase.id, attempt, reason: e.message, code: codeForRunnerError(e), maxAttempts, infraRetry: infraRetries + 1 });
+        await sleep(INFRA_SPAWN_BACKOFF_MS[infraRetries++]);
+        attempt--;
+        continue;
+      }
       if (e.code === "NO_API_KEY" || e.code === "NO_SDK" || e.code === "CLI_BACKEND_FAILED") throw e;
       // forge#2377: no `usage` field here — the runner threw, so no result
       // (and therefore no usage data) was ever produced for this attempt.
       // Do not fabricate a value; omitting the field (rather than a stale
       // `lastUsage` from a prior attempt) keeps this event's usage
       // trustworthy as "usage actually observed on this attempt".
-      await recordAttempt(attempt, null, t0, null);
-      appendEvent(dir, issue, { event: "PHASE_FAILED", phase: phase.id, attempt, reason: e.message, code: codeForRunnerError(e), maxAttempts });
+      // forge#4037: a wall-clock kill (CLI_TIMEOUT) consumes this attempt; its usage is recorded when the CLI emitted any.
+      const killUsage = e.code === "CLI_TIMEOUT" ? e.usage ?? null : null;
+      if (killUsage) addUsage(killUsage);
+      await recordAttempt(attempt, killUsage, t0, null);
+      appendEvent(dir, issue, { event: "PHASE_FAILED", phase: phase.id, attempt, reason: e.message, code: codeForRunnerError(e), maxAttempts, ...(killUsage ? { usage: killUsage } : {}) });
       lastCode = OUTCOME_CODES.ENGINE_ERROR;
       continue;
     }
@@ -1116,7 +1322,7 @@ async function runPhaseWithRetry(phase, state, ctx) {
       // forge#3521: keep outcome.outputs (e.g. the PR number) for persistence and
       // forward `handoff` so only an explicit opt-in escalation is honoured; a
       // no-signal failure carries no handoff and stops at needs-human.
-      return { status: "blocked", detail: outcome.detail, outputs: outcome.outputs, handoff: outcome.handoff, code: lastCode, usage: cumulativeUsage, last_attempt_usage: lastUsage };
+      return { status: "blocked", detail: outcome.detail, outputs: outcome.outputs, handoff: outcome.handoff, ...(outcome.wait ? { wait: outcome.wait } : {}), code: lastCode, usage: cumulativeUsage, last_attempt_usage: lastUsage };
     }
   }
   // Exhausted transient retries → escalate (spec §7).

package/bin/forgedock.mjs

@@ -3972,7 +3972,7 @@ switch (command) {
       // an uncaught throw, which already set exitCode = 1 below. Now that
       // runIssue() resolves cleanly with a terminalReason instead of throwing,
       // preserve that same non-zero exit code for this reason too.
-      if (result && (result.terminalReason === "needs-human" || result.terminalReason === "engine-error" || result.terminalReason === "not-retryable")) exitCode = 1;
+      if (result && (result.terminalReason === "needs-human" || result.terminalReason === "engine-error" || result.terminalReason === "not-retryable" || result.terminalReason === "dependency-open" || result.terminalReason === "dependency-unreadable")) exitCode = 1;
       // forge#3574: a lease-held deferral did NOT run the issue; exit 75
       // (EX_TEMPFAIL) so orchestrators neither count it as done nor as a failure.
       if (result && result.terminalReason === "deferred") exitCode = 75;