forgedock 1.49.0 → 1.51.0

package/bin/lean/live-execution.mjs

@@ -0,0 +1,136 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Live execution control of a running batch (#5138): `forgedock orchestrate --set-max-remote <id> N` and
+ * `--set-placement <id> remote|hybrid|local`, like `--set-parallel` for K. The override lives in the batch's `execution`
+ * file (batch.mjs `readExecution`/`setExecution`, JSON `{maxRemote?, mode?, provider?}`); the runner re-reads it on every
+ * admission through the getters `liveRemote` puts on the `remote` object orchestrate gets (`max`, `fallbackLocal`, `place`).
+ * Running lanes are never stopped: a lower `max_remote` only holds new remote admissions. A batch that started without a
+ * remote provider (`provider` not true: mode local, or `max_remote` 0) refuses `remote`/`hybrid`, since its provider,
+ * ledger and execution budget are built once at start; start a new batch for that.
+ */
+
+export const LIVE_MODES = Object.freeze(["remote", "hybrid", "local"]);
+const LIGHT_WEIGHT = Object.freeze({ heavy: false, signals: [] });
+
+/** Validate a patch `{maxRemote?, mode?, provider?}`; throws on a bad value. @returns the cleaned patch. */
+export function validateExecutionPatch(patch = {}) {
+  const out = {};
+  if (patch.maxRemote !== undefined) {
+    const n = typeof patch.maxRemote === "string" && /^\d+$/.test(patch.maxRemote.trim()) ? parseInt(patch.maxRemote, 10) : patch.maxRemote;
+    if (!Number.isInteger(n) || n < 0) throw new Error("max_remote must be a non-negative integer");
+    out.maxRemote = n;
+  }
+  if (patch.mode !== undefined) {
+    if (!LIVE_MODES.includes(patch.mode)) throw new Error(`placement must be one of ${LIVE_MODES.join("|")}`);
+    out.mode = patch.mode;
+  }
+  if (patch.provider !== undefined) {
+    if (typeof patch.provider !== "boolean") throw new Error("provider must be a boolean");
+    out.provider = patch.provider;
+  }
+  return out;
+}
+
+/** The fields of a stored override that are valid (anything else is dropped); null when there is none. */
+export function sanitizeOverride(o) {
+  if (!o || typeof o !== "object" || Array.isArray(o)) return null;
+  const out = {};
+  if (Number.isInteger(o.maxRemote) && o.maxRemote >= 0) out.maxRemote = o.maxRemote;
+  if (LIVE_MODES.includes(o.mode)) out.mode = o.mode;
+  if (typeof o.provider === "boolean") out.provider = o.provider;
+  return out;
+}
+
+/** The pinned forge.yaml `exec` with the live override's `maxRemote` and `mode` applied. */
+export function effectiveExecution(exec, override) {
+  const o = sanitizeOverride(override) ?? {};
+  return { ...(exec ?? {}), ...(o.maxRemote !== undefined ? { maxRemote: o.maxRemote } : {}), ...(o.mode !== undefined ? { mode: o.mode } : {}) };
+}
+
+/**
+ * `--set-max-remote <id> N` / `--set-placement <id> MODE` in argv → `{id, patch}`, or null when neither flag is there.
+ * Throws on a missing id or a bad value.
+ */
+export function parseSetExecution(argv = []) {
+  const at = (flag) => argv.indexOf(flag);
+  let id = null;
+  const patch = {};
+  for (const [flag, key] of [["--set-max-remote", "maxRemote"], ["--set-placement", "mode"]]) {
+    const i = at(flag);
+    if (i < 0) continue;
+    const bid = argv[i + 1], val = argv[i + 2];
+    if (!bid || val === undefined) throw new Error(`usage: forgedock orchestrate ${flag} <id> ${key === "mode" ? LIVE_MODES.join("|") : "N"}`);
+    if (id !== null && id !== bid) throw new Error("--set-max-remote and --set-placement must name the same batch");
+    id = bid;
+    patch[key] = key === "maxRemote" ? (/^\d+$/.test(val) ? parseInt(val, 10) : NaN) : val;
+  }
+  return id === null ? null : { id, patch: validateExecutionPatch(patch) };
+}
+
+/**
+ * Apply a patch to `batch` (batch.mjs openBatch): refused for a missing batch, and `remote`/`hybrid` for a batch whose
+ * `execution` file does not record `provider: true`. @returns the stored override.
+ */
+export function applySetExecution(batch, patch) {
+  if (!batch.exists()) throw new Error(`no batch ${batch.id}`);
+  const p = validateExecutionPatch(patch);
+  if ((p.mode === "remote" || p.mode === "hybrid") && batch.readExecution()?.provider !== true) {
+    throw new Error(`batch ${batch.id} has no remote provider; placement ${p.mode} needs one: start a new batch with forge.yaml execution`);
+  }
+  return batch.setExecution(p);
+}
+
+/** Text for a stored override: `max_remote 2, placement hybrid`. */
+export function formatOverride(o) {
+  const s = sanitizeOverride(o) ?? {};
+  return [s.maxRemote !== undefined ? `max_remote ${s.maxRemote}` : null, s.mode ? `placement ${s.mode}` : null].filter(Boolean).join(", ") || "no change";
+}
+
+/**
+ * Hybrid validation weight (#4256) that is computed only when first needed: `initial` (already computed) or `compute()`
+ * started on the first `get()`; until it resolves (or when it fails) a light weight is used.
+ */
+export function lazyWeight({ initial = null, compute = null } = {}) {
+  let w = initial, started = initial !== null;
+  return {
+    get: () => {
+      if (!started && typeof compute === "function") {
+        started = true;
+        Promise.resolve().then(compute).then((r) => { w = r ?? LIGHT_WEIGHT; }, () => { w = LIGHT_WEIGHT; });
+      }
+      return w ?? LIGHT_WEIGHT;
+    },
+  };
+}
+
+/**
+ * Put live `max`, `fallbackLocal` and `place` getters on `base` (the `remote` object orchestrate gets; never spread it
+ * afterwards, a spread freezes the getters). `place(exec, n)` places one lane under the effective exec (execution.mjs
+ * `placeLane` with the remote or hybrid facts); mode `local` places every new lane on the host. `onChange(exec)` is
+ * called once each time the effective `{mode, maxRemote}` differs from the last one seen (the pinned one first).
+ */
+export function liveRemote(base, { exec, readOverride = () => null, place, onChange = null }) {
+  const keyOf = (e) => `${e.mode}|${e.maxRemote}`;
+  let last = keyOf(effectiveExecution(exec, null));
+  const current = () => {
+    let o = null;
+    try { o = readOverride() ?? null; } catch { o = null; }
+    const e = effectiveExecution(exec, o);
+    const k = keyOf(e);
+    if (k !== last) { last = k; try { onChange?.(e); } catch { /* advisory */ } }
+    return e;
+  };
+  return Object.defineProperties(base, {
+    max: { get: () => current().maxRemote, enumerable: true, configurable: true },
+    fallbackLocal: { get: () => { const e = current(); return e.mode === "hybrid" && Number.isInteger(e.maxRemote) && e.maxRemote >= 1; }, enumerable: true, configurable: true },
+    place: {
+      value: (n) => {
+        const e = current();
+        if (e.mode === "local") return { where: "local", reason: "live placement: local" };
+        return place(e, n);
+      },
+      enumerable: true, configurable: true, writable: true,
+    },
+  });
+}

package/bin/lean/live-execution.test.mjs

@@ -0,0 +1,83 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { tmpDir } from "../tests/_tmp.mjs";
+import { openBatch } from "./batch.mjs";
+import { validateExecutionPatch, sanitizeOverride, effectiveExecution, parseSetExecution, applySetExecution, formatOverride, lazyWeight, liveRemote, LIVE_MODES } from "./live-execution.mjs";
+
+describe("#5138 live execution control", () => {
+  it("validates patches: max_remote a non-negative integer, mode remote|hybrid|local", () => {
+    assert.deepEqual(LIVE_MODES, ["remote", "hybrid", "local"]);
+    assert.deepEqual(validateExecutionPatch({ maxRemote: 0, mode: "local" }), { maxRemote: 0, mode: "local" });
+    assert.deepEqual(validateExecutionPatch({ maxRemote: "3" }), { maxRemote: 3 });
+    for (const bad of [{ maxRemote: -1 }, { maxRemote: 1.5 }, { maxRemote: "x" }]) assert.throws(() => validateExecutionPatch(bad), /non-negative integer/);
+    assert.throws(() => validateExecutionPatch({ mode: "cloud" }), /placement must be one of/);
+    assert.throws(() => validateExecutionPatch({ provider: "yes" }), /boolean/);
+    assert.deepEqual(sanitizeOverride({ maxRemote: -2, mode: "x", provider: true }), { provider: true });
+    assert.equal(sanitizeOverride(null), null);
+  });
+
+  it("merges the override over the pinned exec", () => {
+    const exec = { mode: "remote", maxRemote: 4, provider: "container" };
+    assert.deepEqual(effectiveExecution(exec, null), exec);
+    assert.deepEqual(effectiveExecution(exec, { maxRemote: 1, mode: "hybrid" }), { mode: "hybrid", maxRemote: 1, provider: "container" });
+    assert.equal(formatOverride({ maxRemote: 2, mode: "hybrid" }), "max_remote 2, placement hybrid");
+  });
+
+  it("parses --set-max-remote / --set-placement", () => {
+    assert.equal(parseSetExecution(["--set-parallel", "b", "2"]), null);
+    assert.deepEqual(parseSetExecution(["--set-max-remote", "b1", "2"]), { id: "b1", patch: { maxRemote: 2 } });
+    assert.deepEqual(parseSetExecution(["--set-placement", "b1", "hybrid", "--set-max-remote", "b1", "0"]), { id: "b1", patch: { maxRemote: 0, mode: "hybrid" } });
+    assert.throws(() => parseSetExecution(["--set-max-remote", "b1", "-1"]), /non-negative/);
+    assert.throws(() => parseSetExecution(["--set-placement", "b1"]), /usage/);
+    assert.throws(() => parseSetExecution(["--set-placement", "b1", "x"]), /placement must be/);
+    assert.throws(() => parseSetExecution(["--set-placement", "b1", "local", "--set-max-remote", "b2", "1"]), /same batch/);
+  });
+
+  it("applySetExecution: missing batch and a provider-less batch going remote/hybrid are refused", () => {
+    const root = tmpDir("live-exec-");
+    assert.throws(() => applySetExecution(openBatch(root, "nope"), { mode: "local" }), /no batch nope/);
+    const b = openBatch(root, "b1");
+    b.writePlan({ argv: ["1"] });
+    for (const mode of ["remote", "hybrid"]) assert.throws(() => applySetExecution(b, { mode }), /no remote provider/);
+    assert.deepEqual(applySetExecution(b, { mode: "local", maxRemote: 0 }), { mode: "local", maxRemote: 0 });
+    b.setExecution({ provider: true });
+    assert.deepEqual(applySetExecution(b, { mode: "hybrid" }), { mode: "hybrid", maxRemote: 0, provider: true });
+    assert.deepEqual(b.readExecution(), { mode: "hybrid", maxRemote: 0, provider: true });
+  });
+
+  it("liveRemote getters follow the override file; local places on the host; onChange once per change", () => {
+    let o = null;
+    const changes = [], placed = [];
+    const r = liveRemote({ onBudget: "stop" }, { exec: { mode: "remote", maxRemote: 2 }, readOverride: () => o, onChange: (e) => changes.push(`${e.mode}/${e.maxRemote}`),
+      place: (e, n) => { placed.push([e.mode, n]); return { where: "remote", reason: `mode ${e.mode}` }; } });
+    assert.equal(r.max, 2);
+    assert.equal(r.fallbackLocal, false);
+    assert.deepEqual(r.place(1), { where: "remote", reason: "mode remote" });
+    o = { maxRemote: 5 };
+    assert.equal(r.max, 5);
+    o = { maxRemote: 5, mode: "hybrid" };
+    assert.equal(r.fallbackLocal, true);
+    r.place(2);
+    o = { maxRemote: 5, mode: "local" };
+    assert.deepEqual(r.place(3), { where: "local", reason: "live placement: local" });
+    assert.deepEqual(placed, [["remote", 1], ["hybrid", 2]]);
+    assert.deepEqual(changes, ["remote/5", "hybrid/5", "local/5"]);
+    assert.equal({ ...r }.onBudget, "stop");
+  });
+
+  it("lazyWeight is light until the computed weight resolves, and computes once", async () => {
+    let calls = 0;
+    const w = lazyWeight({ compute: async () => { calls++; return { heavy: true, signals: ["tsc"] }; } });
+    assert.deepEqual(w.get(), { heavy: false, signals: [] });
+    await new Promise((r) => setImmediate(r));
+    assert.deepEqual(w.get(), { heavy: true, signals: ["tsc"] });
+    w.get();
+    assert.equal(calls, 1);
+    assert.deepEqual(lazyWeight({ initial: { heavy: true, signals: [] } }).get(), { heavy: true, signals: [] });
+    const failing = lazyWeight({ compute: () => { throw new Error("scan"); } });
+    failing.get(); await new Promise((r) => setImmediate(r));
+    assert.deepEqual(failing.get(), { heavy: false, signals: [] });
+  });
+});

package/bin/lean/orchestrate-blocked.test.mjs

@@ -120,3 +120,50 @@ describe("#5048: the pool is unreachable only after a failed capacity probe", ()
     assert.deepEqual(notes, ["down", "clear"]);
   });
 });
+
+describe("#5162: failed roots hold their subtree; only real cycles are cycles", () => {
+  const memStore = () => { let st = null; return { load: () => st, save: (s) => { st = JSON.parse(JSON.stringify(s)); }, get: () => st }; };
+  const chain = { 1: "", 2: "**Depends on**: #1", 3: "**Depends on**: #2", 4: "**Depends on**: #1, #3" };
+  const outcome = (rows, n) => rows.find((r) => r.issue === n);
+
+  it("a DAG whose engine-error root ends marks no row dependency cycle; every dependent is held and names the root", async () => {
+    const rows = await orchestrate({ issues: [1, 2, 3, 4], parallel: 2, pollMs: 5, readBody: async (n) => chain[n], engineErrorRetries: 0, transientRetries: 0,
+      runOne: async () => ({ terminalReason: "engine-error", detail: "boom" }) });
+    assert.ok(!rows.some((r) => /dependency cycle/.test(r.detail ?? "")), JSON.stringify(rows));
+    for (const n of [2, 3, 4]) { assert.equal(outcome(rows, n).outcome, "held"); assert.match(outcome(rows, n).detail, /^blocked by #1 \(engine-error\)$/); }
+  });
+
+  it("a genuine cycle is still labelled dependency cycle; its downstream names a member", async () => {
+    const bodies = { 1: "**Depends on**: #2", 2: "**Depends on**: #1", 3: "**Depends on**: #2" };
+    const rows = await orchestrate({ issues: [1, 2, 3], parallel: 2, pollMs: 5, readBody: async (n) => bodies[n], runOne: async () => merged });
+    assert.deepEqual(rows.map((r) => r.outcome), ["skipped", "skipped", "skipped"]);
+    assert.equal(outcome(rows, 1).detail, "dependency cycle");
+    assert.equal(outcome(rows, 2).detail, "dependency cycle");
+    assert.match(outcome(rows, 3).detail, /^blocked by #[12] \(dependency cycle\)$/);
+  });
+
+  it("dependents of a needs-human root end held; after the label is removed a resume admits root then dependents in order", async () => {
+    const store = memStore();
+    const first = await orchestrate({ issues: [1, 2, 3, 4], parallel: 2, pollMs: 5, store, readBody: async (n) => chain[n], runOne: async () => ({ terminalReason: "needs-human", code: "ci.red", detail: "red" }) });
+    assert.equal(outcome(first, 1).outcome, "needs-human");
+    for (const n of [2, 3, 4]) assert.deepEqual([outcome(first, n).outcome, outcome(first, n).detail], ["held", "blocked by #1 (needs-human)"]);
+    const started = [];
+    const rows = await orchestrate({ issues: [1, 2, 3, 4], parallel: 3, pollMs: 5, store, readBody: async (n) => chain[n], issueLabels: async () => [],
+      runOne: async (n) => { started.push(n); await new Promise((r) => setTimeout(r, 5)); return merged; } });
+    assert.deepEqual(started, [1, 2, 3, 4]);
+    assert.ok(rows.every((r) => r.outcome === "merged"), JSON.stringify(rows));
+  });
+
+  it("a resume with the needs-human label still on the root keeps the subtree held", async () => {
+    const store = memStore();
+    await orchestrate({ issues: [1, 2], parallel: 1, pollMs: 5, store, readBody: async (n) => chain[n], runOne: async () => ({ terminalReason: "needs-human", detail: "x" }) });
+    const rows = await orchestrate({ issues: [1, 2], parallel: 1, pollMs: 5, store, readBody: async (n) => chain[n], issueLabels: async () => ["needs-human"], runOne: async () => merged });
+    assert.deepEqual(rows.map((r) => r.outcome), ["needs-human", "held"]);
+  });
+
+  it("the report lists a held subtree with its root and one --requeue command", async () => {
+    const { formatOutcomeLists } = await import("./orchestrate.mjs");
+    const lines = formatOutcomeLists([{ issue: 1, outcome: "engine-error" }, { issue: 2, outcome: "held", root: 1 }, { issue: 3, outcome: "held", root: 1 }]);
+    assert.ok(lines.some((l) => /held subtree: #1 \(engine-error\) holds #2 #3; release: forgedock orchestrate --requeue <batch> 1/.test(l)), lines.join("\n"));
+  });
+});

package/bin/lean/orchestrate.mjs

@@ -20,8 +20,10 @@
  * A seam calls `onLive()` before it waits on a live lane (#4504); an adopt that resolves first is settled (or requeued on null)
  * before scheduling, so a live lane does not hold up the others.
  * `deferred` is a scheduling fact, not an end: each typed code is requeued with its retry-at (`deferPolicy`), a usage
- * limit holds admissions for that account until its reset, and deferrals other than a wait (`host.busy`,
- * `sandbox.capacity`, #4528; `sandbox.unreachable`, #4788; `rate_limited`, #4784) stop after `maxDeferrals`.
+ * limit (and #5168 a claude login failure, `backend.auth`) holds admissions for that account until its reset, and deferrals other than a wait (`host.busy`,
+ * `sandbox.capacity`, #4528; `sandbox.unreachable`, #4788; `rate_limited`, #4784; `backend.auth`, #5168) stop after `maxDeferrals`.
+ * #5161: an engine-error whose detail is a transport/gateway failure (`isTransientEngineError`) is requeued with backoff
+ * (`transientRetryDelayMs`) up to `transientRetries` times, not counted against `ENGINE_ERROR_RELAUNCHES`; then it ends engine-error.
  * #5112: with `remote.offlineFallback` (`execution.fallback_local`), an issue deferred `remote.capacityFallbackAfter` times (default
  * `CAPACITY_FALLBACK_AFTER`) for `sandbox.capacity` runs on the host when `remote.headroom()` (host-busy guard) allows it.
  * Usage-limit holds (#4190) are keyed by backend identity in `limits` (the host store, shared by every batch on the
@@ -31,6 +33,8 @@
  * runs local in hybrid (`fallbackLocal`, local account not held) or waits for the reset; admission is not held. An `engine-error` is
  * relaunched once; `breaker` ({errors, window_min}) stops admissions after a burst and lets running lanes finish, the
  * rest stay pending (outcome `held`) for a resume. `getParallel()` is read on every pass, so K can change live.
+ * #5138: `remote.max`, `remote.fallbackLocal` and `remote.place` are read on every grant too (engine-cli puts live getters on
+ * them, live-execution.mjs), so `--set-max-remote` / `--set-placement` change a running batch; running lanes are not stopped.
  * `getChildren()` + `autoChildren {mode, pollS, epics}` (#4965, `orchestration.auto_add_children`): with mode `epic`|`all`, at most once per `pollS`
  * it reads one `{list, elsewhere}` and adds the open children (`**Parent**: #P`, sub-issues) of every batch issue and `epics` through the same
  * path as `getAdded` (one decision and one `added` row listing the edges); mode `off` also stops the decomposed-lane join.
@@ -126,6 +130,27 @@ import { joinsDecomposed, addsEpicChildren, selectChildren, edgeText, DEFAULT_PO
 
 // A dependency that finished like this no longer blocks its dependents.
 const SATISFIED = new Set(["merged", "invalid"]);
+// #5162: a dependent of a root that ended like this is held (resumable), not skipped.
+const HOLD_ROOT = new Set(["needs-human", "engine-error", "deferred", "budget-exceeded"]);
+/** #5162: members of a real dependency cycle (a nontrivial SCC or a self-loop) among `nodes`; `edgesOf(n)` lists n's dependencies. */
+export function cyclicNodes(nodes, edgesOf) {
+  const inSet = new Set(nodes), idx = new Map(), low = new Map(), stack = [], on = new Set(), out = new Set();
+  let i = 0;
+  const visit = (v) => {
+    idx.set(v, i); low.set(v, i); i++; stack.push(v); on.add(v);
+    for (const w of edgesOf(v).filter((d) => inSet.has(d))) {
+      if (w === v) out.add(v);
+      if (!idx.has(w)) { visit(w); low.set(v, Math.min(low.get(v), low.get(w))); } else if (on.has(w)) low.set(v, Math.min(low.get(v), idx.get(w)));
+    }
+    if (low.get(v) === idx.get(v)) {
+      const comp = [];
+      for (let w; w !== v;) { w = stack.pop(); on.delete(w); comp.push(w); }
+      if (comp.length > 1) for (const c of comp) out.add(c);
+    }
+  };
+  for (const n of nodes) if (!idx.has(n)) visit(n);
+  return out;
+}
 const WATCH_EVERY_MS = 60_000; // #4595
 // Times one issue may be requeued behind a dependency before its dependency-open row is final.
 export const MAX_REQUEUES = 2;
@@ -144,6 +169,15 @@ export const CAPACITY_FALLBACK_AFTER = 3;
 export const RATE_LIMIT_FALLBACK_S = 15 * 60;
 export const DEFER_BACKOFF_S = 120;
 export const DEFER_BACKOFF_MAX_S = 30 * 60;
+// #4685 + #5161: an engine-error detail that names a link or gateway failure (curl's "returned error: 502", HTTP/1.1 503, ...).
+export const TRANSPORT_TEXT = /fetch failed|ECONNREFUSED|ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENOTFOUND|socket hang up|HTTP 50[234]|returned error: 5\d\d|HTTP\/\d(\.\d)? 5\d\d|Bad Gateway|Service Unavailable|overloaded/i;
+/** #5161: a transient engine failure (model gateway 5xx, connection reset): retried with backoff, not the single relaunch. */
+export const isTransientEngineError = (detail) => TRANSPORT_TEXT.test(String(detail ?? ""));
+// #5161: transient engine-error retries per issue (forge.yaml `orchestration.transient_retries`) and their backoff: 30s, 2m, 8m, then 15m.
+export const DEFAULT_TRANSIENT_RETRIES = 4;
+export const TRANSIENT_BACKOFF_S = 30;
+export const TRANSIENT_BACKOFF_MAX_S = 15 * 60;
+export const transientRetryDelayMs = (k) => Math.min(TRANSIENT_BACKOFF_S * 4 ** (Math.max(1, k) - 1), TRANSIENT_BACKOFF_MAX_S) * 1000;
 // A dependency-open whose dependencies already finished satisfied (GitHub not yet closed): wait before relaunching.
 export const DEP_REQUEUE_DELAY_S = 60;
 
@@ -202,6 +236,8 @@ export function deferPolicy(r, { now, deferrals = 0, maxDeferrals = DEFAULT_MAX_
     const at = posNum(r?.resetAtEpochMs) && r.resetAtEpochMs > now ? r.resetAtEpochMs : now + (posNum(r?.retryInSeconds) ?? RATE_LIMIT_FALLBACK_S) * 1000;
     return { delayMs: at - now, hold: true };
   }
+  // #5168: a failed claude login is an account condition, not an engine fault: held like a usage limit, never capped.
+  if (code === "backend.auth") return { delayMs: (posNum(r?.retryInSeconds) ?? RATE_LIMIT_FALLBACK_S) * 1000, hold: true };
   const k = deferrals + 1;
   if (k > maxDeferrals) return { capped: true };
   if (posNum(r?.retryInSeconds)) return { delayMs: r.retryInSeconds * 1000 };
@@ -265,7 +301,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, integration = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null, preflight = null,
   recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, issueState = null, getCancelled = null, getHeld = null, getBlocked = null, trusted = null, getGates = null, stopLane = null,
   estimate = null, starveMs = DEFAULT_STARVE_MS, setupConcurrency = DEFAULT_SETUP_CONCURRENCY, signal = null, dispatchCheck = null, syncParked = null, syncEveryMs = DEFAULT_SYNC_EVERY_MS, lifecycle = null,
-  getChildren = null, autoChildren = null, baseWait = null, rotator = null }) {
+  getChildren = null, autoChildren = null, baseWait = null, rotator = null, transientRetries = DEFAULT_TRANSIENT_RETRIES }) {
   const saved = store?.load?.() ?? null;
   const order = saved ? [...saved.order] : [];
   const set = new Set(order);
@@ -282,6 +318,8 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   const capDefers = toMap(saved?.capDeferrals); // #5112: issue -> `sandbox.capacity` deferrals (uncapped, so not in `deferrals`)
   const capFellBack = new Set(); // #5112: issues already reported as a capacity fallback (one decision each)
   const engineErrors = toMap(saved?.engineErrors);
+  const transients = toMap(saved?.transientRetries); // #5161: issue -> transient engine-error retries taken
+  const transientMax = Number.isInteger(transientRetries) && transientRetries >= 0 ? transientRetries : DEFAULT_TRANSIENT_RETRIES;
   const checked = toMap(saved?.prefilter); // issue -> pre-filter result
   const estimates = toMap(saved?.estimates); // #4409: issue -> estimated lane minutes; only with `estimate`
   const enqueued = toMap(saved?.enqueued); // #4409: issue -> when it last entered the queue (starvation guard)
@@ -319,7 +357,9 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   const integHeld = new Map(); // issue -> reason (red integration streak on its milestone lane, #4667) holding it this pass
   const reg = sched?.registry ?? null;
   const priorities = toMap(saved?.priorities); // issue -> priority number; only with the registry
-  const remoteMax = remote && Number.isInteger(remote.max) && remote.max >= 0 ? remote.max : 0;
+  // #5138: read on every grant, so `--set-max-remote` reaches a running batch (a getter on `remote`); running lanes are never stopped.
+  const remoteMax = () => { const m = remote?.max; return Number.isInteger(m) && m >= 0 ? m : 0; };
+  const remoteCapped = new Set(); // #5138: issues waiting for a max_remote slot; polled so a raised live max_remote admits them
   let poolOfflineSince = null, poolStallNoted = false; // #4788: when sampled online first read 0, and whether the stall was reported
   const capWait = makeCapWait({ now }); // #4909: a lane refused for capacity waits for a free slot, recorded as queue time
   let capAt, capAdmits = 0; // #4540: provider capacity sample and the remote admissions made since it
@@ -365,7 +405,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   const save = () => {
     if (!store || halted) return;
     store.save({ v: 1, ...(setup ? { phase: "setup", setup: { ...setup } } : {}), order, deps: toObj(deps), lanes: toObj(lanes), crossSeen: [...crossSeen], results: toObj(results), running: toObj(runRecs),
-      pending, notBefore: toObj(notBefore), requeues: toObj(requeues), deferrals: toObj(deferrals), engineErrors: toObj(engineErrors), ...(quota.size ? { quota: toObj(quota) } : {}), ...(capDefers.size ? { capDeferrals: toObj(capDefers) } : {}),
+      pending, notBefore: toObj(notBefore), requeues: toObj(requeues), deferrals: toObj(deferrals), engineErrors: toObj(engineErrors), ...(transients.size ? { transientRetries: toObj(transients) } : {}), ...(quota.size ? { quota: toObj(quota) } : {}), ...(capDefers.size ? { capDeferrals: toObj(capDefers) } : {}),
       prefilter: toObj(checked), errorTimes, breaker: tripped, ...(budget ? { classes: toObj(classes), dispatch: { ...dispatch } } : {}),
       ...(reg ? { priorities: toObj(priorities) } : {}), ...(sandboxUse.size ? { sandbox: toObj(sandboxUse) } : {}),
       ...(recoveries.size || recovering.size ? { recoveries: toObj(recoveries), recovering: toObj(recovering), recoveryClasses: toObj(classUsed) } : {}),
@@ -601,11 +641,11 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
         } else if (cap && Number.isInteger(cap.free)) {
           if (cap.at !== capAt) { capAt = cap.at; capAdmits = 0; }
           if (capWait.blocked(n, cap)) { notBefore.set(n, now() + Math.max(pollMs, 1)); return false; }
-          if (capAdmits >= cap.free && remoteRun.size < remoteMax) { notBefore.set(n, now() + Math.max(pollMs, 1)); return false; }
+          if (capAdmits >= cap.free && remoteRun.size < remoteMax()) { notBefore.set(n, now() + Math.max(pollMs, 1)); return false; }
         }
-        if (p.where === "remote" && remoteRun.size < remoteMax) { capWait.admitted(n); remoteRun.add(n); capAdmits++; placed.set(n, p); return true; }
-        if (p.where === "remote" && !remote.fallbackLocal) return false;
-        if (p.where === "remote") p = { where: "local", reason: `fallback to local: max_remote ${remoteMax} reached (${p.reason})` }; // #4256: hybrid runs it on the host
+        if (p.where === "remote" && remoteRun.size < remoteMax()) { remoteCapped.delete(n); capWait.admitted(n); remoteRun.add(n); capAdmits++; placed.set(n, p); return true; }
+        if (p.where === "remote" && !remote.fallbackLocal) { remoteCapped.add(n); return false; }
+        if (p.where === "remote") p = { where: "local", reason: `fallback to local: max_remote ${remoteMax()} reached (${p.reason})` }; // #4256: hybrid runs it on the host
       } else {
         noteBudget(adm.reason);
         if (remote.onBudget !== "local" && !remote.fallbackLocal) { budgetStop = adm.reason; return false; }
@@ -629,15 +669,19 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     results.set(n, { issue: n, outcome: "skipped", pr: null, cost_usd: 0, minutes: 0, detail, ...(trigger !== null && dispatchCheck ? { waiting: true } : {}) });
     update({ ...results.get(n), ...(trigger !== null ? { trigger } : {}) });
   };
+  // #5162: a dependent of a failed root ends held (trigger = the root, not the parent) and is released with the root.
+  const holdBehind = (n, root, rootOutcome) => {
+    results.set(n, { issue: n, outcome: "held", pr: null, cost_usd: 0, minutes: 0, detail: `blocked by #${root} (${rootOutcome})`, root, ...(dispatchCheck ? { waiting: true } : {}) });
+    update({ ...results.get(n), trigger: root });
+  };
   const requeue = (n, delayMs) => { pending.push(n); enqueued.set(n, now()); if (delayMs > 0) notBefore.set(n, now() + delayMs); else notBefore.delete(n); };
-  const TRANSPORT_TEXT = /fetch failed|ECONNREFUSED|ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENOTFOUND|socket hang up|HTTP 50[234]/i;
   const TRANSPORT_COOLDOWN_MS = 5 * 60000;
   let transportErrors = 0; // #4685: engine-errors in the current window that were link failures
   const noteEngineError = (detail = "") => {
     if (!breaker) return;
     const t = now(), windowMs = breaker.window_min * 60000;
     errorTimes = [...errorTimes.filter((x) => t - x < windowMs), t];
-    transportErrors = TRANSPORT_TEXT.test(String(detail)) ? Math.min(transportErrors + 1, errorTimes.length) : 0;
+    transportErrors = isTransientEngineError(detail) ? Math.min(transportErrors + 1, errorTimes.length) : 0;
     if (!tripped && errorTimes.length >= breaker.errors) {
       tripped = { at: t, ...(transportErrors >= errorTimes.length ? { transport: true } : {}), detail: `circuit breaker: ${errorTimes.length} engine-errors within ${breaker.window_min} min; admissions stopped` };
       update({ issue: null, outcome: "breaker", detail: tripped.detail });
@@ -753,7 +797,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
             resumeAt, rotations: (q.rotations ?? 0) + (rq?.rotated ? 1 : 0) });
           requeue(n, delayMs); done(); save();
           const iso = new Date(resumeAt).toISOString();
-          update({ ...row, outcome: "paused", quota: { resumeAt: iso }, detail: `paused (quota) until ${iso}: ${r?.code ?? "deferred"}${r?.detail ? ` ${r.detail}` : ""}` });
+          update({ ...row, outcome: "paused", quota: { resumeAt: iso }, detail: `paused (${r?.code === "backend.auth" ? "backend.auth" : "quota"}) until ${iso}: ${r?.code ?? "deferred"}${r?.detail ? ` ${r.detail}` : ""}` });
           return;
         }
         requeue(n, capGated ? 0 : delayMs); done(); save();
@@ -764,7 +808,20 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     }
     if (row.outcome === "engine-error") {
       noteEngineError(r?.detail);
-      if ((engineErrors.get(n) ?? 0) < ENGINE_ERROR_RELAUNCHES) {
+      // #5161: a transient failure (gateway 5xx, reset) waits out the outage with backoff instead of the single immediate relaunch.
+      if (transientMax > 0 && isTransientEngineError(row.detail)) {
+        const k = (transients.get(n) ?? 0) + 1;
+        if (k <= transientMax) {
+          const delayMs = transientRetryDelayMs(k), secs = Math.round(delayMs / 1000);
+          transients.set(n, k);
+          requeue(n, delayMs); done(); save();
+          decided({ issue: n, what: `retry after transient engine-error (${k}/${transientMax}) in ${secs}s`, why: `transient: ${firstLine(row.detail) || "engine-error"}`,
+            alternative: "end the issue engine-error and skip its dependents" });
+          update({ ...row, detail: `transient retry ${k}/${transientMax} in ${secs}s: ${row.detail ?? ""}` });
+          return;
+        }
+        row.detail = `gave up after ${transientMax} transient retries: ${row.detail ?? ""}`;
+      } else if ((engineErrors.get(n) ?? 0) < ENGINE_ERROR_RELAUNCHES) {
         engineErrors.set(n, (engineErrors.get(n) ?? 0) + 1);
         requeue(n, 0); done(); save();
         decided({ issue: n, what: "relaunch once after engine-error", why: firstLine(row.detail) || "engine-error", alternative: "end the issue engine-error and skip its dependents" });
@@ -1035,12 +1092,13 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       results.delete(e.issue);
       for (let i = 0; i < moved.length; i++) { // dependents skipped behind a requeued issue (fixpoint)
         for (const [m, r] of [...results]) {
-          if (r?.outcome !== "skipped" || Number(/^dependency #(\d+)/.exec(r.detail ?? "")?.[1]) !== moved[i]) continue;
+          const heldBehind = r?.outcome === "held" && r.root === moved[i]; // #5162
+          if (!heldBehind && (r?.outcome !== "skipped" || Number(/^dependency #(\d+)/.exec(r.detail ?? "")?.[1]) !== moved[i])) continue;
           priors.set(m, r); results.delete(m); moved.push(m);
         }
       }
       for (const m of moved) {
-        for (const x of [requeues, deferrals, capDefers, engineErrors, recoveries, recovering]) x.delete(m);
+        for (const x of [requeues, deferrals, capDefers, engineErrors, transients, recoveries, recovering]) x.delete(m);
         relabelled.delete(m);
         if (m === e.issue) requeuedInfo.set(m, { reason: e.reason, by: e.by, at: e.at, prior: [...(requeuedInfo.get(m)?.prior ?? []), priors.get(m)] });
         if (!pending.includes(m)) requeue(m, 0);
@@ -1178,7 +1236,45 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   const readmitted = new Set(); // "n:b": re-admit n on blocker b once; a lane that re-skips on b stays skipped
   const blockerOf = (n) => { const r = results.get(n); const m = r?.outcome === "skipped" ? /^dependency #(\d+) /.exec(r.detail ?? "") : null; return m ? Number(m[1]) : null; };
   const watchable = () => !!dispatchCheck && [...results.keys()].some((n) => blockerOf(n) !== null);
+  // #5162: held rows behind `root` return to pending (and the root too when `withRoot`: requeued, counters reset), parents first via the edges.
+  const releaseHeld = (root, withRoot, why) => {
+    const moved = withRoot && results.has(root) ? [root] : [];
+    if (moved.length) results.delete(root);
+    for (const [m, r] of [...results]) if (r?.outcome === "held" && r.root === root) { results.delete(m); moved.push(m); }
+    for (const m of moved) {
+      for (const x of [requeues, deferrals, capDefers, engineErrors, transients, recoveries, recovering]) x.delete(m);
+      relabelled.delete(m);
+      if (!pending.includes(m)) requeue(m, 0);
+      update({ issue: m, outcome: "readmitted", detail: m === root ? why : `dependency #${root} ${why}`, trigger: root });
+    }
+    if (moved.length) { save(); decided({ issue: root, what: `released held subtree of #${root}: ${moved.filter((m) => m !== root).map((m) => `#${m}`).join(", ") || "none"}`, why, alternative: "leave it held until a --requeue" }); }
+  };
+  const restoredHeld = new Set([...results.values()].filter((r) => r?.outcome === "held" && Number.isInteger(r.root)).map((r) => r.root)); // roots held by an earlier run of this batch
+  const watchHeldRoots = async () => {
+    for (const root of new Set([...results.values()].filter((r) => r?.outcome === "held" && Number.isInteger(r.root)).map((r) => r.root))) {
+      const ro = results.get(root)?.outcome;
+      if (ro === undefined || running.has(root) || settled.has(root)) continue;
+      if (restoredHeld.has(root) && ["needs-human", "budget-exceeded"].includes(ro) && typeof issueLabels === "function") { // label removed: requeue the root, its subtree follows
+        let names = null;
+        try { names = await issueLabels(root); } catch { names = null; }
+        if (Array.isArray(names) && !names.includes("needs-human")) { releaseHeld(root, true, "needs-human label removed"); continue; }
+      }
+      if (!dispatchCheck) continue;
+      let g = null;
+      try { g = (await dispatchCheck(root)) ?? null; } catch { g = null; }
+      if (g?.final) { // closed as not planned: terminal for its held dependents
+        settled.add(root);
+        for (const [m, r] of [...results]) if (r?.outcome === "held" && r.root === root && r.waiting) { const { waiting, ...rest } = r; results.set(m, rest); update({ ...rest, trigger: root }); }
+        continue;
+      }
+      if (g && SATISFIED.has(g.outcome)) { // merged or closed outside the batch
+        results.set(root, { ...results.get(root), outcome: g.outcome, pr: Number.isInteger(g.pr) ? g.pr : results.get(root).pr, detail: g.detail ?? results.get(root).detail });
+        update(results.get(root)); releaseHeld(root, false, "merged or closed");
+      }
+    }
+  };
   const watchBlockers = async (final) => {
+    if (!cancelled && !tripped && (final || lastWatch === null || now() - lastWatch >= WATCH_EVERY_MS) && [...results.values()].some((r) => r?.outcome === "held" && Number.isInteger(r.root))) await watchHeldRoots();
     if (!watchable() || cancelled || tripped) return;
     const t0 = now();
     if (!final && lastWatch !== null && t0 - lastWatch < WATCH_EVERY_MS) return;
@@ -1236,7 +1332,12 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
           const n = pending[i];
           if (results.has(n)) { pending.splice(i, 1); continue; }
           const bad = edges(n).find((d) => results.has(d) && !SATISFIED.has(results.get(d).outcome));
-          if (bad !== undefined) { pending.splice(i, 1); skip(n, `dependency #${bad} ended ${results.get(bad).outcome}`, bad); again = true; }
+          if (bad !== undefined) {
+            pending.splice(i, 1); again = true;
+            const br = results.get(bad), root = br.root ?? bad, ro = results.get(root)?.outcome ?? br.outcome;
+            if (HOLD_ROOT.has(ro)) holdBehind(n, root, ro); // #5162
+            else skip(n, `dependency #${bad} ended ${br.outcome}`, bad);
+          }
         }
       }
       const t = now();
@@ -1345,6 +1446,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       if (holdUntil && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // #4446: a cleared hold (forgedock limits clear) admits on the next poll
       if ((getParallel || getHeld) && pending.length) wake = Math.min(wake ?? pollMs, pollMs);
       if (offered.size && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // a slot another batch frees
+      if (pending.some((n) => remoteCapped.has(n))) wake = Math.min(wake ?? pollMs, pollMs); // #5138: a live --set-max-remote raise
       if ((getAdded || getRequeued || (getChildren && addsEpicChildren(autoChildren?.mode))) && running.size) wake = Math.min(wake ?? pollMs, pollMs); // #4408/#4550: an add or requeue while only lanes run
       if ((pfAll || pfRemote) && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // #4866
       if (baseWait && pending.length && baseHeld.size) wake = Math.min(wake ?? pollMs, pollMs); // #5031: a landed commit releases on the next tick
@@ -1366,8 +1468,21 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       else if (pending.length && budgetHeld) { if (await drainAdds()) continue; break; } // maintenance the budget does not admit now: held for a later batch
       else if (pending.length && opHeldHit.size && !pending.some((n) => !opHeldHit.has(n) && isReady(n))) { if (await drainAdds()) continue; break; } // #4592: held by the operator: pending for a --resume
       else if (pending.length && (driftHeld.size || integHeld.size)) { if (await drainAdds()) continue; break; } // held by a drift flag (block_dispatch) or a red integration streak (#4667): pending for a later batch
+      else if (pending.some((n) => remoteCapped.has(n))) await nap(wake ?? pollMs); // #5138: live max_remote 0: wait for a raise or another placement
       else if (pending.length) { // nothing runnable, nothing running, nothing waiting on time: a dependency cycle
-        for (const n of pending.splice(0)) skip(n, "dependency cycle");
+        // #5162: only real cycles are labelled so; the rest name the cycle member (or failed root) that blocks them
+        const cyc = cyclicNodes(pending, edges), inP = new Set(pending);
+        const blocker = (n, seen = new Set()) => {
+          for (const d of edges(n)) {
+            if (cyc.has(d)) return d;
+            if (inP.has(d) && !seen.has(d)) { seen.add(d); const b = blocker(d, seen); if (b !== null) return b; }
+          }
+          return null;
+        };
+        for (const n of pending.splice(0)) {
+          const b = cyc.has(n) ? null : blocker(n);
+          skip(n, cyc.has(n) ? "dependency cycle" : b !== null ? `blocked by #${b} (dependency cycle)` : "blocked: no dependency path is runnable");
+        }
         save();
       }
     }
@@ -1438,6 +1553,9 @@ export function formatOutcomeLists(rows) {
   const one = (x) => String(x ?? "").split("\n")[0].slice(0, OUTCOME_LINE_MAX);
   const requeued = (rows ?? []).filter((r) => r?.requeued); // #4550: operator --requeue after a decision; the row is the latest outcome
   if (requeued.length) out.push(`requeued: ${requeued.map((r) => `#${r.issue} (${one(r.requeued.reason)})`).join(" ")}`);
+  const heldBy = new Map(); // #5162: held subtrees by root, one --requeue command each
+  for (const r of rows ?? []) if (r?.outcome === "held" && Number.isInteger(r.root)) heldBy.set(r.root, [...(heldBy.get(r.root) ?? []), r.issue]);
+  for (const [root, xs] of heldBy) { const ro = (rows ?? []).find((r) => r?.issue === root)?.outcome ?? "failed"; out.push(`held subtree: #${root} (${ro}) holds ${xs.map((x) => `#${x}`).join(" ")}; release: forgedock orchestrate --requeue <batch> ${root}`); }
   const rec = (rows ?? []).filter((r) => r?.recovered);
   if (rec.length) out.push(`recovered: ${rec.map((r) => `#${r.issue} (${r.recovered})`).join(" ")}`);
   for (const r of rows ?? []) {

package/bin/lean/prefilter.mjs

@@ -455,11 +455,12 @@ export async function prefilterIssue({ issue, repo, lane, cwd, io, model = DEFAU
       return { ...out, detail: "acceptance probe failed after promotion (FORGE:PROBE): no already-fixed close; lane" };
     }
     const sig = await collectSignals({ iss, repo, lane, io, cwd, selfLogins });
-    const close = async (kind, body, detail) => {
+    // #5152: `completed` for a delivered outcome (CI green again, ALREADY-FIXED); only an INVALID verdict is dropped scope
+    const close = async (kind, body, detail, reason = "completed") => {
       let note = "";
       if (!dryRun) {
         // #4315: close first: the marker is posted only on an issue that was closed, so it always means "reopened" later
-        await io.gh(["issue", "close", String(issue), "-R", repo, "--reason", "not planned"]);
+        await io.gh(["issue", "close", String(issue), "-R", repo, "--reason", reason]);
         // #4314: the issue IS closed from here on: the result is closed:true whatever the comment does, so no lane starts on it.
         // The comment is retried once, then a marker-only comment (a second, tiny payload) keeps the #4305 reopen guard working.
         const comment = (text) => io.gh(["issue", "comment", String(issue), "-R", repo, "--body", text]);
@@ -498,7 +499,7 @@ export async function prefilterIssue({ issue, repo, lane, cwd, io, model = DEFAU
     if (!v || v.verdict === "CONFIRMED" || !v.evidence.length) return keep(`candidate (${why}) ${v ? (v.verdict === "CONFIRMED" ? "confirmed" : "without evidence") : "unparseable verdict"}: lane`);
     const ev = v.evidence.map((x) => `- ${sanitizeMd(x).replace(/\s+/g, " ").slice(0, 400)}`).join("\n");
     return await close("validated", `## Pre-filter: ${v.verdict}\n**Signal:** ${why}\n**Summary:** ${sanitizeMd(v.summary).replace(/\s+/g, " ").slice(0, 600) || "not reported"}\n\n**Evidence:**\n${ev}\n\nValidated by a single read-only check (${model}), without a lane. Reopen if this is wrong.`,
-      `${v.verdict.toLowerCase()} (${why})`);
+      `${v.verdict.toLowerCase()} (${why})`, v.verdict === "INVALID" ? "not planned" : "completed");
   } catch (e) {
     return { ...out, detail: `pre-filter error (${String(e?.message ?? e).slice(0, 120)}): lane` };
   }