forgedock 1.43.1 → 1.44.0

package/bin/lean/orchestrate-children.test.mjs

@@ -18,7 +18,7 @@ async function run({ mode, list, pollS = 0.001, elsewhere = [], epics = [], bodi
     runOne: async (n) => { started.push(n); if (n === 1) await g; return merged; }, ...extra });
   return { started, log, decisions, calls: () => calls, open, done };
 }
-const L = (...kids) => [{ number: 1, body: "", labels: [] }, ...kids.map((n) => ({ number: n, body: "**Parent**: #1", labels: [] }))];
+const L = (...kids) => [{ number: 1, body: "", labels: [], authorAssociation: "MEMBER" }, ...kids.map((n) => ({ number: n, body: "**Parent**: #1", labels: [], authorAssociation: "MEMBER" }))];
 
 describe("#4965 auto_add_children", () => {
   it("epic: adds open children of a batch issue, one decision and one added row listing them", async () => {
@@ -40,8 +40,20 @@ describe("#4965 auto_add_children", () => {
       assert.deepEqual(r.started, [1]);
     }
   });
+  it("#4982 an untrusted author's child is not added and logs one skipped row", async () => {
+    const list = [...L(2), { number: 3, body: "**Parent**: #1", labels: [], authorAssociation: "NONE" }];
+    const r = await run({ mode: "epic", list });
+    await until(() => r.started.includes(2) && r.log.some((x) => x.outcome === "skipped"));
+    await new Promise((x) => setTimeout(x, 40));
+    r.open(); await r.done;
+    assert.ok(!r.started.includes(3));
+    const sk = r.log.filter((x) => x.outcome === "skipped");
+    assert.equal(sk.length, 1);
+    assert.equal(sk[0].issue, 3);
+    assert.match(sk[0].detail, /#3 .*untrusted author/);
+  });
   it("needs-human children and children live in another batch are not added", async () => {
-    const list = [...L(2, 3), { number: 4, body: "**Parent**: #1", labels: [{ name: "needs-human" }] }];
+    const list = [...L(2, 3), { number: 4, body: "**Parent**: #1", labels: [{ name: "needs-human" }], authorAssociation: "MEMBER" }];
     const r = await run({ mode: "all", list, elsewhere: [3] });
     await until(() => r.started.includes(2));
     await new Promise((x) => setTimeout(x, 40));
@@ -49,10 +61,10 @@ describe("#4965 auto_add_children", () => {
     assert.deepEqual([...r.started].sort(), [1, 2]);
   });
   it("an --epic root outside the batch adds its children; late children join on a later pass", async () => {
-    let list = [{ number: 50, body: "", labels: [] }, { number: 51, body: "**Parent**: #50", labels: [] }];
+    let list = [{ number: 50, body: "", labels: [], authorAssociation: "MEMBER" }, { number: 51, body: "**Parent**: #50", labels: [], authorAssociation: "MEMBER" }];
     const r = await run({ mode: "epic", epics: [50], list: () => list });
     await until(() => r.started.includes(51));
-    list = [...list, { number: 52, body: "**Parent**: #50", labels: [] }];
+    list = [...list, { number: 52, body: "**Parent**: #50", labels: [], authorAssociation: "MEMBER" }];
     await until(() => r.started.includes(52));
     r.open(); await r.done;
     assert.equal(r.log.filter((x) => x.outcome === "added").length, 2);

package/bin/lean/orchestrate.mjs

@@ -115,6 +115,7 @@ import { formatParkedLine, postDeployOnly } from "./prefilter.mjs";
 import { laneSize, SIZES } from "./budget.mjs";
 import { requeueKey, REQUEUABLE } from "./batch.mjs";
 import { makeCapWait } from "./capwait.mjs";
+import { mapLimit } from "./pmap.mjs";
 import { joinsDecomposed, addsEpicChildren, selectChildren, edgeText, DEFAULT_POLL_S } from "./autochildren.mjs";
 
 // A dependency that finished like this no longer blocks its dependents.
@@ -144,6 +145,8 @@ export const EST_DEFAULT_MIN = Object.freeze({ S: 10, M: 25, L: 50 });
 export const POST_DEPLOY_EST_MIN = 2;
 // A ready issue queued longer than this joins the head band (starvation guard; only with an estimator).
 export const DEFAULT_STARVE_MS = 30 * 60000;
+/** #4963: issues read at once while a batch sets up. */
+export const DEFAULT_SETUP_CONCURRENCY = 6;
 // Merged runs of one repo and size needed before their median replaces the default.
 export const HISTORY_MIN_SAMPLES = 3;
 
@@ -253,8 +256,8 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   store = null, adopt = null, getParallel = null, getAdded = null, getRequeued = null, breaker = null, maxDeferrals = DEFAULT_MAX_DEFERRALS, sleep = cancellableSleep, pollMs = 5000,
   classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, integration = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null, preflight = null,
   recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, issueState = null, getCancelled = null, getHeld = null, getBlocked = null, trusted = null, getGates = null, stopLane = null,
-  estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null, dispatchCheck = null, syncParked = null, syncEveryMs = DEFAULT_SYNC_EVERY_MS, lifecycle = null,
-  getChildren = null, autoChildren = null }) {
+  estimate = null, starveMs = DEFAULT_STARVE_MS, setupConcurrency = DEFAULT_SETUP_CONCURRENCY, signal = null, dispatchCheck = null, syncParked = null, syncEveryMs = DEFAULT_SYNC_EVERY_MS, lifecycle = null,
+  getChildren = null, autoChildren = null, baseWait = null }) {
   const saved = store?.load?.() ?? null;
   const order = saved ? [...saved.order] : [];
   const set = new Set(order);
@@ -336,10 +339,11 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   const decided = (d) => { try { Promise.resolve(decide?.(d)).catch(() => {}); } catch { /* best effort */ } };
   const firstLine = (x) => String(x ?? "").split("\n")[0].slice(0, 300);
 
+  let setup = null; // #4963: {read, total} while the initial per-issue reads run; written with phase "setup" so a slow start is visible
   let halted = false; // #4454: stopped by `signal`; the batch's state is no longer this process's to write
   const save = () => {
     if (!store || halted) return;
-    store.save({ v: 1, order, deps: toObj(deps), lanes: toObj(lanes), crossSeen: [...crossSeen], results: toObj(results), running: toObj(runRecs),
+    store.save({ v: 1, ...(setup ? { phase: "setup", setup: { ...setup } } : {}), order, deps: toObj(deps), lanes: toObj(lanes), crossSeen: [...crossSeen], results: toObj(results), running: toObj(runRecs),
       pending, notBefore: toObj(notBefore), requeues: toObj(requeues), deferrals: toObj(deferrals), engineErrors: toObj(engineErrors),
       prefilter: toObj(checked), errorTimes, breaker: tripped, ...(budget ? { classes: toObj(classes), dispatch: { ...dispatch } } : {}),
       ...(reg ? { priorities: toObj(priorities) } : {}), ...(sandboxUse.size ? { sandbox: toObj(sandboxUse) } : {}),
@@ -377,11 +381,13 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     let body = "";
     try { body = await readBody(n); } catch { /* unreadable: no ordering info, work-on's own check still applies */ }
     if (drift?.holds) bodies.set(n, body);
-    deps.set(n, await declared(n, body));
+    const d = await declared(n, body);
     await readEstimate(n, body);
     await classifyOne(n);
     await readPriority(n);
     if (resolveLane) { let l; try { l = await resolveLane(n); } catch (e) { l = { ok: false, code: "lane.unresolved", detail: e?.message ?? String(e), transient: true }; } lanes.set(n, l); }
+    // deps marks the issue read (a resume skips it): set it last, so a concurrent setup save never persists a half-read issue (#4963)
+    deps.set(n, d);
   };
   const edges = (n) => (deps.get(n) ?? []).filter((d) => set.has(d));
   // A milestone issue waiting on a staging fix (or the reverse) never sees that fix on its own base: surface the edge.
@@ -412,7 +418,14 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   // New issues (a fresh batch, or extra issues passed to a resume) join the queue.
   for (const n of issues) if (!set.has(n)) { set.add(n); order.push(n); pending.push(n); }
   for (const n of pending) if (!enqueued.has(n)) enqueued.set(n, now());
-  for (const n of order) if (!deps.has(n)) await readDeps(n);
+  // #4963: bounded-concurrency setup reads; each issue's result lands in Maps keyed by issue, so `order` and the edges are unchanged.
+  const unread = order.filter((n) => !deps.has(n));
+  if (unread.length) {
+    setup = { read: 0, total: unread.length };
+    save();
+    await mapLimit(unread, setupConcurrency, async (n) => { await readDeps(n); setup.read++; save(); });
+    setup = null;
+  }
   if (estimate) for (const n of order) { // a resume that enabled the estimator
     if (estimates.has(n) || results.has(n)) continue;
     let b = ""; try { b = await readBody(n); } catch { b = ""; }
@@ -446,6 +459,11 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       .sort((a, b) => a.p - b.p || a.b - b.b || a.s - b.s || b.w - a.w || a.i - b.i).map((x) => x.n);
   };
   const entryOf = (n) => ({ key: keyOf(n), batch: sched?.batch ?? batchId ?? null, repo: sched?.repo ?? null, issue: n, priority: prioOf(n), ...(sched?.exclusive ? { exclusive: true } : {}) });
+  // #4964: remote lanes are published to the registry for `sched show` only; they never take a slot
+  const endRemote = (n) => { if (remoteRun.delete(n)) reg?.dropRemote?.(keyOf(n)); };
+  const noteRemote = (n, h) => { reg?.noteRemote?.(keyOf(n), { ...entryOf(n), sandbox: h?.id ?? null, agent: h?.agent ?? null }); };
+  let poolDown = false;
+  const notePool = (down, reason) => { if (down === poolDown) return; poolDown = down; reg?.notePool?.(sched?.batch ?? batchId ?? "?", down ? { reason } : null); };
   const releaseSlot = (n) => { if (reg && slotted.delete(n)) reg.release(keyOf(n)); };
   const offerFresh = (ns) => { const fresh = ns.filter((n) => !offered.has(n)); if (!fresh.length) return; reg.offer(fresh.map(entryOf)); for (const n of fresh) offered.add(n); };
   const withdrawExcept = (keep) => {
@@ -500,6 +518,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
         // #4540: the provider's free slots (`remote.capacity() -> {free, at}`, sync, null = unknown) cap admissions per sample `at`; the issue queues here instead of a 503.
         let cap = null;
         try { cap = remote.capacity?.() ?? null; } catch { cap = null; }
+        if (typeof remote.capacity === "function") notePool(cap === null, "capacity probe failed");
         // #4788: zero live agents for longer than `remote.poolOfflineMin` (default 10): one loud batch event, none while online > 0.
         if (cap && cap.online === 0) {
           poolOfflineSince ??= now();
@@ -585,7 +604,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       try { remote?.onUsage?.(n, r.sandbox, rec); } catch { /* the batch total still counts it */ }
     }
     if (sandboxUse.has(n)) row.sandbox = { ...sandboxUse.get(n), ...(sandboxUse.get(n).by_agent ? { by_agent: { ...sandboxUse.get(n).by_agent } } : {}) };
-    const done = () => { running.delete(n); runRecs.delete(n); releaseSlot(n); remoteRun.delete(n); };
+    const done = () => { running.delete(n); runRecs.delete(n); releaseSlot(n); endRemote(n); };
     if (row.outcome === "dependency-open" && depList.length) {
       // #4387: a parked dependency is not pulled in again (fail-open on an unreadable label).
       let parked;
@@ -633,6 +652,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     if (row.outcome === "deferred") {
       const pol = deferPolicy(r, { now: now(), deferrals: deferrals.get(n) ?? 0, maxDeferrals });
       let capGated = false; // #4909: the pool reports free slots, so the slot (not the retry-in timer) gates the relaunch
+      if (r?.code === "sandbox.unreachable") notePool(true, r.detail ?? "sandbox unreachable");
       if (r?.code === "sandbox.capacity") {
         capWait.refused(n);
         try { capGated = Number.isInteger(remote?.capacity?.()?.free); } catch { capGated = false; }
@@ -781,6 +801,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
         try {
           r = await runOne(n, lane, { remote: !!rec.remote, ...(rec.placement ? { placement: rec.placement } : {}), ...(recovering.has(n) ? { recover: { code: recovering.get(n).code, attempt: recovering.get(n).attempt, final: recovering.get(n).attempt >= (recovering.get(n).max ?? recoveryMax) } } : {}), onStart: (info) => {
             Object.assign(rec, info ?? {}); save();
+            if (rec.remote && remoteRun.has(n)) noteRemote(n, rec.handle);
             if (cancelled) stop(n, rec); // #4376: launched while the cancel arrived
             if (reg && slotted.has(n) && info?.pid) reg.setLanePid(keyOf(n), info.pid);
           } });
@@ -805,18 +826,18 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   for (const a of adopting) a.live = a.first ? await a.first : false;
   for (const { n, rec, p, live } of adopting) {
     if (live) {
-      if (rec.remote) remoteRun.add(n);
+      if (rec.remote) { remoteRun.add(n); noteRemote(n, rec.handle); }
       else if (reg) { reg.adoptSlot(keyOf(n), { ...entryOf(n), lanePid: rec.pid }); slotted.add(n); }
       track(n, p.then((r) => {
         if (r) return settle(n, r, rec, checked.get(n) ?? null);
         if (halted) return; // #4454: kept for the next resume
-        running.delete(n); releaseSlot(n); remoteRun.delete(n); requeueLost(n); save(); // ended without a result after all
+        running.delete(n); releaseSlot(n); endRemote(n); requeueLost(n); save(); // ended without a result after all
       }));
       continue;
     }
     const r = await p;
     if (!r) { requeueLost(n); continue; }
-    if (rec.remote) remoteRun.add(n);
+    if (rec.remote) { remoteRun.add(n); noteRemote(n, rec.handle); }
     else if (reg) { reg.adoptSlot(keyOf(n), { ...entryOf(n), lanePid: rec.pid }); slotted.add(n); } // released by the settle
     track(n, Promise.resolve(r).then((x) => settle(n, x, rec, checked.get(n) ?? null)));
     await running.get(n); // settled before scheduling
@@ -832,7 +853,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   }
   save();
 
-  const heldDetail = (n) => cancelled ? "cancelled by operator" : opHeld.has(n) ? "held by operator" : tripped?.detail ?? (integHeld.has(n) ? `integration: ${integHeld.get(n)}` : (driftHeld.has(n) ? `drift: ${driftHeld.get(n)} flagged` : budgetStop ? `execution budget: ${budgetStop}` : budgetHeld ? "maintenance share" : "not run"));
+  const heldDetail = (n) => cancelled ? "cancelled by operator" : opHeld.has(n) ? "held by operator" : baseHeld.has(n) ? baseHeld.get(n) : tripped?.detail ?? (integHeld.has(n) ? `integration: ${integHeld.get(n)}` : (driftHeld.has(n) ? `drift: ${driftHeld.get(n)} flagged` : budgetStop ? `execution budget: ${budgetStop}` : budgetHeld ? "maintenance share" : "not run"));
   // #4315: a recovery queued but never launched (the batch stopped: breaker, budget, hold; or a resume that did not relaunch a
   // lost run) left the issue without needs-human, since a non-final attempt does not relabel. Relabel it once, with the full reason.
   // #4318: also on a throw out of the loop and on a signal (`onExit`); `cause` overrides the held reason. Runs once per process.
@@ -875,6 +896,12 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   };
   // #4965: epic and late children (`orchestration.auto_add_children` epic|all): at most one `getChildren()` list call per poll window.
   let nextChildScan = 0, childScanFailed = false;
+  const skippedKids = new Set(); // #4982: one row per untrusted child, not one per scan
+  const skipChild = ({ number, reason }) => {
+    if (skippedKids.has(number)) return;
+    skippedKids.add(number);
+    update({ issue: number, outcome: "skipped", detail: `#${number} not auto-added: ${reason} (orchestration.auto_add_children)` });
+  };
   const drainChildren = async () => {
     if (!getChildren || !addsEpicChildren(autoChildren?.mode)) return 0;
     const t = now();
@@ -886,7 +913,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       return 0;
     }
     childScanFailed = false;
-    const fresh = selectChildren({ list, roots: [...set, ...(autoChildren.epics ?? [])], skip: [...set, ...(elsewhere ?? [])] })
+    const fresh = selectChildren({ list, roots: [...set, ...(autoChildren.epics ?? [])], skip: [...set, ...(elsewhere ?? [])], onSkip: skipChild })
       .filter((n) => { try { return !(reg?.holder && reg.holder(entryOf(n)) != null); } catch { return true; } });
     if (!fresh.length) return 0;
     for (const n of pending) {
@@ -970,6 +997,21 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   // #4773: re-read the issues carrying the blocked label; one decision and row per change. A failed read keeps the last set.
   let blocked = new Set();
   const blockedHit = new Set(); // blocked issues that were pending this pass
+  // #5031: `baseWait(n)` (async, default off) returns a detail string while a dependency's merge commit is not yet on n's base, else null.
+  // The issue is admitted but not dispatched; it is re-checked every pass until released (a commit that landed stays landed). Fail-open.
+  const baseHeld = new Map(), baseHit = new Set(), baseDone = new Set();
+  const refreshBaseWaits = async () => {
+    if (!baseWait) return;
+    for (const n of pending) {
+      if (baseDone.has(n) || results.has(n)) continue;
+      let why = null;
+      try { why = (await baseWait(n)) ?? null; } catch { why = null; }
+      if (why && !baseHeld.has(n)) update({ issue: n, outcome: "held", detail: why });
+      if (why) { baseHeld.set(n, why); continue; }
+      if (baseHeld.delete(n)) { decided({ issue: n, what: "released: dependency commit is on the base", why: "the awaited merge commit landed", alternative: "keep waiting" }); update({ issue: n, outcome: "unblocked", detail: "dependency commit landed" }); }
+      baseDone.add(n);
+    }
+  };
   const refreshBlocked = async () => {
     if (!getBlocked) return;
     let got;
@@ -1106,8 +1148,10 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       integHeld.clear();
       opHeldHit.clear();
       blockedHit.clear();
+      baseHit.clear();
       refreshHeld();
       await refreshBlocked();
+      await refreshBaseWaits();
       checkGates();
       // #4866: lane preflight. A failing check holds admission (all lanes, or only remote ones by its scope), says why once per batch,
       // and is re-checked every pass so the batch resumes on its own; it is not an engine-error, so the breaker never sees it.
@@ -1125,6 +1169,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
         if (pfAll || (pfRemote && isRemote(n))) return false;
         if (opHeld.has(n)) { opHeldHit.add(n); return false; }
         if (blocked.has(n)) { blockedHit.add(n); return false; } // #4773
+        if (baseHeld.has(n)) { baseHit.add(n); return false; } // #5031
         if (!((notBefore.get(n) ?? 0) <= t && edges(n).every((d) => results.has(d) && SATISFIED.has(results.get(d).outcome)))) return false;
         if (integration?.holds) {
           let why = null;
@@ -1204,6 +1249,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       if (offered.size && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // a slot another batch frees
       if ((getAdded || getRequeued || (getChildren && addsEpicChildren(autoChildren?.mode))) && running.size) wake = Math.min(wake ?? pollMs, pollMs); // #4408/#4550: an add or requeue while only lanes run
       if ((pfAll || pfRemote) && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // #4866
+      if (baseWait && pending.length && baseHeld.size) wake = Math.min(wake ?? pollMs, pollMs); // #5031: a landed commit releases on the next tick
       if (getBlocked && pending.length && blockedHit.size) wake = Math.min(wake ?? pollMs, pollMs); // #4773: a removed label releases on the next tick
       if (getHeld && running.size) wake = Math.min(wake ?? pollMs, pollMs); // #4592: a release while only lanes run
       if (syncParked && running.size && parkedPrs().length) wake = Math.min(wake ?? syncEveryMs, Math.max(0, syncEveryMs - (t - (lastParkedSync ?? t)))); // #4525
@@ -1217,6 +1263,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       else if (pending.length && offered.size) await nap(wake); // waiting for a host slot
       else if (pending.length && (pfAll || pfRemote)) await nap(Math.min(wake ?? pollMs, pollMs)); // #4866: a failing lane preflight is re-checked on the next poll
       else if (pending.length && blockedHit.size) await nap(wake ?? pollMs); // #4773: only blocked issues wait: poll for the label's removal
+      else if (pending.length && baseHit.size) await nap(wake ?? pollMs); // #5031: only issues waiting for a base commit: poll for it
       else if (pending.length && waits.length) await nap(Math.min(wake, getCancelled ? pollMs : Infinity)); // wake is poll-capped by a hold or a live K; #4454: nap wakes on a stop
       else if (pending.length && budgetHeld) { if (await drainAdds()) continue; break; } // maintenance the budget does not admit now: held for a later batch
       else if (pending.length && opHeldHit.size && !pending.some((n) => !opHeldHit.has(n) && isReady(n))) { if (await drainAdds()) continue; break; } // #4592: held by the operator: pending for a --resume

package/bin/lean/pmap.mjs

@@ -0,0 +1,17 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/** Map `items` through async `fn` with at most `limit` in flight; results keep input order. A throw rejects after the in-flight calls settle. */
+export async function mapLimit(items, limit, fn) {
+  const list = [...items], out = new Array(list.length);
+  const width = Math.max(1, Math.min(Math.floor(Number(limit)) || 1, list.length));
+  let next = 0, failed = false, error;
+  const worker = async () => {
+    while (!failed && next < list.length) {
+      const i = next++;
+      try { out[i] = await fn(list[i], i); } catch (e) { if (!failed) { failed = true; error = e; } }
+    }
+  };
+  await Promise.all(Array.from({ length: width }, worker));
+  if (failed) throw error;
+  return out;
+}

package/bin/lean/pmap.test.mjs

@@ -0,0 +1,27 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { mapLimit } from "./pmap.mjs";
+
+describe("mapLimit", () => {
+  it("bounds concurrency and keeps input order", async () => {
+    let live = 0, peak = 0;
+    const out = await mapLimit([5, 1, 4, 2, 3, 6], 3, async (n) => {
+      live++; peak = Math.max(peak, live);
+      await new Promise((r) => setTimeout(r, n * 3));
+      live--; return n * 10;
+    });
+    assert.deepEqual(out, [50, 10, 40, 20, 30, 60]);
+    assert.equal(peak, 3);
+  });
+  it("handles empty input and a bad limit as serial", async () => {
+    assert.deepEqual(await mapLimit([], 4, async () => 1), []);
+    let live = 0, peak = 0;
+    await mapLimit([1, 2, 3], 0, async () => { live++; peak = Math.max(peak, live); await new Promise((r) => setTimeout(r, 2)); live--; });
+    assert.equal(peak, 1);
+  });
+  it("rejects with the first error", async () => {
+    await assert.rejects(mapLimit([1, 2, 3], 2, async (n) => { if (n === 2) throw new Error("boom"); return n; }), /boom/);
+  });
+});

package/bin/lean/prodwatch-triage.mjs

@@ -37,6 +37,12 @@ export const signalRef = (fp) => `<!-- FORGE:SIGNAL_REF fp=${fp} -->`;
 /** The run of FORGE marker comments at the very end of a body: the only place the engine writes markers, so the only place they are trusted. */
 export const trailerOf = (body) => /(?:<!--\s*FORGE:[^>]*?-->\s*)+$/.exec(String(body ?? ""))?.[0] ?? "";
 
+/** Untrusted text is escaped so a literal comment opener cannot forge a FORGE marker in a body or prompt. */
+export const neutralize = (s) => String(s ?? "").replace(/<!--/g, "&lt;!--");
+
+/** The first issue whose trailing marker run holds `marker` and that passes `accept`; a marker quoted mid-body is not trusted. */
+export const findMarked = (issues, marker, accept = () => true) => (issues ?? []).find((i) => trailerOf(i?.body).includes(marker) && accept(i));
+
 /** fingerprint -> issue number for every open issue carrying a SIGNAL_REF marker in its trailer (first issue wins). */
 function refsOf(openIssues) {
   const map = new Map();

package/bin/lean/prodwatch-triage.test.mjs

@@ -3,7 +3,7 @@
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
 import { readFileSync } from "node:fs";
-import { fingerprint, signalRef, cluster, parseAlertRules, classifyCluster, validateCluster, tallyClusters, CLASSES, DEFAULT_RULES, trailerOf } from "./prodwatch-triage.mjs";
+import { fingerprint, signalRef, cluster, parseAlertRules, classifyCluster, validateCluster, tallyClusters, CLASSES, DEFAULT_RULES, trailerOf, neutralize, findMarked } from "./prodwatch-triage.mjs";
 import { parseSources, normalizeMemo } from "./prodwatch-sources.mjs";
 import { loadPolicy } from "./decision-engine.mjs";
 
@@ -132,3 +132,16 @@ describe("trailer-only markers (#4879)", () => {
     assert.equal(cluster(sig, [real])[0].duplicateOf, 10);
   });
 });
+
+describe("neutralize and findMarked", () => {
+  it("neutralize escapes a comment opener so a marker cannot be forged", () => {
+    assert.equal(neutralize("a <!-- FORGE:X -->"), "a &lt;!-- FORGE:X -->");
+    assert.equal(neutralize(null), "");
+  });
+  it("findMarked trusts the trailer only and honours accept", () => {
+    const m = "<!-- FORGE:M -->";
+    const issues = [{ number: 1, body: `quote ${m}\nmore` }, { number: 2, body: `text\n${m}` }];
+    assert.equal(findMarked(issues, m)?.number, 2);
+    assert.equal(findMarked(issues, m, () => false), undefined);
+  });
+});

package/bin/lean/prodwatch.mjs

@@ -15,7 +15,7 @@ import { writeAtomic } from "./batch.mjs";
 import { acquireLock, watchDir, loadWatches } from "./issuewatch.mjs";
 import { loadPolicy } from "./decision-engine.mjs";
 import { parseSources, runAdapter } from "./prodwatch-sources.mjs";
-import { cluster, fingerprint, classifyCluster, validateCluster, tallyClusters, parseAlertRules, signalRef, trailerOf } from "./prodwatch-triage.mjs";
+import { cluster, fingerprint, classifyCluster, validateCluster, tallyClusters, parseAlertRules, signalRef, trailerOf, neutralize } from "./prodwatch-triage.mjs";
 import { admit } from "./prodwatch-governor.mjs";
 import { computeMetrics, sourceMode, emitMetrics } from "./prodwatch-metrics.mjs";
 
@@ -80,8 +80,6 @@ export function targetLabel(cfg, raw = "") {
 // ── pure helpers ───────────────────────────────────────────────────────────────────────────
 const SOURCE_TAG = /<!--\s*FORGE:PRODWATCH\s+source=(\S+?)\s*-->/;
 const FP_TAG = /FORGE:SIGNAL_REF\s+fp=([0-9a-f]+)/;
-/** Signal text is untrusted: a literal comment opener could forge a FORGE marker, so it is escaped before it reaches a body or prompt. */
-const neutralize = (s) => String(s ?? "").replace(/<!--/g, "&lt;!--");
 const oneLine = (s, n) => String(s ?? "").replace(/\s+/g, " ").trim().slice(0, n);
 
 /** Rows for computeMetrics from the prod-watch issues (open and closed) that carry our markers: GitHub is the register. */

package/bin/lean/recovery.test.mjs

@@ -9,7 +9,7 @@ import { tmpDir } from "../tests/_tmp.mjs";
 import { orchestrate, formatOutcomeLists, OUTCOME_LINE_MAX } from "./orchestrate.mjs";
 import { RECOVER_HOW } from "./lean.mjs";
 import { recoveryEvidence } from "./render.mjs";
-import { RECOVERABLE_CODES } from "../engine/outcome-codes.mjs";
+import { RECOVERABLE_CODES, classifyNeedsHuman } from "../engine/outcome-codes.mjs";
 import { runFromCli, orchestrateFromCli } from "../engine-cli.mjs";
 // orchestrateFromCli opens the host lane registry (#4191): keep it in a temp dir.
 process.env.FORGEDOCK_SCHED_DIR = tmpDir("recovery-sched-");
@@ -96,12 +96,29 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
     assert.ok(formatOutcomeLists(rows).some((l) => /^unrecovered: #1 \(budget\.exceeded: Implement turn budget spent/.test(l)));
   });
 
+  it("conformance gaps are recoverable: one recovery lane each, regression/guard_broken/scope_creep/e2e_unverified stay terminal, notes name the defer route", async () => {
+    const codes = ["conformance.unimplemented", "conformance.omission", "conformance.unjustified_deferral"];
+    for (const c of codes) {
+      assert.equal(classifyNeedsHuman({ outcome: "needs-human", code: c }), "recoverable", c);
+      assert.ok(RECOVERABLE_CODES.includes(c) && RECOVER_HOW[c], c);
+    }
+    for (const c of ["conformance.regression", "conformance.guard_broken", "conformance.scope_creep", "conformance.e2e_unverified"]) {
+      assert.equal(classifyNeedsHuman({ outcome: "needs-human", code: c }), "terminal", c);
+    }
+    for (const c of codes.slice(1)) assert.match(RECOVER_HOW[c], /`defer`[\s\S]*why|why[\s\S]*follow-up/i);
+    assert.match(RECOVER_HOW["conformance.omission"], /add-requirement/);
+    const { rows, calls } = await batch({ 1: [NH("conformance.omission", "R2 not covered", { branch: "feat/issue-1-lean" }), NH("conformance.omission", "again")] });
+    assert.deepEqual(calls.filter((c) => c.n === 1).map((c) => c.recover?.code ?? null), [null, "conformance.omission"]);
+    assert.equal(calls.filter((c) => c.n === 1).length, 2, "no second recovery beyond max_attempts");
+    assert.equal(byIssue(rows)[1].outcome, "needs-human");
+  });
+
   it("human codes (product-decision, spend) are never recovered, still skip dependents and are the only escalations, asked once while other lanes keep running", async () => {
     const { rows, calls, decisions } = await batch({
       1: [NH("builder.blocked", "Builder is blocked (product-decision): pick a pricing model")],
       2: [],
       3: [NH("builder.blocked", "Builder is blocked (spend): needs a paid plan")],
-      4: [NH("conformance.omission", "R2 not covered")],
+      4: [NH("conformance.regression", "R2 broke")],
       5: [{ terminalReason: "merged" }] }, { 2: "**Depends on**: #1" });
     const r = byIssue(rows);
     assert.equal(calls.filter((c) => c.n === 1).length, 1); assert.equal(calls.filter((c) => c.n === 3).length, 1);
@@ -114,7 +131,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
     assert.match(esc, /#1 \(builder\.blocked\): Builder is blocked \(product-decision\)/);
     assert.match(esc, /#3 \(builder\.blocked\): Builder is blocked \(spend\)/);
     assert.doesNotMatch(esc, /#4/, "the needs-human list holds only human codes");
-    assert.ok(lines.some((l) => /^unrecovered: #4 \(conformance\.omission: R2 not covered\)$/.test(l)));
+    assert.ok(lines.some((l) => /^unrecovered: #4 \(conformance\.regression: R2 broke\)$/.test(l)));
   });
 
   it("base.conflict: a recovery base-sync with a clean result re-queues the issue; the report lists recovered issues separately", async () => {
@@ -224,7 +241,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
   it("/orchestrate spec step 3 lists the recoverable codes, asks no confirmation on them, and escalates once at the end", async () => {
     const spec = readFileSync(new URL("../../commands/orchestrate.md", import.meta.url), "utf8");
     const step3 = spec.slice(spec.indexOf("## 3. Launch lanes"), spec.indexOf("## 4. Report"));
-    for (const c of ["`ci.red`", "`base.conflict`", "`ci-pending` on a conflicting", "`contract.invalid` caused by an issue edited mid-run"]) assert.ok(step3.includes(c), c);
+    for (const c of ["`ci.red`", "`base.conflict`", "`ci-pending` on a conflicting", "`contract.invalid` caused by an issue edited mid-run", "`conformance.unimplemented`", "`conformance.omission`", "`conformance.unjustified_deferral`"]) assert.ok(step3.includes(c), c);
     assert.match(step3, /\*\*Do not ask for confirmation on them\.\*\*/);
     assert.match(step3, /FORGE:DECISION/);
     assert.match(step3, /one question list[\s\S]*once, at the end of the batch/);

package/bin/lean/retry.mjs

@@ -32,6 +32,17 @@ export async function retrying(fn, { transient, permanent = null, attempts = 4,
   throw last;
 }
 
+/** Poll `probe` until it returns a truthy value (returned) or `timeoutMs` has passed (null). The probe runs at least once. */
+export async function pollUntil(probe, { timeoutMs, pollMs, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), now = () => Date.now() }) {
+  const t0 = now();
+  for (;;) {
+    const v = await probe();
+    if (v) return v;
+    if (now() - t0 > timeoutMs) return null;
+    await sleep(pollMs);
+  }
+}
+
 /**
  * #4507: run `call(env)` with the lane's gh cache env; on a gh parse error (corrupt cache) reset the cache and run it
  * once more. Without a cache it still retries once. Any other error is rethrown unchanged.

package/bin/lean/retry.test.mjs

@@ -0,0 +1,16 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { pollUntil } from "./retry.mjs";
+
+test("pollUntil returns the first truthy probe value, sleeping between tries", async () => {
+  let n = 0, t = 0; const sleeps = [];
+  const v = await pollUntil(() => (++n >= 3 ? "ok" : null), { timeoutMs: 100, pollMs: 7, sleep: async (ms) => { sleeps.push(ms); t += ms; }, now: () => t });
+  assert.equal(v, "ok"); assert.deepEqual(sleeps, [7, 7]);
+});
+
+test("pollUntil returns null once the timeout passes, probing at least once", async () => {
+  let t = 0, n = 0;
+  const v = await pollUntil(() => { n++; return null; }, { timeoutMs: 20, pollMs: 10, sleep: async (ms) => { t += ms; }, now: () => t });
+  assert.equal(v, null); assert.ok(n >= 3);
+  assert.equal(await pollUntil(() => null, { timeoutMs: -1, pollMs: 1, sleep: async () => {}, now: () => 0 }), null);
+});

package/bin/lean/sched.mjs

@@ -144,12 +144,14 @@ export function openSched(dir = schedDir(), { now = () => Date.now(), pid = proc
     }
     const obj = (x) => (x && typeof x === "object" && !Array.isArray(x) ? x : {});
     const weights = Object.fromEntries(Object.entries(obj(o?.weights)).filter(([, n]) => posInt(n)));
-    return { v: 1, cap: posInt(o?.cap), weights, seq: Number.isInteger(o?.seq) ? o.seq : 0, slots: obj(o?.slots), queue: obj(o?.queue) };
+    return { v: 1, cap: posInt(o?.cap), weights, seq: Number.isInteger(o?.seq) ? o.seq : 0, slots: obj(o?.slots), queue: obj(o?.queue), remote: obj(o?.remote), pool: obj(o?.pool) };
   };
   // Drop queued entries of dead dispatchers, and slots whose dispatcher and lane are both gone.
   const reclaim = (s) => {
     for (const [k, e] of Object.entries(s.queue)) if (!alive(e.pid)) delete s.queue[k];
     for (const [k, e] of Object.entries(s.slots)) if (!alive(e.pid) && !(e.lanePid && alive(e.lanePid))) delete s.slots[k];
+    for (const [k, e] of Object.entries(s.remote)) if (!alive(e?.pid)) delete s.remote[k]; // #4964
+    for (const [k, e] of Object.entries(s.pool)) if (!alive(e?.pid)) delete s.pool[k];
   };
   const mutate = (fn) => {
     lock();
@@ -200,7 +202,18 @@ export function openSched(dir = schedDir(), { now = () => Date.now(), pid = proc
     /** Queue entries `[{key, batch, repo, issue, priority}]` (their FIFO position is kept on a re-offer). */
     offer: (entries) => { try { mutate((s) => { for (const e of entries) enqueue(s, e.key, e); }); } catch { /* fail-open */ } },
     withdraw: (keys) => { if (!keys.length) return; try { mutate((s) => { for (const k of keys) delete s.queue[k]; }); } catch { /* fail-open */ } },
-    release: (key) => { try { mutate((s) => { delete s.slots[key]; delete s.queue[key]; }); } catch { /* fail-open */ } },
+    release: (key) => { try { mutate((s) => { delete s.slots[key]; delete s.queue[key]; delete s.remote[key]; }); } catch { /* fail-open */ } },
+    /** #4964: publish a running remote (pool) lane for `sched show`; it never takes a host slot. */
+    noteRemote: (key, entry = {}) => {
+      try {
+        mutate((s) => { s.remote[key] = { pid, batch: entry.batch ?? null, repo: entry.repo ?? null, issue: entry.issue ?? null, sandbox: entry.sandbox ?? null, agent: entry.agent ?? null, at: now() }; });
+      } catch { /* fail-open */ }
+    },
+    dropRemote: (key) => { try { mutate((s) => { delete s.remote[key]; }); } catch { /* fail-open */ } },
+    /** #4964: this batch's pool is unreachable (`{reason}`), or null to clear it. */
+    notePool: (batch, state) => {
+      try { mutate((s) => { if (state) s.pool[batch] = { pid, state: "unreachable", reason: state.reason ?? null, at: now() }; else delete s.pool[batch]; }); } catch { /* fail-open */ }
+    },
     setLanePid: (key, lanePid) => { try { mutate((s) => { if (s.slots[key] && posInt(lanePid)) s.slots[key].lanePid = lanePid; }); } catch { /* fail-open */ } },
     /** Record a lane this dispatcher re-attached to: it runs already, so the cap does not gate it. */
     adoptSlot: (key, entry = {}) => {
@@ -231,6 +244,16 @@ export function openSched(dir = schedDir(), { now = () => Date.now(), pid = proc
       weights: { ...s.weights },
       slots: Object.entries(s.slots).map(([key, e]) => ({ key, ...e })),
       queue: grantOrder(s).map(([key, e]) => ({ key, ...e })),
+      remote: Object.entries(s.remote).map(([key, e]) => ({ key, ...e })),
+      pool: Object.entries(s.pool).map(([batch, e]) => ({ batch, ...e })),
     })),
   };
 }
+
+/** #4964: `sched show` lines for remote (pool) lanes and an unreachable pool; empty when there are none. */
+export function remoteLines(st) {
+  const out = (st?.remote ?? []).map((e) => `  remote #${e.issue} ${e.repo ?? "-"} batch ${e.batch ?? "-"} sandbox ${e.sandbox ?? "-"} agent ${e.agent ?? "-"}  dispatcher ${e.pid}`);
+  const down = st?.pool ?? [];
+  if (down.length) out.push(`pool: unreachable (${down.map((e) => `batch ${e.batch}${e.reason ? `: ${e.reason}` : ""}`).join("; ")})`);
+  return out;
+}

package/bin/lean/sched.test.mjs

@@ -5,7 +5,7 @@ import assert from "node:assert/strict";
 import { writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
-import { grantOrder, openSched, priorityFromLabels } from "./sched.mjs";
+import { grantOrder, openSched, priorityFromLabels, remoteLines } from "./sched.mjs";
 import { orchestrate, estimateLaneMinutes, laneHistory, formatEstimateSummary, EST_DEFAULT_MIN, POST_DEPLOY_EST_MIN } from "./orchestrate.mjs";
 import { schedFromCli } from "../engine-cli.mjs";
 
@@ -512,3 +512,60 @@ describe("live local lane cap (#4777)", () => {
     assert.equal(peak, 2);
   });
 });
+
+describe("sched show lists remote (pool) lanes (#4964)", () => {
+  const show = async (sched) => { const lines = []; const l = console.log; console.log = (m) => lines.push(String(m)); try { const st = await schedFromCli(["show"], { sched }); return { st, lines }; } finally { console.log = l; } };
+
+  it("remote lanes get their own rows and a remote N count, and take no host slot or cap", async () => {
+    const { handle } = world();
+    const a = handle(501), b = handle(502);
+    a.setCap(1);
+    a.noteRemote("B#2", { batch: "B", repo: "o/r", issue: 2, sandbox: "fdrsbx-B-2", agent: "legion" });
+    a.noteRemote("B#3", { batch: "B", repo: "o/r", issue: 3, sandbox: "fdrsbx-B-3" });
+    const { st, lines } = await show(b);
+    assert.equal(st.slots.length, 0);
+    assert.equal(st.remote.length, 2);
+    assert.ok(lines.some((m) => /running 0, queued 0, remote 2$/.test(m)), lines.join("\n"));
+    assert.ok(lines.includes("  remote #2 o/r batch B sandbox fdrsbx-B-2 agent legion  dispatcher 501"), lines.join("\n"));
+    assert.equal(b.tryAcquire("C#1", {}, null).ok, true, "cap 1 is still free: remote lanes hold no slot");
+    a.dropRemote("B#2");
+    assert.deepEqual(b.list().remote.map((r) => r.key), ["B#3"]);
+    a.release("B#3");
+    assert.equal(b.list().remote.length, 0);
+  });
+
+  it("a dead dispatcher's remote lane and pool notice are reclaimed", () => {
+    const { handle, live } = world();
+    const a = handle(511), b = handle(512);
+    a.noteRemote("B#2", { batch: "B", issue: 2 });
+    a.notePool("B", { reason: "capacity probe failed" });
+    assert.equal(b.list().remote.length, 1);
+    live.delete(511);
+    assert.deepEqual([b.list().remote.length, b.list().pool.length], [0, 0]);
+  });
+
+  it("an unreachable pool is one line; none once cleared", async () => {
+    const { handle } = world();
+    const a = handle(521);
+    a.notePool("B", { reason: "capacity probe failed" });
+    const down = (await show(a)).lines.filter((m) => m.startsWith("pool:"));
+    assert.deepEqual(down, ["pool: unreachable (batch B: capacity probe failed)"]);
+    a.notePool("B", null);
+    assert.equal((await show(a)).lines.filter((m) => m.startsWith("pool:")).length, 0);
+    assert.deepEqual(remoteLines({}), []);
+  });
+
+  it("orchestrate publishes a remote lane while it runs, drops it after, and never acquires a slot", async () => {
+    const { handle } = world();
+    const reg = handle(531);
+    let seen = null, acquired = 0;
+    const spy = { ...reg, tryAcquire: (...x) => { acquired++; return reg.tryAcquire(...x); } };
+    await orchestrate({ issues: [7], parallel: 1, pollMs: 5, trusted: () => true, readBody: async () => "", sched: { registry: spy, batch: "B", repo: "o/r" },
+      remote: { max: 1, isRemote: () => true, capacity: () => ({ free: 1, online: 1 }) },
+      runOne: async (n, lane, ctx) => { ctx.onStart?.({ handle: { id: "fdrsbx-B-7", agent: "legion" }, remote: true }); seen = reg.list(); return { terminalReason: "merged" }; } });
+    assert.equal(acquired, 0);
+    assert.deepEqual(seen.remote.map((r) => [r.issue, r.sandbox, r.agent]), [[7, "fdrsbx-B-7", "legion"]]);
+    assert.equal(seen.slots.length, 0);
+    assert.equal(reg.list().remote.length, 0);
+  });
+});