forgedock 1.43.1 → 1.45.0

package/bin/lean/orchestrate.mjs

@@ -115,6 +115,7 @@ import { formatParkedLine, postDeployOnly } from "./prefilter.mjs";
 import { laneSize, SIZES } from "./budget.mjs";
 import { requeueKey, REQUEUABLE } from "./batch.mjs";
 import { makeCapWait } from "./capwait.mjs";
+import { mapLimit } from "./pmap.mjs";
 import { joinsDecomposed, addsEpicChildren, selectChildren, edgeText, DEFAULT_POLL_S } from "./autochildren.mjs";
 
 // A dependency that finished like this no longer blocks its dependents.
@@ -144,6 +145,8 @@ export const EST_DEFAULT_MIN = Object.freeze({ S: 10, M: 25, L: 50 });
 export const POST_DEPLOY_EST_MIN = 2;
 // A ready issue queued longer than this joins the head band (starvation guard; only with an estimator).
 export const DEFAULT_STARVE_MS = 30 * 60000;
+/** #4963: issues read at once while a batch sets up. */
+export const DEFAULT_SETUP_CONCURRENCY = 6;
 // Merged runs of one repo and size needed before their median replaces the default.
 export const HISTORY_MIN_SAMPLES = 3;
 
@@ -253,8 +256,8 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   store = null, adopt = null, getParallel = null, getAdded = null, getRequeued = null, breaker = null, maxDeferrals = DEFAULT_MAX_DEFERRALS, sleep = cancellableSleep, pollMs = 5000,
   classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, integration = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null, preflight = null,
   recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, issueState = null, getCancelled = null, getHeld = null, getBlocked = null, trusted = null, getGates = null, stopLane = null,
-  estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null, dispatchCheck = null, syncParked = null, syncEveryMs = DEFAULT_SYNC_EVERY_MS, lifecycle = null,
-  getChildren = null, autoChildren = null }) {
+  estimate = null, starveMs = DEFAULT_STARVE_MS, setupConcurrency = DEFAULT_SETUP_CONCURRENCY, signal = null, dispatchCheck = null, syncParked = null, syncEveryMs = DEFAULT_SYNC_EVERY_MS, lifecycle = null,
+  getChildren = null, autoChildren = null, baseWait = null }) {
   const saved = store?.load?.() ?? null;
   const order = saved ? [...saved.order] : [];
   const set = new Set(order);
@@ -336,10 +339,11 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   const decided = (d) => { try { Promise.resolve(decide?.(d)).catch(() => {}); } catch { /* best effort */ } };
   const firstLine = (x) => String(x ?? "").split("\n")[0].slice(0, 300);
 
+  let setup = null; // #4963: {read, total} while the initial per-issue reads run; written with phase "setup" so a slow start is visible
   let halted = false; // #4454: stopped by `signal`; the batch's state is no longer this process's to write
   const save = () => {
     if (!store || halted) return;
-    store.save({ v: 1, order, deps: toObj(deps), lanes: toObj(lanes), crossSeen: [...crossSeen], results: toObj(results), running: toObj(runRecs),
+    store.save({ v: 1, ...(setup ? { phase: "setup", setup: { ...setup } } : {}), order, deps: toObj(deps), lanes: toObj(lanes), crossSeen: [...crossSeen], results: toObj(results), running: toObj(runRecs),
       pending, notBefore: toObj(notBefore), requeues: toObj(requeues), deferrals: toObj(deferrals), engineErrors: toObj(engineErrors),
       prefilter: toObj(checked), errorTimes, breaker: tripped, ...(budget ? { classes: toObj(classes), dispatch: { ...dispatch } } : {}),
       ...(reg ? { priorities: toObj(priorities) } : {}), ...(sandboxUse.size ? { sandbox: toObj(sandboxUse) } : {}),
@@ -377,11 +381,13 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     let body = "";
     try { body = await readBody(n); } catch { /* unreadable: no ordering info, work-on's own check still applies */ }
     if (drift?.holds) bodies.set(n, body);
-    deps.set(n, await declared(n, body));
+    const d = await declared(n, body);
     await readEstimate(n, body);
     await classifyOne(n);
     await readPriority(n);
     if (resolveLane) { let l; try { l = await resolveLane(n); } catch (e) { l = { ok: false, code: "lane.unresolved", detail: e?.message ?? String(e), transient: true }; } lanes.set(n, l); }
+    // deps marks the issue read (a resume skips it): set it last, so a concurrent setup save never persists a half-read issue (#4963)
+    deps.set(n, d);
   };
   const edges = (n) => (deps.get(n) ?? []).filter((d) => set.has(d));
   // A milestone issue waiting on a staging fix (or the reverse) never sees that fix on its own base: surface the edge.
@@ -412,7 +418,14 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   // New issues (a fresh batch, or extra issues passed to a resume) join the queue.
   for (const n of issues) if (!set.has(n)) { set.add(n); order.push(n); pending.push(n); }
   for (const n of pending) if (!enqueued.has(n)) enqueued.set(n, now());
-  for (const n of order) if (!deps.has(n)) await readDeps(n);
+  // #4963: bounded-concurrency setup reads; each issue's result lands in Maps keyed by issue, so `order` and the edges are unchanged.
+  const unread = order.filter((n) => !deps.has(n));
+  if (unread.length) {
+    setup = { read: 0, total: unread.length };
+    save();
+    await mapLimit(unread, setupConcurrency, async (n) => { await readDeps(n); setup.read++; save(); });
+    setup = null;
+  }
   if (estimate) for (const n of order) { // a resume that enabled the estimator
     if (estimates.has(n) || results.has(n)) continue;
     let b = ""; try { b = await readBody(n); } catch { b = ""; }
@@ -446,6 +459,11 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       .sort((a, b) => a.p - b.p || a.b - b.b || a.s - b.s || b.w - a.w || a.i - b.i).map((x) => x.n);
   };
   const entryOf = (n) => ({ key: keyOf(n), batch: sched?.batch ?? batchId ?? null, repo: sched?.repo ?? null, issue: n, priority: prioOf(n), ...(sched?.exclusive ? { exclusive: true } : {}) });
+  // #4964: remote lanes are published to the registry for `sched show` only; they never take a slot
+  const endRemote = (n) => { if (remoteRun.delete(n)) reg?.dropRemote?.(keyOf(n)); };
+  const noteRemote = (n, h) => { reg?.noteRemote?.(keyOf(n), { ...entryOf(n), sandbox: h?.id ?? null, agent: h?.agent ?? null }); };
+  let poolDown = false;
+  const notePool = (down, reason) => { if (down === poolDown) return; poolDown = down; reg?.notePool?.(sched?.batch ?? batchId ?? "?", down ? { reason } : null); };
   const releaseSlot = (n) => { if (reg && slotted.delete(n)) reg.release(keyOf(n)); };
   const offerFresh = (ns) => { const fresh = ns.filter((n) => !offered.has(n)); if (!fresh.length) return; reg.offer(fresh.map(entryOf)); for (const n of fresh) offered.add(n); };
   const withdrawExcept = (keep) => {
@@ -498,8 +516,10 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       const adm = admitRemote();
       if (adm.ok) {
         // #4540: the provider's free slots (`remote.capacity() -> {free, at}`, sync, null = unknown) cap admissions per sample `at`; the issue queues here instead of a 503.
-        let cap = null;
-        try { cap = remote.capacity?.() ?? null; } catch { cap = null; }
+        let cap = null, probeFailed = false;
+        try { cap = remote.capacity?.() ?? null; } catch { probeFailed = true; }
+        // #5048: only a failed probe (a throw) marks the pool down; null is unknown (first pass, no capacity()) and leaves the state; a sample clears it.
+        if (probeFailed) notePool(true, "capacity probe failed"); else if (cap !== null) notePool(false);
         // #4788: zero live agents for longer than `remote.poolOfflineMin` (default 10): one loud batch event, none while online > 0.
         if (cap && cap.online === 0) {
           poolOfflineSince ??= now();
@@ -585,7 +605,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       try { remote?.onUsage?.(n, r.sandbox, rec); } catch { /* the batch total still counts it */ }
     }
     if (sandboxUse.has(n)) row.sandbox = { ...sandboxUse.get(n), ...(sandboxUse.get(n).by_agent ? { by_agent: { ...sandboxUse.get(n).by_agent } } : {}) };
-    const done = () => { running.delete(n); runRecs.delete(n); releaseSlot(n); remoteRun.delete(n); };
+    const done = () => { running.delete(n); runRecs.delete(n); releaseSlot(n); endRemote(n); };
     if (row.outcome === "dependency-open" && depList.length) {
       // #4387: a parked dependency is not pulled in again (fail-open on an unreadable label).
       let parked;
@@ -633,6 +653,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     if (row.outcome === "deferred") {
       const pol = deferPolicy(r, { now: now(), deferrals: deferrals.get(n) ?? 0, maxDeferrals });
       let capGated = false; // #4909: the pool reports free slots, so the slot (not the retry-in timer) gates the relaunch
+      if (r?.code === "sandbox.unreachable") notePool(true, r.detail ?? "sandbox unreachable");
       if (r?.code === "sandbox.capacity") {
         capWait.refused(n);
         try { capGated = Number.isInteger(remote?.capacity?.()?.free); } catch { capGated = false; }
@@ -781,6 +802,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
         try {
           r = await runOne(n, lane, { remote: !!rec.remote, ...(rec.placement ? { placement: rec.placement } : {}), ...(recovering.has(n) ? { recover: { code: recovering.get(n).code, attempt: recovering.get(n).attempt, final: recovering.get(n).attempt >= (recovering.get(n).max ?? recoveryMax) } } : {}), onStart: (info) => {
             Object.assign(rec, info ?? {}); save();
+            if (rec.remote && remoteRun.has(n)) noteRemote(n, rec.handle);
             if (cancelled) stop(n, rec); // #4376: launched while the cancel arrived
             if (reg && slotted.has(n) && info?.pid) reg.setLanePid(keyOf(n), info.pid);
           } });
@@ -805,18 +827,18 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   for (const a of adopting) a.live = a.first ? await a.first : false;
   for (const { n, rec, p, live } of adopting) {
     if (live) {
-      if (rec.remote) remoteRun.add(n);
+      if (rec.remote) { remoteRun.add(n); noteRemote(n, rec.handle); }
       else if (reg) { reg.adoptSlot(keyOf(n), { ...entryOf(n), lanePid: rec.pid }); slotted.add(n); }
       track(n, p.then((r) => {
         if (r) return settle(n, r, rec, checked.get(n) ?? null);
         if (halted) return; // #4454: kept for the next resume
-        running.delete(n); releaseSlot(n); remoteRun.delete(n); requeueLost(n); save(); // ended without a result after all
+        running.delete(n); releaseSlot(n); endRemote(n); requeueLost(n); save(); // ended without a result after all
       }));
       continue;
     }
     const r = await p;
     if (!r) { requeueLost(n); continue; }
-    if (rec.remote) remoteRun.add(n);
+    if (rec.remote) { remoteRun.add(n); noteRemote(n, rec.handle); }
     else if (reg) { reg.adoptSlot(keyOf(n), { ...entryOf(n), lanePid: rec.pid }); slotted.add(n); } // released by the settle
     track(n, Promise.resolve(r).then((x) => settle(n, x, rec, checked.get(n) ?? null)));
     await running.get(n); // settled before scheduling
@@ -832,7 +854,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   }
   save();
 
-  const heldDetail = (n) => cancelled ? "cancelled by operator" : opHeld.has(n) ? "held by operator" : tripped?.detail ?? (integHeld.has(n) ? `integration: ${integHeld.get(n)}` : (driftHeld.has(n) ? `drift: ${driftHeld.get(n)} flagged` : budgetStop ? `execution budget: ${budgetStop}` : budgetHeld ? "maintenance share" : "not run"));
+  const heldDetail = (n) => cancelled ? "cancelled by operator" : opHeld.has(n) ? "held by operator" : baseHeld.has(n) ? baseHeld.get(n) : tripped?.detail ?? (integHeld.has(n) ? `integration: ${integHeld.get(n)}` : (driftHeld.has(n) ? `drift: ${driftHeld.get(n)} flagged` : budgetStop ? `execution budget: ${budgetStop}` : budgetHeld ? "maintenance share" : "not run"));
   // #4315: a recovery queued but never launched (the batch stopped: breaker, budget, hold; or a resume that did not relaunch a
   // lost run) left the issue without needs-human, since a non-final attempt does not relabel. Relabel it once, with the full reason.
   // #4318: also on a throw out of the loop and on a signal (`onExit`); `cause` overrides the held reason. Runs once per process.
@@ -875,6 +897,12 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   };
   // #4965: epic and late children (`orchestration.auto_add_children` epic|all): at most one `getChildren()` list call per poll window.
   let nextChildScan = 0, childScanFailed = false;
+  const skippedKids = new Set(); // #4982: one row per untrusted child, not one per scan
+  const skipChild = ({ number, reason }) => {
+    if (skippedKids.has(number)) return;
+    skippedKids.add(number);
+    update({ issue: number, outcome: "skipped", detail: `#${number} not auto-added: ${reason} (orchestration.auto_add_children)` });
+  };
   const drainChildren = async () => {
     if (!getChildren || !addsEpicChildren(autoChildren?.mode)) return 0;
     const t = now();
@@ -886,7 +914,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       return 0;
     }
     childScanFailed = false;
-    const fresh = selectChildren({ list, roots: [...set, ...(autoChildren.epics ?? [])], skip: [...set, ...(elsewhere ?? [])] })
+    const fresh = selectChildren({ list, roots: [...set, ...(autoChildren.epics ?? [])], skip: [...set, ...(elsewhere ?? [])], onSkip: skipChild })
       .filter((n) => { try { return !(reg?.holder && reg.holder(entryOf(n)) != null); } catch { return true; } });
     if (!fresh.length) return 0;
     for (const n of pending) {
@@ -970,6 +998,21 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   // #4773: re-read the issues carrying the blocked label; one decision and row per change. A failed read keeps the last set.
   let blocked = new Set();
   const blockedHit = new Set(); // blocked issues that were pending this pass
+  // #5031: `baseWait(n)` (async, default off) returns a detail string while a dependency's merge commit is not yet on n's base, else null.
+  // The issue is admitted but not dispatched; it is re-checked every pass until released (a commit that landed stays landed). Fail-open.
+  const baseHeld = new Map(), baseHit = new Set(), baseDone = new Set();
+  const refreshBaseWaits = async () => {
+    if (!baseWait) return;
+    for (const n of pending) {
+      if (baseDone.has(n) || results.has(n)) continue;
+      let why = null;
+      try { why = (await baseWait(n)) ?? null; } catch { why = null; }
+      if (why && !baseHeld.has(n)) update({ issue: n, outcome: "held", detail: why });
+      if (why) { baseHeld.set(n, why); continue; }
+      if (baseHeld.delete(n)) { decided({ issue: n, what: "released: dependency commit is on the base", why: "the awaited merge commit landed", alternative: "keep waiting" }); update({ issue: n, outcome: "unblocked", detail: "dependency commit landed" }); }
+      baseDone.add(n);
+    }
+  };
   const refreshBlocked = async () => {
     if (!getBlocked) return;
     let got;
@@ -1106,8 +1149,10 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       integHeld.clear();
       opHeldHit.clear();
       blockedHit.clear();
+      baseHit.clear();
       refreshHeld();
       await refreshBlocked();
+      await refreshBaseWaits();
       checkGates();
       // #4866: lane preflight. A failing check holds admission (all lanes, or only remote ones by its scope), says why once per batch,
       // and is re-checked every pass so the batch resumes on its own; it is not an engine-error, so the breaker never sees it.
@@ -1125,6 +1170,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
         if (pfAll || (pfRemote && isRemote(n))) return false;
         if (opHeld.has(n)) { opHeldHit.add(n); return false; }
         if (blocked.has(n)) { blockedHit.add(n); return false; } // #4773
+        if (baseHeld.has(n)) { baseHit.add(n); return false; } // #5031
         if (!((notBefore.get(n) ?? 0) <= t && edges(n).every((d) => results.has(d) && SATISFIED.has(results.get(d).outcome)))) return false;
         if (integration?.holds) {
           let why = null;
@@ -1204,6 +1250,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       if (offered.size && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // a slot another batch frees
       if ((getAdded || getRequeued || (getChildren && addsEpicChildren(autoChildren?.mode))) && running.size) wake = Math.min(wake ?? pollMs, pollMs); // #4408/#4550: an add or requeue while only lanes run
       if ((pfAll || pfRemote) && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // #4866
+      if (baseWait && pending.length && baseHeld.size) wake = Math.min(wake ?? pollMs, pollMs); // #5031: a landed commit releases on the next tick
       if (getBlocked && pending.length && blockedHit.size) wake = Math.min(wake ?? pollMs, pollMs); // #4773: a removed label releases on the next tick
       if (getHeld && running.size) wake = Math.min(wake ?? pollMs, pollMs); // #4592: a release while only lanes run
       if (syncParked && running.size && parkedPrs().length) wake = Math.min(wake ?? syncEveryMs, Math.max(0, syncEveryMs - (t - (lastParkedSync ?? t)))); // #4525
@@ -1217,6 +1264,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       else if (pending.length && offered.size) await nap(wake); // waiting for a host slot
       else if (pending.length && (pfAll || pfRemote)) await nap(Math.min(wake ?? pollMs, pollMs)); // #4866: a failing lane preflight is re-checked on the next poll
       else if (pending.length && blockedHit.size) await nap(wake ?? pollMs); // #4773: only blocked issues wait: poll for the label's removal
+      else if (pending.length && baseHit.size) await nap(wake ?? pollMs); // #5031: only issues waiting for a base commit: poll for it
       else if (pending.length && waits.length) await nap(Math.min(wake, getCancelled ? pollMs : Infinity)); // wake is poll-capped by a hold or a live K; #4454: nap wakes on a stop
       else if (pending.length && budgetHeld) { if (await drainAdds()) continue; break; } // maintenance the budget does not admit now: held for a later batch
       else if (pending.length && opHeldHit.size && !pending.some((n) => !opHeldHit.has(n) && isReady(n))) { if (await drainAdds()) continue; break; } // #4592: held by the operator: pending for a --resume

package/bin/lean/pmap.mjs

@@ -0,0 +1,17 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/** Map `items` through async `fn` with at most `limit` in flight; results keep input order. A throw rejects after the in-flight calls settle. */
+export async function mapLimit(items, limit, fn) {
+  const list = [...items], out = new Array(list.length);
+  const width = Math.max(1, Math.min(Math.floor(Number(limit)) || 1, list.length));
+  let next = 0, failed = false, error;
+  const worker = async () => {
+    while (!failed && next < list.length) {
+      const i = next++;
+      try { out[i] = await fn(list[i], i); } catch (e) { if (!failed) { failed = true; error = e; } }
+    }
+  };
+  await Promise.all(Array.from({ length: width }, worker));
+  if (failed) throw error;
+  return out;
+}

package/bin/lean/pmap.test.mjs

@@ -0,0 +1,27 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { mapLimit } from "./pmap.mjs";
+
+describe("mapLimit", () => {
+  it("bounds concurrency and keeps input order", async () => {
+    let live = 0, peak = 0;
+    const out = await mapLimit([5, 1, 4, 2, 3, 6], 3, async (n) => {
+      live++; peak = Math.max(peak, live);
+      await new Promise((r) => setTimeout(r, n * 3));
+      live--; return n * 10;
+    });
+    assert.deepEqual(out, [50, 10, 40, 20, 30, 60]);
+    assert.equal(peak, 3);
+  });
+  it("handles empty input and a bad limit as serial", async () => {
+    assert.deepEqual(await mapLimit([], 4, async () => 1), []);
+    let live = 0, peak = 0;
+    await mapLimit([1, 2, 3], 0, async () => { live++; peak = Math.max(peak, live); await new Promise((r) => setTimeout(r, 2)); live--; });
+    assert.equal(peak, 1);
+  });
+  it("rejects with the first error", async () => {
+    await assert.rejects(mapLimit([1, 2, 3], 2, async (n) => { if (n === 2) throw new Error("boom"); return n; }), /boom/);
+  });
+});

package/bin/lean/postdeploy-check.test.mjs

@@ -96,7 +96,9 @@ describe("post-deploy named checks (#4626)", () => {
     assert.ok(validateContract(c, Q).problems.some((p) => /needs a check id/.test(p)));
     assert.ok(!validateContract({ ...c, requirements: [{ ...c.requirements[0], check: "smoke" }] }, Q).problems.some((p) => /check id/.test(p)));
     const frozen = { ...c, requirements: [{ id: "R1", quote: Q, disposition: "deliver" }] };
-    assert.ok(applyAmendments(frozen, [{ kind: "post-deploy", requirement: "R1", why: "w" }], Q).problems.some((p) => /check id/.test(p)));
+    const deferred = applyAmendments(frozen, [{ kind: "post-deploy", requirement: "R1", why: "w" }], Q); // #5020: no check id defers to the orchestrator
+    assert.deepEqual(deferred.problems, []);
+    assert.equal(deferred.contract.requirements[0].disposition, "defer");
     const ok = applyAmendments(frozen, [{ kind: "post-deploy", requirement: "R1", why: "w", check: "smoke" }], Q);
     assert.equal(ok.contract.requirements[0].check, "smoke");
     const pc = precheck({ contract: { ...c, requirements: [{ ...c.requirements[0], check: "ghost" }] }, result: { requirements: [] }, changedFiles: [], checkIds: new Set(["smoke"]) });

package/bin/lean/preflight.mjs

@@ -101,6 +101,12 @@ function moduleNeeds(exe, args) {
  * probe (tool) a module probe depends on: skip the module probe when its interpreter is missing. A name `repoHas(name)` holds
  * or a deliverable's change text names (the change adds it) is not probed; paths and `$...` words are never probed.
  */
+/** #5020: the contract's runnable commands: each validate entry plus the repro_test, which fails the same ways. */
+export function contractCommands(contract) {
+  const out = (Array.isArray(contract?.validate) ? contract.validate : []).filter((v) => v?.id && typeof v.command === "string");
+  const t = contract?.repro_test;
+  return t && typeof t === "object" && typeof t.command === "string" && t.command.trim() ? [...out, { id: "repro_test", command: t.command }] : out;
+}
 export function preflightProbes(contract, { repoHas = () => false, cap = PREFLIGHT_CAP } = {}) {
   const changes = (Array.isArray(contract?.deliverables) ? contract.deliverables : []).map((d) => String(d?.change ?? "")).join("\n");
   const named = (name) => new RegExp(`(^|[^\\w@/.-])${name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?![\\w/-])`).test(changes);
@@ -115,8 +121,7 @@ export function preflightProbes(contract, { repoHas = () => false, cap = PREFLIG
       : `node -e ${sq(NODE_PROBE)} ${sq(tool)} >/dev/null 2>&1`;
     byKey.set(key, { ids: [id], tool, kind, interp, needs: kind === "command" ? null : interp, probe });
   };
-  for (const v of Array.isArray(contract?.validate) ? contract.validate : []) {
-    if (!v?.id || typeof v.command !== "string") continue;
+  for (const v of contractCommands(contract)) {
     for (const seg of shellSegments(v.command)) {
       let i = 0;
       while (i < seg.length && (/^[A-Za-z_]\w*=/.test(seg[i]) || ["!", "time", "exec", "command", "env"].includes(seg[i]))) i++;
@@ -238,7 +243,7 @@ const porcelain = (out) => new Map(String(out ?? "").split("\n").filter((l) => l
  */
 export async function runPreflight({ contract, run, git, env = {}, repoHas = () => false, knownPath = () => false, skipIds = [], now = () => Date.now(), perCommandMs = RUN_PROBE_MS, budgetMs = RUN_BUDGET_MS }) {
   const misses = [], skipped = [], reverted = [];
-  const todo = (Array.isArray(contract?.validate) ? contract.validate : []).filter((v) => v?.id && typeof v.command === "string" && !skipIds.includes(v.id));
+  const todo = contractCommands(contract).filter((v) => !skipIds.includes(v.id));
   if (!todo.length) return { misses, skipped, reverted };
   const status = async () => porcelain(await git(["status", "--porcelain", "--untracked-files=all"]).catch(() => ""));
   const before = await status(), deadline = now() + budgetMs;

package/bin/lean/preflight.test.mjs

@@ -4,7 +4,7 @@ import { describe, it } from "node:test";
 import assert from "node:assert/strict";
 import { mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
-import { testRunnerProblems, shellSegments, pythonImports, nodeImports, preflightProbes, preflightProblems, runProbeMiss, unsetEnvVars, runPreflight, knownToContract } from "./preflight.mjs";
+import { testRunnerProblems, shellSegments, pythonImports, nodeImports, preflightProbes, preflightProblems, runProbeMiss, unsetEnvVars, runPreflight, knownToContract, contractCommands } from "./preflight.mjs";
 import { runCheckCommand } from "./effects.mjs";
 import { repoTracks } from "./conformance.mjs";
 import { tmpDir } from "../tests/_tmp.mjs";
@@ -120,3 +120,15 @@ describe("testRunnerProblems (#4857)", () => {
     assert.deepEqual(testRunnerProblems(C("pytest services/worker/t.py"), null), []);
   });
 });
+
+describe("#5020 repro_test is probed like a validate command", () => {
+  const c = { validate: [{ id: "V1", command: "node --test a.test.mjs" }], repro_test: { path: "t.py", id: "t", command: "madeuptool --run t.py" } };
+  it("contractCommands lists validate entries then the repro_test", () => {
+    assert.deepEqual(contractCommands(c).map((x) => x.id), ["V1", "repro_test"]);
+    assert.deepEqual(contractCommands({ repro_test: "none" }), []);
+  });
+  it("preflightProbes probes the repro command's tool", () => {
+    const { probes } = preflightProbes(c);
+    assert.ok(probes.some((p) => p.tool === "madeuptool" && p.ids.includes("repro_test")));
+  });
+});

package/bin/lean/prodwatch-triage.mjs

@@ -37,6 +37,12 @@ export const signalRef = (fp) => `<!-- FORGE:SIGNAL_REF fp=${fp} -->`;
 /** The run of FORGE marker comments at the very end of a body: the only place the engine writes markers, so the only place they are trusted. */
 export const trailerOf = (body) => /(?:<!--\s*FORGE:[^>]*?-->\s*)+$/.exec(String(body ?? ""))?.[0] ?? "";
 
+/** Untrusted text is escaped so a literal comment opener cannot forge a FORGE marker in a body or prompt. */
+export const neutralize = (s) => String(s ?? "").replace(/<!--/g, "&lt;!--");
+
+/** The first issue whose trailing marker run holds `marker` and that passes `accept`; a marker quoted mid-body is not trusted. */
+export const findMarked = (issues, marker, accept = () => true) => (issues ?? []).find((i) => trailerOf(i?.body).includes(marker) && accept(i));
+
 /** fingerprint -> issue number for every open issue carrying a SIGNAL_REF marker in its trailer (first issue wins). */
 function refsOf(openIssues) {
   const map = new Map();

package/bin/lean/prodwatch-triage.test.mjs

@@ -3,7 +3,7 @@
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
 import { readFileSync } from "node:fs";
-import { fingerprint, signalRef, cluster, parseAlertRules, classifyCluster, validateCluster, tallyClusters, CLASSES, DEFAULT_RULES, trailerOf } from "./prodwatch-triage.mjs";
+import { fingerprint, signalRef, cluster, parseAlertRules, classifyCluster, validateCluster, tallyClusters, CLASSES, DEFAULT_RULES, trailerOf, neutralize, findMarked } from "./prodwatch-triage.mjs";
 import { parseSources, normalizeMemo } from "./prodwatch-sources.mjs";
 import { loadPolicy } from "./decision-engine.mjs";
 
@@ -132,3 +132,16 @@ describe("trailer-only markers (#4879)", () => {
     assert.equal(cluster(sig, [real])[0].duplicateOf, 10);
   });
 });
+
+describe("neutralize and findMarked", () => {
+  it("neutralize escapes a comment opener so a marker cannot be forged", () => {
+    assert.equal(neutralize("a <!-- FORGE:X -->"), "a &lt;!-- FORGE:X -->");
+    assert.equal(neutralize(null), "");
+  });
+  it("findMarked trusts the trailer only and honours accept", () => {
+    const m = "<!-- FORGE:M -->";
+    const issues = [{ number: 1, body: `quote ${m}\nmore` }, { number: 2, body: `text\n${m}` }];
+    assert.equal(findMarked(issues, m)?.number, 2);
+    assert.equal(findMarked(issues, m, () => false), undefined);
+  });
+});

package/bin/lean/prodwatch.mjs

@@ -15,7 +15,7 @@ import { writeAtomic } from "./batch.mjs";
 import { acquireLock, watchDir, loadWatches } from "./issuewatch.mjs";
 import { loadPolicy } from "./decision-engine.mjs";
 import { parseSources, runAdapter } from "./prodwatch-sources.mjs";
-import { cluster, fingerprint, classifyCluster, validateCluster, tallyClusters, parseAlertRules, signalRef, trailerOf } from "./prodwatch-triage.mjs";
+import { cluster, fingerprint, classifyCluster, validateCluster, tallyClusters, parseAlertRules, signalRef, trailerOf, neutralize } from "./prodwatch-triage.mjs";
 import { admit } from "./prodwatch-governor.mjs";
 import { computeMetrics, sourceMode, emitMetrics } from "./prodwatch-metrics.mjs";
 
@@ -80,8 +80,6 @@ export function targetLabel(cfg, raw = "") {
 // ── pure helpers ───────────────────────────────────────────────────────────────────────────
 const SOURCE_TAG = /<!--\s*FORGE:PRODWATCH\s+source=(\S+?)\s*-->/;
 const FP_TAG = /FORGE:SIGNAL_REF\s+fp=([0-9a-f]+)/;
-/** Signal text is untrusted: a literal comment opener could forge a FORGE marker, so it is escaped before it reaches a body or prompt. */
-const neutralize = (s) => String(s ?? "").replace(/<!--/g, "&lt;!--");
 const oneLine = (s, n) => String(s ?? "").replace(/\s+/g, " ").trim().slice(0, n);
 
 /** Rows for computeMetrics from the prod-watch issues (open and closed) that carry our markers: GitHub is the register. */

package/bin/lean/prompts.mjs

@@ -246,7 +246,7 @@ export function contextPackNote(pack) {
 /** #4857: validate-command rules for the plan prompt (the implement session resumes it), kept out of the shared templates (their token caps are guarded). */
 export function unrunnableValidateNote(testRunners = null) {
   const map = testRunners && typeof testRunners === "object" ? Object.entries(testRunners).map(([k, v]) => `${k}/ -> ${v}`).join("; ") : "";
-  return `\n\nValidate and repro commands must pass bash -n (no prose notes), select tests that exist, and run from the right cwd${map ? ` (forge.yaml verification.test_runners: ${map})` : ""}. An unrunnable validate (missing tool, wrong cwd, no tests selected) is never a block or "human" product-decision: run an equivalent through the repo's toolchain and record {"kind":"substitute-validate","validate":"V#","command":"<runnable equivalent>","why":""} in ${RESULT_FILE} amendments.`;
+  return `\n\nValidate and repro commands must pass bash -n (no prose notes), select tests that exist, and run from the right cwd${map ? ` (forge.yaml verification.test_runners: ${map})` : ""}. An unrunnable validate (missing tool, wrong cwd, no tests selected) is never a block or "human" product-decision: run an equivalent through the repo's toolchain and record {"kind":"substitute-validate","validate":"V#","command":"<runnable equivalent>","why":""} in ${RESULT_FILE} amendments. A post-deploy requirement needs a registered check id; one that is a manual or production-credential action with no runnable check is disposition "defer" with bucket "owner:<action>" (the engine also converts a check-less post-deploy to a defer to the orchestrator). forge.yaml validate.recipes rewrite matching commands at freeze.`;
 }
 
 export function reproPlanNote(on) {

package/bin/lean/recovery.test.mjs

@@ -9,7 +9,7 @@ import { tmpDir } from "../tests/_tmp.mjs";
 import { orchestrate, formatOutcomeLists, OUTCOME_LINE_MAX } from "./orchestrate.mjs";
 import { RECOVER_HOW } from "./lean.mjs";
 import { recoveryEvidence } from "./render.mjs";
-import { RECOVERABLE_CODES } from "../engine/outcome-codes.mjs";
+import { RECOVERABLE_CODES, classifyNeedsHuman } from "../engine/outcome-codes.mjs";
 import { runFromCli, orchestrateFromCli } from "../engine-cli.mjs";
 // orchestrateFromCli opens the host lane registry (#4191): keep it in a temp dir.
 process.env.FORGEDOCK_SCHED_DIR = tmpDir("recovery-sched-");
@@ -96,12 +96,29 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
     assert.ok(formatOutcomeLists(rows).some((l) => /^unrecovered: #1 \(budget\.exceeded: Implement turn budget spent/.test(l)));
   });
 
+  it("conformance gaps are recoverable: one recovery lane each, regression/guard_broken/scope_creep/e2e_unverified stay terminal, notes name the defer route", async () => {
+    const codes = ["conformance.unimplemented", "conformance.omission", "conformance.unjustified_deferral"];
+    for (const c of codes) {
+      assert.equal(classifyNeedsHuman({ outcome: "needs-human", code: c }), "recoverable", c);
+      assert.ok(RECOVERABLE_CODES.includes(c) && RECOVER_HOW[c], c);
+    }
+    for (const c of ["conformance.regression", "conformance.guard_broken", "conformance.scope_creep", "conformance.e2e_unverified"]) {
+      assert.equal(classifyNeedsHuman({ outcome: "needs-human", code: c }), "terminal", c);
+    }
+    for (const c of codes.slice(1)) assert.match(RECOVER_HOW[c], /`defer`[\s\S]*why|why[\s\S]*follow-up/i);
+    assert.match(RECOVER_HOW["conformance.omission"], /add-requirement/);
+    const { rows, calls } = await batch({ 1: [NH("conformance.omission", "R2 not covered", { branch: "feat/issue-1-lean" }), NH("conformance.omission", "again")] });
+    assert.deepEqual(calls.filter((c) => c.n === 1).map((c) => c.recover?.code ?? null), [null, "conformance.omission"]);
+    assert.equal(calls.filter((c) => c.n === 1).length, 2, "no second recovery beyond max_attempts");
+    assert.equal(byIssue(rows)[1].outcome, "needs-human");
+  });
+
   it("human codes (product-decision, spend) are never recovered, still skip dependents and are the only escalations, asked once while other lanes keep running", async () => {
     const { rows, calls, decisions } = await batch({
       1: [NH("builder.blocked", "Builder is blocked (product-decision): pick a pricing model")],
       2: [],
       3: [NH("builder.blocked", "Builder is blocked (spend): needs a paid plan")],
-      4: [NH("conformance.omission", "R2 not covered")],
+      4: [NH("conformance.regression", "R2 broke")],
       5: [{ terminalReason: "merged" }] }, { 2: "**Depends on**: #1" });
     const r = byIssue(rows);
     assert.equal(calls.filter((c) => c.n === 1).length, 1); assert.equal(calls.filter((c) => c.n === 3).length, 1);
@@ -114,7 +131,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
     assert.match(esc, /#1 \(builder\.blocked\): Builder is blocked \(product-decision\)/);
     assert.match(esc, /#3 \(builder\.blocked\): Builder is blocked \(spend\)/);
     assert.doesNotMatch(esc, /#4/, "the needs-human list holds only human codes");
-    assert.ok(lines.some((l) => /^unrecovered: #4 \(conformance\.omission: R2 not covered\)$/.test(l)));
+    assert.ok(lines.some((l) => /^unrecovered: #4 \(conformance\.regression: R2 broke\)$/.test(l)));
   });
 
   it("base.conflict: a recovery base-sync with a clean result re-queues the issue; the report lists recovered issues separately", async () => {
@@ -224,7 +241,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
   it("/orchestrate spec step 3 lists the recoverable codes, asks no confirmation on them, and escalates once at the end", async () => {
     const spec = readFileSync(new URL("../../commands/orchestrate.md", import.meta.url), "utf8");
     const step3 = spec.slice(spec.indexOf("## 3. Launch lanes"), spec.indexOf("## 4. Report"));
-    for (const c of ["`ci.red`", "`base.conflict`", "`ci-pending` on a conflicting", "`contract.invalid` caused by an issue edited mid-run"]) assert.ok(step3.includes(c), c);
+    for (const c of ["`ci.red`", "`base.conflict`", "`ci-pending` on a conflicting", "`contract.invalid` caused by an issue edited mid-run", "`conformance.unimplemented`", "`conformance.omission`", "`conformance.unjustified_deferral`"]) assert.ok(step3.includes(c), c);
     assert.match(step3, /\*\*Do not ask for confirmation on them\.\*\*/);
     assert.match(step3, /FORGE:DECISION/);
     assert.match(step3, /one question list[\s\S]*once, at the end of the batch/);

package/bin/lean/retry.mjs

@@ -32,6 +32,17 @@ export async function retrying(fn, { transient, permanent = null, attempts = 4,
   throw last;
 }
 
+/** Poll `probe` until it returns a truthy value (returned) or `timeoutMs` has passed (null). The probe runs at least once. */
+export async function pollUntil(probe, { timeoutMs, pollMs, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), now = () => Date.now() }) {
+  const t0 = now();
+  for (;;) {
+    const v = await probe();
+    if (v) return v;
+    if (now() - t0 > timeoutMs) return null;
+    await sleep(pollMs);
+  }
+}
+
 /**
  * #4507: run `call(env)` with the lane's gh cache env; on a gh parse error (corrupt cache) reset the cache and run it
  * once more. Without a cache it still retries once. Any other error is rethrown unchanged.

package/bin/lean/retry.test.mjs

@@ -0,0 +1,16 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { pollUntil } from "./retry.mjs";
+
+test("pollUntil returns the first truthy probe value, sleeping between tries", async () => {
+  let n = 0, t = 0; const sleeps = [];
+  const v = await pollUntil(() => (++n >= 3 ? "ok" : null), { timeoutMs: 100, pollMs: 7, sleep: async (ms) => { sleeps.push(ms); t += ms; }, now: () => t });
+  assert.equal(v, "ok"); assert.deepEqual(sleeps, [7, 7]);
+});
+
+test("pollUntil returns null once the timeout passes, probing at least once", async () => {
+  let t = 0, n = 0;
+  const v = await pollUntil(() => { n++; return null; }, { timeoutMs: 20, pollMs: 10, sleep: async (ms) => { t += ms; }, now: () => t });
+  assert.equal(v, null); assert.ok(n >= 3);
+  assert.equal(await pollUntil(() => null, { timeoutMs: -1, pollMs: 1, sleep: async () => {}, now: () => 0 }), null);
+});

package/bin/lean/sandbox-provider.contract.test.mjs

@@ -189,6 +189,16 @@ describe("remote adapter (#4254)", () => {
     await assert.rejects(() => q.status(h), /ECONNRESET/); // #4672: a transport error propagates; providerLanes retries it for the unreachable window
   });
 
+  it("#5053 httpTransport: a 401 on status is transient and names its hop; a launch 401 is not transient", async () => {
+    const reply = (status, body) => ({ status, ok: status < 300, text: async () => JSON.stringify(body), headers: { get: () => null } });
+    const t = httpTransport({ endpoint: "https://pool.example", token: TOKEN, fetchFn: async () => reply(401, { error: "unauthorized", hop: "agent", agent: "a1", reason: "agent auth: router token rejected by agent" }) });
+    await assert.rejects(t.status("x"), (e) => e.transient === true && e.hop === "agent" && e.reason === "router token rejected by agent" && /agent a1 auth: router token rejected/.test(e.message) && !e.message.includes(TOKEN));
+    await assert.rejects(t.result("x"), (e) => e.transient === true && e.status === 401);
+    await assert.rejects(t.launch({ id: "x" }), (e) => e.transient !== true && e.status === 401);
+    const r = httpTransport({ endpoint: "https://pool.example", fetchFn: async () => reply(401, { hop: "router" }) });
+    await assert.rejects(r.status("x"), /router auth/);
+  });
+
   it("httpTransport: bearer token, JSON protocol, 404 = gone/destroyed, 429 retryable; the token never in errors", async () => {
     const calls = [];
     const reply = (status, body, headers = {}) => ({ status, ok: status >= 200 && status < 300, text: async () => (body === undefined ? "" : JSON.stringify(body)), headers: { get: (k) => headers[k] ?? null } });
@@ -320,10 +330,11 @@ describe("pool provider (#4546)", () => {
 
 describe("remote capacity (GET /status, #4540)", () => {
   const mk = (res) => httpTransport({ endpoint: "https://pool.example/v1", fetchFn: async (url) => { if (res instanceof Error) throw res; if (url.endsWith("/agents")) return { status: 404, ok: false, text: async () => "", headers: { get: () => null } }; assert.equal(url, "https://pool.example/v1/status"); return { status: res.status, ok: res.status < 400, text: async () => res.body ?? "", headers: { get: () => null } }; } });
-  it("free integer is reported; 404, malformed, negative and transport errors are unknown (null)", async () => {
+  it("free integer is reported; 404, malformed and negative are unknown (null); an unreachable router throws (#5048)", async () => {
     assert.deepEqual(await mk({ status: 200, body: '{"free":3}' }).capacity(), { free: 3 });
     assert.deepEqual(await mk({ status: 200, body: '{"free":0}' }).capacity(), { free: 0 });
-    for (const r of [{ status: 404 }, { status: 200, body: "nope" }, { status: 200, body: '{"free":"2"}' }, { status: 200, body: '{"free":-1}' }, new Error("down")]) assert.equal(await mk(r).capacity(), null);
+    for (const r of [{ status: 404 }, { status: 200, body: "nope" }, { status: 200, body: '{"free":"2"}' }, { status: 200, body: '{"free":-1}' }]) assert.equal(await mk(r).capacity(), null);
+    await assert.rejects(() => mk(new Error("down")).capacity(), (e) => e.unreachable === true && /pool router unreachable/.test(e.message));
   });
   it("#4788: no online on /status: derived from /agents (ok && !paused); an unreadable /agents stays unknown", async () => {
     const mkA = (status, agents) => httpTransport({ endpoint: "https://pool.example/v1", fetchFn: async (url) => {
@@ -337,12 +348,12 @@ describe("remote capacity (GET /status, #4540)", () => {
     assert.deepEqual(await mkA(200, down).capacity(), { online: 0, total: 1 });
     assert.equal(await mkA(404, "nope").capacity(), null);
   });
-  it("remoteProvider.capacity delegates, and is null without a transport capability or on a throw", async () => {
+  it("remoteProvider.capacity delegates, is null without a transport capability, and passes a transport error on (#5048)", async () => {
     const batch = openBatch(join(tmpDir("cap-"), "batches"), "bt");
     const p = (transport) => remoteProvider({ batch, repo: "o/r", transport, env: {}, credentials: () => null });
     assert.deepEqual(await p({ capacity: async () => ({ free: 1 }) }).capacity(), { free: 1 });
     assert.equal(await p({}).capacity(), null);
-    assert.equal(await p({ capacity: async () => { throw new Error("x"); } }).capacity(), null);
+    await assert.rejects(() => p({ capacity: async () => { throw new Error("x"); } }).capacity(), /x/);
   });
 });