forgedock 1.29.0 → 1.30.0

package/bin/lean/orchestrate.mjs

@@ -51,17 +51,24 @@
  * and instead of stopping when the budget refuses. The final `{where, reason}` is kept on the run record and the row as
  * `placement` and passed to `runOne` as `ctx.placement` (the lane records it in its FORGE:STATE).
  *
- * In-batch recovery (#4267), off unless `recovery: {maxAttempts}` is passed: a row `classifyNeedsHuman` calls
+ * In-batch recovery (#4267), on by default: this module recovers only when `recovery: {maxAttempts}` is passed, and the
+ * CLI always passes it (forge.yaml `orchestration.recovery.max_attempts`, default 1; 0 disables): a row `classifyNeedsHuman` calls
  * recoverable (needs-human `ci.red` / `base.conflict` / `builder.block_untyped`, `contract.invalid` after a mid-run issue
  * edit, `ci-pending` whose PR `conflicting(issue, pr)` says conflicts) is not final while its persisted `recoveries` stay
- * under `maxAttempts`: it is requeued as a recovery lane (`ctx.recover = {code, attempt}`) under the same admission as any
- * lane, so its dependents stay queued. A merged recovery row carries `recovered: <code>`; any other end is final with
- * `recovery` evidence. Human codes (`builder.blocked`) are never recovered. Every operational decision the runner takes
+ * under `maxAttempts`: it is requeued as a recovery lane (`ctx.recover = {code, attempt, final}`, `final` when attempt ==
+ * `maxAttempts`) under the same admission as any lane, so its dependents stay queued. Only a final lane relabels needs-human
+ * itself; a non-final attempt that ends needs-human/ci-pending without a successor is relabelled here, once (#4302). A recovery
+ * still queued when the batch exits (breaker, budget, hold, or a resume that does not relaunch it) is relabelled once with the
+ * full persisted lane reason (#4315); also when the loop throws, and through `onExit(fn)` (the caller's signal handler
+ * awaits `fn(cause)`), each once (#4318). `relabelNeedsHuman` may return `false`/"failed" (not recorded, a resume retries)
+ * or "skipped" (closed or leased issue: recorded); any other result counts as relabelled.
+ * A merged recovery row carries `recovered: <code>`; any other end is final with `recovery` evidence. Human codes (`builder.blocked`) are never recovered. Every operational decision the runner takes
  * (recovery, engine-error relaunch, dependency requeue, sub-issues added) goes to `decide({issue, what, why, alternative})`,
  * fire-and-forget: the runner never waits on it.
  */
 import { parseDependsOn } from "../engine/phases.mjs";
 import { classifyNeedsHuman, recoveryCodeOf } from "../engine/outcome-codes.mjs";
+import { recoveryEvidence } from "./render.mjs";
 
 // A dependency that finished like this no longer blocks its dependents.
 const SATISFIED = new Set(["merged", "invalid"]);
@@ -155,7 +162,7 @@ const toMap = (o) => new Map(Object.entries(o ?? {}).map(([k, v]) => [Number(k),
 export async function orchestrate({ issues, parallel = 3, runOne, readBody, prefilter = null, resolveLane = null, now = () => Date.now(), onUpdate = () => {}, onCrossLane = () => {},
   store = null, adopt = null, getParallel = null, breaker = null, maxDeferrals = DEFAULT_MAX_DEFERRALS, sleep = cancellableSleep, pollMs = 5000,
   classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null,
-  recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null }) {
+  recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null }) {
   const saved = store?.load?.() ?? null;
   const order = saved ? [...saved.order] : [];
   const set = new Set(order);
@@ -206,7 +213,8 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   let budgetStop = null, budgetHit = null; // this pass's refusal under on_budget stop; the last refusal reason
   const recoveryMax = Number.isInteger(recovery?.maxAttempts) && recovery.maxAttempts > 0 ? recovery.maxAttempts : 0;
   const recoveries = toMap(saved?.recoveries); // issue -> recovery lanes launched (#4267)
-  const recovering = toMap(saved?.recovering); // issue -> {code, attempt} of the recovery lane queued or running
+  const recovering = toMap(saved?.recovering); // issue -> {code, attempt, detail, outcome, ...} of the recovery lane queued or running
+  const relabelled = new Set(saved?.relabelled ?? []); // #4315: queued recoveries relabelled needs-human when a batch stopped
   // #4267: decisions are recorded, never awaited: a failing or slow recorder cannot block the batch.
   const decided = (d) => { try { Promise.resolve(decide?.(d)).catch(() => {}); } catch { /* best effort */ } };
   const firstLine = (x) => String(x ?? "").split("\n")[0].slice(0, 300);
@@ -217,7 +225,8 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       pending, notBefore: toObj(notBefore), requeues: toObj(requeues), deferrals: toObj(deferrals), engineErrors: toObj(engineErrors),
       prefilter: toObj(checked), errorTimes, breaker: tripped, ...(budget ? { classes: toObj(classes), dispatch: { ...dispatch } } : {}),
       ...(reg ? { priorities: toObj(priorities) } : {}), ...(sandboxUse.size ? { sandbox: toObj(sandboxUse) } : {}),
-      ...(recoveries.size || recovering.size ? { recoveries: toObj(recoveries), recovering: toObj(recovering) } : {}) });
+      ...(recoveries.size || recovering.size ? { recoveries: toObj(recoveries), recovering: toObj(recovering) } : {}),
+      ...(relabelled.size ? { relabelled: [...relabelled] } : {}) });
   };
   const update = (row) => { store?.log?.(row); onUpdate(row); };
   const classifyOne = async (n) => {
@@ -338,8 +347,11 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
 
   // Apply one lane's result: requeue it (deferred, dependency-open, engine-error) or record its final row.
   const settle = async (n, r, rec, pf) => {
-    const row = { issue: n, outcome: r?.terminalReason ?? "engine-error", pr: r?.pr ?? null, cost_usd: (r?.cost_usd ?? 0) + (pf?.cost_usd ?? 0), minutes: Math.round((now() - rec.startedAt) / 6000) / 10, detail: r?.detail, ...(r?.code ? { code: r.code } : {}), ...(pf ? { prefilter: pf } : {}),
+    // #4304: the pre-filter runs once per issue; a recovery pass's row adds the earlier attempt's cost, which already holds it.
+    const pfCost = recovering.has(n) ? 0 : (pf?.cost_usd ?? 0);
+    const row = { issue: n, outcome: r?.terminalReason ?? "engine-error", pr: r?.pr ?? null, cost_usd: (r?.cost_usd ?? 0) + pfCost, minutes: Math.round((now() - rec.startedAt) / 6000) / 10, detail: r?.detail, ...(r?.code ? { code: r.code } : {}), ...(pf ? { prefilter: pf } : {}),
       ...(r?.waits && typeof r.waits === "object" ? { waits: r.waits } : {}), ...(Number.isInteger(r?.heavyQueueMax) ? { heavyQueueMax: r.heavyQueueMax } : {}),
+      ...(Number.isFinite(r?.precheckMs) ? { precheck_ms: r.precheckMs } : {}), ...(Number.isInteger(r?.precheckDeferred) && r.precheckDeferred > 0 ? { precheck_deferred: r.precheckDeferred } : {}), // #4331
       ...(rec?.placement ? { placement: rec.placement } : {}) };
     const depList = (Array.isArray(r?.dependencies) ? r.dependencies : []).filter((d) => Number.isSafeInteger(d) && d > 0 && d !== n);
     const subs = (Array.isArray(r?.subissues) ? r.subissues : []).filter((d) => Number.isSafeInteger(d) && d > 0 && d !== n);
@@ -399,6 +411,8 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     }
     // #4267: a recovery pass ends here; a recoverable row under the attempt cap is requeued as a recovery lane.
     const prior = recovering.get(n) ?? null;
+    // #4318: the full persisted lane reason (capped by the comment writer), not its first 300 chars
+    const evidence = () => recoveryEvidence({ code: prior.code, attempt: prior.attempt, outcome: row.outcome, resultCode: row.code, detail: row.detail, full: true });
     if (prior) {
       recovering.delete(n);
       // #4298: the row reports the whole issue: earlier attempts' cost and minutes are summed into the recovery's.
@@ -409,7 +423,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       // A recovery that failed for good keeps needs-human. The lane relabels needs-human/ci-pending itself; engine-error (after its
       // relaunch) and deferred (no longer requeued) end here, so the runner does it, once. invalid/decomposed/dependency-*/merged never.
       if (["engine-error", "deferred"].includes(row.outcome) && typeof relabelNeedsHuman === "function") {
-        try { await relabelNeedsHuman({ issue: n, evidence: `Recovery pass for \`${prior.code}\` (attempt ${prior.attempt}) did not merge: it ended \`${row.outcome}\`${row.code ? ` (\`${row.code}\`)` : ""}${firstLine(row.detail) ? `: ${firstLine(row.detail)}` : ""}. The issue stays \`needs-human\`.` }); } catch { /* best effort */ }
+        try { await relabelNeedsHuman({ issue: n, evidence: evidence() }); } catch { /* best effort */ }
       }
     }
     if (recoveryMax && (row.outcome === "needs-human" || row.outcome === "ci-pending")) {
@@ -421,7 +435,9 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       const used = recoveries.get(n) ?? 0;
       if (classifyNeedsHuman(facts) === "recoverable" && used < recoveryMax) {
         const code = recoveryCodeOf(facts), attempt = used + 1;
-        recoveries.set(n, attempt); recovering.set(n, { code, attempt, cost_usd: Number(row.cost_usd) || 0, minutes: Number(row.minutes) || 0 });
+        // #4315: the full lane detail is kept (never firstLine) for the relabel when the batch stops before this attempt
+        recoveries.set(n, attempt); recovering.set(n, { code, attempt, cost_usd: Number(row.cost_usd) || 0, minutes: Number(row.minutes) || 0,
+          detail: String(row.detail ?? ""), outcome: row.outcome, ...(row.code ? { result_code: row.code } : {}), prev_attempt: prior?.attempt ?? 0 });
         requeue(n, 0); done(); save();
         decided({ issue: n, what: `recovery lane for ${code} (attempt ${attempt}/${recoveryMax}); dependents stay queued`, why: firstLine(row.detail) || code,
           alternative: "leave needs-human for the operator and skip the dependents" });
@@ -429,6 +445,10 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
         return;
       }
     }
+    // #4302: a non-final recovery attempt left needs-human/ci-pending unlabelled for its successor; none is launched, so relabel once.
+    if (prior && prior.attempt < recoveryMax && ["needs-human", "ci-pending"].includes(row.outcome) && typeof relabelNeedsHuman === "function") {
+      try { await relabelNeedsHuman({ issue: n, evidence: evidence() }); } catch { /* best effort */ }
+    }
     results.set(n, row);
     if (row.outcome === "merged" && drift?.onMerged && r?.structure && String(rec?.lane ?? r?.lane ?? "").startsWith("milestone/")) {
       save(); update(row);
@@ -449,6 +469,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       return;
     }
     const lane = ln?.lane;
+    if (relabelled.delete(n)) save(); // #4315: the recovery lane launches after all; it removes needs-human itself
     const pl = placed.get(n);
     placed.delete(n);
     const rec = { startedAt: now(), ...(lane ? { lane } : {}), ...(remoteRun.has(n) ? { remote: true } : {}), ...(pl ? { placement: pl } : {}) };
@@ -461,7 +482,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       if (pf?.closed) r = { terminalReason: "invalid", cost_usd: 0, detail: `pre-filter: ${pf.detail}` };
       else {
         try {
-          r = await runOne(n, lane, { remote: !!rec.remote, ...(rec.placement ? { placement: rec.placement } : {}), ...(recovering.has(n) ? { recover: { code: recovering.get(n).code, attempt: recovering.get(n).attempt } } : {}), onStart: (info) => {
+          r = await runOne(n, lane, { remote: !!rec.remote, ...(rec.placement ? { placement: rec.placement } : {}), ...(recovering.has(n) ? { recover: { code: recovering.get(n).code, attempt: recovering.get(n).attempt, final: recovering.get(n).attempt >= recoveryMax } } : {}), onStart: (info) => {
             Object.assign(rec, info ?? {}); save();
             if (reg && slotted.has(n) && info?.pid) reg.setLanePid(keyOf(n), info.pid);
           } });
@@ -494,86 +515,114 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   }
   save();
 
-  const K = () => { let k = null; try { k = getParallel?.(); } catch { k = null; } return Number.isInteger(k) && k >= 1 ? k : parallel; };
-  while (pending.length || running.size) {
-    // Resolve skips first: a dependency that finished unsatisfied (or never can) skips its dependents.
-    for (let i = pending.length - 1; i >= 0; i--) {
-      const n = pending[i];
-      if (results.has(n)) { pending.splice(i, 1); continue; }
-      const bad = edges(n).find((d) => results.has(d) && !SATISFIED.has(results.get(d).outcome));
-      if (bad !== undefined) { pending.splice(i, 1); skip(n, `dependency #${bad} ended ${results.get(bad).outcome}`); }
+  const heldDetail = (n) => tripped?.detail ?? (driftHeld.has(n) ? `drift: ${driftHeld.get(n)} flagged` : budgetStop ? `execution budget: ${budgetStop}` : budgetHeld ? "maintenance share" : "not run");
+  // #4315: a recovery queued but never launched (the batch stopped: breaker, budget, hold; or a resume that did not relaunch a
+  // lost run) left the issue without needs-human, since a non-final attempt does not relabel. Relabel it once, with the full reason.
+  // #4318: also on a throw out of the loop and on a signal (`onExit`); `cause` overrides the held reason. Runs once per process.
+  let exitRelabel = null;
+  const relabelQueued = (cause = null) => (exitRelabel ??= (async () => {
+    for (const [n, q] of recovering) {
+      if (running.has(n) || results.has(n) || relabelled.has(n) || typeof relabelNeedsHuman !== "function") continue;
+      const why = cause ?? (heldDetail(n) === "not run" ? "batch stopped" : heldDetail(n));
+      const evidence = recoveryEvidence({ code: q.code, attempt: q.prev_attempt, outcome: q.outcome, resultCode: q.result_code, detail: q.detail, full: true,
+        queued: q.attempt, cause: why, recoveryMax });
+      let res;
+      try { res = await relabelNeedsHuman({ issue: n, evidence }); } catch { res = "failed"; }
+      if (res !== false && res !== "failed") relabelled.add(n); // a failed label add is retried by a resume
     }
-    const t = now();
-    const holdUntil = heldUntil();
-    let started = 0;
-    driftHeld.clear();
-    const isReady = (n) => {
-      if (!((notBefore.get(n) ?? 0) <= t && edges(n).every((d) => results.has(d) && SATISFIED.has(results.get(d).outcome)))) return false;
-      if (!drift?.holds) return true;
-      let pkg = null;
-      try { pkg = drift.holds(n, { body: bodies.get(n) ?? "", lane: lanes.get(n)?.lane ?? null }); } catch { pkg = null; }
-      if (pkg) driftHeld.set(n, pkg);
-      return !pkg;
-    };
-    budgetHeld = false;
-    // #4183: a hook (owned-resource GC near the host ceiling) before lanes are admitted; its errors never stop the batch.
-    if (beforeAdmit && !tripped && !holdUntil && running.size < K() && pending.some(isReady)) { try { await beforeAdmit(); } catch { /* ignored */ } }
-    budgetStop = null;
-    const wanted = new Set(); // issues that stay offered in the host registry after this pass
-    if (!tripped && !holdUntil && budget) {
-      while (running.size < K()) {
-        const ready = pending.filter(isReady);
-        const n = pickNext(ready, { classes, share: budget.share, whenIdle: budget.whenIdle, started: dispatch.started, maint: dispatch.maint,
-          featuresPending: pending.some((p) => !clsOf(classes, p).maintenance) });
-        if (n === null) { budgetHeld = ready.length > 0; break; }
-        if (!grant(n, wanted)) break;
-        pending.splice(pending.indexOf(n), 1);
-        if (lanes.get(n)?.ok !== false) { dispatch.started++; if (clsOf(classes, n).maintenance) dispatch.maint++; }
-        start(n); started++;
+    save();
+  })());
+  if (typeof onExit === "function") { try { onExit(relabelQueued); } catch { /* best effort */ } }
+
+  const K = () => { let k = null; try { k = getParallel?.(); } catch { k = null; } return Number.isInteger(k) && k >= 1 ? k : parallel; };
+  try {
+    while (pending.length || running.size) {
+      // Resolve skips first: a dependency that finished unsatisfied (or never can) skips its dependents.
+      for (let i = pending.length - 1; i >= 0; i--) {
+        const n = pending[i];
+        if (results.has(n)) { pending.splice(i, 1); continue; }
+        const bad = edges(n).find((d) => results.has(d) && !SATISFIED.has(results.get(d).outcome));
+        if (bad !== undefined) { pending.splice(i, 1); skip(n, `dependency #${bad} ended ${results.get(bad).outcome}`); }
       }
-    } else if (!tripped && !holdUntil && reg) {
-      // Priority first, pending order within a priority; the free per-batch K is offered so the host queue keeps FIFO.
-      const ready = pending.filter(isReady).map((n, i) => [n, i]).sort((a, b) => prioOf(a[0]) - prioOf(b[0]) || a[1] - b[1]).map(([n]) => n);
-      const want = ready.filter((n) => lanes.get(n)?.ok !== false && !isRemote(n)).slice(0, Math.max(0, K() - running.size));
-      offerFresh(want);
-      for (const n of want) wanted.add(n);
-      for (const n of ready) {
-        if (running.size >= K() || !grant(n, wanted)) break;
-        pending.splice(pending.indexOf(n), 1); start(n); started++;
+      const t = now();
+      const holdUntil = heldUntil();
+      let started = 0;
+      driftHeld.clear();
+      const isReady = (n) => {
+        if (!((notBefore.get(n) ?? 0) <= t && edges(n).every((d) => results.has(d) && SATISFIED.has(results.get(d).outcome)))) return false;
+        if (!drift?.holds) return true;
+        let pkg = null;
+        try { pkg = drift.holds(n, { body: bodies.get(n) ?? "", lane: lanes.get(n)?.lane ?? null }); } catch { pkg = null; }
+        if (pkg) driftHeld.set(n, pkg);
+        return !pkg;
+      };
+      budgetHeld = false;
+      // #4183: a hook (owned-resource GC near the host ceiling) before lanes are admitted; its errors never stop the batch.
+      if (beforeAdmit && !tripped && !holdUntil && running.size < K() && pending.some(isReady)) { try { await beforeAdmit(); } catch { /* ignored */ } }
+      budgetStop = null;
+      const wanted = new Set(); // issues that stay offered in the host registry after this pass
+      if (!tripped && !holdUntil && budget) {
+        while (running.size < K()) {
+          const ready = pending.filter(isReady);
+          const n = pickNext(ready, { classes, share: budget.share, whenIdle: budget.whenIdle, started: dispatch.started, maint: dispatch.maint,
+            featuresPending: pending.some((p) => !clsOf(classes, p).maintenance) });
+          if (n === null) { budgetHeld = ready.length > 0; break; }
+          if (!grant(n, wanted)) break;
+          pending.splice(pending.indexOf(n), 1);
+          if (lanes.get(n)?.ok !== false) { dispatch.started++; if (clsOf(classes, n).maintenance) dispatch.maint++; }
+          start(n); started++;
+        }
+      } else if (!tripped && !holdUntil && reg) {
+        // Priority first, pending order within a priority; the free per-batch K is offered so the host queue keeps FIFO.
+        const ready = pending.filter(isReady).map((n, i) => [n, i]).sort((a, b) => prioOf(a[0]) - prioOf(b[0]) || a[1] - b[1]).map(([n]) => n);
+        const want = ready.filter((n) => lanes.get(n)?.ok !== false && !isRemote(n)).slice(0, Math.max(0, K() - running.size));
+        offerFresh(want);
+        for (const n of want) wanted.add(n);
+        for (const n of ready) {
+          if (running.size >= K() || !grant(n, wanted)) break;
+          pending.splice(pending.indexOf(n), 1); start(n); started++;
+        }
+      } else if (!tripped && !holdUntil) {
+        for (let i = 0; i < pending.length && running.size < K();) {
+          const n = pending[i];
+          if (isReady(n) && grant(n, new Set())) { pending.splice(i, 1); start(n); started++; }
+          else i++;
+        }
       }
-    } else if (!tripped && !holdUntil) {
-      for (let i = 0; i < pending.length && running.size < K();) {
-        const n = pending[i];
-        if (isReady(n) && grant(n, new Set())) { pending.splice(i, 1); start(n); started++; }
-        else i++;
+      if (reg) withdrawExcept(wanted);
+      if (tripped && !running.size) break; // breaker: let running lanes finish, keep the rest pending for a resume
+      if (budgetStop && !running.size) break; // execution budget (stop): the same, the rest end held
+      // Wake for a delayed issue, the end of a usage-limit hold, or (with a live K) a poll.
+      const waits = pending.map((n) => notBefore.get(n) ?? 0).filter((x) => x > t);
+      if (holdUntil) waits.push(holdUntil);
+      let wake = waits.length ? Math.max(0, Math.min(...waits) - t) : null;
+      if (getParallel && pending.length) wake = Math.min(wake ?? pollMs, pollMs);
+      if (offered.size && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // a slot another batch frees
+      if (tripped) wake = null;
+      if (running.size) {
+        if (wake === null) await Promise.race(running.values());
+        else { const ac = new AbortController(); await Promise.race([...running.values(), sleep(wake, ac.signal)]); ac.abort(); }
+      } else if (started) continue; // only lane-refused rows settled (synchronously): re-resolve skips first
+      else if (pending.length && offered.size) await sleep(wake); // waiting for a host slot
+      else if (pending.length && waits.length) await sleep(Math.max(0, Math.min(...waits) - t));
+      else if (pending.length && budgetHeld) break; // maintenance the budget does not admit now: held for a later batch
+      else if (pending.length && driftHeld.size) break; // held by a drift flag (block_dispatch): pending for a later batch
+      else if (pending.length) { // nothing runnable, nothing running, nothing waiting on time: a dependency cycle
+        for (const n of pending.splice(0)) skip(n, "dependency cycle");
+        save();
       }
     }
-    if (reg) withdrawExcept(wanted);
-    if (tripped && !running.size) break; // breaker: let running lanes finish, keep the rest pending for a resume
-    if (budgetStop && !running.size) break; // execution budget (stop): the same, the rest end held
-    // Wake for a delayed issue, the end of a usage-limit hold, or (with a live K) a poll.
-    const waits = pending.map((n) => notBefore.get(n) ?? 0).filter((x) => x > t);
-    if (holdUntil) waits.push(holdUntil);
-    let wake = waits.length ? Math.max(0, Math.min(...waits) - t) : null;
-    if (getParallel && pending.length) wake = Math.min(wake ?? pollMs, pollMs);
-    if (offered.size && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // a slot another batch frees
-    if (tripped) wake = null;
-    if (running.size) {
-      if (wake === null) await Promise.race(running.values());
-      else { const ac = new AbortController(); await Promise.race([...running.values(), sleep(wake, ac.signal)]); ac.abort(); }
-    } else if (started) continue; // only lane-refused rows settled (synchronously): re-resolve skips first
-    else if (pending.length && offered.size) await sleep(wake); // waiting for a host slot
-    else if (pending.length && waits.length) await sleep(Math.max(0, Math.min(...waits) - t));
-    else if (pending.length && budgetHeld) break; // maintenance the budget does not admit now: held for a later batch
-    else if (pending.length && driftHeld.size) break; // held by a drift flag (block_dispatch): pending for a later batch
-    else if (pending.length) { // nothing runnable, nothing running, nothing waiting on time: a dependency cycle
-      for (const n of pending.splice(0)) skip(n, "dependency cycle");
-      save();
-    }
+  } catch (e) {
+    // #4318: an exception out of the loop still relabels the queued recoveries before it propagates
+    try { await relabelQueued(`batch failed: ${firstLine(e?.message ?? e)}`); } catch { /* best effort */ }
+    throw e;
   }
   if (reg) withdrawExcept(new Set());
+  await relabelQueued();
   save();
-  return order.map((n) => results.get(n) ?? { issue: n, outcome: "held", pr: null, cost_usd: 0, minutes: 0, detail: tripped?.detail ?? (driftHeld.has(n) ? `drift: ${driftHeld.get(n)} flagged` : budgetStop ? `execution budget: ${budgetStop}` : budgetHeld ? "maintenance share" : "not run") });
+  return order.map((n) => results.get(n) ?? { issue: n, outcome: "held", pr: null, cost_usd: 0, minutes: 0, detail: heldDetail(n),
+    ...(recovering.has(n) ? { recovery: { code: recovering.get(n).code, attempts: recoveries.get(n) ?? recovering.get(n).attempt, outcome: recovering.get(n).outcome ?? "needs-human",
+      ...(recovering.get(n).result_code ? { result_code: recovering.get(n).result_code } : {}), detail: firstLine(recovering.get(n).detail ?? recovering.get(n).code) } } : {}) });
 }
 
 /** Max chars of one row's detail in the batch-report outcome lists (#4298). */

package/bin/lean/precheck.mjs

@@ -0,0 +1,55 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Pre-PR declared check commands (#4240), one round (#4331): the commands run concurrently up to the host worker cap
+ * (the caller holds one heavy-step slot for the whole round), each timed, inside one total budget. A command that never
+ * started before the budget ran out, or that the budget cut short, is left to CI (`deferred-to-ci`), never a failure.
+ * Repo-agnostic: commands are opaque `bash -c` strings run through the injected `run`.
+ */
+
+/** rc of a command killed by its timeout (effects.mjs runCheckCommand). */
+const TIMEOUT_RC = 124;
+
+/**
+ * @param {{commands:string[], parallelism?:number, budgetMs:number, perCommandMs:number, now?:() => number,
+ *   run:(a:{command:string, timeoutMs:number}) => Promise<{rc:number, output?:string}>}} p
+ * @returns {Promise<{results:{command:string, rc:number, output:string, ms:number}[], deferred:string[], ms:number}>}
+ *   results in input order (commands that ran and were not cut by the budget); deferred in input order
+ */
+export async function runChecks({ commands, parallelism = 1, budgetMs, perCommandMs, now = () => Date.now(), run }) {
+  const list = Array.isArray(commands) ? commands : [];
+  const workers = Math.max(1, Math.floor(Number(parallelism)) || 1);
+  const start = now(), deadline = start + (Number(budgetMs) > 0 ? Number(budgetMs) : 0);
+  const per = Number(perCommandMs) > 0 ? Number(perCommandMs) : Infinity;
+  const slots = new Array(list.length).fill(null);
+  let next = 0;
+  async function worker() {
+    while (next < list.length) {
+      const i = next++, command = list[i], left = deadline - now();
+      if (left <= 0) { slots[i] = { deferred: true }; continue; }
+      const timeoutMs = Math.min(per, left), t = now();
+      let r;
+      try { r = await run({ command, timeoutMs }); } catch (e) { r = { rc: 127, output: String(e?.message ?? e) }; }
+      const rc = Number.isInteger(r?.rc) ? r.rc : 1;
+      // the budget, not the command's own timeout, cut it short: CI runs it
+      if (rc === TIMEOUT_RC && timeoutMs < per) { slots[i] = { deferred: true }; continue; }
+      slots[i] = { command, rc, output: String(r?.output ?? ""), ms: Math.max(0, now() - t) };
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(workers, list.length) }, worker));
+  const results = [], deferred = [];
+  slots.forEach((s, i) => { if (s?.deferred) deferred.push(list[i]); else if (s) results.push(s); });
+  return { results, deferred, ms: Math.max(0, now() - start) };
+}
+
+/**
+ * #4331 batch-report line: pre-PR check wall time summed over the rows' `precheck_ms`, and the commands deferred to CI
+ * (`precheck_deferred`). Null when no lane ran pre-PR checks.
+ */
+export function formatPrecheckSummary(rows) {
+  const ran = (rows ?? []).filter((r) => Number.isFinite(r?.precheck_ms) && r.precheck_ms >= 0);
+  if (!ran.length) return null;
+  const ms = ran.reduce((a, r) => a + r.precheck_ms, 0);
+  const deferred = ran.reduce((a, r) => a + (Number.isInteger(r.precheck_deferred) ? r.precheck_deferred : 0), 0);
+  return `precheck: ${(ms / 60000).toFixed(1)}m total over ${ran.length} lane${ran.length === 1 ? "" : "s"}${deferred ? `, ${deferred} deferred to CI` : ""}`;
+}

package/bin/lean/precheck.test.mjs

@@ -0,0 +1,93 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpDir } from "../tests/_tmp.mjs";
+import { runChecks, formatPrecheckSummary } from "./precheck.mjs";
+import { readPrecheckBudget } from "../engine-cli.mjs";
+import { orchestrate } from "./orchestrate.mjs";
+
+/** A fake run: each call takes `cost[command]` ms of the shared clock (default 1000) and returns rc[command] (default 0). */
+function harness({ cost = {}, rc = {} } = {}) {
+  const h = { t: 0, live: 0, peak: 0, calls: [] };
+  h.now = () => h.t;
+  h.run = async ({ command, timeoutMs }) => {
+    h.calls.push([command, timeoutMs]); h.peak = Math.max(h.peak, ++h.live);
+    await new Promise((r) => setImmediate(r));
+    const c = cost[command] ?? 1000;
+    h.t += Math.min(c, timeoutMs); h.live--;
+    return c > timeoutMs ? { rc: 124, output: "timed out" } : { rc: rc[command] ?? 0, output: `${command} out` };
+  };
+  return h;
+}
+
+describe("runChecks (#4331)", () => {
+  it("runs up to `parallelism` commands at once and returns results in input order with ms", async () => {
+    const h = harness();
+    const r = await runChecks({ commands: ["a", "b", "c", "d", "e"], parallelism: 3, budgetMs: 600000, perCommandMs: 300000, now: h.now, run: h.run });
+    assert.equal(h.peak, 3);
+    assert.deepEqual(r.results.map((x) => x.command), ["a", "b", "c", "d", "e"]);
+    assert.ok(r.results.every((x) => x.rc === 0 && Number.isFinite(x.ms) && x.ms >= 0));
+    assert.deepEqual(r.deferred, []);
+    assert.ok(r.ms > 0);
+  });
+
+  it("parallelism 1 is serial; 0, NaN or missing fall back to 1", async () => {
+    for (const p of [1, 0, NaN, undefined, "x"]) {
+      const h = harness();
+      await runChecks({ commands: ["a", "b", "c"], parallelism: p, budgetMs: 600000, perCommandMs: 300000, now: h.now, run: h.run });
+      assert.equal(h.peak, 1, String(p));
+    }
+  });
+
+  it("past the budget the remaining commands are deferred to CI, not failed; a running one gets what is left", async () => {
+    const h = harness();
+    const r = await runChecks({ commands: ["a", "b", "c"], parallelism: 1, budgetMs: 1500, perCommandMs: 300000, now: h.now, run: h.run });
+    assert.deepEqual(h.calls, [["a", 1500], ["b", 500]]);
+    assert.deepEqual(r.results.map((x) => [x.command, x.rc]), [["a", 0]], "b was cut by the budget (rc 124): deferred, not a failure");
+    assert.deepEqual(r.deferred, ["b", "c"]);
+  });
+
+  it("a command's own timeout (not the budget) stays a failure; a throw is rc 127", async () => {
+    const h = harness({ cost: { slow: 5000 } });
+    const r = await runChecks({ commands: ["slow", "boom", "bad"], parallelism: 2, budgetMs: 600000, perCommandMs: 2000, now: h.now,
+      run: (a) => (a.command === "boom" ? Promise.reject(new Error("spawn failed")) : a.command === "bad" ? Promise.resolve({ rc: 2, output: "lint" }) : h.run(a)) });
+    assert.deepEqual(r.results.map((x) => [x.command, x.rc]), [["slow", 124], ["boom", 127], ["bad", 2]]);
+    assert.match(r.results[1].output, /spawn failed/);
+    assert.deepEqual(r.deferred, []);
+  });
+
+  it("no commands: nothing runs", async () => {
+    let n = 0;
+    const r = await runChecks({ commands: [], parallelism: 4, budgetMs: 1000, perCommandMs: 1000, now: () => 0, run: async () => { n++; return { rc: 0 }; } });
+    assert.deepEqual([n, r.results, r.deferred, r.ms], [0, [], [], 0]);
+  });
+});
+
+describe("formatPrecheckSummary (#4331)", () => {
+  it("sums precheck_ms over the lanes that ran checks and counts deferrals; null when none did", () => {
+    assert.equal(formatPrecheckSummary([{ precheck_ms: 90000 }, { precheck_ms: 30000, precheck_deferred: 2 }, { issue: 3 }]), "precheck: 2.0m total over 2 lanes, 2 deferred to CI");
+    assert.equal(formatPrecheckSummary([{ precheck_ms: 6000 }]), "precheck: 0.1m total over 1 lane");
+    assert.equal(formatPrecheckSummary([{ issue: 1 }]), null);
+    assert.equal(formatPrecheckSummary(undefined), null);
+  });
+  it("orchestrate rows carry the lane's precheckMs/precheckDeferred as precheck_ms/precheck_deferred for the report line", async () => {
+    const rows = await orchestrate({ issues: [1, 2, 3], parallel: 1, readBody: async () => "",
+      runOne: async (n) => ({ terminalReason: "merged", ...({ 1: { precheckMs: 90000, precheckDeferred: 1 }, 2: { precheckMs: 30000, precheckDeferred: 0 } })[n] }) });
+    assert.deepEqual(rows.map((r) => [r.issue, r.precheck_ms, r.precheck_deferred]), [[1, 90000, 1], [2, 30000, undefined], [3, undefined, undefined]]);
+    assert.equal(formatPrecheckSummary(rows), "precheck: 2.0m total over 2 lanes, 1 deferred to CI");
+  });
+});
+
+describe("readPrecheckBudget: forge.yaml lean.precheck.max_minutes (#4331)", () => {
+  const withYaml = (text, fn) => { const d = tmpDir("precheck-"); writeFileSync(join(d, "forge.yaml"), text); return fn(d); };
+  it("reads the nested block and the flat key; invalid or absent is undefined", () => {
+    assert.equal(withYaml("lean:\n  precheck:\n    max_minutes: 7\n", readPrecheckBudget), 7);
+    assert.equal(withYaml("lean:\n  precheck_max_minutes: 15\n  repro_test: off\n", readPrecheckBudget), 15);
+    for (const bad of ["0", "-1", "abc", "121"]) assert.equal(withYaml(`lean:\n  precheck_max_minutes: ${bad}\n`, readPrecheckBudget), undefined, bad);
+    assert.equal(withYaml("verify:\n  mode: local\n", readPrecheckBudget), undefined);
+    assert.equal(readPrecheckBudget(tmpDir("precheck-")), undefined);
+  });
+});