forgedock 1.40.0 → 1.42.0

package/bin/lean/lean.test.mjs

@@ -13,9 +13,9 @@ import { makeLedger } from "./ledger.mjs";
 import { writeIndex } from "./guards.mjs";
 import { parseVerify } from "./verify.mjs";
 import { worktreeFingerprint } from "./precheck.mjs";
-import { renderInvestigation, renderPlan, renderImplementation, renderReview, renderPrBody, renderTrajectory, renderStructure, renderContract, contractJson, parseContractComment, sanitizeMd, capComment, section, cell, NR, MAX_COMMENT_CHARS } from "./render.mjs";
+import { renderInvestigation, renderPlan, renderImplementation, renderReview, renderPrBody, renderTrajectory, renderStructure, renderContract, renderAmendments, parseAmendmentsComment, contractJson, parseContractComment, sanitizeMd, capComment, section, cell, NR, MAX_COMMENT_CHARS } from "./render.mjs";
 import { orchestrate, parseDeps, formatTable, pickNext } from "./orchestrate.mjs";
-import { REVIEWER_SYSTEM, REVIEW_RUBRIC, approxTokens, builderPlanPrompt, builderImplementPrompt, reviewerPrompt, hotspotPlanNote, hotspotBuildRule } from "./prompts.mjs";
+import { REVIEWER_SYSTEM, REVIEW_RUBRIC, approxTokens, builderPlanPrompt, builderImplementPrompt, reviewerPrompt, decisionNote, hotspotPlanNote, hotspotBuildRule } from "./prompts.mjs";
 import { parseReview, unnamedEdges, structureNote, applyAmendments, validateContract, scopeText, DECISION_SOURCES } from "./conformance.mjs";
 import { mapRecord } from "../engine/telemetry-seam.mjs";
 import { readLog as readRepoLog } from "../engine/runlog.mjs";
@@ -24,7 +24,7 @@ const readLog = (dir, issue, repo = "o/r") => readRepoLog(dir, issue, { repo });
 import { makeRunTmp, wrapLaneLimits, laneCapNotice, systemdRunAvailable, makeLeanRunner, requiredChecksFor, procsInDir, makeLeanIo, makeGhCache } from "./effects.mjs";
 import { retrying, ghCacheRetry, refreshAuthOnce, GH_AUTH_RE, PUSH_TRANSIENT_RE } from "./retry.mjs";
 import { checkAdmission, hostParallelism } from "./admission.mjs";
-import { OUTCOME_CODES, classifyNeedsHuman } from "../engine/outcome-codes.mjs";
+import { OUTCOME_CODES, ENV_VALIDATE_TIMEOUT, classifyNeedsHuman } from "../engine/outcome-codes.mjs";
 import * as refactorMod from "./refactor.mjs";
 import { readAdmission, readConformanceMode, readReviewIncomplete, readEnvConfig, readContextPack } from "../engine-cli.mjs";
 import { prepareToolchain } from "./toolchain.mjs";
@@ -1342,6 +1342,18 @@ describe("conformance shadow mode (the default)", () => {
       assert.ok(ran[0].env && typeof ran[0].env === "object", "the rerun gets the lane env");
     }
   });
+  it("#4906 a validate rerun that times out (rc 124) ends env.validate_timeout: recoverable, no builder resume, never conformance.unimplemented", async () => {
+    const v = [{ id: "V1", command: "node --test test/foo.test.js", exit: 124 }];
+    for (const conformance of ["shadow", "enforce"]) {
+      const w = world({ validation: v });
+      w.io.runCommand = async () => ({ rc: 124, output: "(timed out after 1s)" });
+      const r = await run(w, { conformance });
+      assert.equal(r.code, ENV_VALIDATE_TIMEOUT, conformance); assert.equal(r.terminalReason, "needs-human");
+      assert.equal(classifyNeedsHuman(r), "recoverable"); assert.equal(w.merges.length, 0);
+      assert.equal(builders(w).length, 2, "plan + implement only"); assert.equal(reviewers(w).length, 0);
+      assert.match(nhBody(w), /timed out \(exit 124\) even after one longer retry/);
+    }
+  });
   it("#4323 a rerun that fails for a repo reason stays conformance.unimplemented; one that passes merges", async () => {
     const v = [{ id: "V1", command: "node --test test/foo.test.js", exit: 1 }];
     const w = world({ validation: v, resumeValidation: v });
@@ -3800,6 +3812,45 @@ describe("batch robustness (#4112-#4115)", () => {
     }
   });
 
+  const NONFF = "To github.com:o/r.git\n ! [rejected] feat/issue-7-lean -> feat/issue-7-lean (non-fast-forward)\nerror: failed to push some refs";
+  const ownerProbe = (w, { open = [], body = "" } = {}) => { // answers #4920's lane-owner lookups only
+    const g0 = w.io.gh;
+    w.io.gh = async (a, o) => {
+      if (a[0] === "pr" && a[1] === "list" && a[a.indexOf("--limit") + 1] === "5") return JSON.stringify(open);
+      if (a[0] === "issue" && a[1] === "view" && a[a.length - 1] === "body") return JSON.stringify({ body });
+      return g0(a, o);
+    };
+    return w;
+  };
+  it("#4920 non-ff push with another lane's open PR ends deferred (lease held), not needs-human", async () => {
+    const w = ownerProbe(world({ pushErrors: [NONFF] }), { open: [{ number: 4919 }] });
+    const r = await run(w);
+    assert.equal(r.terminalReason, "deferred");
+    assert.equal(r.code, "lease.held");
+    assert.match(r.detail, /non-fast-forward.*PR #4919 is open on feat\/issue-7-lean/);
+  });
+  it("#4920 non-ff push while another lane holds a live lease ends deferred", async () => {
+    const lease = { v: 1, issue: 7, lease: { by: "core_sbx_1_ab12", until: Date.now() + 600000, host: "sbx", pid: 1 } };
+    const w = ownerProbe(world({ pushErrors: [NONFF] }), { body: `x\n<!-- FORGE:STATE\n${JSON.stringify(lease)}\n-->` });
+    const r = await run(w);
+    assert.equal([r.terminalReason, r.code].join("/"), "deferred/lease.held");
+  });
+  it("#4920 a non-ff push nobody else owns, and any other permanent rejection, stay needs-human", async () => {
+    assert.equal((await run(ownerProbe(world({ pushErrors: [NONFF] })))).terminalReason, "needs-human");
+    assert.equal((await run(ownerProbe(world({ pushErrors: ["remote: error: GH006: Protected branch update failed"] }), { open: [{ number: 4919 }] }))).terminalReason, "needs-human");
+  });
+  it("#4920 two sandbox lanes (pid 1) get distinct lease holders, and a lease another lane wrote meanwhile defers this lane", async () => {
+    const ids = [];
+    for (let i = 0; i < 2; i++) { const w = world(); w.leaseWrites = []; w.io.projector = { readState: async () => null, writeState: async (_i, st) => { w.leaseWrites.push(st); } }; await run(w, { pid: 1, hostName: "sbx" }); ids.push(w.leaseWrites.find((x) => x.lease)?.lease.by); }
+    assert.ok(ids[0] && ids[1] && ids[0] !== ids[1], `${ids}`);
+    assert.match(ids[0], /^core_sbx_1_[0-9a-f]{6}$/);
+    const w = world(); w.leaseWrites = []; let reads = 0;
+    w.io.projector = { readState: async () => (++reads === 1 ? null : { v: 1, lease: { by: "core_sbx_1_other", until: Date.now() + 600000 } }), writeState: async (_i, st) => { w.leaseWrites.push(st); } };
+    const r = await run(w, { pid: 1, hostName: "sbx" });
+    assert.equal([r.terminalReason, r.code].join("/"), "deferred/lease-held");
+    assert.equal(w.builderCalls?.length ?? 0, 0);
+  });
+
   it("#4118 a rerun reuses a local lane branch with unpushed commits instead of resetting it to the base", async () => {
     const w = world();
     const git0 = w.io.git;
@@ -4073,6 +4124,41 @@ describe("batch robustness (#4112-#4115)", () => {
     assert.ok(!w.comments.some((c) => c.includes("FORGE:CONTRACT")), "not re-posted");
     assert.match(traj(w)[0], /frozen contract [0-9a-f]{12} recovered from the issue/);
   });
+  it("recovery lane replays accepted supersedes amendment (#4899)", async () => {
+    const sup = { kind: "supersedes", guard: "#4785", why: "behaviour changed on purpose" };
+    const state = { contract: contractHash(contractJson(CONTRACT)), amendments: [sup] };
+    const mk = (st) => ({ body: `<!-- FORGE:AMENDMENT -->\n${renderAmendments([{ kind: "supersedes", target: "#4785", why: sup.why }], {}, st)}`, author_association: "OWNER", user: { type: "User", login: "op" } });
+    const w = world({ prList: [], remote: "abc\trefs/heads/feat/issue-7-lean", issueComments: [frozenComment(), mk(state)] });
+    assert.equal((await run(w, { recover: { code: "ci.red", attempt: 1 } })).terminalReason, "merged");
+    const posted = w.comments.find((c) => c.includes("FORGE:AMENDMENT-STATE"));
+    assert.ok(posted, "the recovery lane re-applies the prior supersedes and posts it");
+    assert.deepEqual(parseAmendmentsComment(posted).amendments, [sup]);
+    // a state bound to another contract hash is ignored
+    const w2 = world({ prList: [], remote: "abc\trefs/heads/feat/issue-7-lean", issueComments: [frozenComment(), mk({ ...state, contract: "000000000000" })] });
+    await run(w2, { recover: { code: "ci.red", attempt: 1 } });
+    assert.ok(!w2.comments.some((c) => c.includes("FORGE:AMENDMENT-STATE")));
+  });
+  it("#4899 recovery lane keeps enforcing a guard that was not superseded", async () => {
+    const mkGuard = (issue) => ({ v: 1, issue, pr: issue * 10, merge_sha: "abc1234", tests: [{ path: "src/foo.js", ids: [] }], hunks: [], why: `guard ${issue}` });
+    const sup = { kind: "supersedes", guard: "#4785", why: "behaviour changed on purpose" };
+    const prior = { body: `<!-- FORGE:AMENDMENT -->\n${renderAmendments([{ kind: "supersedes", target: "#4785", why: sup.why }], {}, { contract: contractHash(contractJson(CONTRACT)), amendments: [sup] })}`, author_association: "OWNER", user: { type: "User", login: "op" } };
+    const dir = tmpDir("guard-recover-");
+    const w = world({ prList: [], remote: "abc\trefs/heads/feat/issue-7-lean", issueComments: [frozenComment(), prior], baseFiles: { ":src/foo.js": "old" } });
+    w.io.runCommand = async () => ({ rc: 1, output: "guard test failed" });
+    await run(w, { dir, worktreeRoot: dir, recover: { code: "ci.red", attempt: 1 }, loadGuards: async () => [mkGuard(4785), mkGuard(9)] });
+    const text = JSON.stringify([w.comments, w.calls.filter((c) => c[0] === "runner").map((c) => c[4])]);
+    assert.match(text, /conformance\.guard_broken.{0,400}?guard\\":\\"#9\\"/, "precheck reports #9 as guard_broken");
+    assert.doesNotMatch(text, /conformance\.guard_broken.{0,400}?guard\\":\\"#4785\\"/, "precheck does not report the superseded #4785");
+  });
+  it("#4899 amendment state round-trips; precheck still enforces a guard that was not superseded", () => {
+    const am = [{ kind: "supersedes", guard: "#1", why: "w" }];
+    assert.deepEqual(parseAmendmentsComment(renderAmendments([{ kind: "supersedes", target: "#1", why: "w" }], {}, { contract: "abc", amendments: am })), { contract: "abc", amendments: am });
+    assert.equal(parseAmendmentsComment(renderAmendments([], {})), null);
+    const frozen = { requirements: [], deliverables: [] };
+    const eff = applyAmendments(frozen, am).contract;
+    assert.deepEqual((eff.supersedes ?? []).map((s) => s.guard), ["#1"]);
+    assert.equal(applyAmendments(frozen, []).contract.supersedes, undefined);
+  });
   const scoped = (body = ISSUE_BODY) => contractHash(scopeText({ title: "foo breaks", body: `${body}\n\n<!-- FORGE:STATE {"v":1} -->` }, []));
   it("#4265 a rerun recovers a contract frozen from the current issue scope (engine annotations aside)", async () => {
     const c = frozenComment(CONTRACT, (b) => b.replace(/\(sha256 ([0-9a-f]{12})\)/, `(sha256 $1, issue ${scoped()})`));
@@ -6552,6 +6638,42 @@ describe("resume digest in builder prompts (#4400)", () => {
     assert.ok(p.indexOf("Recovery pass (ci.red)") >= 0 && p.indexOf("Recovery pass (ci.red)") < p.indexOf("Resume digest of the last run"), p);
   });
 
+  describe("#4901 FORGE:DECISION after the contract froze", () => {
+    const dec = (t, who = BOT) => ({ body: `<!-- FORGE:DECISION {"v":1} -->\n## Decision\n${t}`, ...who });
+    const CONTRACT_C = { body: frozen().body, ...BOT };
+    const NOBODY = { author: { login: "rando" }, authorAssociation: "NONE" };
+    const withDecisions = (viewComments) => world({ prList: [], remote: "abc\trefs/heads/feat/issue-7-lean", issueComments: [frozen()], issueBodies: [body], viewComments });
+
+    it("reviewerPrompt keeps trusted FORGE:DECISION comments, drops other FORGE: records and untrusted decisions", () => {
+      const p = reviewerPrompt({ issue: { number: 7, title: "t", body: "b" }, contract: CONTRACT, diff: "", comments: [dec("deliver the nightly timer"), dec("evil override", NOBODY),
+        { body: "<!-- FORGE:STATE\n{}\n-->", ...BOT }, { body: "<!-- FORGE:TRAJECTORY -->\nt", ...BOT }, { body: "<!-- FORGE:FACTS -->\nf", ...BOT }] });
+      assert.match(p, /deliver the nightly timer/);
+      assert.doesNotMatch(p, /evil override|FORGE:STATE|FORGE:TRAJECTORY|FORGE:FACTS/);
+    });
+
+    it("decisionNote: only trusted decisions after the last contract; recovery records skipped", () => {
+      const n = decisionNote([dec("old, before contract"), CONTRACT_C, dec("declare supersedes X"), dec("untrusted", NOBODY), { body: "<!-- FORGE:DECISION -->\n## Decision: recovery pass for `ci.red` (attempt 1)", ...BOT }]);
+      assert.match(n, /declare supersedes X/);
+      assert.doesNotMatch(n, /old, before contract|untrusted|recovery pass/);
+      assert.equal(decisionNote([CONTRACT_C]), "");
+    });
+
+    it("a requeued lane with a frozen contract gives the builder a newer trusted decision, not an untrusted or older one", async () => {
+      const w = withDecisions([dec("older decision"), CONTRACT_C, dec("deliver the nightly timer"), dec("untrusted override", NOBODY), { body: factsRec, ...BOT }]);
+      await run(w);
+      const p = builders(w)[0][4];
+      assert.match(p, /Binding decisions posted after the contract froze/);
+      assert.match(p, /deliver the nightly timer/);
+      assert.doesNotMatch(p, /untrusted override|older decision/);
+    });
+
+    it("context_pack off still passes the decision", async () => {
+      const w = withDecisions([CONTRACT_C, dec("deliver the nightly timer")]);
+      await run(w, { contextPack: { mode: "off" } });
+      assert.match(builders(w)[0][4], /deliver the nightly timer/);
+    });
+  });
+
   it("a first run gets no digest; context_pack off gives none; the contract quotes are unchanged", async () => {
     const first = world({ issueBodies: [body], viewComments: [{ body: factsRec, ...BOT }] });
     await run(first);
@@ -6898,3 +7020,51 @@ describe("block rules: a check timeout is not a block (#4589)", () => {
     assert.match(blockCorrectionPrompt("x"), /times out \(rc 124\) is host load, never a block/);
   });
 });
+
+describe("per-kind conformance modes (engine.conformance_kinds)", () => {
+  const CREEP = ["src/foo.js", "src/billing.js", "test/foo.test.js"];
+  const shadowNote = (w) => w.comments.find((c) => c.includes("<!-- FORGE:REVIEW -->")) ?? "";
+  const blockedNothingMerged = (w) => { assert.equal(w.prs.length, 0); assert.equal(w.merges.length, 0); assert.ok(w.edits.some((e) => e.includes("needs-human"))); };
+  it("a shadow kind in an enforce lane is recorded, not blocking; the lane merges", async () => {
+    const w = world({ changed: CREEP, reviews: [conf({}, CONTRACT, CREEP)] });
+    const r = await run(w, { conformance: "enforce", conformanceKinds: { "conformance.scope_creep": "shadow" } });
+    assert.equal(r.terminalReason, "merged");
+    assert.match(shadowNote(w), /would BLOCK[\s\S]*conformance\.scope_creep[\s\S]*src\/billing\.js/);
+  });
+
+  it("an enforce kind in a shadow lane blocks (scope creep is recorded-only in shadow by default)", async () => {
+    const w = world({ changed: CREEP });
+    const r = await run(w, { conformance: "shadow", conformanceKinds: { "conformance.scope_creep": "enforce" } });
+    assert.equal(r.code, "conformance.scope_creep"); assert.equal(reviewers(w).length, 0);
+    blockedNothingMerged(w);
+  });
+
+  it("an enforce lane without kinds still blocks scope creep (unchanged default)", async () => {
+    const w = world({ changed: CREEP });
+    assert.equal((await run(w, { conformance: "enforce", conformanceKinds: {} })).code, "conformance.scope_creep");
+  });
+});
+
+describe("mutation check (engine.conformance_mutation, #4913)", () => {
+  const MC = { ...CONTRACT, deliverables: [...CONTRACT.deliverables, { id: "D2", path: "test/foo.test.js", change: "t", requirements: ["R1"] }] };
+  const setup = (rc) => {
+    const w = world({ contract: MC, result: { status: "fixed", summary: "s", evidence: [], tests: [], validation: [{ id: "V1", exit: 0 }] } });
+    w.cmds = []; w.io.runCommand = async (a) => { w.cmds.push(a.command); return { rc: a.command.includes("--test") && w.cmds.length > 0 ? rc : 0, output: "" }; };
+    return w;
+  };
+  const applied = (w) => w.calls.filter((c) => c[0] === "git" && c[1] === "apply").length;
+  it("off (default): no revert, no extra command", async () => {
+    const w = setup(0); await run(w, { conformance: "shadow" });
+    assert.equal(applied(w), 0); assert.ok(!w.calls.some((c) => c[0] === "git" && c[1] === "checkout" && c[2] === "HEAD"));
+  });
+  it("report: a vacuous test is recorded (shadow kind), the lane still merges, the tree is restored", async () => {
+    const w = setup(0); const r = await run(w, { conformance: "enforce", conformanceMutation: "report" });
+    assert.equal(r.terminalReason, "merged"); assert.equal(applied(w), 1);
+    const seq = w.calls.filter((c) => c[0] === "git" && (c[1] === "apply" || (c[1] === "reset" && c[3] === "--hard"))).map((c) => c[1]);
+    assert.deepEqual(seq, ["apply", "reset"]);
+  });
+  it("report with the kind set to enforce blocks on a survivor", async () => {
+    const w = setup(0); const r = await run(w, { conformance: "enforce", conformanceMutation: "report", conformanceKinds: { "conformance.mutation_survived": "enforce" } });
+    assert.equal(r.code, "conformance.mutation_survived"); assert.equal(w.merges.length, 0);
+  });
+});

package/bin/lean/orchestrate.mjs

@@ -111,6 +111,7 @@ import { renderAlerts } from "../stats/render.mjs";
 import { formatParkedLine, postDeployOnly } from "./prefilter.mjs";
 import { laneSize, SIZES } from "./budget.mjs";
 import { requeueKey, REQUEUABLE } from "./batch.mjs";
+import { makeCapWait } from "./capwait.mjs";
 
 // A dependency that finished like this no longer blocks its dependents.
 const SATISFIED = new Set(["merged", "invalid"]);
@@ -245,7 +246,7 @@ const toMap = (o) => new Map(Object.entries(o ?? {}).map(([k, v]) => [Number(k),
  */
 export async function orchestrate({ issues, parallel = 3, runOne, readBody, prefilter = null, resolveLane = null, now = () => Date.now(), onUpdate = () => {}, onCrossLane = () => {},
   store = null, adopt = null, getParallel = null, getAdded = null, getRequeued = null, breaker = null, maxDeferrals = DEFAULT_MAX_DEFERRALS, sleep = cancellableSleep, pollMs = 5000,
-  classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, integration = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null,
+  classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, integration = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null, preflight = null,
   recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, issueState = null, getCancelled = null, getHeld = null, getBlocked = null, trusted = null, getGates = null, stopLane = null,
   estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null, dispatchCheck = null, syncParked = null, syncEveryMs = DEFAULT_SYNC_EVERY_MS, lifecycle = null }) {
   const saved = store?.load?.() ?? null;
@@ -271,6 +272,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   const holds = limits ?? memoryLimits(now);
   const heldUntil = () => { try { const id = identity(); const u = holds.holdUntil(id); return Number.isFinite(u) && u > now() ? u : null; } catch { return null; } };
   let tripped = saved?.breaker ?? null;
+  let pfAll = false, pfRemote = false, pfNoted = new Set(); // #4866: lane preflight holds
   let opHeld = new Set(); // #4592: operator-held issues this pass (the batch `hold` file is the authority)
   const opHeldHit = new Set(); // held issues that were pending this pass
   // #4376: operator cancel (the batch `cancel` file through `getCancelled`); persisted, but the file stays the authority.
@@ -289,6 +291,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   const priorities = toMap(saved?.priorities); // issue -> priority number; only with the registry
   const remoteMax = remote && Number.isInteger(remote.max) && remote.max >= 0 ? remote.max : 0;
   let poolOfflineSince = null, poolStallNoted = false; // #4788: when sampled online first read 0, and whether the stall was reported
+  const capWait = makeCapWait({ now }); // #4909: a lane refused for capacity waits for a free slot, recorded as queue time
   let capAt, capAdmits = 0; // #4540: provider capacity sample and the remote admissions made since it
   const remoteRun = new Set(); // remote lanes running now (#4253): admitted against remoteMax, never the host cap
   let budgetLocal = false; // the execution budget refused a remote lane under on_budget local: the rest run on the host
@@ -505,9 +508,10 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
           p = { where: "local", reason: `fallback to local: no pool agent online (${p.reason})` };
         } else if (cap && Number.isInteger(cap.free)) {
           if (cap.at !== capAt) { capAt = cap.at; capAdmits = 0; }
+          if (capWait.blocked(n, cap)) { notBefore.set(n, now() + Math.max(pollMs, 1)); return false; }
           if (capAdmits >= cap.free && remoteRun.size < remoteMax) { notBefore.set(n, now() + Math.max(pollMs, 1)); return false; }
         }
-        if (p.where === "remote" && remoteRun.size < remoteMax) { remoteRun.add(n); capAdmits++; placed.set(n, p); return true; }
+        if (p.where === "remote" && remoteRun.size < remoteMax) { capWait.admitted(n); remoteRun.add(n); capAdmits++; placed.set(n, p); return true; }
         if (p.where === "remote" && !remote.fallbackLocal) return false;
         if (p.where === "remote") p = { where: "local", reason: `fallback to local: max_remote ${remoteMax} reached (${p.reason})` }; // #4256: hybrid runs it on the host
       } else {
@@ -554,8 +558,9 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     // #4304: the pre-filter runs once per issue; a recovery pass's row adds the earlier attempt's cost, which already holds it.
     const pfCost = recovering.has(n) ? 0 : (pf?.cost_usd ?? 0);
     const row = { issue: n, outcome: r?.terminalReason ?? "engine-error", pr: r?.pr ?? null, cost_usd: (r?.cost_usd ?? 0) + pfCost, minutes: Math.round((now() - rec.startedAt) / 6000) / 10, detail: r?.detail, ...(r?.code ? { code: r.code } : {}), ...(pf ? { prefilter: pf } : {}),
-      ...(r?.waits && typeof r.waits === "object" ? { waits: r.waits } : {}), ...(Number.isInteger(r?.heavyQueueMax) ? { heavyQueueMax: r.heavyQueueMax } : {}),
+      ...(() => { const w = capWait.take(n, r?.waits); return w && typeof w === "object" ? { waits: w } : {}; })(), ...(Number.isInteger(r?.heavyQueueMax) ? { heavyQueueMax: r.heavyQueueMax } : {}),
       ...(Number.isFinite(r?.wall_ms) ? { wall_ms: r.wall_ms } : {}), ...(Number.isFinite(r?.model_ms) ? { model_ms: r.model_ms } : {}), // #4451
+      ...(phasesOf(r, rec) ? { phases: phasesOf(r, rec) } : {}), // #4908
       ...(Number.isFinite(r?.precheckMs) ? { precheck_ms: r.precheckMs } : {}), ...(Number.isInteger(r?.precheckDeferred) && r.precheckDeferred > 0 ? { precheck_deferred: r.precheckDeferred } : {}), // #4331
       ...(typeof r?.firstPass === "boolean" ? { first_pass: r.firstPass } : {}), // #4362
       ...(r?.turns && typeof r.turns === "object" ? { turns: r.turns } : {}), // #4392
@@ -621,6 +626,11 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     }
     if (row.outcome === "deferred") {
       const pol = deferPolicy(r, { now: now(), deferrals: deferrals.get(n) ?? 0, maxDeferrals });
+      let capGated = false; // #4909: the pool reports free slots, so the slot (not the retry-in timer) gates the relaunch
+      if (r?.code === "sandbox.capacity") {
+        capWait.refused(n);
+        try { capGated = Number.isInteger(remote?.capacity?.()?.free); } catch { capGated = false; }
+      }
       if (!pol.capped) {
         if (!pol.host) deferrals.set(n, (deferrals.get(n) ?? 0) + 1);
         // A usage limit is held per account; the hold, not a per-issue delay, gates the retry, so a lane on another
@@ -629,7 +639,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
         if (pol.hold) {
           try { holds.set(r?.backendId ?? identity(), now() + pol.delayMs, `batch ${batchId ?? "?"} #${n}`); delayMs = 0; } catch { /* fail-open */ }
         }
-        requeue(n, delayMs); done(); save();
+        requeue(n, capGated && !pol.hold ? 0 : delayMs); done(); save();
         update({ ...row, detail: `requeued in ${Math.round(pol.delayMs / 1000)}s${pol.hold ? " (admissions held)" : ""}: ${r?.code ?? "deferred"}${r?.detail ? ` ${r.detail}` : ""}` });
         return;
       }
@@ -740,8 +750,9 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     if (relabelled.delete(n)) save(); // #4315: the recovery lane launches after all; it removes needs-human itself
     const pl = placed.get(n);
     placed.delete(n);
+    const queueMs = Number.isFinite(enqueued.get(n)) ? Math.max(0, now() - enqueued.get(n)) : null; // #4908: queue_wait in the phase ledger
     heads.delete(n); enqueued.delete(n); busyNoted.delete(n);
-    const rec = { startedAt: now(), ...(lane ? { lane } : {}), ...(remoteRun.has(n) ? { remote: true } : {}), ...(pl ? { placement: pl } : {}) };
+    const rec = { startedAt: now(), ...(queueMs != null ? { queueMs } : {}), ...(lane ? { lane } : {}), ...(remoteRun.has(n) ? { remote: true } : {}), ...(pl ? { placement: pl } : {}) };
     runRecs.set(n, rec); notBefore.delete(n); save();
     track(n, (async () => {
       // Pre-filter once per issue (a requeued issue is not re-checked); fail-open: an error means a normal lane.
@@ -751,13 +762,13 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       let gone = null;
       if (dispatchCheck && !cancelled && !pf?.closed && !pf?.parked) {
         try { gone = (await dispatchCheck(n)) ?? null; } catch { gone = null; }
-        if (!["merged", "invalid"].includes(gone?.outcome)) gone = null;
+        if (!["merged", "invalid", "deferred"].includes(gone?.outcome)) gone = null; // #4920: deferred = another lane owns the issue (lease or open PR on GitHub)
       }
       if (halted) return; // #4454: stopped before launch; no pid, so a resume queues it again
       let r;
       if (cancelled) r = { terminalReason: "deferred", code: "run.interrupted", cost_usd: 0, detail: "cancelled before launch" }; // #4376
       else if (pf?.closed) r = { terminalReason: "invalid", cost_usd: 0, detail: `pre-filter: ${pf.detail}` };
-      else if (gone) r = { terminalReason: gone.outcome, pr: Number.isInteger(gone.pr) ? gone.pr : null, cost_usd: 0, detail: `skipped at dispatch: ${gone.detail ?? gone.outcome}` };
+      else if (gone) r = { terminalReason: gone.outcome, ...(gone.outcome === "deferred" && gone.code ? { code: gone.code } : {}), pr: Number.isInteger(gone.pr) ? gone.pr : null, cost_usd: 0, detail: `skipped at dispatch: ${gone.detail ?? gone.outcome}` };
       // #4389: a parked post-deploy follow-up starts no lane; it ends awaiting-promotion|awaiting-precondition, re-checked next batch.
       else if (pf?.parked) r = { terminalReason: pf.kind, cost_usd: 0, detail: `pre-filter: ${pf.detail}` };
       else {
@@ -1059,7 +1070,20 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       refreshHeld();
       await refreshBlocked();
       checkGates();
+      // #4866: lane preflight. A failing check holds admission (all lanes, or only remote ones by its scope), says why once per batch,
+      // and is re-checked every pass so the batch resumes on its own; it is not an engine-error, so the breaker never sees it.
+      pfAll = false; pfRemote = false;
+      if (preflight && !tripped && !cancelled && !holdUntil && pending.length) {
+        let fails = [];
+        try { fails = (await preflight()) ?? []; } catch { fails = []; }
+        for (const f of fails) { if (f.scope === "remote") pfRemote = true; else pfAll = true; }
+        const keys = new Set(fails.map((f) => `${f.code}|${f.detail}`));
+        for (const f of fails) if (!pfNoted.has(`${f.code}|${f.detail}`)) update({ issue: null, outcome: "preflight", code: f.code, detail: `${f.code}: ${f.detail}; admissions paused until it clears` });
+        if (pfNoted.size && !keys.size) update({ issue: null, outcome: "preflight", detail: "preflight cleared; admissions resumed" });
+        pfNoted = keys;
+      }
       const isReady = (n) => {
+        if (pfAll || (pfRemote && isRemote(n))) return false;
         if (opHeld.has(n)) { opHeldHit.add(n); return false; }
         if (blocked.has(n)) { blockedHit.add(n); return false; } // #4773
         if (!((notBefore.get(n) ?? 0) <= t && edges(n).every((d) => results.has(d) && SATISFIED.has(results.get(d).outcome)))) return false;
@@ -1140,6 +1164,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       if ((getParallel || getHeld) && pending.length) wake = Math.min(wake ?? pollMs, pollMs);
       if (offered.size && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // a slot another batch frees
       if ((getAdded || getRequeued) && running.size) wake = Math.min(wake ?? pollMs, pollMs); // #4408/#4550: an add or requeue while only lanes run
+      if ((pfAll || pfRemote) && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // #4866
       if (getBlocked && pending.length && blockedHit.size) wake = Math.min(wake ?? pollMs, pollMs); // #4773: a removed label releases on the next tick
       if (getHeld && running.size) wake = Math.min(wake ?? pollMs, pollMs); // #4592: a release while only lanes run
       if (syncParked && running.size && parkedPrs().length) wake = Math.min(wake ?? syncEveryMs, Math.max(0, syncEveryMs - (t - (lastParkedSync ?? t)))); // #4525
@@ -1151,6 +1176,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
         else { const ac = new AbortController(); await race([...running.values(), sleep(wake, ac.signal)]); ac.abort(); }
       } else if (started) continue; // only lane-refused rows settled (synchronously): re-resolve skips first
       else if (pending.length && offered.size) await nap(wake); // waiting for a host slot
+      else if (pending.length && (pfAll || pfRemote)) await nap(Math.min(wake ?? pollMs, pollMs)); // #4866: a failing lane preflight is re-checked on the next poll
       else if (pending.length && blockedHit.size) await nap(wake ?? pollMs); // #4773: only blocked issues wait: poll for the label's removal
       else if (pending.length && waits.length) await nap(Math.min(wake, getCancelled ? pollMs : Infinity)); // wake is poll-capped by a hold or a live K; #4454: nap wakes on a stop
       else if (pending.length && budgetHeld) { if (await drainAdds()) continue; break; } // maintenance the budget does not admit now: held for a later batch
@@ -1240,6 +1266,15 @@ export function formatTable(rows) {
  * #4169 batch-report line: wait time per phase summed over the rows' lane `waits` ({phase: ms}), and the heavy-step
  * semaphore queue: `now` live from `heavyStats.queued`, `max` the longest queue a lane of this batch saw. Null when nothing waited or queued.
  */
+/**
+ * #4908: the lane's phase ledger for its batch row, plus `queue_wait` (the admission wait the batch saw; it precedes the lane
+ * clock, so the other phases still sum to `wall_ms`). Null without a ledger.
+ */
+export function phasesOf(r, rec) {
+  if (!r?.phases || typeof r.phases !== "object" || !Object.keys(r.phases).length) return null;
+  return { ...(Number.isFinite(rec?.queueMs) && rec.queueMs > 0 ? { queue_wait: { ms: Math.round(rec.queueMs) } } : {}), ...r.phases };
+}
+
 export function formatWaitSummary(rows, heavyStats = null) {
   const by = {};
   for (const r of rows ?? []) for (const [p, ms] of Object.entries(r?.waits ?? {})) if (Number.isFinite(ms) && ms >= 0) by[p] = (by[p] ?? 0) + ms;

package/bin/lean/phaseledger.mjs

@@ -0,0 +1,116 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Per-lane phase ledger (#4908): where a lane's wall time goes. Engine phases only, nothing repo-specific.
+ * The lane marks the phase it enters; entering one closes the previous, so the lane phases sum to the lane's wall time.
+ * `queue_wait` (and, for a remote lane, the sandbox start the dispatcher saw) happens before the lane clock starts, so the
+ * batch runner adds them to the row; they are not part of the lane-wall sum.
+ * Row shape: `phases: { <id>: { ms, model_ms?, usd? } }`. FORGE:TELEMETRY values are scalars, so `flattenPhases` gives
+ * `phase_<id>_ms|_model_ms|_usd` keys for it and `unflattenPhases` reads them back.
+ */
+
+export const LANE_PHASES = Object.freeze(["queue_wait", "sandbox_start", "deps_install", "validate", "plan", "implement", "precheck", "review", "ci_wait", "base_sync", "merge"]);
+const isNum = (x) => typeof x === "number" && Number.isFinite(x);
+
+function pct(values, p) {
+  const v = values.filter(isNum).sort((a, b) => a - b);
+  if (!v.length) return null;
+  if (p === 50) return v.length % 2 ? v[(v.length - 1) / 2] : (v[v.length / 2 - 1] + v[v.length / 2]) / 2;
+  return v[Math.max(0, Math.ceil((v.length * p) / 100) - 1)];
+}
+
+/** Sequential ledger starting in `first` at `start`. `now` is the lane clock. */
+export function createLedger(now, first = "sandbox_start", start = now()) {
+  let cur = LANE_PHASES.includes(first) ? first : null, since = start;
+  const ms = {}, model = {}, usd = {};
+  const close = (t) => { if (cur) ms[cur] = (ms[cur] ?? 0) + Math.max(0, t - since); };
+  const enter = (phase) => {
+    if (!LANE_PHASES.includes(phase) || phase === cur) return;
+    const t = now(); close(t); cur = phase; since = t;
+  };
+  return {
+    enter,
+    /** Run `fn` inside `phase`, then return to the phase that was current. */
+    async during(phase, fn) {
+      const prev = cur; enter(phase);
+      try { return await fn(); } finally { if (prev) enter(prev); }
+    },
+    current: () => cur,
+    addModel(phase, modelMs, cost) {
+      const id = LANE_PHASES.includes(phase) ? phase : cur;
+      if (!id) return;
+      if (isNum(modelMs) && modelMs > 0) model[id] = (model[id] ?? 0) + modelMs;
+      if (isNum(cost) && cost > 0) usd[id] = (usd[id] ?? 0) + cost;
+    },
+    /** The ledger as of now (the open phase counted up to now); does not end the current phase. */
+    snapshot() {
+      const t = now(), total = { ...ms };
+      if (cur) total[cur] = (total[cur] ?? 0) + Math.max(0, t - since);
+      const out = {};
+      for (const id of LANE_PHASES) {
+        if (!(id in total) && !(id in model) && !(id in usd)) continue;
+        out[id] = { ms: Math.round(total[id] ?? 0), ...(model[id] ? { model_ms: Math.round(model[id]) } : {}), ...(usd[id] ? { usd: Math.round(usd[id] * 1e6) / 1e6 } : {}) };
+      }
+      return out;
+    },
+  };
+}
+
+/** Sum of the lane phases' durations (queue_wait excluded: it precedes the lane clock). */
+export const laneSum = (phases) => Object.entries(phases ?? {}).reduce((s, [id, v]) => (id === "queue_wait" || !isNum(v?.ms) ? s : s + v.ms), 0);
+
+/** True when the lane phases sum to within `tol` of `wallMs`. */
+export const withinWall = (phases, wallMs, tol = 0.05) => isNum(wallMs) && wallMs >= 0 && Math.abs(laneSum(phases) - wallMs) <= Math.max(1, wallMs * tol);
+
+export function flattenPhases(phases) {
+  const out = {};
+  for (const [id, v] of Object.entries(phases ?? {})) {
+    if (!LANE_PHASES.includes(id) || !v) continue;
+    if (isNum(v.ms)) out[`phase_${id}_ms`] = v.ms;
+    if (isNum(v.model_ms)) out[`phase_${id}_model_ms`] = v.model_ms;
+    if (isNum(v.usd)) out[`phase_${id}_usd`] = v.usd;
+  }
+  return out;
+}
+
+export function unflattenPhases(rec) {
+  const out = {};
+  for (const id of LANE_PHASES) {
+    const ms = rec?.[`phase_${id}_ms`], mm = rec?.[`phase_${id}_model_ms`], usd = rec?.[`phase_${id}_usd`];
+    if (!isNum(ms)) continue;
+    out[id] = { ms, ...(isNum(mm) ? { model_ms: mm } : {}), ...(isNum(usd) ? { usd } : {}) };
+  }
+  return out;
+}
+
+/**
+ * p50/p90 per phase over the given `phases` objects, and the phase with the largest share of the summed time.
+ * `{ n: 0, phases: {}, largest: null }` when none carries a ledger.
+ */
+export function summarizePhases(list) {
+  const by = {}, lanes = (list ?? []).filter((p) => p && typeof p === "object" && Object.keys(p).length);
+  for (const p of lanes) for (const [id, v] of Object.entries(p)) if (LANE_PHASES.includes(id) && isNum(v?.ms) && v.ms >= 0) (by[id] ??= []).push(v.ms);
+  const phases = {};
+  let total = 0;
+  for (const id of LANE_PHASES) {
+    if (!by[id]) continue;
+    const sum = by[id].reduce((a, b) => a + b, 0);
+    total += sum;
+    phases[id] = { n: by[id].length, p50: pct(by[id], 50), p90: pct(by[id], 90), sum };
+  }
+  let largest = null;
+  for (const [id, v] of Object.entries(phases)) if (!largest || v.sum > phases[largest].sum) largest = id;
+  return { n: lanes.length, phases, largest: largest ? { phase: largest, share: total > 0 ? phases[largest].sum / total : 0 } : null };
+}
+
+const min = (ms) => `${(ms / 60000).toFixed(1)}m`;
+
+/** Report line(s) for p50/p90 per phase and the largest phase; `n/a` when no lane carried a ledger. Never throws. */
+export function formatLedgerSummary(rows) {
+  try {
+    const s = summarizePhases((rows ?? []).map((r) => r?.phases));
+    if (!s.n) return "phases (wall time): n/a";
+    const parts = Object.entries(s.phases).map(([id, v]) => `${id} ${min(v.p50)}/${min(v.p90)}`);
+    return `phases (p50/p90 per lane, ${s.n} lane${s.n === 1 ? "" : "s"}): ${parts.join(", ")}; largest: ${s.largest.phase} (${Math.round(s.largest.share * 100)}% of ledger time)`;
+  } catch { return "phases (wall time): n/a"; }
+}

package/bin/lean/phaseledger.test.mjs

@@ -0,0 +1,72 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { phasesOf } from "./orchestrate.mjs";
+import { aggregate } from "../stats/aggregate.mjs";
+import { LANE_PHASES, createLedger, laneSum, withinWall, flattenPhases, unflattenPhases, summarizePhases, formatLedgerSummary } from "./phaseledger.mjs";
+
+const clock = () => { let t = 1000; const f = () => t; f.tick = (ms) => { t += ms; }; return f; };
+
+describe("phase ledger (#4908)", () => {
+  it("lane phases sum to within 5% of wall and only reached phases appear", async () => {
+    const now = clock(), t0 = now(), L = createLedger(now);
+    now.tick(2000); L.enter("deps_install");
+    now.tick(70000); L.enter("plan");
+    now.tick(60000); L.enter("implement");
+    now.tick(120000);
+    await L.during("validate", async () => { now.tick(30000); });
+    now.tick(10000); L.enter("ci_wait"); now.tick(5000); L.enter("merge"); now.tick(1000);
+    L.addModel("implement", 100000, 1.5);
+    const p = L.snapshot(), wall = now() - t0;
+    assert.deepEqual(Object.keys(p), ["sandbox_start", "deps_install", "validate", "plan", "implement", "ci_wait", "merge"]);
+    assert.equal(p.implement.ms, 130000);
+    assert.deepEqual([p.implement.model_ms, p.implement.usd], [100000, 1.5]);
+    assert.ok(withinWall(p, wall));
+    assert.equal(laneSum(p), wall);
+    assert.ok(!withinWall({ plan: { ms: 1000 } }, 100000));
+  });
+
+  it("queue_wait is outside the lane sum; unknown phases are ignored", () => {
+    const now = clock(), L = createLedger(now);
+    L.enter("nope"); now.tick(500);
+    const p = { ...L.snapshot(), queue_wait: { ms: 99999 } };
+    assert.equal(laneSum(p), 500);
+  });
+
+  it("flatten/unflatten round trip for FORGE:TELEMETRY scalars", () => {
+    const p = { plan: { ms: 5, model_ms: 3, usd: 0.25 }, merge: { ms: 7 } };
+    const flat = flattenPhases(p);
+    assert.deepEqual(flat, { phase_plan_ms: 5, phase_plan_model_ms: 3, phase_plan_usd: 0.25, phase_merge_ms: 7 });
+    assert.deepEqual(unflattenPhases(flat), p);
+  });
+
+  it("summary gives p50/p90 per phase and the largest phase; n/a without ledger data", () => {
+    const rows = [{ phases: { plan: { ms: 60000 }, implement: { ms: 300000 } } }, { phases: { plan: { ms: 120000 }, implement: { ms: 900000 } } }, { issue: 3 }, { phases: {} }];
+    const s = summarizePhases(rows.map((r) => r.phases));
+    assert.equal(s.n, 2);
+    assert.equal(s.phases.plan.p50, 90000);
+    assert.equal(s.phases.implement.p90, 900000);
+    assert.equal(s.largest.phase, "implement");
+    const line = formatLedgerSummary(rows);
+    assert.match(line, /implement 10\.0m\/15\.0m/);
+    assert.match(line, /largest: implement/);
+    assert.equal(formatLedgerSummary([{ issue: 1 }]), "phases (wall time): n/a");
+    assert.equal(formatLedgerSummary(null), "phases (wall time): n/a");
+    assert.equal(LANE_PHASES.length, 11);
+  });
+
+  it("the batch row adds queue_wait to the lane ledger and stays null without one (#4908)", () => {
+    const lane = { plan: { ms: 10 }, merge: { ms: 5 } };
+    assert.deepEqual(phasesOf({ phases: lane }, { queueMs: 300 }), { queue_wait: { ms: 300 }, ...lane });
+    assert.deepEqual(phasesOf({ phases: lane }, {}), lane);
+    assert.equal(phasesOf({ wall_ms: 5 }, { queueMs: 1 }), null);
+  });
+
+  it("forgedock stats aggregates the ledger and tolerates runs without one", () => {
+    const run = (ledger) => ({ source: "local", repo: "o/r", version: "1.0.0", issue: 1, run: "r", finished: true, reason: "merged", code: null, remediated: false, usageEvents: 1, startTs: null, endTs: null, wallMs: 100, phases: {}, builderAttempts: [], ledger });
+    const t = aggregate({ runs: [run({ plan: { ms: 40 }, implement: { ms: 60 } }), run({}), run(undefined)] }).groups?.[0]?.time;
+    assert.equal(t?.ledger.n, 1);
+    assert.equal(t.ledger.largest.phase, "implement");
+  });
+});

package/bin/lean/prodwatch.mjs

@@ -120,6 +120,15 @@ const isSummary = (i) => trailerOf(i?.body).includes(SUMMARY_TAG);
 
 const issueNumber = (out) => Number(/(\d+)\s*$/.exec(String(out ?? "").trim())?.[1]) || null;
 
+const LIST_FIELDS = "number,state,stateReason,body,createdAt,closedAt";
+
+/** Every open issue carrying `label`: `gh api --paginate --slurp` walks all pages (`gh issue list` has no cursor); PRs are dropped and items mapped to the `issue list` shape. */
+async function listOpen(io, label) {
+  const pages = JSON.parse(String(await io.gh(["api", "--paginate", "--slurp", `repos/{owner}/{repo}/issues?labels=${encodeURIComponent(label)}&state=open&per_page=100`])));
+  return pages.flat().filter((i) => !i.pull_request)
+    .map((i) => ({ number: i.number, state: "OPEN", stateReason: i.state_reason ?? null, body: i.body ?? "", createdAt: i.created_at, closedAt: i.closed_at ?? null }));
+}
+
 // ── one round ──────────────────────────────────────────────────────────────────────────────
 /**
  * One poll round. `state` is the cache `{filed:[{at,fp,issue}], verified:{fp:at}, paged:{fp:at}, digest:{date,issue,items}}`; it is mutated
@@ -132,8 +141,10 @@ export async function roundOnce({ cfg, io, state = {}, dryRun = false, label = D
   const write = async (what, fn) => { if (dryRun) return null; try { return await fn(); } catch (e) { rep.errors.push(`${what}: ${e.message}`); return null; } };
 
   let issues;
-  try { issues = JSON.parse(String(await io.gh(["issue", "list", "--label", label, "--state", "all", "--limit", "200", "--json", "number,state,stateReason,body,createdAt,closedAt"]))); }
+  try { issues = await listOpen(io, label); }
   catch (e) { return { ...rep, errors: [`issue list failed, nothing filed: ${e.message}`] }; }
+  try { issues = issues.concat(JSON.parse(String(await io.gh(["issue", "list", "--label", label, "--state", "closed", "--limit", "200", "--json", LIST_FIELDS]))).filter((i) => i.state === "CLOSED")); }
+  catch (e) { rep.errors.push(`closed issue list failed, metrics and verify skip closed issues: ${e.message}`); }
   const openAll = issues.filter((i) => i.state === "OPEN"), openSummary = openAll.find(isSummary) ?? null;
   const open = openAll.filter((i) => !isSummary(i)), rows = metricRows(issues.filter((i) => !isSummary(i)));
   rep.metrics = computeMetrics(rows);