forgedock 1.33.0 → 1.35.0

package/bin/lean/lean.test.mjs

@@ -118,6 +118,8 @@ function world(o = {}) {
       if (a[0] === "write-tree") return w.salvageTrees.length ? w.salvageTrees.shift() : "";
       if (a[0] === "rev-parse" && a[1] === "HEAD^{tree}") return "0".repeat(40);
       if (a.includes("commit-tree")) return "c".repeat(40);
+      if (a[0] === "diff" && a.includes("--diff-filter=A")) return (w.added ?? []).join("\0"); // #4590: files the branch added vs the base
+      if (a[0] === "rm" && a[1] === "--cached" && w.added) w.added = w.added.filter((f) => !a.includes(f));
       if (a[0] === "diff" && a.includes("--diff-filter=U")) return w.conflictFiles.join("\n");
       if (a[0] === "diff" && a[1] === "--check") return "";
       if (a[0] === "grep") return w.markers ? "src/foo.js:3:<<<<<<< HEAD" : "";
@@ -129,6 +131,7 @@ function world(o = {}) {
       if (a[0] === "diff" && a[1] === "--numstat") return w.numstat ?? "";
       if (a[0] === "ls-files") return [...new Set(["src/foo.js", "src/bar.js", "test/foo.test.js", ...w.changed])].join("\n");
       if (a[0] === "diff") return "diff --git a/src/foo.js b/src/foo.js\n+fix";
+      if (a[0] === "merge-base" && a[1] === "--is-ancestor") { if ((o.shipped ?? []).includes(String(a[2]).replace(/^origin\//, ""))) return ""; throw new Error("not an ancestor"); } // #4593
       if (a[0] === "ls-remote" && String(a.at(-1)).startsWith("refs/heads/")) { const b = a.at(-1).slice("refs/heads/".length); return (o.branchesOnOrigin ?? ["staging", "main"]).includes(b) ? `abc\trefs/heads/${b}\n` : ""; }
       if (a[0] === "ls-remote") return w.remote;
       if (a[0] === "show" && w.baseFiles) { if (!(a[1] in w.baseFiles)) throw new Error(`fatal: path not in ${a[1]}`); return w.baseFiles[a[1]]; }
@@ -1618,9 +1621,9 @@ describe("hotspot prevention (forge#4195): plan facts, build rule, unexempted gr
     assert.equal(hotspotBuildRule(facts([], { file_lines: 1000, growth_lines: 50 })).trim(), "Don't grow a file over 1000 lines by more than 50 lines. Put new code next to it.");
     assert.equal(hotspotBuildRule(null), ""); assert.equal(hotspotBuildRule(facts([], { enabled: false })), "");
     const plan = builderPlanPrompt(7) + hotspotPlanNote(facts([HOT, { ...HOT, path: "a/b.py" }, { ...HOT, path: "c/d.py" }]));
-    // 1850: #4404 (verify out_of_band) and #4396 (split size rule) grew the plan prompt to ~1803 tokens on CI paths.
-    assert.ok(approxTokens(plan) <= 1850, `plan prompt with hotspots is ${approxTokens(plan)} tokens`);
-    assert.ok(approxTokens(builderImplementPrompt() + hotspotBuildRule(facts([]))) <= 1500);
+    // 1850: #4404 (verify out_of_band) and #4396 (split size rule) grew the plan prompt to ~1803 tokens on CI paths; 1900: #4589 (check timeout is not a block, one block-rule line).
+    assert.ok(approxTokens(plan) <= 1900, `plan prompt with hotspots is ${approxTokens(plan)} tokens`);
+    assert.ok(approxTokens(builderImplementPrompt() + hotspotBuildRule(facts([]))) <= 1550); // 1550: #4591 (never hand-name migration files, one rule line)
   });
 
   it("#4519 generated_paths: both prompts name the globs (even with the gate off), and an amendment adding one blocks", async () => {
@@ -2103,6 +2106,9 @@ describe("prompts", () => {
     const p = reviewerPrompt({ issue: { number: 7, title: "t", body: "b" }, comments: [{ body: "please also cover bar" }, { body: "<!-- FORGE:STARTED -->\nx" }], contract: CONTRACT, result: { status: "fixed" }, files: ["src/foo.js"], diff: "x".repeat(2000) });
     for (const t of ["<issue_body>", "<issue_comments>\nplease also cover bar\n</issue_comments>", "<contract>", '"R1"', "<builder_result>", "<changed_files>\nsrc/foo.js", "<diff>"]) assert.ok(p.includes(t), t);
     assert.doesNotMatch(p, /FORGE:STARTED/);
+    const am = reviewerPrompt({ issue: { number: 7, title: "t", body: "b" }, contract: { ...CONTRACT, requirements: [{ ...CONTRACT.requirements[0], amendment: { text: "reveal once", why: "hash-only token" } }] }, diff: "" });
+    assert.ok(am.includes("reveal once") && am.includes("hash-only token"));
+    assert.match(REVIEWER_SYSTEM, /amendment\.text/);
     assert.ok(approxTokens(REVIEWER_SYSTEM) + approxTokens(reviewerPrompt({ issue: { number: 7, title: "t", body: "b" }, contract: CONTRACT, diff: "" })) <= 1800);
   });
 
@@ -2243,6 +2249,51 @@ describe("orchestrate", () => {
     assert.deepEqual(rows.map((r) => [r.issue, r.outcome, r.detail]), [[9, "needs-human", "ci.red"], [1, "skipped", "dependency #9 needs-human"]]);
   });
 
+  it("#4595 a dependent skipped for a needs-human blocker is re-admitted when the blocker merges meanwhile; its dependents follow", async () => {
+    const log = [];
+    let merged = false;
+    const rows = await orchestrate({ issues: [9, 1, 2], parallel: 1,
+      readBody: async (n) => ({ 1: "**Depends on**: #9", 2: "**Depends on**: #1" })[n] ?? "",
+      dispatchCheck: async (n) => (n === 9 && merged ? { outcome: "merged", pr: 90, detail: "PR #90 already merged" } : null),
+      runOne: async (n) => { log.push(n); if (n === 9) { merged = true; return { terminalReason: "needs-human", detail: "ci.red" }; } return { terminalReason: "merged", pr: 100 + n }; } });
+    assert.deepEqual(log, [9, 1, 2]);
+    assert.deepEqual(rows.map((r) => [r.issue, r.outcome]), [[9, "merged"], [1, "merged"], [2, "merged"]]);
+    assert.equal(rows[0].pr, 90);
+  });
+
+  it("#4595 an out-of-set parked blocker that closes re-admits the skipped issue", async () => {
+    const log = [];
+    let closed = false;
+    const rows = await orchestrate({ issues: [1], parallel: 1, readBody: async () => "",
+      issueLabels: async () => ["needs-human"],
+      dispatchCheck: async (n) => (n === 9 && closed ? { outcome: "invalid", detail: "issue closed" } : null),
+      runOne: async (n) => { log.push(n); if (log.length === 1) { closed = true; return { terminalReason: "dependency-open", dependencies: [9] }; } return { terminalReason: "merged" }; } });
+    assert.deepEqual(log, [1, 1]);
+    assert.deepEqual(rows.map((r) => [r.issue, r.outcome]), [[1, "merged"]]);
+  });
+
+  it("#4614 an out-of-set blocker that dispatchCheck reports merged but the lane still sees open re-admits once, then stays skipped", async () => {
+    const log = [];
+    let checks = 0;
+    const rows = await orchestrate({ issues: [20], parallel: 1, readBody: async () => "",
+      issueLabels: async () => ["needs-human"],
+      dispatchCheck: async (n) => { checks++; return n === 10 ? { outcome: "merged", pr: 77, detail: "PR #77 already merged" } : null; },
+      runOne: async (n) => { log.push(n); if (log.length > 5) throw new Error("re-admission loop"); return { terminalReason: "dependency-open", dependencies: [10] }; } });
+    assert.deepEqual(log, [20, 20], "one re-admission, no loop");
+    assert.ok(checks <= 3, `bounded dispatch checks (${checks})`);
+    assert.deepEqual(rows.map((r) => [r.issue, r.outcome]), [[20, "skipped"]]);
+  });
+
+  it("#4595 an unreadable or still-open blocker leaves the dependent skipped; no dispatchCheck changes nothing", async () => {
+    for (const dispatchCheck of [async () => { throw new Error("gh down"); }, async () => null, undefined]) {
+      const log = [];
+      const rows = await orchestrate({ issues: [9, 1], parallel: 1, readBody: async (n) => (n === 1 ? "**Depends on**: #9" : ""), ...(dispatchCheck ? { dispatchCheck } : {}),
+        runOne: async (n) => { log.push(n); return { terminalReason: "needs-human", detail: "ci.red" }; } });
+      assert.deepEqual(log, [9]);
+      assert.deepEqual(rows.map((r) => [r.issue, r.outcome]), [[9, "needs-human"], [1, "skipped"]]);
+    }
+  });
+
   it("#4124 a repeated dependency-open stops after the requeue cap", async () => {
     const h = harness({}, { 1: "dependency-open" });
     const clock = { t: 0 }, starts = [];
@@ -3724,6 +3775,28 @@ describe("lane resolution in core work-on (#4125)", () => {
   });
 });
 
+describe("shipped lane follows the open PR base (#4593)", () => {
+  const pr = (base) => [{ number: 77, headRefOid: "aaaa1111aaaa1111", baseRefName: base }];
+  const go = (o) => run(world({ milestone: "Old", branchesOnOrigin: ["staging", "milestone/old", "milestone/new"], remote: "abc\trefs/heads/feat/issue-7-lean", ...o }), { lane: undefined });
+  it("proceeds on the PR base when the issue's lane is shipped", async () => {
+    const w = world({ milestone: "Old", branchesOnOrigin: ["staging", "milestone/old", "milestone/new"], shipped: ["milestone/old"], prList: pr("milestone/new"), remote: "abc\trefs/heads/feat/issue-7-lean" });
+    const r = await run(w, { lane: undefined });
+    assert.notEqual(r.terminalReason, "needs-human");
+    assert.equal(r.terminalReason, "merged");
+    assert.ok(w.calls.some((c) => c[0] === "git" && c.includes("origin/milestone/new..refs/heads/feat/issue-7-lean")), "work is compared against the PR base");
+  });
+  it("stays needs-human when the lane is not shipped, the base is shipped, missing, or the default branch", async () => {
+    for (const [o, code] of [
+      [{ prList: pr("milestone/new") }, "lane.mismatch"],
+      [{ shipped: ["milestone/old", "milestone/new"], prList: pr("milestone/new") }, "lane.mismatch"],
+      [{ shipped: ["milestone/old"], prList: pr("milestone/gone") }, "lane.mismatch"],
+      [{ shipped: ["milestone/old"], prList: pr("main") }, "lane.default_branch"]]) {
+      const r = await go(o);
+      assert.deepEqual([r.terminalReason, r.code], ["needs-human", code], JSON.stringify(o));
+    }
+  });
+});
+
 describe("lane entry points (#4125)", () => {
   const quiet = async (fn) => { const l = console.log; console.log = () => {}; try { return await fn(); } finally { console.log = l; } };
   const deps = () => ({ dir: tmpDir("core-cli-"), emitTelemetry: async () => {}, leanIo: {}, io: { gh: async (a) => (a.includes("defaultBranchRef") ? "main\n" : "o/r\n") },
@@ -3964,7 +4037,7 @@ describe("verify-by-CI (#4167): ci mode runs the heavy checks on CI and feeds fa
     assert.equal(readVerify(d), undefined);
     writeFileSync(`${d}/forge.yaml`, "verify:\n  mode: ci\n  workflow: verify.yml\n  skip_paths: \"*.md,docs/**\"\n  max_rounds: 3\n  precheck_timeout_seconds: 60\n  e2e_label: full-ci\n  e2e_checks: \"e2e (1),e2e (2)\"\n");
     const e2e = { mode: "auto", label: "full-ci", workflow: null, pathsInput: null, checks: ["e2e (1)", "e2e (2)"], paths: ["e2e/**", "**/e2e/**", "*.e2e.*", "*.e2e-spec.*", "cypress/**", "playwright/**"], blocking: "changed" };
-    assert.deepEqual(readVerify(d), { mode: "ci", workflow: "verify.yml", pathsInput: null, skipPaths: ["*.md", "docs/**"], maxRounds: 3, precheckTimeoutSeconds: 60, precheckTotalSeconds: 600, checkRun: false, checks: null, postMerge: null, duplicateExports: true, e2e });
+    assert.deepEqual(readVerify(d), { mode: "ci", workflow: "verify.yml", pathsInput: null, skipPaths: ["*.md", "docs/**"], maxRounds: 3, precheckTimeoutSeconds: 60, precheckTotalSeconds: 600, checkRun: false, checks: null, postMerge: null, flaky: [], duplicateExports: true, invariants: [], e2e });
   });
 
   describe("#4387 a no-commit CI fix round reruns the failed jobs once", () => {
@@ -4003,6 +4076,31 @@ describe("verify-by-CI (#4167): ci mode runs the heavy checks on CI and feeds fa
       assert.deepEqual(w.reran, ["77"]); assert.equal(w.merges.length, 0);
       assert.deepEqual(flakeEvents(dir).map((e) => e.result), ["fail"]);
     });
+
+    /** #4586: count the builder CI-fix prompts of a world. */
+    const fixPrompts = (w) => { const n = { n: 0 }; const inner = w.io.runner; w.io.runner = async (c) => { if (/CI failed on the pushed branch/.test(c.prompt)) n.n++; return inner(c); }; return n; };
+
+    it("#4586 a flake that goes green on the rerun spends no fix round and never resumes the builder", async () => {
+      const w = flaky("success"), n = fixPrompts(w), dir = tmpDir("core-ci-");
+      const r = await run(w, { verify: PUSH, dir });
+      assert.equal(r.terminalReason, "merged"); assert.equal(n.n, 0); assert.deepEqual(w.reran, ["77"]);
+    });
+
+    it("#4586 a failure that repeats on the same head after the rerun counts one round and reaches the builder", async () => {
+      const w = flaky("failure"), n = fixPrompts(w), dir = tmpDir("core-ci-");
+      const r = await run(w, { verify: PUSH, dir });
+      assert.equal(r.code, "ci.red"); assert.ok(n.n >= 1);
+      assert.equal(readLog(dir, 7).filter((e) => e.event === "CI_FIX_ROUND").length, 1, "one counted round");
+      assert.deepEqual(w.reran, ["77"], "one rerun per head, none after the round");
+    });
+
+    it("#4586 verify.flaky: a listed failing job is reported, never reruns, never feeds the builder, and the lane merges", async () => {
+      const w = flaky("failure"), n = fixPrompts(w), dir = tmpDir("core-ci-");
+      w.io.checkStates = async () => [{ name: "test", state: "success", url: RUN, head: w.head }];
+      const r = await run(w, { verify: { ...PUSH, flaky: ["test"] }, dir });
+      assert.equal(r.terminalReason, "merged"); assert.equal(n.n, 0); assert.deepEqual(w.reran, []);
+      assert.deepEqual(readLog(dir, 7).filter((e) => e.event === "CI_FLAKY_REPORTED").map((e) => e.jobs), [["test"]]);
+    });
   });
 });
 
@@ -4392,6 +4490,25 @@ describe("local-mode CI fix rounds at merge and declared pre-PR check commands (
     assert.ok(log.findIndex((e) => e.event === "PRECHECK_COMMANDS") < log.findIndex((e) => e.event === "PR_OPENED"));
   });
 
+  it("#4585 verify.invariants run in the pre-PR round even with no learned commands, and the prompts carry the checklist", async () => {
+    const w = world(); const ran = [];
+    w.io.runCommand = async ({ command }) => { ran.push(command); return { rc: 0, output: "" }; };
+    const r = await run(w, { verify: { invariants: ["bash scripts/inv.sh"] } });
+    assert.equal(r.terminalReason, "merged");
+    assert.deepEqual(ran, ["bash scripts/inv.sh"]);
+    const b = builders(w);
+    assert.match(b[0][4], /Repo-wide invariants[\s\S]*bash scripts\/inv\.sh/); assert.match(b[1][4], /Invariants checklist[\s\S]*bash scripts\/inv\.sh/);
+    const w2 = world(); let n = 0; w2.io.runCommand = async () => { n++; return { rc: 0, output: "" }; };
+    await run(w2); assert.equal(n, 0, "no invariants: nothing runs");
+  });
+
+  it("#4585 a failing invariant resumes the builder like a failed check command", async () => {
+    const w = world(); let n = 0;
+    w.io.runCommand = async () => (n++ ? { rc: 0, output: "" } : { rc: 1, output: "registry missing table x" });
+    await run(w, { verify: { invariants: ["bash scripts/inv.sh"] } });
+    assert.ok(n >= 2, "the invariant is rerun after the builder fix round");
+  });
+
   it("pre-PR: a passing set adds no builder turn; no commands configured runs nothing", async () => {
     const w = world(); let n = 0;
     w.io.runCommand = async () => { n++; return { rc: 0, output: "" }; };
@@ -4430,7 +4547,7 @@ describe("#4364 in-session forgedock check: plan hand-off and self-check record"
   it("writes check-plan.json beside the contract and the implement prompt asks for forgedock check", async () => {
     const w = world(); w.io.runCommand = async () => ({ rc: 0, output: "" });
     assert.equal((await run(w, { precheck: { commands: [" make lint ", ""], maxMinutes: 5 }, verify: { mode: "hybrid" } })).terminalReason, "merged");
-    assert.deepEqual(JSON.parse(w.files[`${WT}/.forge-lean/check-plan.json`]), { v: 1, base: "origin/staging", mode: "hybrid", learned: ["make lint"], maxMinutes: 5, precheckTotalSeconds: 600, precheckTimeoutSeconds: 300 });
+    assert.deepEqual(JSON.parse(w.files[`${WT}/.forge-lean/check-plan.json`]), { v: 1, base: "origin/staging", mode: "hybrid", learned: ["make lint"], invariants: [], maxMinutes: 5, precheckTotalSeconds: 600, precheckTimeoutSeconds: 300 });
     assert.match(builders(w)[1][4], /forgedock\.mjs check` in the worktree[\s\S]*report the problem as blocked/);
   });
 
@@ -5082,6 +5199,23 @@ describe("enumerating tests + sequence numbers at hand-off (#4322)", () => {
     assert.equal(builders(w).length, 2);
   });
 
+  const HAND = ["db/2026_x.sql: not produced by the migration generator (no journal entry added for it)"];
+  it("#4591 a hand-named migration resumes the builder once with the generator rule; a remainder is an anomaly; clean costs no round", async () => {
+    const w = hworld(); const d = [HAND, []];
+    w.io.migrationGenerator = async (x) => { w.genArgs = x; return d.length > 1 ? d.shift() : d[0]; };
+    assert.equal((await run(w)).terminalReason, "merged");
+    const b = builders(w);
+    assert.equal(b.length, 3);
+    assert.match(b[2][4], /db\/2026_x\.sql[\s\S]*Never name or number migration files by hand/);
+    assert.equal(w.genArgs.cwd, WT);
+    const w2 = hworld(); w2.io.migrationGenerator = async () => HAND;
+    assert.equal((await run(w2)).terminalReason, "merged");
+    assert.match(traj(w2)[0], /Hand-named migration\(s\) remain \(hand-off\)/);
+    const w3 = hworld(); w3.io.migrationGenerator = async () => [];
+    await run(w3);
+    assert.equal(builders(w3).length, 2);
+  });
+
   it("no enumerating tests and no collisions: no new plan step, no extra builder call (prompts unchanged)", async () => {
     const w = hworld();
     assert.equal((await run(w)).terminalReason, "merged");
@@ -5184,6 +5318,31 @@ describe("enumerating tests + sequence numbers at hand-off (#4322)", () => {
       assert.match(w.comments.find((c) => c.includes("<!-- FORGE:BASE_SYNC -->")), /deterministic: base side taken[\s\S]*renumbered deterministically[\s\S]*not required \(deterministic renumber \+ base merge\)/);
     });
 
+    it("#4588: base-sync conflicts the engine resolves (union / JSON map / generated) need no builder, re-review or sequence resolver", async () => {
+      const w = dworld({ seq: [[], []], prView: [CONFLICTING, OKSTATE], mergeResults: ["conflict"], conflictFiles: ["src/index.ts", "i18n/status.json"] }), asked = [];
+      w.io.conflictResolve = async (x) => { asked.push(x); w.head = "eeee0000eeee0000"; w.conflictFiles = []; w.mergeInProgress = false; return { resolved: x.files, remaining: [] }; };
+      assert.equal((await run(w, { baseMerge: { unionGlobs: ["src/index.ts"], jsonGlobs: ["i18n/status.json"], generatedGlobs: [], generatedCommand: "" } })).terminalReason, "merged");
+      assert.deepEqual(asked[0].files, ["src/index.ts", "i18n/status.json"]);
+      assert.equal(asked[0].cfg.jsonGlobs[0], "i18n/status.json");
+      assert.equal(w.resolveArgs, undefined, "sequence resolver not asked");
+      assert.equal(builders(w).length, 2, "plan + implement only");
+      assert.equal(reviewers(w).length, 1, "no re-review");
+      assert.match(w.comments.find((c) => c.includes("<!-- FORGE:BASE_SYNC -->")), /deterministic: 2 union \/ JSON-map \/ generated file/);
+    });
+
+    it("#4588: files the engine cannot resolve go to the sequence resolver, and the builder when that fails; without config nothing changes", async () => {
+      const w = dworld({ prView: [CONFLICTING, OKSTATE], mergeResults: ["conflict"], conflictFiles: ["src/index.ts", "db/meta/_journal.json"], resolve: { unsupported: "x" } });
+      w.io.conflictResolve = async () => ({ resolved: ["src/index.ts"], remaining: ["db/meta/_journal.json"] });
+      assert.equal((await run(w, { baseMerge: { unionGlobs: ["src/index.ts"] } })).terminalReason, "merged");
+      assert.deepEqual(w.resolveArgs.files, ["db/meta/_journal.json"]);
+      assert.ok(builders(w).length >= 3, "the builder finishes the merge");
+      const n = dworld({ prView: [CONFLICTING, OKSTATE], mergeResults: ["conflict"], conflictFiles: ["db/meta/_journal.json"] });
+      let called = false; n.io.conflictResolve = async () => { called = true; return { resolved: [], remaining: [] }; };
+      await run(n);
+      assert.equal(called, false, "no baseMerge config: resolver not consulted");
+      assert.deepEqual(n.resolveArgs.files, ["db/meta/_journal.json"]);
+    });
+
     it("a schema-changing migration without lean.sequence.regenerate falls back to the builder renumber and the trajectory records why", async () => {
       const w = dworld({ seq: [COLL, []], det: [{ unsupported: "schema-changing migration db/0007_usage.sql needs lean.sequence.regenerate (its snapshot cannot be cloned)" }] });
       assert.equal((await run(w, { sequenceRegenerate: null })).terminalReason, "merged");
@@ -6222,3 +6381,36 @@ describe("#4450 light lane: one builder session, derived contract, upgrade to th
     assert.equal(events(dir, "LIGHT_DONE").length + events(dir, "LIGHT_UPGRADE").length, 0);
   });
 });
+
+describe("tool state never rides the lane branch (#4590)", () => {
+  const untracks = (w) => w.calls.filter((c) => c[0] === "git" && c[1] === "rm" && c[2] === "--cached");
+  const subjects = (w) => w.calls.filter((c) => c[0] === "git" && c[1] === "commit" && c.some((x) => /untrack lane tool state/.test(x)));
+
+  it("a branch that added .forge-lean/ files and tool caches is untracked and committed before the PR push", async () => {
+    const dir = tmpDir("toolstate-");
+    const w = world({}); w.added = ["src/foo.js", ".forge-lean/result.json", "apps/worker/.eslintcache", "packages/a/tsconfig.tsbuildinfo"];
+    assert.equal((await run(w, { dir })).terminalReason, "merged");
+    const rm = untracks(w);
+    assert.ok(rm.length >= 1, "git rm --cached ran");
+    for (const f of [".forge-lean/result.json", "apps/worker/.eslintcache", "packages/a/tsconfig.tsbuildinfo"]) assert.ok(rm[0].includes(f), f);
+    assert.ok(!rm[0].includes("src/foo.js"), "product files stay");
+    assert.equal(subjects(w).length, 1, "one untrack commit; the next push finds the branch clean");
+    const firstPush = w.calls.findIndex((c) => c[0] === "git" && c[1] === "push");
+    assert.ok(w.calls.findIndex((c) => c === subjects(w)[0]) < firstPush, "committed before the push");
+    assert.ok(readLog(dir, 7).some((e) => e.event === "TOOLSTATE_UNTRACKED"));
+  });
+
+  it("a clean branch makes no untrack commit", async () => {
+    const w = world({}); w.added = ["src/foo.js"];
+    assert.equal((await run(w)).terminalReason, "merged");
+    assert.equal(untracks(w).length, 0);
+    assert.equal(subjects(w).length, 0);
+  });
+});
+
+describe("block rules: a check timeout is not a block (#4589)", () => {
+  it("tells the builder rc 124 is host load, never a human block", async () => {
+    const { blockCorrectionPrompt } = await import("./prompts.mjs");
+    assert.match(blockCorrectionPrompt("x"), /times out \(rc 124\) is host load, never a block/);
+  });
+});

package/bin/lean/orchestrate.mjs

@@ -7,6 +7,7 @@
  * `decomposed` run adds its sub-issues and its dependents wait on them instead (#4124). A dependency that is parked
  * (ended needs-human in this batch, or an out-of-set issue labelled needs-human per `issueLabels(n)`) is never pulled in:
  * the issue ends `skipped` (`dependency #N needs-human`) and its dependents with it (#4387).
+ * A skipped dependent whose blocker later closes or merges outside its lane is re-admitted (#4595, needs `dispatchCheck`).
  * No investigation, synthesis or agent spawning. `runOne(issue)` and `readBody(issue)` are injected.
  * Optional `prefilter(issue, lane)` (#4123) runs in the lane slot before `runOne`: when it closed the issue, no lane starts.
  * When it parked the issue (#4389, `parked: true`), no lane starts either and the row ends with the parked kind.
@@ -28,6 +29,8 @@
  * `getAdded()` (#4408, `forgedock orchestrate --add`) is read on every pass too: issues it returns that are not in the batch
  * join it through `addIssues` (the still-pending issues' `Depends on` are re-read first, so edges hold both ways), with one
  * batch decision and one `added` row; while it is set, lanes running with an empty queue still wake every `pollMs`.
+ * `getHeld()` (#4592, `--hold`/`--release`) is read on every pass: the issues it returns are never admitted (a running lane is not stopped);
+ * when only held issues (and their dependents) remain and nothing runs, the batch ends with them `held` ("held by operator"), and a `--resume <id> <issue...>` naming them (or `--release`) re-admits them.
  *
  * Maintenance budget (#4197), off unless `maintenance: {share, whenIdle}` is passed: `classify(issue)` gives each issue
  * `{maintenance, priority}` (persisted as `classes`) and admission follows `pickNext`: P0/P1 features, then a maintenance
@@ -104,6 +107,7 @@ import { laneSize, SIZES } from "./budget.mjs";
 
 // A dependency that finished like this no longer blocks its dependents.
 const SATISFIED = new Set(["merged", "invalid"]);
+const WATCH_EVERY_MS = 60_000; // #4595
 // Times one issue may be requeued behind a dependency before its dependency-open row is final.
 export const MAX_REQUEUES = 2;
 // Deferrals other than a busy resource (UNCAPPED_DEFERRALS) one issue may take before its deferred row is final.
@@ -232,7 +236,7 @@ const toMap = (o) => new Map(Object.entries(o ?? {}).map(([k, v]) => [Number(k),
 export async function orchestrate({ issues, parallel = 3, runOne, readBody, prefilter = null, resolveLane = null, now = () => Date.now(), onUpdate = () => {}, onCrossLane = () => {},
   store = null, adopt = null, getParallel = null, getAdded = null, breaker = null, maxDeferrals = DEFAULT_MAX_DEFERRALS, sleep = cancellableSleep, pollMs = 5000,
   classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null,
-  recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, getCancelled = null, stopLane = null,
+  recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, getCancelled = null, getHeld = null, stopLane = null,
   estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null, dispatchCheck = null, syncParked = null, syncEveryMs = DEFAULT_SYNC_EVERY_MS }) {
   const saved = store?.load?.() ?? null;
   const order = saved ? [...saved.order] : [];
@@ -257,6 +261,8 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   const holds = limits ?? memoryLimits(now);
   const heldUntil = () => { try { const id = identity(); const u = holds.holdUntil(id); return Number.isFinite(u) && u > now() ? u : null; } catch { return null; } };
   let tripped = saved?.breaker ?? null;
+  let opHeld = new Set(); // #4592: operator-held issues this pass (the batch `hold` file is the authority)
+  const opHeldHit = new Set(); // held issues that were pending this pass
   // #4376: operator cancel (the batch `cancel` file through `getCancelled`); persisted, but the file stays the authority.
   let cancelled = getCancelled ? (saved?.cancelled ?? null) : null;
   const budget = maintenance && Number(maintenance.share) > 0 ? { share: Number(maintenance.share), whenIdle: !!maintenance.whenIdle } : null;
@@ -736,7 +742,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   }
   save();
 
-  const heldDetail = (n) => cancelled ? "cancelled by operator" : tripped?.detail ?? (driftHeld.has(n) ? `drift: ${driftHeld.get(n)} flagged` : budgetStop ? `execution budget: ${budgetStop}` : budgetHeld ? "maintenance share" : "not run");
+  const heldDetail = (n) => cancelled ? "cancelled by operator" : opHeld.has(n) ? "held by operator" : tripped?.detail ?? (driftHeld.has(n) ? `drift: ${driftHeld.get(n)} flagged` : budgetStop ? `execution budget: ${budgetStop}` : budgetHeld ? "maintenance share" : "not run");
   // #4315: a recovery queued but never launched (the batch stopped: breaker, budget, hold; or a resume that did not relaunch a
   // lost run) left the issue without needs-human, since a non-final attempt does not relabel. Relabel it once, with the full reason.
   // #4318: also on a throw out of the loop and on a signal (`onExit`); `cause` overrides the held reason. Runs once per process.
@@ -783,6 +789,24 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     stopSent.add(n);
     try { Promise.resolve(stopLane(n, rec)).catch(() => {}); } catch { /* best effort */ }
   }
+  // #4592: re-read the hold set; one decision and one row when it changes.
+  const refreshHeld = () => {
+    if (!getHeld) return;
+    let got = [];
+    try { got = getHeld() ?? []; } catch { got = opHeld.size ? [...opHeld] : []; }
+    const next = new Set((Array.isArray(got) ? got : []).filter((n) => Number.isSafeInteger(n) && n > 0));
+    const fmt = (ns) => ns.map((n) => `#${n}`).join(", ");
+    const held = [...next].filter((n) => !opHeld.has(n)), freed = [...opHeld].filter((n) => !next.has(n));
+    opHeld = next;
+    if (held.length) {
+      decided({ issue: null, what: `held ${fmt(held)} (forgedock orchestrate --hold)`, why: "operator request", alternative: "cancel the batch" });
+      update({ issue: null, outcome: "hold", detail: fmt(held) });
+    }
+    if (freed.length) {
+      decided({ issue: null, what: `released ${fmt(freed)} (forgedock orchestrate --release)`, why: "operator request", alternative: "resume the batch" });
+      update({ issue: null, outcome: "release", detail: fmt(freed) });
+    }
+  };
   const checkCancel = () => {
     if (!getCancelled) return;
     let c = null;
@@ -811,25 +835,71 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     halted = true;
     return Object.assign(new Error("batch stopped: schedd shutting down"), { code: "batch.stopped" });
   };
+  // #4595: a skipped row whose blocker (`dependency #B ...`) was finished outside the lane is re-admitted: B is re-checked
+  // through `dispatchCheck` (at most every WATCH_EVERY_MS, and once before the batch would end); fail-open.
+  let lastWatch = null;
+  const readmitted = new Set(); // "n:b": re-admit n on blocker b once; a lane that re-skips on b stays skipped
+  const blockerOf = (n) => { const r = results.get(n); const m = r?.outcome === "skipped" ? /^dependency #(\d+) /.exec(r.detail ?? "") : null; return m ? Number(m[1]) : null; };
+  const watchable = () => !!dispatchCheck && [...results.keys()].some((n) => blockerOf(n) !== null);
+  const watchBlockers = async (final) => {
+    if (!watchable() || cancelled || tripped) return;
+    const t0 = now();
+    if (!final && lastWatch !== null && t0 - lastWatch < WATCH_EVERY_MS) return;
+    lastWatch = t0;
+    const resolved = new Set();
+    const roots = new Set([...results.keys()].filter((n) => !readmitted.has(`${n}:${blockerOf(n)}`)).map(blockerOf).filter((b) => b !== null && blockerOf(b) === null));
+    for (const b of roots) {
+      let g = null;
+      try { g = (await dispatchCheck(b)) ?? null; } catch { g = null; }
+      if (!g || !SATISFIED.has(g.outcome)) continue;
+      resolved.add(b);
+      const row = results.get(b);
+      if (row) { results.set(b, { ...row, outcome: g.outcome, pr: Number.isInteger(g.pr) ? g.pr : row.pr, detail: g.detail ?? row.detail }); update(results.get(b)); }
+    }
+    const back = new Set();
+    for (let grew = true; grew;) {
+      grew = false;
+      for (const n of results.keys()) {
+        const b = blockerOf(n);
+        if (b !== null && !back.has(n) && !readmitted.has(`${n}:${b}`) && (resolved.has(b) || back.has(b))) { back.add(n); grew = true; }
+      }
+    }
+    for (const n of back) {
+      const b = blockerOf(n);
+      readmitted.add(`${n}:${b}`);
+      results.delete(n);
+      pending.push(n);
+      decided({ issue: n, what: `re-admitted: dependency #${b} merged or closed after this issue was skipped`, why: "the blocker was finished outside this batch lane",
+        alternative: "leave it skipped until the next batch" });
+    }
+    if (back.size) save();
+  };
   const K = () => { let k = null; try { k = getParallel?.(); } catch { k = null; } return Number.isInteger(k) && k >= 1 ? k : parallel; };
   try {
     for (;;) {
       if (signal?.aborted) throw halt();
       await drainAdds();
       checkCancel();
+      await watchBlockers(!pending.length && !running.size);
       if (!pending.length && !running.size) break;
       // Resolve skips first: a dependency that finished unsatisfied (or never can) skips its dependents.
-      for (let i = pending.length - 1; i >= 0; i--) {
-        const n = pending[i];
-        if (results.has(n)) { pending.splice(i, 1); continue; }
-        const bad = edges(n).find((d) => results.has(d) && !SATISFIED.has(results.get(d).outcome));
-        if (bad !== undefined) { pending.splice(i, 1); skip(n, `dependency #${bad} ended ${results.get(bad).outcome}`); }
+      for (let again = true; again;) { // #4595: to a fixpoint, so a chain's tail is skipped on its blocker, not as a "cycle"
+        again = false;
+        for (let i = pending.length - 1; i >= 0; i--) {
+          const n = pending[i];
+          if (results.has(n)) { pending.splice(i, 1); continue; }
+          const bad = edges(n).find((d) => results.has(d) && !SATISFIED.has(results.get(d).outcome));
+          if (bad !== undefined) { pending.splice(i, 1); skip(n, `dependency #${bad} ended ${results.get(bad).outcome}`); again = true; }
+        }
       }
       const t = now();
       const holdUntil = heldUntil();
       let started = 0;
       driftHeld.clear();
+      opHeldHit.clear();
+      refreshHeld();
       const isReady = (n) => {
+        if (opHeld.has(n)) { opHeldHit.add(n); return false; }
         if (!((notBefore.get(n) ?? 0) <= t && edges(n).every((d) => results.has(d) && SATISFIED.has(results.get(d).outcome)))) return false;
         if (!drift?.holds) return true;
         let pkg = null;
@@ -898,10 +968,12 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       if (holdUntil) waits.push(holdUntil);
       let wake = waits.length ? Math.max(0, Math.min(...waits) - t) : null;
       if (holdUntil && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // #4446: a cleared hold (forgedock limits clear) admits on the next poll
-      if (getParallel && pending.length) wake = Math.min(wake ?? pollMs, pollMs);
+      if ((getParallel || getHeld) && pending.length) wake = Math.min(wake ?? pollMs, pollMs);
       if (offered.size && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // a slot another batch frees
+      if (getHeld && running.size) wake = Math.min(wake ?? pollMs, pollMs); // #4592: a release while only lanes run
       if (getAdded && running.size) wake = Math.min(wake ?? pollMs, pollMs); // #4408: an add while only lanes run
       if (syncParked && running.size && parkedPrs().length) wake = Math.min(wake ?? syncEveryMs, Math.max(0, syncEveryMs - (t - (lastParkedSync ?? t)))); // #4525
+      if (running.size && watchable()) wake = Math.min(wake ?? WATCH_EVERY_MS, WATCH_EVERY_MS); // #4595: re-check the blockers of skipped rows
       if (tripped) wake = null;
       if (getCancelled && running.size && !cancelled) wake = Math.min(wake ?? pollMs, pollMs); // #4376: notice a cancel while lanes run
       if (running.size) {
@@ -911,6 +983,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       else if (pending.length && offered.size) await nap(wake); // waiting for a host slot
       else if (pending.length && waits.length) await nap(Math.min(wake, getCancelled ? pollMs : Infinity)); // wake is poll-capped by a hold or a live K; #4454: nap wakes on a stop
       else if (pending.length && budgetHeld) { if (await drainAdds()) continue; break; } // maintenance the budget does not admit now: held for a later batch
+      else if (pending.length && opHeldHit.size && !pending.some((n) => !opHeldHit.has(n) && isReady(n))) { if (await drainAdds()) continue; break; } // #4592: held by the operator: pending for a --resume
       else if (pending.length && driftHeld.size) { if (await drainAdds()) continue; break; } // held by a drift flag (block_dispatch): pending for a later batch
       else if (pending.length) { // nothing runnable, nothing running, nothing waiting on time: a dependency cycle
         for (const n of pending.splice(0)) skip(n, "dependency cycle");

package/bin/lean/precheck.mjs

@@ -10,7 +10,8 @@
  * which the builder runs in-session (`selfCheck`); `worktreeFingerprint` tells whether that run saw the final tree.
  */
 import { createHash } from "node:crypto";
-import { scopeChecks, selectCommands } from "./testmap.mjs";
+import { loadavg, cpus } from "node:os";
+import { scopeChecks, selectCommands, mergeChecks } from "./testmap.mjs";
 import { LOG_CAP } from "./verify.mjs";
 
 /** #4364: the engine's check plan for `forgedock check` (written beside contract.json) and the CLI's last run record. */
@@ -22,6 +23,43 @@ export const CHECK_LOG_DIR = ".forge-lean/logs";
 /** rc of a command killed by its timeout (effects.mjs runCheckCommand). */
 const TIMEOUT_RC = 124;
 
+/** Ceiling of a load-scaled retry timeout. */
+const MAX_RETRY_MS = 3600000;
+
+/** Host load per core, from the OS; the default `load` of `runWithTimeoutRetry`. */
+const hostLoad = () => ({ load1: loadavg()[0], cores: cpus().length || 1 });
+
+/**
+ * #4589: the timeout of the one retry after a command hit its own timeout: `perMs * max(2, ceil(load1 / cores))`, at most
+ * 3600 s and never below `perMs`. Deterministic; a timed-out command has no measured duration, so load scales it.
+ */
+export function retryTimeoutMs({ perMs, load1, cores }) {
+  const per = Number(perMs);
+  if (!(per > 0) || !Number.isFinite(per)) return per;
+  const factor = Math.max(2, Math.ceil((Number(load1) || 0) / (Number(cores) > 0 ? Number(cores) : 1)));
+  return Math.max(per, Math.min(per * factor, MAX_RETRY_MS));
+}
+
+/**
+ * #4589: a check timeout (rc 124) is a busy host, not a failing command: run `run(timeoutMs)`, and when it timed out on its
+ * own, once more with `retryTimeoutMs`. `left` (ms still available in a round budget) caps the retry; a retry the budget cut
+ * short is `cut` (left to CI). A repeated timeout keeps rc 124 and says it was infrastructure. Other rcs are never retried.
+ * @param {(timeoutMs:number) => Promise<{rc:number, output?:string}>} run
+ * @returns {Promise<{r:{rc:number, output?:string}, retried:boolean, cut:boolean}>}
+ */
+export async function runWithTimeoutRetry(run, timeoutMs, { left = () => Infinity, load = hostLoad } = {}) {
+  const r = await run(timeoutMs);
+  if (r?.rc !== TIMEOUT_RC || !(timeoutMs > 0) || !Number.isFinite(timeoutMs)) return { r, retried: false, cut: false };
+  let want = timeoutMs;
+  try { want = retryTimeoutMs({ perMs: timeoutMs, ...load() }); } catch { /* no load reading: the minimum factor */ want = timeoutMs * 2; }
+  const t2 = Math.min(want, left());
+  if (!(t2 > timeoutMs)) return { r, retried: false, cut: t2 < want }; // no budget left to retry in: CI runs it
+  const r2 = await run(t2);
+  const cut = r2?.rc === TIMEOUT_RC && t2 < want;
+  if (r2?.rc === TIMEOUT_RC && !cut) return { r: { ...r2, output: `${String(r2.output ?? "")}\n(infra timeout: host load, not a code failure; retried once with ${Math.round(t2 / 1000)}s)` }, retried: true, cut };
+  return { r: r2, retried: true, cut };
+}
+
 /**
  * @param {{commands:string[], parallelism?:number, budgetMs:number, perCommandMs:number, now?:() => number,
  *   acquire?:() => Promise<{release?:Function}|null>,
@@ -30,7 +68,7 @@ const TIMEOUT_RC = 124;
  * @returns {Promise<{results:{command:string, rc:number, output:string, ms:number}[], deferred:string[], ms:number}>}
  *   results in input order (commands that ran and were not cut by the budget); deferred in input order
  */
-export async function runChecks({ commands, parallelism = 1, budgetMs, perCommandMs, now = () => Date.now(), acquire, run }) {
+export async function runChecks({ commands, parallelism = 1, budgetMs, perCommandMs, now = () => Date.now(), acquire, run, load }) {
   const list = Array.isArray(commands) ? commands : [];
   const workers = Math.max(1, Math.floor(Number(parallelism)) || 1);
   const k = Math.max(1, Math.min(workers, list.length)), share = Math.max(1, Math.floor(workers / k));
@@ -44,7 +82,13 @@ export async function runChecks({ commands, parallelism = 1, budgetMs, perComman
     if (left <= 0) return { deferred: true };
     const timeoutMs = Math.min(per, left), t = now();
     let r;
-    try { r = await run({ command, timeoutMs, workers: share }); } catch (e) { r = { rc: 127, output: String(e?.message ?? e) }; }
+    try {
+      if (timeoutMs >= per) { // #4589: its own timeout, not the budget's: one longer retry
+        const x = await runWithTimeoutRetry((ms) => run({ command, timeoutMs: ms, workers: share }), timeoutMs, { left: () => deadline - now(), ...(load ? { load } : {}) });
+        if (x.cut) return { deferred: true };
+        r = x.r;
+      } else r = await run({ command, timeoutMs, workers: share });
+    } catch (e) { r = { rc: 127, output: String(e?.message ?? e) }; }
     const rc = Number.isInteger(r?.rc) ? r.rc : 1;
     // the budget, not the command's own timeout, cut it short: CI runs it
     if (rc === TIMEOUT_RC && timeoutMs < per) return { deferred: true };
@@ -80,15 +124,19 @@ export function formatPrecheckSummary(rows) {
 /**
  * #4364: the pre-PR commands and budget for a round: `scopeChecks` over the trimmed learned commands (a mapped learned
  * command runs only when its globs match a changed file); budget lean.precheck.max_minutes, else
- * verify.precheck_total_seconds (default 600 s); per command verify.precheck_timeout_seconds.
- * @returns {{commands:string[], source:string, added:number, dropped:string[], selected:number, budgetMs:number, perCommandMs:number}}
+ * verify.precheck_total_seconds (default 600 s); per command verify.precheck_timeout_seconds. #4585: `invariants`
+ * (verify.invariants) are appended unscoped after the selected commands, deduped; the test map never drops them.
+ * @returns {{commands:string[], source:string, added:number, dropped:string[], selected:number, invariants?:number, budgetMs:number, perCommandMs:number}}
  */
-export function planPrecheck({ testMap = null, files = [], learned = [], maxMinutes, verify = {} }) {
+export function planPrecheck({ testMap = null, files = [], learned = [], maxMinutes, verify = {}, invariants = verify?.invariants }) {
   const list = (Array.isArray(learned) ? learned : []).map((c) => String(c ?? "").trim()).filter(Boolean);
-  const m = scopeChecks(testMap, files, list);
+  const s = scopeChecks(testMap, files, list);
+  const inv = (Array.isArray(invariants) ? invariants : []).map((c) => String(c ?? "").trim()).filter(Boolean);
+  const withInv = mergeChecks(inv, s.commands);
+  const m = inv.length ? { ...s, commands: withInv.commands } : s;
   const max = Number(maxMinutes);
   const budgetMs = (max > 0 ? max * 60 : Number(verify.precheckTotalSeconds) || 600) * 1000;
-  return { ...m, selected: testMap ? selectCommands(testMap, files).length : 0, budgetMs, perCommandMs: (Number(verify.precheckTimeoutSeconds) || 300) * 1000 };
+  return { ...m, ...(inv.length ? { invariants: inv.length } : {}), selected: testMap ? selectCommands(testMap, files).length : 0, budgetMs, perCommandMs: (Number(verify.precheckTimeoutSeconds) || 300) * 1000 };
 }
 
 const lines = (out) => String(out ?? "").split("\n").map((x) => x.trim()).filter(Boolean);