forgedock 1.31.0 → 1.32.0

package/bin/lean/lightlane.test.mjs

@@ -0,0 +1,237 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { deriveLightContract, runLightStep, runLightTests, parseNumstat, apiShapeReasons, upgradeNote, lightBuildPrompt, renderLiteReceipt } from "./lightlane.mjs";
+import { validateContract } from "./conformance.mjs";
+
+const TITLE = "foo crashes on null input";
+const ISSUE = `${TITLE}\n\nfoo crashes on null input in src/foo.js.\n\n- foo must handle a null argument without throwing.\n- Existing callers of foo keep working.`;
+const RT = { path: "test/foo.test.js", id: "foo handles null", command: "node --test test/foo.test.js" };
+const RESULT = { status: "fixed", summary: "guard null", requirement: "foo must handle a null argument without throwing.", repro_test: RT, tests: [{ command: "node --test test/foo.test.js", result: "pass" }, { command: "node --test test/bar.test.js", result: "pass" }] };
+const FILES = ["src/foo.js", "test/foo.test.js"];
+const validate = (c) => validateContract(c, ISSUE, { repro: true });
+
+describe("deriveLightContract (#4450)", () => {
+  it("quotes the builder's verbatim sentence, delivers the changed non-test files, validates the repro and the targeted tests", () => {
+    const c = deriveLightContract({ title: TITLE, issueText: ISSUE, result: RESULT, files: [...FILES, ".forge-lean/result.json"] });
+    assert.deepEqual(c.requirements, [{ id: "R1", quote: "foo must handle a null argument without throwing.", disposition: "deliver" }]);
+    assert.deepEqual(c.deliverables.map((d) => d.path), ["src/foo.js"]);
+    assert.deepEqual(c.validate, [{ id: "V1", command: "node --test test/foo.test.js" }, { id: "V2", command: "node --test test/bar.test.js" }]);
+    assert.deepEqual(c.repro_test, RT);
+    assert.equal(validate(c).ok, true, validate(c).problems.join("; "));
+  });
+  it("a sentence that is not in the issue falls back to the title; no repro test is repro_test none with a reason", () => {
+    const c = deriveLightContract({ title: TITLE, issueText: ISSUE, result: { ...RESULT, requirement: "make foo better in every way possible", repro_test: undefined }, files: FILES });
+    assert.equal(c.requirements[0].quote, TITLE);
+    assert.equal(c.repro_test, "none");
+    assert.ok(c.repro_test_reason);
+    assert.deepEqual(c.validate.map((v) => v.command), ["node --test test/foo.test.js", "node --test test/bar.test.js"]);
+  });
+  it("a test-only diff has no deliverable, so the derived contract is invalid", () => {
+    const c = deriveLightContract({ title: TITLE, issueText: ISSUE, result: RESULT, files: ["test/foo.test.js"] });
+    assert.equal(validate(c).ok, false);
+  });
+});
+
+describe("parseNumstat / apiShapeReasons / upgradeNote", () => {
+  it("numstat rows, binary as 0, renames to the new path, the engine dir skipped", () => {
+    assert.deepEqual(parseNumstat("3\t1\tsrc/foo.js\n-\t-\timg.png\n2\t0\tsrc/{a.js => b.js}\n1\t1\told.js => new.js\n4\t0\t.forge-lean/result.json\n").files, [
+      { path: "src/foo.js", added: 3, deleted: 1 }, { path: "img.png", added: 0, deleted: 0 }, { path: "src/b.js", added: 2, deleted: 0 }, { path: "new.js", added: 1, deleted: 1 }]);
+  });
+  it("none is no reason; changes and unknown files are reasons; a missing result is unknown", () => {
+    assert.deepEqual(apiShapeReasons({ status: "none", changes: [], unknown: [] }), []);
+    assert.deepEqual(apiShapeReasons({ status: "changed", changes: [{ file: "a.py", symbol: "ichunked", change: "generator-function" }], unknown: [] }), ["a.py: ichunked generator-function"]);
+    assert.deepEqual(apiShapeReasons({ status: "unknown", changes: [], unknown: [{ file: "x.go", lang: "go", reason: "unsupported-language" }] }), ["x.go: unknown (go), unsupported-language"]);
+    assert.equal(apiShapeReasons(null).length, 1);
+  });
+  it("the upgrade note lists the reasons, escapes markup and is empty without reasons", () => {
+    assert.equal(upgradeNote([]), "");
+    const n = upgradeNote([{ code: "diff_lines", detail: "90 non-test lines (max 40) <!-- x -->" }]);
+    assert.match(n, /Light-lane upgrade/);
+    assert.match(n, /- diff_lines: 90 non-test lines \(max 40\)/);
+    assert.ok(!n.includes("<!--"));
+  });
+  it("the light prompt asks for test first, fix, targeted tests and the derivation fields", () => {
+    const p = lightBuildPrompt(7);
+    assert.match(p, /failing regression test first/);
+    assert.match(p, /"requirement":/);
+    assert.match(p, /"repro_test":/);
+    assert.match(p, /"status":"upgrade"/);
+    assert.match(p, /Host rules/);
+  });
+});
+
+/** A light step whose effects are fakes; `o` overrides any of them. */
+function step(o = {}) {
+  const calls = { build: 0, fix: 0, tests: 0, checkpoint: 0 };
+  const ctx = {
+    title: TITLE, issueText: ISSUE, riskPaths: [], cfg: { mode: "on" }, validate,
+    build: async () => { calls.build++; },
+    readResult: async () => o.result ?? RESULT,
+    checkpoint: async () => { calls.checkpoint++; },
+    changedFiles: async () => o.files ?? FILES,
+    numstat: async () => o.numstat ?? "5\t1\tsrc/foo.js\n20\t0\ttest/foo.test.js\n",
+    runTests: async (cmds) => { calls.tests++; const rc = (o.testRcs ?? [])[calls.tests - 1] ?? 0; return cmds.map((command) => ({ command, rc, output: rc ? "not ok 1" : "" })); },
+    fixTests: async () => { calls.fix++; },
+    reproOf: async () => o.repro ?? { base: "FAILED", head: "RAN" },
+    apiShape: async () => (o.api === undefined ? { status: "none", changes: [], unknown: [] } : o.api),
+    ...(o.ctx ?? {}),
+  };
+  return { ctx, calls };
+}
+const codes = (r) => r.reasons.map((x) => x.code);
+
+describe("runLightStep (#4450)", () => {
+  it("nothing triggers: one build, the derived contract, no fix round", async () => {
+    const { ctx, calls } = step();
+    const r = await runLightStep(ctx);
+    assert.equal(r.upgrade, false, JSON.stringify(r.reasons));
+    assert.equal(calls.build, 1); assert.equal(calls.fix, 0); assert.equal(calls.checkpoint, 1);
+    assert.deepEqual(r.contract.deliverables.map((d) => d.path), ["src/foo.js"]);
+  });
+  const cases = [
+    ["diff_lines", { numstat: "60\t0\tsrc/foo.js\n" }],
+    ["diff_files", { numstat: "1\t0\tsrc/a.js\n1\t0\tsrc/b.js\n1\t0\tsrc/c.js\n", files: ["src/a.js", "src/b.js", "src/c.js", "test/foo.test.js"] }],
+    ["repro_not_red", { repro: { base: "PASSED", head: "RAN" } }],
+    ["tests_failed_twice", { testRcs: [1, 1] }],
+    ["risk_path", { numstat: "1\t0\tsrc/auth/login.js\n", files: ["src/auth/login.js", "test/foo.test.js"] }],
+    ["api_shape", { api: { status: "changed", changes: [{ file: "src/foo.js", symbol: "foo", change: "signature" }], unknown: [] } }],
+    ["api_shape", { api: null }],
+    ["not_fixed", { result: { status: "upgrade", summary: "too big" } }],
+    ["no_commit", { files: [] }],
+    ["contract_invalid", { files: ["test/foo.test.js"], numstat: "3\t0\ttest/foo.test.js\n" }],
+    ["light_error", { ctx: { numstat: async () => { throw new Error("git broke"); } } }],
+  ];
+  for (const [code, o] of cases) {
+    it(`upgrades on ${code}${o.api === null ? " (no detector result)" : ""}`, async () => {
+      const r = await runLightStep(step(o).ctx);
+      assert.equal(r.upgrade, true);
+      assert.ok(codes(r).includes(code), codes(r).join(","));
+      assert.equal(r.contract, null);
+    });
+  }
+  it("one failed test run gets one fix round and stays light when the rerun passes", async () => {
+    const { ctx, calls } = step({ testRcs: [1, 0] });
+    const r = await runLightStep(ctx);
+    assert.equal(r.upgrade, false); assert.equal(calls.fix, 1); assert.equal(r.testFailures, 1);
+  });
+});
+
+describe("#4451 light prechecks run concurrently, capped by host parallelism", () => {
+  /** fake commands that each wait for an explicit release; records start/end order and the max in flight */
+  const fake = () => {
+    const log = [], gates = new Map();
+    let live = 0, max = 0;
+    const run = ({ command }) => new Promise((resolve) => {
+      live++; max = Math.max(max, live); log.push(`start ${command}`);
+      gates.set(command, () => { live--; log.push(`end ${command}`); resolve({ rc: command === "b" ? 1 : 0, output: command }); });
+    });
+    const tick = () => new Promise((r) => setImmediate(r));
+    return { run, log, gates, tick, max: () => max };
+  };
+  it("parallelism >= count: every command starts before the first finishes; results in input order", async () => {
+    const f = fake();
+    const p = runLightTests({ commands: ["a", "b", "c"], parallelism: 4, run: f.run });
+    await f.tick();
+    assert.deepEqual(f.log, ["start a", "start b", "start c"]);
+    for (const c of ["c", "a", "b"]) { f.gates.get(c)(); await f.tick(); }
+    const r = await p;
+    assert.deepEqual(r, [{ command: "a", rc: 0, output: "a" }, { command: "b", rc: 1, output: "b" }, { command: "c", rc: 0, output: "c" }]);
+    assert.equal(f.max(), 3);
+  });
+  it("parallelism 2 of 4: never more than 2 in flight; the next starts only when a slot frees", async () => {
+    const f = fake();
+    const p = runLightTests({ commands: ["a", "b", "c", "d"], parallelism: 2, run: f.run });
+    await f.tick();
+    assert.deepEqual(f.log, ["start a", "start b"]);
+    f.gates.get("b")(); await f.tick();
+    assert.deepEqual(f.log.slice(2), ["end b", "start c"]);
+    for (const c of ["a", "c"]) { f.gates.get(c)(); await f.tick(); }
+    f.gates.get("d")();
+    assert.equal((await p).length, 4);
+    assert.equal(f.max(), 2);
+  });
+  it("parallelism 1 runs them one after another; a timeout (rc 124) stays a failure, never deferred", async () => {
+    const order = [];
+    let t = 0; // a clock that advances between commands (a wall clock does too, in a slow run)
+    const r = await runLightTests({ commands: ["x", "y"], parallelism: 1, now: () => (t += 1000), run: async ({ command }) => { order.push(command); return { rc: command === "y" ? 124 : 0, output: "" }; } });
+    assert.deepEqual(order, ["x", "y"]);
+    assert.deepEqual(r.map((x) => x.rc), [0, 124]);
+  });
+  it("each held command takes one acquire slot and releases it", async () => {
+    let held = 0, peak = 0;
+    const acquire = async () => { held++; peak = Math.max(peak, held); return { release: () => { held--; } }; };
+    await runLightTests({ commands: ["a", "b", "c"], parallelism: 2, acquire, run: async () => ({ rc: 0, output: "" }) });
+    assert.equal(held, 0); assert.ok(peak <= 2);
+  });
+  it("runLightStep starts the repro, numstat and API-shape gates before any of them resolves", async () => {
+    const started = [], pending = [];
+    const later = (name, v) => () => new Promise((res) => { started.push(name); pending.push(() => res(v)); });
+    const { ctx } = step({ ctx: { reproOf: later("repro", { base: "FAILED", head: "RAN" }), numstat: later("numstat", "5\t1\tsrc/foo.js\n"), apiShape: later("api", { status: "none", changes: [], unknown: [] }) } });
+    const p = runLightStep(ctx);
+    for (let i = 0; i < 20 && started.length < 3; i++) await new Promise((r) => setImmediate(r));
+    assert.deepEqual([...started].sort(), ["api", "numstat", "repro"], "all three in flight together");
+    pending.forEach((f) => f());
+    assert.equal((await p).upgrade, false);
+  });
+});
+
+describe("renderLiteReceipt (#4452)", () => {
+  const ELIG = { mode: "on", verdict: "eligible", eligible: true, light: true, reasons: [] };
+
+  it("renders diff stat, red/green result, gates, true cost and eligibility reason for a light lane", async () => {
+    const r = await runLightStep(step().ctx);
+    const body = renderLiteReceipt({ lr: r, eligibility: ELIG, costUsd: 0.4321, contractHash: "c0ffee12" });
+    assert.match(body, /^## Light lane receipt/);
+    assert.match(body, /\*\*Outcome:\*\* light/);
+    assert.match(body, /\*\*Diff:\*\* 2 file\(s\), \+25\/-1 \(non-test: 1 file\(s\), 6 line\(s\)\)/);
+    assert.match(body, /\*\*Red\/green:\*\* base FAILED → head RAN \(red on base\)/);
+    assert.match(body, /\*\*Gates:\*\*\n  - targeted tests: 2 command\(s\), passed/);
+    assert.match(body, /`node --test test\/foo\.test\.js`/);
+    assert.match(body, /API shape: unchanged/);
+    assert.match(body, /contract: derived, valid \(`c0ffee12`\)/);
+    assert.match(body, /\*\*Cost:\*\* \$0\.4321 \(true per-session cost, #4407 deltas\)/);
+    assert.match(body, /\*\*Eligibility:\*\* eligible \(mode on\): repro or expected behaviour/);
+    assert.ok(!/Upgrade reasons/.test(body));
+    assert.ok(!/<!--|undefined|NaN/.test(body), body);
+  });
+
+  it("an upgraded lane lists the gates that fired and the upgrade reasons; free text is sanitised", async () => {
+    const api = { status: "changed", changes: [{ file: "src/foo.js", symbol: "foo", change: "sync-async" }], unknown: [] };
+    const r = await runLightStep(step({ api, testRcs: [1, 1], repro: { base: "PASSED", head: "RAN" } }).ctx);
+    assert.equal(r.upgrade, true);
+    const elig = { mode: "on", verdict: "eligible", reasons: [{ code: "no_repro", detail: "x <!-- y --> <b>" }] };
+    const body = renderLiteReceipt({ lr: r, eligibility: elig, costUsd: 1.5 });
+    assert.match(body, /^## Light lane upgraded to the full lane/);
+    assert.match(body, /\*\*Outcome:\*\* upgraded/);
+    assert.match(body, /base PASSED → head RAN \(not red on base\)/);
+    assert.match(body, /targeted tests: 2 command\(s\), failed twice/);
+    assert.match(body, /API shape: 1 change\(s\)\/unknown\n {4}- src\/foo\.js: foo sync-async/);
+    assert.match(body, /contract: not derived \(upgraded\)/);
+    assert.match(body, /\*\*Upgrade reasons:\*\*[\s\S]*- api_shape: [\s\S]*- repro_not_red: [\s\S]*- tests_failed_twice: /);
+    assert.match(body, /- no_repro: x y &lt;b>/);
+    assert.ok(!/<!--|--!?>/.test(body));
+  });
+
+  it("an early upgrade (no commit) renders n/a, never undefined", async () => {
+    const r = await runLightStep(step({ files: [] }).ctx);
+    const body = renderLiteReceipt({ lr: r, eligibility: null, costUsd: null });
+    assert.match(body, /\*\*Diff:\*\* n\/a/);
+    assert.match(body, /\*\*Red\/green:\*\* n\/a/);
+    assert.match(body, /targeted tests: not run/);
+    assert.match(body, /API shape: not checked/);
+    assert.match(body, /\*\*Cost:\*\* n\/a/);
+    assert.match(body, /- no_commit: /);
+    assert.ok(!/undefined|NaN/.test(body), body);
+  });
+
+  it("runLightStep returns the receipt inputs alongside its #4450 fields", async () => {
+    const r = await runLightStep(step().ctx);
+    assert.deepEqual(r.diff, { files: [{ path: "src/foo.js", added: 5, deleted: 1 }, { path: "test/foo.test.js", added: 20, deleted: 0 }] });
+    assert.deepEqual(r.apiShape, []);
+    assert.deepEqual(r.testCommands, ["node --test test/foo.test.js", "node --test test/bar.test.js"]);
+    assert.equal(r.testRuns, 2);
+    for (const k of ["upgrade", "reasons", "contract", "files", "repro", "testFailures"]) assert.ok(k in r, k);
+  });
+});

package/bin/lean/orchestrate.mjs

@@ -41,6 +41,13 @@
  * `priorities`) ranks it host-wide, and the batch's ready issues are tried in priority order. Ready issues it cannot
  * start yet stay offered in the registry (polled every `pollMs`) and are withdrawn once no longer ready; a settled lane
  * releases its slot, an adopted one re-records it.
+ * Per-issue lock (#4481), on with `sched.exclusive` (registry entries ask `exclusive: true`): when another batch holds a live slot for the same repo and issue (`tryAcquire` `conflict`, or
+ * `registry.holder` for a lane placed remote), the issue is not started: it is retried after `pollMs` with one `deferred`
+ * row naming that batch (no deferral count, no breaker), and admission goes on with the next ready issue.
+ *
+ * Dispatch check (#4481), off unless `dispatchCheck(issue)` is passed: it runs before EVERY lane start (first and
+ * requeued, unlike the once-per-issue pre-filter) and returns null (run) or `{outcome: "merged", pr}` / `{outcome:
+ * "invalid"}` for an issue already closed or whose PR merged; the row ends with that outcome and no lane starts. Fail-open.
  *
  * Remote lanes (#4253), off unless `remote: {max, isRemote(issue)}` is passed (execution.mjs): a lane `isRemote` places
  * in a sandbox takes no host registry slot; it starts only while fewer than `max` remote lanes of this batch run (the
@@ -217,7 +224,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   store = null, adopt = null, getParallel = null, getAdded = null, breaker = null, maxDeferrals = DEFAULT_MAX_DEFERRALS, sleep = cancellableSleep, pollMs = 5000,
   classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null,
   recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, getCancelled = null, stopLane = null,
-  estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null }) {
+  estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null, dispatchCheck = null }) {
   const saved = store?.load?.() ?? null;
   const order = saved ? [...saved.order] : [];
   const set = new Set(order);
@@ -380,7 +387,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     return ready.map((n) => ({ n, p: prioOf(n), b: heads.has(n) || starved(n) ? 0 : 1, s: starved(n) ? enqueued.get(n) : 0, w: weight(n), i: idx.get(n) ?? 0 }))
       .sort((a, b) => a.p - b.p || a.b - b.b || a.s - b.s || b.w - a.w || a.i - b.i).map((x) => x.n);
   };
-  const entryOf = (n) => ({ key: keyOf(n), batch: sched?.batch ?? batchId ?? null, repo: sched?.repo ?? null, issue: n, priority: prioOf(n) });
+  const entryOf = (n) => ({ key: keyOf(n), batch: sched?.batch ?? batchId ?? null, repo: sched?.repo ?? null, issue: n, priority: prioOf(n), ...(sched?.exclusive ? { exclusive: true } : {}) });
   const releaseSlot = (n) => { if (reg && slotted.delete(n)) reg.release(keyOf(n)); };
   const offerFresh = (ns) => { const fresh = ns.filter((n) => !offered.has(n)); if (!fresh.length) return; reg.offer(fresh.map(entryOf)); for (const n of fresh) offered.add(n); };
   const withdrawExcept = (keep) => {
@@ -407,10 +414,28 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     if (budgetHit === null) update({ issue: null, outcome: "budget", detail: `execution budget reached: ${reason}; ${remote?.onBudget === "local" || remote?.fallbackLocal ? "remaining lanes run locally" : "no further remote lanes admitted"}` });
     budgetHit = reason;
   };
+  // #4481: another batch runs a lane for `n` now: retry after a poll, one row (and decision) per other batch.
+  const busyNoted = new Map(); // issue -> the other batch last reported
+  const busy = (n, other, wanted) => {
+    const delayMs = Math.max(pollMs, 1);
+    notBefore.set(n, now() + delayMs); offered.delete(n); wanted.delete(n);
+    if (busyNoted.get(n) !== other) {
+      busyNoted.set(n, other); save();
+      decided({ issue: n, what: `deferred: batch ${other} runs a lane for #${n}`, why: "one lane per issue on the host (two batches include it)", alternative: "run a second lane on the same branch" });
+      update({ issue: n, outcome: "deferred", pr: null, cost_usd: 0, minutes: 0, detail: `#${n} has a live lane in batch ${other}; retried in ${Math.round(delayMs / 1000)}s` });
+    }
+    return "busy";
+  };
   // A host slot for `n` (a lane-refused issue runs nothing and needs none); refused: `n` stays queued in the registry.
+  // true: start it; false: no capacity now; "busy" (#4481): another batch runs `n`, try the next issue.
   const grant = (n, wanted) => {
     if (lanes.get(n)?.ok === false) return true;
     let p = placeOf(n);
+    if (p?.where === "remote" && reg?.holder) {
+      let other = null;
+      try { other = reg.holder(entryOf(n)) ?? null; } catch { other = null; }
+      if (other !== null) return busy(n, other, wanted);
+    }
     if (p?.where === "remote") {
       const adm = admitRemote();
       if (adm.ok) {
@@ -426,6 +451,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     }
     if (!reg) { if (p) placed.set(n, p); return true; }
     const r = reg.tryAcquire(keyOf(n), entryOf(n), sched.cap);
+    if (!r?.ok && r?.conflict != null) return busy(n, r.conflict, wanted);
     if (!r?.ok) { offered.add(n); wanted.add(n); return false; }
     offered.delete(n); slotted.add(n);
     if (p) placed.set(n, p);
@@ -452,6 +478,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     const pfCost = recovering.has(n) ? 0 : (pf?.cost_usd ?? 0);
     const row = { issue: n, outcome: r?.terminalReason ?? "engine-error", pr: r?.pr ?? null, cost_usd: (r?.cost_usd ?? 0) + pfCost, minutes: Math.round((now() - rec.startedAt) / 6000) / 10, detail: r?.detail, ...(r?.code ? { code: r.code } : {}), ...(pf ? { prefilter: pf } : {}),
       ...(r?.waits && typeof r.waits === "object" ? { waits: r.waits } : {}), ...(Number.isInteger(r?.heavyQueueMax) ? { heavyQueueMax: r.heavyQueueMax } : {}),
+      ...(Number.isFinite(r?.wall_ms) ? { wall_ms: r.wall_ms } : {}), ...(Number.isFinite(r?.model_ms) ? { model_ms: r.model_ms } : {}), // #4451
       ...(Number.isFinite(r?.precheckMs) ? { precheck_ms: r.precheckMs } : {}), ...(Number.isInteger(r?.precheckDeferred) && r.precheckDeferred > 0 ? { precheck_deferred: r.precheckDeferred } : {}), // #4331
       ...(typeof r?.firstPass === "boolean" ? { first_pass: r.firstPass } : {}), // #4362
       ...(r?.turns && typeof r.turns === "object" ? { turns: r.turns } : {}), // #4392
@@ -604,17 +631,24 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
     if (relabelled.delete(n)) save(); // #4315: the recovery lane launches after all; it removes needs-human itself
     const pl = placed.get(n);
     placed.delete(n);
-    heads.delete(n); enqueued.delete(n);
+    heads.delete(n); enqueued.delete(n); busyNoted.delete(n);
     const rec = { startedAt: now(), ...(lane ? { lane } : {}), ...(remoteRun.has(n) ? { remote: true } : {}), ...(pl ? { placement: pl } : {}) };
     runRecs.set(n, rec); notBefore.delete(n); save();
     track(n, (async () => {
       // Pre-filter once per issue (a requeued issue is not re-checked); fail-open: an error means a normal lane.
       if (prefilter && !checked.has(n)) { let pf = null; try { pf = await prefilter(n, lane); } catch { pf = null; } checked.set(n, pf); save(); }
       const pf = checked.get(n) ?? null;
+      // #4481: every start re-checks GitHub: a closed issue or a merged PR ends the row without a lane (fail-open).
+      let gone = null;
+      if (dispatchCheck && !cancelled && !pf?.closed && !pf?.parked) {
+        try { gone = (await dispatchCheck(n)) ?? null; } catch { gone = null; }
+        if (!["merged", "invalid"].includes(gone?.outcome)) gone = null;
+      }
       if (halted) return; // #4454: stopped before launch; no pid, so a resume queues it again
       let r;
       if (cancelled) r = { terminalReason: "deferred", code: "run.interrupted", cost_usd: 0, detail: "cancelled before launch" }; // #4376
       else if (pf?.closed) r = { terminalReason: "invalid", cost_usd: 0, detail: `pre-filter: ${pf.detail}` };
+      else if (gone) r = { terminalReason: gone.outcome, pr: Number.isInteger(gone.pr) ? gone.pr : null, cost_usd: 0, detail: `skipped at dispatch: ${gone.detail ?? gone.outcome}` };
       // #4389: a parked post-deploy follow-up starts no lane; it ends awaiting-promotion|awaiting-precondition, re-checked next batch.
       else if (pf?.parked) r = { terminalReason: pf.kind, cost_usd: 0, detail: `pre-filter: ${pf.detail}` };
       else {
@@ -766,7 +800,9 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
           const n = pickNext(ready, { classes, share: budget.share, whenIdle: budget.whenIdle, started: dispatch.started, maint: dispatch.maint,
             featuresPending: pending.some((p) => !clsOf(classes, p).maintenance) });
           if (n === null) { budgetHeld = ready.length > 0; break; }
-          if (!grant(n, wanted)) break;
+          const g = grant(n, wanted);
+          if (g === "busy") continue; // #4481: not ready any more this pass (notBefore), so `ready` drops it
+          if (!g) break;
           pending.splice(pending.indexOf(n), 1);
           if (lanes.get(n)?.ok !== false) { dispatch.started++; if (clsOf(classes, n).maintenance) dispatch.maint++; }
           start(n); started++;
@@ -778,13 +814,16 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
         offerFresh(want);
         for (const n of want) wanted.add(n);
         for (const n of ready) {
-          if (running.size >= K() || !grant(n, wanted)) break;
+          if (running.size >= K()) break;
+          const g = grant(n, wanted);
+          if (g === "busy") continue; // #4481
+          if (!g) break;
           pending.splice(pending.indexOf(n), 1); start(n); started++;
         }
       } else if (!tripped && !cancelled && !holdUntil) {
         for (const n of rank(pending.filter(isReady))) {
           if (running.size >= K()) break;
-          if (grant(n, new Set())) { pending.splice(pending.indexOf(n), 1); start(n); started++; }
+          if (grant(n, new Set()) === true) { pending.splice(pending.indexOf(n), 1); start(n); started++; }
         }
       }
       if (reg) withdrawExcept(wanted);

package/bin/lean/precheck.test.mjs

@@ -118,6 +118,11 @@ describe("formatPrecheckSummary (#4331)", () => {
     assert.deepEqual(rows.map((r) => [r.issue, r.precheck_ms, r.precheck_deferred]), [[1, 90000, 1], [2, 30000, undefined], [3, undefined, undefined]]);
     assert.equal(formatPrecheckSummary(rows), "precheck: 2.0m total over 2 lanes, 1 deferred to CI");
   });
+  it("#4451: orchestrate rows carry the lane's wall_ms and model_ms, so wall vs model time can be measured", async () => {
+    const rows = await orchestrate({ issues: [1, 2], parallel: 1, readBody: async () => "",
+      runOne: async (n) => ({ terminalReason: "merged", ...({ 1: { wall_ms: 90000, model_ms: 60000 } })[n] }) });
+    assert.deepEqual(rows.map((r) => [r.issue, r.wall_ms, r.model_ms]), [[1, 90000, 60000], [2, undefined, undefined]]);
+  });
   it("#4362: orchestrate rows carry the lane's firstPass as first_pass for the batch first-pass line", async () => {
     const rows = await orchestrate({ issues: [1, 2, 3], parallel: 1, readBody: async () => "",
       runOne: async (n) => ({ terminalReason: "merged", ...({ 1: { firstPass: true }, 2: { firstPass: false } })[n] }) });

package/bin/lean/prefilter.mjs

@@ -12,7 +12,9 @@
  * ALREADY-FIXED/INVALID verdict with evidence closes the issue; everything else, and every error, goes to the normal lane.
  * Post-deploy gate (#4389), with `defaultBranch`: a `FORGE:DEFERRED_FROM: #P` follow-up whose checklist items are all
  * post-deploy is parked (`parked: true`, label `awaiting-promotion`, no lane) while a merge SHA of #P is not an ancestor
- * of origin/<default>, or while forge.yaml `post_deploy.precondition_cmd` exits 1. Anything unresolvable fails open.
+ * of origin/<default>, or while forge.yaml `post_deploy.precondition_cmd` exits 1. With `post_deploy.min_age` (#4425) it also
+ * stays parked (`awaiting-min-age`) until that long (or the longest window an item's why states) after the fix landed on
+ * the default branch. Anything unresolvable fails open.
  * All effects are injected (gh, git, runner, transitionLabel, projector?, execCode?) so it is unit-testable.
  */
 import { execFile } from "node:child_process";
@@ -196,12 +198,52 @@ function execCode(cmd, args, opts) {
     try { execFile(cmd, args, opts, (err) => res(err ? (typeof err.code === "number" ? err.code : -1) : 0)); } catch { res(-1); }
   });
 }
+// #4425: post_deploy.min_age soak window. Deterministic: a strict duration parse, and git for the landing time.
+const UNIT_MS = { m: 60e3, h: 3600e3, d: 86400e3, w: 604800e3 };
+/** forge.yaml `post_deploy.min_age` ("30m", "24h", "7d", "2w"; a bare number is minutes, so "0" is allowed) -> ms, or null. */
+export function parseMinAge(s) {
+  if (s == null) return null;
+  const m = /^(\d+(?:\.\d+)?)\s*([mhdw]?)$/i.exec(String(s).trim());
+  return m ? Math.round(Number(m[1]) * UNIT_MS[(m[2] || "m").toLowerCase()]) : null;
+}
+const WINDOW_RE = /\b(\d+(?:\.\d+)?)\s*(h|hrs?|hours?|d|days?|w|wks?|weeks?)\b/gi;
+const POST_DEPLOY_ITEM_RE = /^`[^`]*`\s+(?:\*\*known-flaw\*\*\s+)?[—-]+\s*post-deploy:(.*)$/;
+/** Per post-deploy checklist item: the largest duration its why states (ms), or null when it states none. */
+export function itemWindowsMs(body) {
+  return noComments(String(body ?? ""), " ").split("\n").map((l) => CHECK_RE.exec(l)?.[1]).filter((x) => x != null)
+    .map((t) => POST_DEPLOY_ITEM_RE.exec(t)).filter(Boolean).map((m) => {
+      const ms = [...m[1].matchAll(WINDOW_RE)].map((w) => Number(w[1]) * UNIT_MS[w[2][0].toLowerCase()]);
+      return ms.length ? Math.round(Math.max(...ms)) : null;
+    });
+}
+/**
+ * Unix seconds when `sha` reached origin/<def>: the oldest first-parent commit of <def> that descends from it (the merge
+ * that brought it in), or the SHA's own commit time when it is on the mainline itself. Null on any git error.
+ */
+export async function landedAt({ sha, def, io, cwd }) {
+  try {
+    // --first-parent with --ancestry-path drops merges that brought the SHA in through a second parent (git 2.55): two walks
+    const range = `${sha}..origin/${def}`;
+    const below = new Set(String(await io.git(["rev-list", "--ancestry-path", range], { cwd })).trim().split(/\s+/).filter(Boolean));
+    const chain = String(await io.git(["log", "--first-parent", "--format=%H %P %ct", range], { cwd })).trim().split("\n").map((l) => l.trim().split(/\s+/));
+    const oldest = chain.filter((c) => below.has(c[0])).at(-1);
+    let own = !oldest;
+    if (!own) {
+      const full = /^[0-9a-f]{40}$/i.test(sha) ? sha : String(await io.git(["rev-parse", `${sha}^{commit}`], { cwd })).trim();
+      own = oldest[1]?.toLowerCase() === full.toLowerCase();
+    }
+    const ct = own ? String(await io.git(["log", "-1", "--format=%ct", sha], { cwd })).trim() : oldest.at(-1);
+    return /^\d+$/.test(ct) ? Number(ct) : null;
+  } catch { return null; }
+}
+const fmtAge = (ms) => (ms % UNIT_MS.d === 0 && ms ? `${ms / UNIT_MS.d}d` : ms % UNIT_MS.h === 0 && ms ? `${ms / UNIT_MS.h}h` : `${Math.round(ms / UNIT_MS.m)}m`);
 /**
  * #4389: never throws, no model call. `defaultBranch` is a name or an (async) function resolving it (and fetching it).
- * @returns {Promise<{state:"not-applicable"|"unresolved"|"awaiting-promotion"|"awaiting-precondition"|"ready", parent:?number, shas:string[], default?:string, detail:string}>}
+ * #4425: `minAge` (ms, opt-in) parks `awaiting-min-age` until max(item window ?? minAge) after the fix landed on <default>.
+ * @returns {Promise<{state:"not-applicable"|"unresolved"|"awaiting-promotion"|"awaiting-min-age"|"awaiting-precondition"|"ready", parent:?number, shas:string[], default?:string, eligibleAt?:string, detail:string}>}
  * Fail-open: no merge SHA, a git answer other than yes/no, an unknown default branch or a hook error give "unresolved".
  */
-export async function postDeployGate({ iss, repo, lane, defaultBranch, cwd, io, preconditionCmd = null }) {
+export async function postDeployGate({ iss, repo, lane, defaultBranch, cwd, io, preconditionCmd = null, minAge = null, now = null }) {
   const parent = deferredFrom(iss?.body);
   const res = (state, detail, extra = {}) => ({ state, parent, shas: [], detail, ...extra });
   if (!parent || !postDeployOnly(iss.body)) return res("not-applicable", "");
@@ -218,6 +260,20 @@ export async function postDeployGate({ iss, repo, lane, defaultBranch, cwd, io,
       if (!on) off.push(sha);
     }
     if (off.length) return res("awaiting-promotion", `fix ${off.map((s) => s.slice(0, 12)).join(", ")} of #${parent} is not on ${def}${lane ? ` (lane ${lane})` : ""}`, { shas, default: def });
+    if (minAge != null) {
+      const required = Math.max(0, ...itemWindowsMs(iss.body).map((w) => w ?? minAge));
+      let landed = 0;
+      for (const sha of shas) {
+        const t = await landedAt({ sha, def, io, cwd });
+        if (t == null) return res("unresolved", `landing time of ${sha.slice(0, 12)} on ${def} unknown`, { shas, default: def });
+        landed = Math.max(landed, t);
+      }
+      const at = now ?? io.now?.() ?? Date.now(), until = landed * 1000 + required;
+      if (at < until) {
+        const eligibleAt = new Date(until).toISOString();
+        return res("awaiting-min-age", `fix of #${parent} on ${def} since ${new Date(landed * 1000).toISOString()}; min age ${fmtAge(required)} until ${eligibleAt}`, { shas, default: def, eligibleAt });
+      }
+    }
     if (preconditionCmd) {
       const code = await (io.execCode ?? execCode)("sh", ["-c", preconditionCmd, "precondition", String(iss.number)],
         { cwd, timeout: 60000, env: { ...process.env, FORGE_ISSUE: String(iss.number) } });
@@ -252,12 +308,12 @@ async function markParked({ issue, repo, io, had, gate }) {
     } catch { /* the comment and the report still show it */ }
   }
   if (had) return;
-  const what = gate.state === "awaiting-promotion" ? "awaiting promotion" : "awaiting its post-deploy precondition";
+  const what = gate.state === "awaiting-promotion" ? "awaiting promotion" : gate.state === "awaiting-min-age" ? "awaiting its post-deploy soak window" : "awaiting its post-deploy precondition";
   const body = `<!-- ${PRECONDITION_MARKER} -->\n## Parked: ${what}\n${sanitizeMd(gate.detail)}.\n\nEvery item here is a post-deploy check, so no lane was started. The next orchestrate or triage run checks again and launches this issue once the precondition holds.`;
   try { await io.gh(["issue", "comment", String(issue), "-R", repo, "--body", capComment(body)]); } catch { /* the label still shows it */ }
 }
 
-/** #4389 batch-report lines: one `awaiting promotion:` line (and one `awaiting precondition:` line) for the parked rows, or "". */
+/** #4389 batch-report lines: one `awaiting promotion:` line (and one `awaiting min age:` / `awaiting precondition:` line) for the parked rows, or "". */
 export function formatParkedLine(rows) {
   const parked = (rows ?? []).filter((r) => r?.prefilter?.parked);
   const uniq = (xs) => [...new Set(xs.filter(Boolean))];
@@ -267,6 +323,11 @@ export function formatParkedLine(rows) {
     const lanes = uniq(promo.map((r) => r.prefilter.lane)), defs = uniq(promo.map((r) => r.prefilter.default));
     out.push(`awaiting promotion: ${promo.map((r) => `#${r.issue}`).join(" ")} (fixes on ${lanes.join(", ") || "the lane"}, not on ${defs.join(", ") || "the default branch"})`);
   }
+  const soak = parked.filter((r) => r.prefilter.kind === "awaiting-min-age"); // #4425
+  if (soak.length) {
+    const defs = uniq(soak.map((r) => r.prefilter.default));
+    out.push(`awaiting min age: ${soak.map((r) => `#${r.issue}`).join(" ")} (fix on ${defs.join(", ") || "the default branch"}, soak window not elapsed)`);
+  }
   const pre = parked.filter((r) => r.prefilter.kind === "awaiting-precondition");
   if (pre.length) out.push(`awaiting precondition: ${pre.map((r) => `#${r.issue}`).join(" ")} (post_deploy.precondition_cmd not ready)`);
   return out.join("\n");
@@ -276,7 +337,7 @@ export function formatParkedLine(rows) {
  * Pre-filter one issue. Never throws.
  * @returns {Promise<{issue:number, closed:boolean, kind:?string, candidate:boolean, verdict:?string, cost_usd:number, detail:string}>}
  */
-export async function prefilterIssue({ issue, repo, lane, cwd, io, model = DEFAULT_PREFILTER_MODEL, dryRun = false, agentId = null, maxMinutes = 5, selfLogins = null, defaultBranch = null, preconditionCmd = null }) {
+export async function prefilterIssue({ issue, repo, lane, cwd, io, model = DEFAULT_PREFILTER_MODEL, dryRun = false, agentId = null, maxMinutes = 5, selfLogins = null, defaultBranch = null, preconditionCmd = null, minAge = null }) {
   const out = { issue, closed: false, kind: null, candidate: false, verdict: null, cost_usd: 0, detail: "" };
   try {
     const iss = JSON.parse(String(await io.gh(["issue", "view", String(issue), "-R", repo, "--json", "number,title,body,state,labels,comments"])));
@@ -292,9 +353,9 @@ export async function prefilterIssue({ issue, repo, lane, cwd, io, model = DEFAU
       if (st?.lease && st.lease.until > Date.now() && st.lease.by !== agentId) return { ...out, detail: `leased by ${st.lease.by}` };
     }
     if (gated) {
-      const gate = await postDeployGate({ iss, repo, lane, defaultBranch, cwd, io, preconditionCmd });
+      const gate = await postDeployGate({ iss, repo, lane, defaultBranch, cwd, io, preconditionCmd, minAge });
       const had = names.includes(AWAITING_PROMOTION_LABEL);
-      if (gate.state === "awaiting-promotion" || gate.state === "awaiting-precondition") {
+      if (["awaiting-promotion", "awaiting-min-age", "awaiting-precondition"].includes(gate.state)) {
         if (!dryRun) await markParked({ issue, repo, io, had, gate });
         return { ...out, parked: true, kind: gate.state, parent: gate.parent, lane, default: gate.default, detail: gate.detail };
       }