forgedock 1.34.0 → 1.36.0

package/bin/lean/postdeploy-monitor.mjs

@@ -0,0 +1,87 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Windowed post-deploy monitoring (#4627). A follow-up whose checks declare `for` (and `every`) stays open with the
+ * `post-deploy-monitoring` label after its first pass. `sweepDue` (run by schedd's timer and `forgedock probe --due`) re-runs
+ * each one when its cadence elapsed or `origin/<default>` moved (a release), with the command read from the default branch
+ * (never an issue). A pass is a FORGE:PROBE record; the window end closes the follow-up; a fail after a pass files one
+ * regression issue naming the fix commit and the last-passing and breaking release, then closes the follow-up. No model call.
+ */
+import { readProbeConfig, resolveDefaultBranch, PROBE_MARKER, OUTPUT_TAIL } from "./probe.mjs";
+import { MONITORING_LABEL, checksAt, itemCheckIds, monitorRecord, runNamedChecks, CHECKS_FILE } from "./postdeploy-check.mjs";
+import { deferredFrom, isTrustedIssueComment } from "./prefilter.mjs";
+
+export const REGRESSION_MARKER = "FORGE:REGRESSION";
+const fenced = (s) => { const f = "`".repeat(Math.max(3, ...[...String(s).matchAll(/`+/g)].map((m) => m[0].length + 1))); return `${f}\n${String(s).trim() || "(no output)"}\n${f}`; };
+const short = (s) => String(s ?? "unknown").slice(0, 12);
+
+/** One monitored follow-up. Outcomes: skipped | unrunnable | not-due | due (dry run) | pass | window-ended | fail. */
+export async function monitorOne({ issue, repo, def, defaultSha, cwd, io, now, dryRun, config, env, spawn }) {
+  const row = (outcome, detail = "", extra = {}) => ({ issue, outcome, detail, ...extra });
+  const iss = JSON.parse(String(await io.gh(["issue", "view", String(issue), "-R", repo, "--json", "number,title,body,state,labels,comments"])));
+  if (String(iss.state).toUpperCase() !== "OPEN") return row("skipped", "not open");
+  const recs = (iss.comments ?? []).filter((c) => isTrustedIssueComment(c)).map((c) => monitorRecord(c?.body)).filter(Boolean);
+  const last = recs[recs.length - 1];
+  const finish = async (comment) => {
+    if (comment) await io.gh(["issue", "comment", String(issue), "-R", repo, "--body", comment]);
+    await io.gh(["issue", "close", String(issue), "-R", repo, "--reason", "completed"]);
+    try { await io.gh(["issue", "edit", String(issue), "-R", repo, "--remove-label", MONITORING_LABEL]); } catch { /* label already gone */ }
+  };
+  if (!last || last.result !== "pass" || !last.until) return row("skipped", last?.result === "fail" ? "already failed (FORGE:PROBE)" : "no windowed pass record");
+  if (now >= last.until) {
+    if (!dryRun) await finish(`<!-- FORGE:MONITOR -->\nPost-deploy monitoring window ended ${new Date(last.until).toISOString()} with no failure; closed.`);
+    return row("window-ended", dryRun ? "dry run: not closed" : "closed", { fix: last.fix });
+  }
+  if (now - last.at < last.everyMs && defaultSha && defaultSha === last.defaultSha) return row("not-due", `next run after ${new Date(last.at + last.everyMs).toISOString()} or a new release`);
+  const ids = itemCheckIds(iss.body);
+  const checks = await checksAt(io.git, [`origin/${def}`], cwd);
+  if (!ids.length || ids.some((id) => !id || !checks[id])) return row("unrunnable", `no runnable check on ${def} (${CHECKS_FILE})`);
+  if (dryRun) return row("due", "dry run: not run", { fix: last.fix });
+  const rows = await runNamedChecks({ ids, checks, cwd, config, env, spawn });
+  const ok = rows.every((r) => r.result === "pass");
+  const rec = { fix: last.fix, default_sha: defaultSha, result: ok ? "pass" : "fail", at: new Date(now).toISOString(), ...(ok ? { window_until: new Date(last.until).toISOString(), every_ms: last.everyMs } : {}) };
+  const sum = rows.map((r) => ({ id: r.id, result: r.result, exit: r.exit, ms: r.ms }));
+  if (ok) {
+    await io.gh(["issue", "comment", String(issue), "-R", repo, "--body", `<!-- ${PROBE_MARKER} -->\n## Post-deploy check: pass\n${JSON.stringify({ ...rec, checks: sum })}\n\nRe-run on \`${def}\` at \`${short(defaultSha)}\`.`]);
+    return row("pass", `checks ${[...new Set(ids)].join(", ")} passed at ${short(defaultSha)}`, { fix: last.fix });
+  }
+  const bad = rows.filter((r) => r.result === "fail");
+  const parent = deferredFrom(iss.body);
+  const title = `Regression: post-deploy check ${bad.map((r) => r.id).join(", ")} failed on ${def}${parent ? ` (fix of #${parent})` : ""}`;
+  const body = [
+    `<!-- ${REGRESSION_MARKER} -->`, `Post-deploy check ${bad.map((r) => `\`${r.id}\``).join(", ")} passed after the fix and now fails.`, "",
+    `- Fix commit: \`${last.fix}\``, `- Last passing release (\`${def}\`): \`${last.defaultSha ?? "unknown"}\``, `- Breaking release (\`${def}\`): \`${defaultSha ?? "unknown"}\``,
+    `- Monitored follow-up: #${issue}${parent ? `; parent: #${parent}` : ""}`, "", ...bad.flatMap((r) => [`**${r.id}** (exit ${r.exit ?? "timeout"})`, fenced(r.output), ""]),
+  ].join("\n").slice(0, OUTPUT_TAIL * 4);
+  const created = String(await io.gh(["issue", "create", "-R", repo, "--title", title, "--body", body]));
+  const num = /\/issues\/(\d+)/.exec(created)?.[1] ?? null;
+  await finish(`<!-- ${PROBE_MARKER} -->\n## Post-deploy check: fail\n${JSON.stringify({ ...rec, regression: num ? Number(num) : null, checks: sum })}\n\nRegression issue ${num ? `#${num}` : "filed"}; monitoring stops.`);
+  return row("fail", `regression ${num ? `#${num}` : "filed"}: ${bad.map((r) => r.id).join(", ")} broke at ${short(defaultSha)} (fix ${short(last.fix)})`, { fix: last.fix, regression: num });
+}
+
+/** Re-run every due monitored follow-up of the repo. Sequential; an error is an `error` row, never a throw. */
+export async function sweepDue({ repo, cwd = process.cwd(), io, dryRun = false, now = () => Date.now(), config = readProbeConfig(cwd), env = process.env, spawn, onResult = null } = {}) {
+  const def = await resolveDefaultBranch({ repo, io, cwd });
+  if (!def) throw new Error("no default branch");
+  const found = JSON.parse(String(await io.gh(["issue", "list", "-R", repo, "--state", "open", "--label", MONITORING_LABEL, "--limit", "100", "--json", "number"])));
+  const list = (Array.isArray(found) ? found : []).map((i) => i.number).filter(Number.isInteger);
+  if (!list.length) return [];
+  try { await io.git(["fetch", "origin", def], { cwd }); } catch { /* offline: the local ref */ }
+  let defaultSha = null;
+  try { defaultSha = String(await io.git(["rev-parse", `origin/${def}`], { cwd })).trim() || null; } catch { /* unknown release */ }
+  const rows = [];
+  for (const issue of list) {
+    try { rows.push(await monitorOne({ issue, repo, def, defaultSha, cwd, io, now: now(), dryRun, config, env, spawn })); } catch (e) {
+      rows.push({ issue, outcome: "error", detail: String(e?.message ?? e).slice(0, 160) });
+    }
+    try { if (!dryRun) await onResult?.(rows[rows.length - 1]); } catch { /* an event sink never fails the sweep */ }
+  }
+  return rows;
+}
+
+/** One line, e.g. `monitoring: pass #1; regression #2; closed #3`; "" when nothing is monitored. */
+export function formatDueSummary(rows) {
+  const g = [["pass", "pass"], ["fail", "regression"], ["window-ended", "window ended"], ["due", "due"], ["not-due", "not due"], ["unrunnable", "unrunnable"], ["error", "error"]];
+  const parts = g.map(([k, l]) => [l, (rows ?? []).filter((r) => r.outcome === k).map((r) => `#${r.issue}`)]).filter(([, a]) => a.length).map(([l, a]) => `${l} ${a.join(" ")}`);
+  return parts.length ? `monitoring: ${parts.join("; ")}` : "";
+}

package/bin/lean/postdeploy-monitor.test.mjs

@@ -0,0 +1,121 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { sweepDue, formatDueSummary } from "./postdeploy-monitor.mjs";
+import { PROBE_MARKER } from "./probe.mjs";
+import { isTrustedIssueComment } from "./prefilter.mjs";
+
+const T0 = Date.parse("2026-10-26T00:00:00Z");
+const H = 3600e3;
+const CHECKS = JSON.stringify({ soak: { run: "echo ok", for: "7d", every: "6h" } });
+const BODY = "## Acceptance Criteria\n\n- [ ] `R1: health row exists` — post-deploy: [check:soak] soak\n\n<!-- FORGE:DEFERRED_FROM: #40 -->\n";
+const rec = (o) => ({ authorAssociation: "OWNER", author: { login: "bot" }, body: `<!-- ${PROBE_MARKER} -->\n${JSON.stringify({ fix: "fix1234567890", default_sha: "rel1", result: "pass", at: new Date(T0).toISOString(), window_until: new Date(T0 + 7 * 24 * H).toISOString(), every_ms: 6 * H, ...o })}` });
+
+function world({ comments = [rec({})], head = "rel1", file = CHECKS, code = 0, state = "OPEN" } = {}) {
+  const w = { gh: [], git: [], spawned: [] };
+  w.io = {
+    gh: async (a) => {
+      w.gh.push(a);
+      const k = a.slice(0, 2).join(" ");
+      if (k === "repo view") return "main\n";
+      if (k === "issue list") return JSON.stringify([{ number: 41 }]);
+      if (k === "issue view") return JSON.stringify({ number: 41, state, title: "", body: BODY, labels: [{ name: "post-deploy-monitoring" }], comments });
+      if (k === "issue create") return "https://github.com/o/r/issues/77\n";
+      return "";
+    },
+    git: async (a) => { w.git.push(a); if (a[0] === "show") { if (file == null) throw new Error("missing"); return file; } if (a[0] === "rev-parse") return `${head}\n`; return ""; },
+  };
+  w.spawn = async (o) => { w.spawned.push(o.run); return { code, stdout: code ? "broke" : "ok", stderr: "" }; };
+  return w;
+}
+const sweep = (w, o = {}) => sweepDue({ repo: "o/r", cwd: "/repo", io: w.io, now: () => T0 + H, config: { env: [], timeoutS: 5 }, env: {}, spawn: w.spawn, ...o });
+const calls = (w, k) => w.gh.filter((a) => a.slice(0, 2).join(" ") === k);
+
+const trustedOk = isTrustedIssueComment(rec({}));
+
+describe("post-deploy windowed monitoring (#4627)", () => {
+  it("fixture comments count as trusted", () => assert.ok(trustedOk));
+
+  it("not due: cadence not elapsed and no new release", async () => {
+    const w = world();
+    const rows = await sweep(w);
+    assert.equal(rows[0].outcome, "not-due"); assert.deepEqual(w.spawned, []);
+  });
+
+  it("due by cadence: re-runs the default-branch command and records a pass; the follow-up stays open", async () => {
+    const w = world();
+    const rows = await sweep(w, { now: () => T0 + 7 * H });
+    assert.equal(rows[0].outcome, "pass"); assert.deepEqual(w.spawned, ["echo ok"]);
+    assert.ok(w.git.some((a) => a.join(" ") === "show origin/main:.forge/post-deploy-checks.json"));
+    assert.equal(calls(w, "issue comment").length, 1);
+    assert.equal(calls(w, "issue close").length, 0); assert.equal(calls(w, "issue create").length, 0);
+  });
+
+  it("due by a new release (origin/main moved) well before the cadence", async () => {
+    const w = world({ head: "rel2" });
+    const rows = await sweep(w);
+    assert.equal(rows[0].outcome, "pass"); assert.deepEqual(w.spawned, ["echo ok"]);
+  });
+
+  it("window end closes the follow-up and drops the label without running", async () => {
+    const w = world();
+    const rows = await sweep(w, { now: () => T0 + 8 * 24 * H });
+    assert.equal(rows[0].outcome, "window-ended"); assert.deepEqual(w.spawned, []);
+    assert.equal(calls(w, "issue close").length, 1);
+    assert.ok(calls(w, "issue edit").some((a) => a.includes("--remove-label")));
+  });
+
+  it("pass then fail files one regression naming the fix commit and both releases, then closes the follow-up", async () => {
+    const w = world({ head: "rel2", code: 1 });
+    const rows = await sweep(w);
+    assert.equal(rows[0].outcome, "fail"); assert.equal(rows[0].regression, "77");
+    const [create] = calls(w, "issue create");
+    const text = create[create.indexOf("--body") + 1];
+    assert.match(create[create.indexOf("--title") + 1], /Regression: post-deploy check soak failed on main \(fix of #40\)/);
+    assert.match(text, /FORGE:REGRESSION/); assert.match(text, /fix1234567890/); assert.match(text, /`rel1`/); assert.match(text, /`rel2`/); assert.match(text, /broke/);
+    assert.equal(calls(w, "issue close").length, 1);
+    assert.ok(!w.gh.some((a) => a[1] === "reopen"));
+    const cm = calls(w, "issue comment")[0];
+    assert.match(cm[cm.indexOf("--body") + 1], /"result":"fail".*"regression":77/s);
+  });
+
+  it("a follow-up whose latest record is a fail is never re-run or re-filed", async () => {
+    const w = world({ head: "rel3", code: 1, comments: [rec({}), rec({ result: "fail", default_sha: "rel2", regression: 77 })] });
+    const rows = await sweep(w);
+    assert.equal(rows[0].outcome, "skipped"); assert.deepEqual(w.spawned, []); assert.equal(calls(w, "issue create").length, 0);
+  });
+
+  it("an untrusted record never counts", async () => {
+    const w = world({ comments: [{ authorAssociation: "NONE", author: { login: "rando" }, body: rec({}).body }] });
+    assert.equal((await sweep(w, { now: () => T0 + 9 * H }))[0].outcome, "skipped");
+    assert.deepEqual(w.spawned, []);
+  });
+
+  it("a check missing on the default branch is unrunnable and never run", async () => {
+    for (const file of [null, JSON.stringify({ other: "echo hi" })]) {
+      const w = world({ file, head: "rel2" });
+      assert.equal((await sweep(w))[0].outcome, "unrunnable"); assert.deepEqual(w.spawned, []);
+    }
+  });
+
+  it("dry run runs nothing and writes nothing", async () => {
+    const w = world({ head: "rel2" });
+    assert.equal((await sweep(w, { dryRun: true }))[0].outcome, "due");
+    assert.deepEqual(w.spawned, []);
+    assert.equal(calls(w, "issue comment").length + calls(w, "issue close").length + calls(w, "issue create").length, 0);
+  });
+
+  it("no monitored follow-up: no git, no run; the summary names outcomes", async () => {
+    const w = world();
+    w.io.gh = async (a) => (a[0] === "repo" ? "main\n" : "[]");
+    assert.deepEqual(await sweep(w), []); assert.equal(w.git.length, 0);
+    assert.equal(formatDueSummary([{ issue: 1, outcome: "pass" }, { issue: 2, outcome: "fail" }]), "monitoring: pass #1; regression #2");
+  });
+
+  it("reports sink rows for pass and fail only when not a dry run", async () => {
+    const seen = [];
+    await sweep(world({ head: "rel2", code: 1 }), { onResult: (r) => seen.push(r.outcome) });
+    assert.deepEqual(seen, ["fail"]);
+  });
+});

package/bin/lean/precheck.mjs

@@ -10,7 +10,8 @@
  * which the builder runs in-session (`selfCheck`); `worktreeFingerprint` tells whether that run saw the final tree.
  */
 import { createHash } from "node:crypto";
-import { scopeChecks, selectCommands } from "./testmap.mjs";
+import { loadavg, cpus } from "node:os";
+import { scopeChecks, selectCommands, mergeChecks } from "./testmap.mjs";
 import { LOG_CAP } from "./verify.mjs";
 
 /** #4364: the engine's check plan for `forgedock check` (written beside contract.json) and the CLI's last run record. */
@@ -22,6 +23,45 @@ export const CHECK_LOG_DIR = ".forge-lean/logs";
 /** rc of a command killed by its timeout (effects.mjs runCheckCommand). */
 const TIMEOUT_RC = 124;
 
+/** Ceiling of a load-scaled retry timeout. */
+const MAX_RETRY_MS = 3600000;
+
+/** Host load per core, from the OS; the default `load` of `runWithTimeoutRetry`. */
+const hostLoad = () => ({ load1: loadavg()[0], cores: cpus().length || 1 });
+
+/**
+ * #4589: the timeout of the one retry after a command hit its own timeout: `perMs * max(2, ceil(load1 / cores))`, at most
+ * 3600 s and never below `perMs`. Deterministic; a timed-out command has no measured duration, so load scales it.
+ */
+export function retryTimeoutMs({ perMs, load1, cores }) {
+  const per = Number(perMs);
+  if (!(per > 0) || !Number.isFinite(per)) return per;
+  const factor = Math.max(2, Math.ceil((Number(load1) || 0) / (Number(cores) > 0 ? Number(cores) : 1)));
+  return Math.max(per, Math.min(per * factor, MAX_RETRY_MS));
+}
+
+/**
+ * #4589: a check timeout (rc 124) is a busy host, not a failing command: run `run(timeoutMs)`, and when it timed out on its
+ * own, once more with `retryTimeoutMs`. `left` (ms still available in a round budget) caps the retry; an exhausted budget, or a retry the budget cut
+ * short, is `cut` (left to CI); a budget with room for no longer retry leaves the first rc 124 a failure (#4620). A repeated timeout keeps rc 124 and says it was infrastructure. Other rcs are never retried.
+ * @param {(timeoutMs:number) => Promise<{rc:number, output?:string}>} run
+ * @returns {Promise<{r:{rc:number, output?:string}, retried:boolean, cut:boolean}>}
+ */
+export async function runWithTimeoutRetry(run, timeoutMs, { left = () => Infinity, load = hostLoad } = {}) {
+  const r = await run(timeoutMs);
+  if (r?.rc !== TIMEOUT_RC || !(timeoutMs > 0) || !Number.isFinite(timeoutMs)) return { r, retried: false, cut: false };
+  let want = timeoutMs;
+  try { want = retryTimeoutMs({ perMs: timeoutMs, ...load() }); } catch { /* no load reading: the minimum factor */ want = timeoutMs * 2; }
+  const room = left();
+  if (!(room > 0)) return { r, retried: false, cut: true }; // the budget is spent: CI runs it
+  const t2 = Math.min(want, room);
+  if (!(t2 > timeoutMs)) return { r, retried: false, cut: false }; // #4620: no room for a longer retry; its own timeout stays a failure
+  const r2 = await run(t2);
+  const cut = r2?.rc === TIMEOUT_RC && t2 < want;
+  if (r2?.rc === TIMEOUT_RC && !cut) return { r: { ...r2, output: `${String(r2.output ?? "")}\n(infra timeout: host load, not a code failure; retried once with ${Math.round(t2 / 1000)}s)` }, retried: true, cut };
+  return { r: r2, retried: true, cut };
+}
+
 /**
  * @param {{commands:string[], parallelism?:number, budgetMs:number, perCommandMs:number, now?:() => number,
  *   acquire?:() => Promise<{release?:Function}|null>,
@@ -30,7 +70,7 @@ const TIMEOUT_RC = 124;
  * @returns {Promise<{results:{command:string, rc:number, output:string, ms:number}[], deferred:string[], ms:number}>}
  *   results in input order (commands that ran and were not cut by the budget); deferred in input order
  */
-export async function runChecks({ commands, parallelism = 1, budgetMs, perCommandMs, now = () => Date.now(), acquire, run }) {
+export async function runChecks({ commands, parallelism = 1, budgetMs, perCommandMs, now = () => Date.now(), acquire, run, load }) {
   const list = Array.isArray(commands) ? commands : [];
   const workers = Math.max(1, Math.floor(Number(parallelism)) || 1);
   const k = Math.max(1, Math.min(workers, list.length)), share = Math.max(1, Math.floor(workers / k));
@@ -44,7 +84,13 @@ export async function runChecks({ commands, parallelism = 1, budgetMs, perComman
     if (left <= 0) return { deferred: true };
     const timeoutMs = Math.min(per, left), t = now();
     let r;
-    try { r = await run({ command, timeoutMs, workers: share }); } catch (e) { r = { rc: 127, output: String(e?.message ?? e) }; }
+    try {
+      if (timeoutMs >= per) { // #4589: its own timeout, not the budget's: one longer retry
+        const x = await runWithTimeoutRetry((ms) => run({ command, timeoutMs: ms, workers: share }), timeoutMs, { left: () => deadline - now(), ...(load ? { load } : {}) });
+        if (x.cut) return { deferred: true };
+        r = x.r;
+      } else r = await run({ command, timeoutMs, workers: share });
+    } catch (e) { r = { rc: 127, output: String(e?.message ?? e) }; }
     const rc = Number.isInteger(r?.rc) ? r.rc : 1;
     // the budget, not the command's own timeout, cut it short: CI runs it
     if (rc === TIMEOUT_RC && timeoutMs < per) return { deferred: true };
@@ -80,15 +126,19 @@ export function formatPrecheckSummary(rows) {
 /**
  * #4364: the pre-PR commands and budget for a round: `scopeChecks` over the trimmed learned commands (a mapped learned
  * command runs only when its globs match a changed file); budget lean.precheck.max_minutes, else
- * verify.precheck_total_seconds (default 600 s); per command verify.precheck_timeout_seconds.
- * @returns {{commands:string[], source:string, added:number, dropped:string[], selected:number, budgetMs:number, perCommandMs:number}}
+ * verify.precheck_total_seconds (default 600 s); per command verify.precheck_timeout_seconds. #4585: `invariants`
+ * (verify.invariants) are appended unscoped after the selected commands, deduped; the test map never drops them.
+ * @returns {{commands:string[], source:string, added:number, dropped:string[], selected:number, invariants?:number, budgetMs:number, perCommandMs:number}}
  */
-export function planPrecheck({ testMap = null, files = [], learned = [], maxMinutes, verify = {} }) {
+export function planPrecheck({ testMap = null, files = [], learned = [], maxMinutes, verify = {}, invariants = verify?.invariants }) {
   const list = (Array.isArray(learned) ? learned : []).map((c) => String(c ?? "").trim()).filter(Boolean);
-  const m = scopeChecks(testMap, files, list);
+  const s = scopeChecks(testMap, files, list);
+  const inv = (Array.isArray(invariants) ? invariants : []).map((c) => String(c ?? "").trim()).filter(Boolean);
+  const withInv = mergeChecks(inv, s.commands);
+  const m = inv.length ? { ...s, commands: withInv.commands } : s;
   const max = Number(maxMinutes);
   const budgetMs = (max > 0 ? max * 60 : Number(verify.precheckTotalSeconds) || 600) * 1000;
-  return { ...m, selected: testMap ? selectCommands(testMap, files).length : 0, budgetMs, perCommandMs: (Number(verify.precheckTimeoutSeconds) || 300) * 1000 };
+  return { ...m, ...(inv.length ? { invariants: inv.length } : {}), selected: testMap ? selectCommands(testMap, files).length : 0, budgetMs, perCommandMs: (Number(verify.precheckTimeoutSeconds) || 300) * 1000 };
 }
 
 const lines = (out) => String(out ?? "").split("\n").map((x) => x.trim()).filter(Boolean);

package/bin/lean/precheck.test.mjs

@@ -6,7 +6,7 @@ import { writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
 import { formatFirstPassSummary } from "./fixround.mjs";
-import { runChecks, formatPrecheckSummary, planPrecheck, selfCheck, selfCheckState, formatCheckTable, worktreeFingerprint, quietFailure } from "./precheck.mjs";
+import { runChecks, retryTimeoutMs, runWithTimeoutRetry, formatPrecheckSummary, planPrecheck, selfCheck, selfCheckState, formatCheckTable, worktreeFingerprint, quietFailure } from "./precheck.mjs";
 import { execFileSync } from "node:child_process";
 import { readPrecheckBudget } from "../engine-cli.mjs";
 import { orchestrate } from "./orchestrate.mjs";
@@ -52,9 +52,9 @@ describe("runChecks (#4331)", () => {
     assert.deepEqual(r.deferred, ["b", "c"]);
   });
 
-  it("a command's own timeout (not the budget) stays a failure; a throw is rc 127", async () => {
+  it("a command's own timeout (not the budget) stays a failure once the retry also times out; a throw is rc 127", async () => {
     const h = harness({ cost: { slow: 5000 } });
-    const r = await runChecks({ commands: ["slow", "boom", "bad"], parallelism: 2, budgetMs: 600000, perCommandMs: 2000, now: h.now,
+    const r = await runChecks({ commands: ["slow", "boom", "bad"], parallelism: 2, budgetMs: 600000, perCommandMs: 2000, now: h.now, load: () => ({ load1: 0, cores: 4 }),
       run: (a) => (a.command === "boom" ? Promise.reject(new Error("spawn failed")) : a.command === "bad" ? Promise.resolve({ rc: 2, output: "lint" }) : h.run(a)) });
     assert.deepEqual(r.results.map((x) => [x.command, x.rc]), [["slow", 124], ["boom", 127], ["bad", 2]]);
     assert.match(r.results[1].output, /spawn failed/);
@@ -145,6 +145,16 @@ describe("readPrecheckBudget: forge.yaml lean.precheck.max_minutes (#4331)", ()
 describe("forgedock check core (#4364)", () => {
   const MAP = { version: 1, entries: [{ globs: ["src/**"], command: "npm test", cwd: "." }, { globs: ["docs/**"], command: "make docs", cwd: "." }] };
 
+  it("planPrecheck #4585: invariants are appended unscoped, deduped, and kept when the map matches nothing", () => {
+    const p = planPrecheck({ testMap: MAP, files: ["README.md"], learned: ["make lint"], verify: { invariants: ["bash scripts/inv.sh", " make lint ", ""] } });
+    assert.deepEqual(p.commands.at(-1), "bash scripts/inv.sh");
+    assert.equal(p.commands.filter((c) => c === "make lint").length, 1);
+    assert.equal(p.invariants, 2);
+    const only = planPrecheck({ files: [], learned: [], verify: { invariants: ["bash scripts/inv.sh"] } });
+    assert.deepEqual(only.commands, ["bash scripts/inv.sh"]);
+    assert.deepEqual(planPrecheck({ testMap: MAP, files: ["README.md"], learned: ["make lint"] }).commands, ["make lint"], "no invariants: unchanged");
+  });
+
   it("planPrecheck: the engine's selection and budget rule (max_minutes wins, else precheck_total_seconds, else 600 s)", () => {
     const p = planPrecheck({ testMap: MAP, files: ["src/a.js"], learned: [" make lint ", "", "make docs"], verify: { precheckTotalSeconds: 120, precheckTimeoutSeconds: 30 } });
     assert.deepEqual(p.commands, ["make lint", "npm test"]); assert.deepEqual(p.dropped, ["make docs"]);
@@ -208,3 +218,67 @@ describe("forgedock check core (#4364)", () => {
     assert.notEqual(await fp(), dirty);
   });
 });
+
+describe("check timeout is infrastructure (#4589)", () => {
+  const idle = () => ({ load1: 0, cores: 4 });
+  it("retryTimeoutMs doubles at least, scales with load per core, caps at 3600 s and never shrinks", () => {
+    assert.equal(retryTimeoutMs({ perMs: 300000, load1: 0, cores: 4 }), 600000);
+    assert.equal(retryTimeoutMs({ perMs: 300000, load1: 13, cores: 4 }), 1200000);
+    assert.equal(retryTimeoutMs({ perMs: 2000000, load1: 40, cores: 4 }), 3600000);
+    assert.equal(retryTimeoutMs({ perMs: 4000000, load1: 0, cores: 4 }), 4000000);
+  });
+  it("runChecks retries a command's own timeout once with a longer one and passes", async () => {
+    const h = harness({ cost: { slow: 400000 } });
+    const r = await runChecks({ commands: ["slow"], budgetMs: Infinity, perCommandMs: 300000, now: h.now, run: h.run, load: idle });
+    assert.deepEqual(h.calls, [["slow", 300000], ["slow", 600000]]);
+    assert.equal(r.results[0].rc, 0);
+    assert.deepEqual(r.deferred, []);
+  });
+  it("a repeated timeout stays rc 124 and is labelled infrastructure", async () => {
+    const h = harness({ cost: { slow: 9e9 } });
+    const r = await runChecks({ commands: ["slow"], budgetMs: Infinity, perCommandMs: 1000, now: h.now, run: h.run, load: idle });
+    assert.equal(h.calls.length, 2);
+    assert.equal(r.results[0].rc, 124);
+    assert.match(r.results[0].output, /infra timeout/);
+  });
+  it("selfCheck is ok when the retry passes", async () => {
+    const h = harness({ cost: { slow: 5000 } });
+    const r = await selfCheck({ contract: { validate: [{ id: "V1", command: "slow" }] }, plan: { commands: [], budgetMs: 0, perCommandMs: 3000 }, now: h.now, run: h.run });
+    assert.equal(r.ok, true);
+  });
+  it("a budget-cut timeout is still deferred, never retried; a non-124 failure is never retried", async () => {
+    const h = harness({ cost: { slow: 9e9 }, rc: { bad: 1 } });
+    const r = await runChecks({ commands: ["slow"], budgetMs: 2000, perCommandMs: 300000, now: h.now, run: h.run, load: idle });
+    assert.equal(h.calls.length, 1);
+    assert.deepEqual(r.deferred, ["slow"]);
+    const h2 = harness({ rc: { bad: 1 } });
+    const r2 = await runChecks({ commands: ["bad"], budgetMs: Infinity, perCommandMs: 1000, now: h2.now, run: h2.run, load: idle });
+    assert.equal(h2.calls.length, 1);
+    assert.equal(r2.results[0].rc, 1);
+  });
+  it("an exhausted budget defers a command that hit its own timeout", async () => {
+    const h = harness({ cost: { slow: 9e9 } });
+    const r = await runChecks({ commands: ["slow"], budgetMs: 1000, perCommandMs: 1000, now: h.now, run: h.run, load: idle });
+    assert.equal(h.calls.length, 1);
+    assert.deepEqual(r.deferred, ["slow"]);
+  });
+  it("default budget: a check hitting its own timeout is reported, not deferred (#4620)", async () => {
+    const plan = planPrecheck({ verify: {}, learned: ["npm test"] });
+    const h = harness({ cost: { "npm test": 9e9 } });
+    const r = await runChecks({ commands: plan.commands, budgetMs: plan.budgetMs, perCommandMs: plan.perCommandMs, now: h.now, run: h.run, load: idle });
+    assert.equal(r.results.length, 1);
+    assert.equal(r.results[0].rc, 124);
+    assert.deepEqual(r.deferred, []);
+  });
+  it("a budget with room left but no longer retry reports the timeout as a failure", async () => {
+    const h = harness({ cost: { slow: 9e9 } });
+    const r = await runChecks({ commands: ["slow"], budgetMs: 1500, perCommandMs: 1000, now: h.now, run: h.run, load: idle });
+    assert.equal(h.calls.length, 1);
+    assert.deepEqual([r.results.map((x) => x.rc), r.deferred], [[124], []]);
+  });
+  it("runWithTimeoutRetry reports retried", async () => {
+    let n = 0;
+    const x = await runWithTimeoutRetry(async (ms) => (++n === 1 ? { rc: 124, output: "t" } : { rc: 0, output: String(ms) }), 1000, { load: idle });
+    assert.deepEqual([x.retried, x.r.rc, x.r.output], [true, 0, "2000"]);
+  });
+});

package/bin/lean/prefilter.mjs

@@ -15,6 +15,8 @@
  * of origin/<default>, or while forge.yaml `post_deploy.precondition_cmd` exits 1. With `post_deploy.min_age` (#4425) it also
  * stays parked (`awaiting-min-age`) until that long (or the longest window an item's why states) after the fix landed on
  * the default branch. Anything unresolvable fails open.
+ * Named checks (#4626): once ready, the items' `[check:<id>]` checks run from the default branch's checks file (no model
+ * call): pass closes the follow-up, fail reopens the parent. An item without a runnable check parks `awaiting-measurement`.
  * All effects are injected (gh, git, runner, transitionLabel, projector?, execCode?) so it is unit-testable.
  */
 import { execFile } from "node:child_process";
@@ -23,6 +25,7 @@ import { PREFILTER_SYSTEM, prefilterPrompt, parsePrefilterVerdict } from "./prom
 import { isTrustedComment } from "../engine/phases.mjs";
 import { APP_SLUG } from "../init-repo.mjs";
 import { probeResultOf } from "./probe.mjs";
+import { AWAITING_MEASUREMENT_LABEL, MONITORING_LABEL, itemCheckIds, measureFollowup } from "./postdeploy-check.mjs";
 
 export const DEFAULT_PREFILTER_MODEL = "claude-haiku-5-5";
 export const PREFILTER_MARKER = "FORGE:PREFILTER";
@@ -300,16 +303,20 @@ export function lazyDefaultBranch({ io, repo, cwd, configured = null }) {
 }
 /** #4389: label and explain a parked issue once (the comment only when the label was not there yet); best effort. */
 async function markParked({ issue, repo, io, had, gate }) {
-  const edit = () => io.gh(["issue", "edit", String(issue), "-R", repo, "--add-label", AWAITING_PROMOTION_LABEL]);
+  const measure = gate.state === "awaiting-measurement"; // #4626
+  const label = measure ? AWAITING_MEASUREMENT_LABEL : AWAITING_PROMOTION_LABEL;
+  const edit = () => io.gh(["issue", "edit", String(issue), "-R", repo, "--add-label", label]);
   try { await edit(); } catch {
     try {
-      await io.gh(["label", "create", AWAITING_PROMOTION_LABEL, "-R", repo, "--force", "--color", "C5DEF5", "--description", "Post-deploy follow-up parked until its parent fix reaches the default branch. Managed by ForgeDock."]);
+      await io.gh(["label", "create", label, "-R", repo, "--force", "--color", "C5DEF5", "--description", measure ? "Post-deploy follow-up with no runnable default-branch check; never given a lane. Managed by ForgeDock." : "Post-deploy follow-up parked until its parent fix reaches the default branch. Managed by ForgeDock."]);
       await edit();
     } catch { /* the comment and the report still show it */ }
   }
   if (had) return;
-  const what = gate.state === "awaiting-promotion" ? "awaiting promotion" : gate.state === "awaiting-min-age" ? "awaiting its post-deploy soak window" : "awaiting its post-deploy precondition";
-  const body = `<!-- ${PRECONDITION_MARKER} -->\n## Parked: ${what}\n${sanitizeMd(gate.detail)}.\n\nEvery item here is a post-deploy check, so no lane was started. The next orchestrate or triage run checks again and launches this issue once the precondition holds.`;
+  const what = measure ? "awaiting measurement" : gate.state === "awaiting-promotion" ? "awaiting promotion" : gate.state === "awaiting-min-age" ? "awaiting its post-deploy soak window" : "awaiting its post-deploy precondition";
+  const tail = measure ? "Every item here is a post-deploy check, and at least one names no runnable check, so no lane was started and none will be. Name each one with `[check:<id>]` and add `<id>` to the checks file on the default branch, or measure it by hand."
+    : "Every item here is a post-deploy check, so no lane was started. The next orchestrate or triage run checks again and launches this issue once the precondition holds.";
+  const body = `<!-- ${PRECONDITION_MARKER} -->\n## Parked: ${what}\n${sanitizeMd(gate.detail)}.\n\n${tail}`;
   try { await io.gh(["issue", "comment", String(issue), "-R", repo, "--body", capComment(body)]); } catch { /* the label still shows it */ }
 }
 
@@ -328,6 +335,12 @@ export function formatParkedLine(rows) {
     const defs = uniq(soak.map((r) => r.prefilter.default));
     out.push(`awaiting min age: ${soak.map((r) => `#${r.issue}`).join(" ")} (fix on ${defs.join(", ") || "the default branch"}, soak window not elapsed)`);
   }
+  const meas = parked.filter((r) => r.prefilter.kind === "awaiting-measurement"); // #4626
+  if (meas.length) out.push(`awaiting measurement: ${meas.map((r) => `#${r.issue}`).join(" ")} (no runnable default-branch check; no lane)`);
+  const failed = parked.filter((r) => r.prefilter.kind === "check-failed");
+  if (failed.length) out.push(`check failed: ${failed.map((r) => `#${r.issue}`).join(" ")} (parent reopened; waiting for a new fix)`);
+  const mon = parked.filter((r) => r.prefilter.kind === "monitoring"); // #4627
+  if (mon.length) out.push(`monitoring: ${mon.map((r) => `#${r.issue}`).join(" ")} (windowed post-deploy checks re-run on later releases)`);
   const pre = parked.filter((r) => r.prefilter.kind === "awaiting-precondition");
   if (pre.length) out.push(`awaiting precondition: ${pre.map((r) => `#${r.issue}`).join(" ")} (post_deploy.precondition_cmd not ready)`);
   return out.join("\n");
@@ -346,6 +359,8 @@ export async function prefilterIssue({ issue, repo, lane, cwd, io, model = DEFAU
     const labelled = names.some((l) => ["needs-human", "workflow:merged"].includes(l));
     // #4389: a post-deploy follow-up is gated even when an earlier lane escalated it (needs-human); nothing else changes.
     const gated = !!defaultBranch && !names.includes("workflow:merged") && deferredFrom(iss.body) != null && postDeployOnly(iss.body);
+    // #4627: a follow-up under windowed monitoring belongs to `sweepDue`, never to a lane or a second measurement
+    if (names.includes(MONITORING_LABEL)) return { ...out, parked: true, kind: "monitoring", detail: "windowed post-deploy checks re-run on later releases" };
     if (labelled && !gated) return { ...out, detail: "labelled, left to the lane" };
     if (io.projector) {
       let st = null;
@@ -362,6 +377,16 @@ export async function prefilterIssue({ issue, repo, lane, cwd, io, model = DEFAU
       if (gate.state === "ready" && had && !dryRun) {
         try { await io.gh(["issue", "edit", String(issue), "-R", repo, "--remove-label", AWAITING_PROMOTION_LABEL]); } catch { /* best effort */ }
       }
+      if (gate.state === "ready") {
+        const m = await measureFollowup({ iss, repo, parent: gate.parent, def: gate.default, shas: gate.shas, cwd, io, dryRun, isTrusted: isTrustedIssueComment, ...(io.spawnCheck ? { spawn: io.spawnCheck } : {}) });
+        if (m.state === "awaiting-measurement") {
+          if (!dryRun) await markParked({ issue, repo, io, had: names.includes(AWAITING_MEASUREMENT_LABEL), gate: { ...gate, state: m.state, detail: m.detail } });
+          return { ...out, parked: true, kind: m.state, parent: gate.parent, lane, default: gate.default, detail: m.detail };
+        }
+        if (m.state === "monitoring") return { ...out, parked: true, kind: "monitoring", parent: gate.parent, lane, default: gate.default, detail: m.detail };
+        if (m.state === "pass") return { ...out, closed: !dryRun, kind: "post-deploy-check", parent: gate.parent, lane, default: gate.default, detail: m.detail };
+        if (m.state === "fail" || m.state === "recorded-fail") return { ...out, parked: true, kind: "check-failed", parent: gate.parent, lane, default: gate.default, detail: m.detail };
+      }
       if (labelled) return { ...out, detail: `labelled, left to the lane (post-deploy gate: ${gate.state}${gate.detail ? `, ${gate.detail}` : ""})` };
     }
     // #4399: an issue reopened by a failed acceptance probe goes to a lane, never back to an already-fixed close

package/bin/lean/prefilter.test.mjs

@@ -397,14 +397,22 @@ describe("post-deploy precondition gate (#4389)", () => {
     assert.deepEqual(w.edits, []); assert.deepEqual(w.comments, []);
   });
 
-  it("an ancestor (after promotion): not parked, the label is removed, and the issue goes on as before", async () => {
+  it("an ancestor (after promotion): the promotion label is removed; with no runnable check it parks awaiting-measurement, never a lane (#4626)", async () => {
     const w = world({ issues: { 41: { body: followup(40, [item("R2", "post-deploy")]), labels: [{ name: "awaiting-promotion" }] } }, mergedPrs: parentPr, ancestry: { [MERGE]: 0 } });
     const r = await gated(w, 41);
-    assert.ok(!r.parked); assert.equal(r.detail, "no already-fixed signal");
-    assert.deepEqual(w.edits, [[41, "--remove-label", "awaiting-promotion"]]);
+    assert.equal(r.parked, true); assert.equal(r.kind, "awaiting-measurement");
+    assert.deepEqual(w.edits, [[41, "--remove-label", "awaiting-promotion"], [41, "--add-label", "awaiting-measurement"]]);
+    assert.equal(w.runner.length, 0);
     const rows = await orchestrate({ issues: [41], readBody: async () => "", prefilter: makePrefilter({ repo: "o/r", cwd: "/repo", io: w.io, defaultBranch: "main" }),
-      runOne: async () => ({ terminalReason: "merged", pr: 7, cost_usd: 1 }) });
-    assert.deepEqual(rows.map((x) => x.outcome), ["merged"]);
+      runOne: async () => { throw new Error("no lane"); } });
+    assert.deepEqual(rows.map((x) => x.outcome), ["awaiting-measurement"]);
+  });
+
+  it("a follow-up under windowed monitoring is parked, never a lane or a second measurement (#4627)", async () => {
+    const w = world({ issues: { 41: { body: followup(40, [item("R2", "[check:smoke] post-deploy")]), labels: [{ name: "post-deploy-monitoring" }] } }, mergedPrs: parentPr, ancestry: { [MERGE]: 0 } });
+    const r = await gated(w, 41);
+    assert.equal(r.parked, true); assert.equal(r.kind, "monitoring"); assert.equal(r.closed, false);
+    assert.deepEqual(w.edits, []); assert.deepEqual(w.comments, []); assert.equal(w.runner.length, 0);
   });
 
   it("an unresolvable parent (no merged PR, no trajectory) or an unknown SHA falls through to the lane", async () => {

package/bin/lean/probe.mjs

@@ -267,7 +267,7 @@ export async function resolveDefaultBranch({ repo, io, cwd = process.cwd() }) {
  * Probe `issues`, or (none given) the issues closed within `sweepDays` that mention `forge-probe`, capped at `limit`.
  * Sequential: probes hit production. Never throws per issue: an error is an `error` row.
  */
-export async function sweepProbes({ repo, cwd = process.cwd(), io, issues = null, limit = 50, sinceDays = null, dryRun = false, config = readProbeConfig(cwd), now = () => Date.now(), env = process.env, spawn } = {}) {
+export async function sweepProbes({ repo, cwd = process.cwd(), io, issues = null, limit = 50, sinceDays = null, dryRun = false, config = readProbeConfig(cwd), now = () => Date.now(), env = process.env, spawn, onResult = null } = {}) {
   if (!config.enabled && !issues?.length) return [];
   const defaultBranch = await resolveDefaultBranch({ repo, io, cwd });
   if (!defaultBranch) throw new Error("no default branch");
@@ -284,6 +284,7 @@ export async function sweepProbes({ repo, cwd = process.cwd(), io, issues = null
     try { rows.push(await probeIssue({ issue, repo, cwd, io, defaultBranch, dryRun, config, env, spawn })); } catch (e) {
       rows.push({ issue, outcome: "error", detail: String(e?.message ?? e).slice(0, 160) });
     }
+    try { if (!dryRun) await onResult?.(rows[rows.length - 1]); } catch { /* an event sink never fails the sweep (#4630) */ }
   }
   return rows;
 }