forgedock 1.35.0 → 1.37.0

package/bin/lean/postdeploy-check.mjs

@@ -0,0 +1,181 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Named post-deploy checks (#4626). A post-deploy requirement names a check id (`[check:<id>]` in its follow-up item);
+ * the command lives in the checks file on the default branch, `.forge/post-deploy-checks.json`:
+ *   { "<id>": { "run": "sh command", "pass": "exit 0 | /regex/flags", "timeout": 120 } }   (a bare string is `run`)
+ * and is read with `git show origin/<default>:<file>`: reviewed default-branch code, never an issue body or comment
+ * (#4468). When the #4389 gate reports a follow-up ready, `measureFollowup` runs every item's check (no model call, the
+ * probe runner's minimal env): all pass -> a FORGE:PROBE record and the follow-up closes; a fail -> the record, and the
+ * parent reopens with the output. An item without a runnable check is `awaiting-measurement`: no lane, ever.
+ */
+import { noComments } from "./render.mjs";
+import { MAX_TIMEOUT_S, OUTPUT_TAIL, PROBE_MARKER, probeEnv, probeResultOf, readProbeConfig, runProbe } from "./probe.mjs";
+import { parseMinAge } from "./prefilter.mjs";
+
+export const CHECKS_FILE = ".forge/post-deploy-checks.json";
+export const CHECK_ID_RE = /^[a-z0-9][a-z0-9._-]{0,63}$/;
+export const AWAITING_MEASUREMENT_LABEL = "awaiting-measurement";
+export const MONITORING_LABEL = "post-deploy-monitoring";
+export const OWNER_ASK_MARKER = "FORGE:OWNER_ASK"; // #4689
+export const DEFAULT_EVERY_MS = 6 * 3600e3;
+const ITEM_RE = /^\s*[-*]\s+\[[ xX]\]\s+(.*)$/;
+const CHECK_TOKEN_RE = /^`[^`]*`\s+(?:\*\*known-flaw\*\*\s+)?[—-]+\s*post-deploy:\s*\[check:\s*([^\]\s]*)\s*\]/;
+
+/**
+ * The checks file's runnable entries `{id: {run, pass, timeoutS, forMs?, everyMs?}}`; malformed input or entries are dropped.
+ * `for` (a window such as "7d") makes the check windowed (#4627); `every` (default 6h) is its cadence and needs a `for`.
+ */
+export function parseChecks(text) {
+  let j;
+  try { j = JSON.parse(String(text ?? "")); } catch { return {}; }
+  if (!j || typeof j !== "object" || Array.isArray(j)) return {};
+  const out = {};
+  for (const [id, v] of Object.entries(j)) {
+    const spec = typeof v === "string" ? { run: v } : v;
+    if (!CHECK_ID_RE.test(id) || !spec || typeof spec.run !== "string" || !spec.run.trim()) continue;
+    let pass = { kind: "exit" };
+    const p = String(spec.pass ?? "").trim();
+    if (p && !/^exit\s+0$/i.test(p)) {
+      const r = /^\/(.+)\/([a-z]*)$/.exec(p);
+      try { if (!r) continue; new RegExp(r[1], r[2]); } catch { continue; }
+      pass = { kind: "regex", re: r[1], flags: r[2] };
+    }
+    const t = spec.timeout == null ? null : Number(spec.timeout);
+    if (t != null && !(Number.isFinite(t) && t > 0)) continue;
+    const win = spec.for == null ? null : parseMinAge(spec.for);
+    const cad = spec.every == null ? null : parseMinAge(spec.every);
+    if ((spec.for != null && !win) || (spec.every != null && !cad) || (spec.every != null && spec.for == null)) continue;
+    out[id] = { run: spec.run.trim(), pass, timeoutS: t == null ? null : Math.min(Math.floor(t) || 1, MAX_TIMEOUT_S), ...(win ? { forMs: win, everyMs: cad ?? DEFAULT_EVERY_MS } : {}) };
+  }
+  return out;
+}
+
+/** Per checklist item of a follow-up body: its `[check:<id>]` id, or null when it names none (or a malformed one). */
+export function itemCheckIds(body) {
+  return noComments(String(body ?? ""), " ").split("\n").map((l) => ITEM_RE.exec(l)?.[1]).filter((t) => t != null)
+    .map((t) => { const m = CHECK_TOKEN_RE.exec(t); return m && CHECK_ID_RE.test(m[1]) ? m[1] : null; });
+}
+
+/** The checks file at each git ref (`git show <ref>:<file>`); a missing file or ref contributes nothing. */
+export async function checksAt(git, refs, cwd) {
+  const out = {};
+  for (const ref of refs) {
+    try { Object.assign(out, parseChecks(String(await git(["show", `${ref}:${CHECKS_FILE}`], { cwd })))); } catch { /* no file at this ref */ }
+  }
+  return out;
+}
+
+const fenced = (s) => { const f = "`".repeat(Math.max(3, ...[...String(s).matchAll(/`+/g)].map((m) => m[0].length + 1))); return `${f}\n${String(s).trim() || "(no output)"}\n${f}`; };
+
+/** Run each distinct check id with the probe runner (minimal env, timeout, masked output): `[{id, result, exit, ms, output}]`. */
+export async function runNamedChecks({ ids, checks, cwd, config, env, spawn }) {
+  const rows = [];
+  for (const id of [...new Set(ids)]) {
+    const r = await runProbe(checks[id], { cwd, env: probeEnv(config.env, env), mask: config.env.map((k) => env[k]), timeoutS: config.timeoutS, ...(spawn ? { spawn } : {}) });
+    rows.push({ id, ...r });
+  }
+  return rows;
+}
+
+/** The windowed-monitoring fields of a FORGE:PROBE record body (`at`, `window_until`, `every_ms`, `regression`, ...), or null. */
+export function monitorRecord(body) {
+  const base = probeResultOf(body);
+  if (!base) return null;
+  const line = String(body).split("\n").find((l) => /^\s*`?\{.*"result"/.test(l));
+  try {
+    const j = JSON.parse(line.trim().replace(/^`|`$/g, ""));
+    const at = Date.parse(j.at), until = Date.parse(j.window_until);
+    return Number.isFinite(at) ? { ...base, at, defaultSha: j.default_sha ?? null, everyMs: Number(j.every_ms) || DEFAULT_EVERY_MS, until: Number.isFinite(until) ? until : null, regression: j.regression ?? null } : null;
+  } catch { return null; }
+}
+
+/** The post-deploy-monitoring label, created on first use; best effort. */
+export async function addMonitoringLabel({ io, repo, issue }) {
+  const edit = () => io.gh(["issue", "edit", String(issue), "-R", repo, "--add-label", MONITORING_LABEL]);
+  try { await edit(); } catch {
+    try { await io.gh(["label", "create", MONITORING_LABEL, "-R", repo, "--force", "--color", "C5DEF5", "--description", "Post-deploy follow-up whose windowed check re-runs on later releases. Managed by ForgeDock."]); await edit(); } catch { /* the record still shows it */ }
+  }
+}
+
+/** #4689: tick every unticked post-deploy checklist item of a follow-up body (the evidence is in the FORGE:PROBE comment). */
+export const tickItems = (body) => String(body ?? "").replace(/^(\s*[-*]\s+)\[ \](\s+`[^`]*`\s+(?:\*\*known-flaw\*\*\s+)?[—-]+\s*post-deploy:)/gm, "$1[x]$2");
+
+/** #4689: one typed `FORGE:OWNER_ASK` comment for the items with no runnable check, posted once; best effort, nothing in a dry run. */
+async function postOwnerAsk({ iss, repo, io, dryRun, isTrusted, ids, missing, def }) {
+  if (dryRun || (iss.comments ?? []).some((c) => isTrusted(c) && String(c?.body ?? "").includes(OWNER_ASK_MARKER))) return;
+  const what = ids.length ? missing.map((m) => (m ? `add \`${m}\` to ${CHECKS_FILE} on ${def}` : "name an item's check with `[check:<id>]`")).join("; ") : "name each item's check with `[check:<id>]` and add it to the checks file";
+  try { await io.gh(["issue", "comment", String(iss.number), "-R", repo, "--body", `<!-- ${OWNER_ASK_MARKER} -->\n## Owner ask\nNo lane was started and none will be. Needed from the owner: ${what}, or measure the item by hand.`]); } catch { /* the label still shows it */ }
+}
+
+/** #4689: one engineering issue per failing check (never a lane): the output, the fix and the follow-up. Best effort. */
+async function fileFailureIssues({ bad, iss, repo, io, parent, def, fix }) {
+  for (const r of bad) {
+    const title = `Post-deploy check ${r.id} fails on ${def}${parent ? ` (fix of #${parent})` : ""}`;
+    const body = `Post-deploy check \`${r.id}\` failed (exit ${r.exit ?? "timeout"}) for fix \`${fix.slice(0, 12)}\` on \`${def}\`. Follow-up: #${iss.number}.\n\n${fenced(r.output)}`.slice(0, OUTPUT_TAIL * 2);
+    const create = (extra) => io.gh(["issue", "create", "-R", repo, "--title", title, "--body", body, ...extra]);
+    try { await create(["--label", "followup:engineering"]); } catch { try { await create([]); } catch { /* the follow-up comment carries the output */ } }
+  }
+}
+
+/**
+ * Measure a ready post-deploy follow-up. Never runs a command that is not in the default branch's checks file.
+ * @returns {Promise<{state:"awaiting-measurement"|"pass"|"fail"|"recorded-fail"|"dry-run"|"monitoring", detail:string}>}
+ * When every item's check declares a window (#4627) a pass records the release and the window end and the follow-up stays
+ * open (`monitoring`); `sweepDue` (postdeploy-monitor.mjs) re-runs it.
+ */
+export async function measureFollowup({ iss, repo, parent, def, shas = [], cwd, io, dryRun = false, isTrusted = () => false, config = readProbeConfig(cwd), env = process.env, spawn, now = () => Date.now() }) {
+  const ids = itemCheckIds(iss.body);
+  const checks = await checksAt(io.git, [`origin/${def}`], cwd);
+  const missing = ids.filter((id) => !id || !checks[id]);
+  if (!ids.length || missing.length) {
+    await postOwnerAsk({ iss, repo, io, dryRun, isTrusted, ids, missing, def });
+    return { state: "awaiting-measurement", detail: `no runnable check on ${def} for ${ids.length ? `${missing.map((m) => m ?? "an item without a check id").join(", ")}` : "any item"} (${CHECKS_FILE})` };
+  }
+  const fix = [...shas].sort().join(",");
+  const prior = (iss.comments ?? []).filter((c) => isTrusted(c)).map((c) => probeResultOf(c?.body)).filter(Boolean).reverse().find((p) => p.fix === fix);
+  if (prior?.result === "fail") return { state: "recorded-fail", detail: `check failed for ${fix.slice(0, 12)} (FORGE:PROBE); waiting for a new fix of #${parent}` };
+  const windowed = ids.every((id) => checks[id].forMs);
+  const monitored = (iss.comments ?? []).filter((c) => isTrusted(c)).map((c) => monitorRecord(c?.body)).filter(Boolean).find((p) => p.fix === fix && p.result === "pass" && p.until);
+  if (windowed && monitored) {
+    await addMonitoringLabel({ io, repo, issue: iss.number }); // retry a label that failed on the first pass; sweepDue selects by it
+    return { state: "monitoring", detail: `windowed checks ${[...new Set(ids)].join(", ")} monitored until ${new Date(monitored.until).toISOString()}` };
+  }
+  if (dryRun) return { state: "dry-run", detail: `checks ${[...new Set(ids)].join(", ")} not run (dry run)` };
+  let defaultSha = null;
+  try { defaultSha = String(await io.git(["rev-parse", `origin/${def}`], { cwd })).trim() || null; } catch { /* recorded as null */ }
+  const rows = [];
+  if (prior?.result !== "pass") {
+    rows.push(...await runNamedChecks({ ids, checks, cwd, config, env, spawn }));
+    const ok = rows.every((r) => r.result === "pass");
+    const at = now();
+    const win = windowed && ok ? { at: new Date(at).toISOString(), window_until: new Date(at + Math.max(...ids.map((id) => checks[id].forMs))).toISOString(), every_ms: Math.min(...ids.map((id) => checks[id].everyMs)) } : {};
+    const body = [
+      `<!-- ${PROBE_MARKER} -->`, `## Post-deploy check: ${ok ? "pass" : "fail"}`,
+      JSON.stringify({ fix, default_sha: defaultSha, result: ok ? "pass" : "fail", ...win, checks: rows.map((r) => ({ id: r.id, result: r.result, exit: r.exit, ms: r.ms })) }), "",
+      `Ran after the fix of #${parent} reached \`${def}\`; commands come from \`${CHECKS_FILE}\` on \`${def}\`.${ok ? (win.at ? ` Monitored until \`${win.window_until}\`, re-run on later releases.` : " The follow-up is closed.") : " The parent is reopened."}`,
+      ...rows.filter((r) => r.result === "fail").flatMap((r) => ["", `**${r.id}** (exit ${r.exit ?? "timeout"})`, fenced(r.output)]),
+    ].join("\n").slice(0, OUTPUT_TAIL * 4);
+    if (!ok) {
+      const bad = rows.filter((r) => r.result === "fail");
+      // Reopen before recording: the FORGE:PROBE fail record parks later passes (recorded-fail), so it must only exist once the parent is open.
+      try { await io.gh(["issue", "reopen", String(parent), "-R", repo]); } catch (e) {
+        return { state: "fail", detail: `check ${bad.map((r) => r.id).join(", ")} failed; reopen of #${parent} failed, no record posted so the next pass retries (${String(e?.message ?? e).slice(0, 120)})` };
+      }
+      await io.gh(["issue", "comment", String(iss.number), "-R", repo, "--body", body]);
+      await fileFailureIssues({ bad, iss, repo, io, parent, def, fix }); // #4689
+      try {
+        await io.gh(["issue", "comment", String(parent), "-R", repo, "--body", `Post-deploy check ${bad.map((r) => `\`${r.id}\``).join(", ")} failed for the fix on \`${def}\` (follow-up #${iss.number}); reopened.\n\n${bad.map((r) => fenced(r.output)).join("\n\n")}`.slice(0, OUTPUT_TAIL * 4)]);
+      } catch { /* the follow-up comment carries the output */ }
+      return { state: "fail", detail: `check ${bad.map((r) => r.id).join(", ")} failed; #${parent} reopened` };
+    }
+    await io.gh(["issue", "comment", String(iss.number), "-R", repo, "--body", body]);
+    if (tickItems(iss.body) !== iss.body) { try { await io.gh(["issue", "edit", String(iss.number), "-R", repo, "--body", tickItems(iss.body)]); } catch { /* the PROBE record is the evidence */ } } // #4689
+  }
+  if (windowed) {
+    await addMonitoringLabel({ io, repo, issue: iss.number });
+    return { state: "monitoring", detail: `checks ${[...new Set(ids)].join(", ")} passed; monitored on later releases` };
+  }
+  await io.gh(["issue", "close", String(iss.number), "-R", repo, "--reason", "completed"]);
+  return { state: "pass", detail: `checks ${[...new Set(ids)].join(", ")} passed` };
+}

package/bin/lean/postdeploy-check.test.mjs

@@ -0,0 +1,206 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { parseChecks, itemCheckIds, measureFollowup, monitorRecord, CHECK_ID_RE, tickItems } from "./postdeploy-check.mjs";
+import { postDeployOnly } from "./prefilter.mjs";
+import { validateContract, applyAmendments, precheck, followupItems } from "./conformance.mjs";
+import { PROBE_MARKER } from "./probe.mjs";
+
+const item = (id, why) => `- [ ] \`${id}: some quote here\` — post-deploy: ${why}`;
+const body = (items) => `## Acceptance Criteria\n\n${items.join("\n")}\n\n<!-- FORGE:DEFERRED_FROM: #40 -->\n`;
+const CHECKS = JSON.stringify({ smoke: { run: "echo ok", timeout: 5 }, bad: "echo nope", "bad id!": "x", rx: { run: "echo up", pass: "/up/" }, norun: {} });
+
+function world({ file = CHECKS, spawn } = {}) {
+  const w = { gh: [], git: [], spawned: [] };
+  w.io = {
+    gh: async (a) => { w.gh.push(a); return ""; },
+    git: async (a) => { w.git.push(a); if (a[0] === "show") { if (file == null) throw new Error("missing"); return file; } return "deadbeef\n"; },
+  };
+  w.spawn = async (o) => { w.spawned.push(o.run); return spawn ? spawn(o) : { code: o.run === "echo nope" ? 1 : 0, stdout: o.run === "echo nope" ? "boom" : "ok", stderr: "" }; };
+  return w;
+}
+const run = (w, bodyText, extra = {}) => measureFollowup({ iss: { number: 41, body: bodyText, comments: [] }, repo: "o/r", parent: 40, def: "main", shas: ["abc"], cwd: "/repo", io: w.io, config: { env: [], timeoutS: 5 }, env: {}, spawn: w.spawn, ...extra });
+
+describe("post-deploy named checks (#4626)", () => {
+  it("parses the checks file; malformed entries are dropped", () => {
+    const c = parseChecks(CHECKS);
+    assert.deepEqual(Object.keys(c).sort(), ["bad", "rx", "smoke"]);
+    assert.equal(c.smoke.timeoutS, 5); assert.equal(c.rx.pass.kind, "regex");
+    assert.deepEqual(parseChecks("not json"), {});
+    assert.ok(CHECK_ID_RE.test("login-smoke") && !CHECK_ID_RE.test("Bad Id"));
+  });
+
+  it("postDeployOnly still matches items with and without a [check:id] token", () => {
+    assert.ok(postDeployOnly(body([item("R1", "[check:smoke] soak"), item("R2", "only in prod")])));
+  });
+  it("extracts [check:id] per item; an item without one yields null", () => {
+    assert.deepEqual(itemCheckIds(body([item("R1", "[check:smoke] soak"), item("R2", "only in prod")])), ["smoke", null]);
+  });
+
+  it("pass: runs the default-branch command, records FORGE:PROBE, closes the follow-up, no reopen", async () => {
+    const w = world();
+    const r = await run(w, body([item("R1", "[check:smoke] x")]));
+    assert.equal(r.state, "pass");
+    assert.deepEqual(w.spawned, ["echo ok"]);
+    assert.ok(w.git.some((a) => a.join(" ") === "show origin/main:.forge/post-deploy-checks.json"));
+    const cm = w.gh.find((a) => a[1] === "comment");
+    assert.match(cm[cm.indexOf("--body") + 1], new RegExp(PROBE_MARKER));
+    assert.ok(w.gh.some((a) => a[1] === "close" && a[2] === "41"));
+    assert.ok(!w.gh.some((a) => a[1] === "reopen"));
+  });
+
+  it("fail: reopens the parent with the output and does not close", async () => {
+    const w = world();
+    const r = await run(w, body([item("R1", "[check:bad] x")]));
+    assert.equal(r.state, "fail");
+    assert.ok(w.gh.some((a) => a[1] === "reopen" && a[2] === "40"));
+    assert.ok(w.gh.some((a) => a[1] === "comment" && a[2] === "40" && a[a.indexOf("--body") + 1].includes("boom")));
+    assert.ok(!w.gh.some((a) => a[1] === "close"));
+  });
+
+  it("no runnable check: awaiting-measurement, nothing runs, no model call", async () => {
+    for (const [file, b] of [[CHECKS, body([item("R1", "[check:ghost] x")])], [null, body([item("R1", "[check:smoke] x")])], [CHECKS, body([item("R1", "x"), item("R2", "[check:smoke] y")])]]) {
+      const w = world({ file });
+      assert.equal((await run(w, b)).state, "awaiting-measurement");
+      assert.deepEqual(w.spawned, []); assert.ok(w.gh.every((a) => a[1] === "comment" && /FORGE:OWNER_ASK/.test(a[a.indexOf("--body") + 1])), "only the typed owner ask (#4689)");
+    }
+  });
+
+  it("a command declared in the issue body never runs", async () => {
+    const w = world();
+    const b = body([item("R1", "[check:ghost] x")]) + "\n```forge-probe\nrun: rm -rf /\n```\n";
+    assert.equal((await run(w, b)).state, "awaiting-measurement");
+    assert.deepEqual(w.spawned, []);
+  });
+
+  it("migration: the six open follow-up shapes (no check id) all park awaiting-measurement", async () => {
+    for (const n of [4415, 4421, 4438, 4488, 4495, 4498]) {
+      const w = world();
+      const r = await run(w, body([item(`R${n % 7}`, "only checkable after the next deploy")]));
+      assert.equal(r.state, "awaiting-measurement", `#${n}`);
+      assert.deepEqual(w.spawned, []);
+    }
+  });
+
+  it("a recorded fail for the same fix is not re-run", async () => {
+    const w = world();
+    const c = { body: `<!-- ${PROBE_MARKER} -->\n${JSON.stringify({ fix: "abc", result: "fail" })}` };
+    const r = await run(w, body([item("R1", "[check:smoke] x")]), { iss: { number: 41, body: body([item("R1", "[check:smoke] x")]), comments: [c] }, isTrusted: () => true });
+    assert.equal(r.state, "recorded-fail"); assert.deepEqual(w.spawned, []);
+  });
+
+  it("contract: post-deploy needs a check id; amendment too; precheck rejects an unknown id; followup item names it", () => {
+    const Q = "The deploy-time health row must exist after promotion.";
+    const c = { requirements: [{ id: "R1", quote: Q, disposition: "post-deploy", why: "w" }], deliverables: [], preserve: [], validate: [{ id: "V1", command: "true" }] };
+    assert.ok(validateContract(c, Q).problems.some((p) => /needs a check id/.test(p)));
+    assert.ok(!validateContract({ ...c, requirements: [{ ...c.requirements[0], check: "smoke" }] }, Q).problems.some((p) => /check id/.test(p)));
+    const frozen = { ...c, requirements: [{ id: "R1", quote: Q, disposition: "deliver" }] };
+    assert.ok(applyAmendments(frozen, [{ kind: "post-deploy", requirement: "R1", why: "w" }], Q).problems.some((p) => /check id/.test(p)));
+    const ok = applyAmendments(frozen, [{ kind: "post-deploy", requirement: "R1", why: "w", check: "smoke" }], Q);
+    assert.equal(ok.contract.requirements[0].check, "smoke");
+    const pc = precheck({ contract: { ...c, requirements: [{ ...c.requirements[0], check: "ghost" }] }, result: { requirements: [] }, changedFiles: [], checkIds: new Set(["smoke"]) });
+    assert.ok(pc.failures.some((f) => f.kind === "check_unknown"));
+    assert.match(followupItems([{ ...c.requirements[0], check: "smoke" }])[0].why, /^post-deploy: \[check:smoke\] w/);
+  });
+});
+
+describe("windowed post-deploy checks (#4627)", () => {
+  const WIN = JSON.stringify({ soak: { run: "echo ok", for: "7d", every: "6h" }, soakd: { run: "echo ok", for: "2d" }, noFor: { run: "echo ok", every: "6h" }, badFor: { run: "echo ok", for: "soon" }, zero: { run: "echo ok", for: "0" }, plain: "echo ok" });
+  const NOW = Date.parse("2026-10-26T00:00:00Z");
+  it("parses for/every; every defaults to 6h; every without for or a bad duration drops the entry", () => {
+    const c = parseChecks(WIN);
+    assert.deepEqual(Object.keys(c).sort(), ["plain", "soak", "soakd"]);
+    assert.equal(c.soak.forMs, 7 * 86400e3); assert.equal(c.soak.everyMs, 6 * 3600e3);
+    assert.equal(c.soakd.everyMs, 6 * 3600e3);
+    assert.equal(c.plain.forMs, undefined);
+  });
+  it("first pass records the release and window, labels, and keeps the follow-up open", async () => {
+    const w = world({ file: WIN });
+    const r = await run(w, body([item("R1", "[check:soak] x")]), { now: () => NOW });
+    assert.equal(r.state, "monitoring");
+    assert.ok(!w.gh.some((a) => a[1] === "close") && !w.gh.some((a) => a[1] === "reopen"));
+    assert.ok(w.gh.some((a) => a[1] === "edit" && a.includes("post-deploy-monitoring")));
+    const cm = w.gh.find((a) => a[1] === "comment");
+    const rec = monitorRecord(cm[cm.indexOf("--body") + 1]);
+    assert.equal(rec.result, "pass"); assert.equal(rec.until, NOW + 7 * 86400e3); assert.equal(rec.everyMs, 6 * 3600e3); assert.equal(rec.defaultSha, "deadbeef");
+  });
+  it("a recorded windowed pass is not re-measured by the pre-filter path", async () => {
+    const w = world({ file: WIN });
+    const rec = { body: `<!-- ${PROBE_MARKER} -->\n${JSON.stringify({ fix: "abc", result: "pass", at: new Date(NOW).toISOString(), window_until: new Date(NOW + 1e9).toISOString() })}` };
+    const b = body([item("R1", "[check:soak] x")]);
+    const r = await run(w, b, { iss: { number: 41, body: b, comments: [rec] }, isTrusted: () => true });
+    assert.equal(r.state, "monitoring"); assert.deepEqual(w.spawned, []);
+  });
+  it("a non-windowed check still closes on pass (part 1 unchanged)", async () => {
+    const w = world({ file: WIN });
+    assert.equal((await run(w, body([item("R1", "[check:plain] x")]))).state, "pass");
+    assert.ok(w.gh.some((a) => a[1] === "close"));
+  });
+  it("a first-run fail of a windowed check reopens the parent as before", async () => {
+    const w = world({ file: JSON.stringify({ soak: { run: "echo nope", for: "7d" } }), spawn: () => ({ code: 1, stdout: "boom", stderr: "" }) });
+    assert.equal((await run(w, body([item("R1", "[check:soak] x")]))).state, "fail");
+    assert.ok(w.gh.some((a) => a[1] === "reopen" && a[2] === "40"));
+  });
+  it("a failed parent reopen posts no fail record, so the next pass re-runs and retries", async () => {
+    const w = world();
+    const gh = w.io.gh;
+    let failReopen = true;
+    w.io.gh = async (a) => { if (a[1] === "reopen" && failReopen) { w.gh.push(a); throw new Error("api down"); } return gh(a); };
+    const b = body([item("R1", "[check:bad] x")]);
+    const r = await run(w, b);
+    assert.equal(r.state, "fail"); assert.match(r.detail, /reopen of #40 failed/);
+    assert.ok(!w.gh.some((a) => a[1] === "comment"));
+    failReopen = false;
+    const r2 = await run(w, b);
+    assert.ok(w.gh.filter((a) => a[1] === "reopen").length === 2 && /#40 reopened/.test(r2.detail));
+    const order = w.gh.map((a) => a[1] === "comment" ? `c${a[2]}` : a[1]);
+    assert.ok(order.indexOf("reopen", 1) < order.indexOf("c41"));
+  });
+  it("a recorded windowed pass retries the monitoring label", async () => {
+    const w = world({ file: WIN });
+    const rec = { body: `<!-- ${PROBE_MARKER} -->\n${JSON.stringify({ fix: "abc", result: "pass", at: new Date(NOW).toISOString(), window_until: new Date(NOW + 1e9).toISOString() })}` };
+    const b = body([item("R1", "[check:soak] x")]);
+    await run(w, b, { iss: { number: 41, body: b, comments: [rec] }, isTrusted: () => true });
+    assert.ok(w.gh.some((a) => a[1] === "edit" && a.includes("post-deploy-monitoring")));
+  });
+});
+
+
+describe("verification outcome (#4689)", () => {
+  it("a pass ticks the post-deploy items with the evidence in the PROBE record, and writes no code", async () => {
+    const w = world();
+    const r = await run(w, body([item("R1", "[check:smoke] x")]));
+    assert.equal(r.state, "pass");
+    const edit = w.gh.find((a) => a[0] === "issue" && a[1] === "edit");
+    const text = edit[edit.indexOf("--body") + 1];
+    assert.match(text, /- \[x\] `R1: some quote here`/);
+    assert.equal(tickItems("- [ ] `R9: other` — defer: code"), "- [ ] `R9: other` — defer: code");
+    const probe = w.gh.find((a) => a[1] === "comment" && /FORGE:PROBE/.test(a[a.indexOf("--body") + 1]));
+    assert.match(probe[probe.indexOf("--body") + 1], /"id":"smoke","result":"pass"/);
+    assert.ok(w.git.every((a) => ["show", "rev-parse"].includes(a[0])), "no git write: " + JSON.stringify(w.git));
+    assert.equal(tickItems("no items"), "no items");
+  });
+
+  it("a fail files one engineering issue per failing check and ticks nothing", async () => {
+    const w = world({ file: JSON.stringify({ bad: "echo nope", worse: "echo nope", smoke: "echo ok" }) });
+    const r = await run(w, body([item("R1", "[check:bad] x"), item("R2", "[check:worse] x"), item("R3", "[check:smoke] x")]));
+    assert.equal(r.state, "fail");
+    const created = w.gh.filter((a) => a[0] === "issue" && a[1] === "create");
+    assert.equal(created.length, 2);
+    assert.ok(created.every((a) => a.includes("followup:engineering")));
+    assert.ok(created[0][created[0].indexOf("--title") + 1].includes("bad"));
+    assert.ok(!w.gh.some((a) => a[1] === "edit" && a.includes("--body")));
+  });
+
+  it("an item with no runnable check posts one typed owner ask, once", async () => {
+    const w = world({ file: null });
+    assert.equal((await run(w, body([item("R1", "[check:smoke] x")]))).state, "awaiting-measurement");
+    const asks = w.gh.filter((a) => a[1] === "comment" && /FORGE:OWNER_ASK/.test(a[a.indexOf("--body") + 1]));
+    assert.equal(asks.length, 1);
+    const seen = { number: 41, body: body([item("R1", "[check:smoke] x")]), comments: [{ body: "<!-- FORGE:OWNER_ASK -->\nask", author: { login: "rapiercraft-forgedock" }, authorAssociation: "CONTRIBUTOR" }] };
+    const w2 = world({ file: null });
+    await run(w2, seen.body, { iss: seen, isTrusted: () => true });
+    assert.equal(w2.gh.filter((a) => a[1] === "comment").length, 0);
+  });
+});

package/bin/lean/postdeploy-monitor.mjs

@@ -0,0 +1,97 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Windowed post-deploy monitoring (#4627). A follow-up whose checks declare `for` (and `every`) stays open with the
+ * `post-deploy-monitoring` label after its first pass. `sweepDue` (run by schedd's timer and `forgedock probe --due`) re-runs
+ * each one when its cadence elapsed or `origin/<default>` moved (a release), with the command read from the default branch
+ * (never an issue). A pass is a FORGE:PROBE record; the window end closes the follow-up; a fail after a pass files one
+ * regression issue naming the fix commit and the last-passing and breaking release, then closes the follow-up. No model call.
+ */
+import { readProbeConfig, resolveDefaultBranch, PROBE_MARKER, OUTPUT_TAIL } from "./probe.mjs";
+import { MONITORING_LABEL, checksAt, itemCheckIds, monitorRecord, runNamedChecks, CHECKS_FILE } from "./postdeploy-check.mjs";
+import { deferredFrom, isTrustedIssueComment } from "./prefilter.mjs";
+
+export const REGRESSION_MARKER = "FORGE:REGRESSION";
+const fenced = (s) => { const f = "`".repeat(Math.max(3, ...[...String(s).matchAll(/`+/g)].map((m) => m[0].length + 1))); return `${f}\n${String(s).trim() || "(no output)"}\n${f}`; };
+/** An existing regression issue (any state) for this follow-up and fix commit, so a retry after a failed finish() reuses it (#4653). */
+export async function findRegression(io, repo, issue, fix) {
+  const found = JSON.parse(String(await io.gh(["issue", "list", "-R", repo, "--state", "all", "--search", `"${REGRESSION_MARKER}" "Monitored follow-up: #${issue}" in:body`, "--limit", "30", "--json", "number,body"])));
+  const hit = (Array.isArray(found) ? found : []).find((i) => {
+    const b = String(i?.body ?? "");
+    return b.includes(`<!-- ${REGRESSION_MARKER} -->`) && new RegExp(`Monitored follow-up: #${issue}(?!\\d)`).test(b) && b.includes(`Fix commit: \`${fix}\``);
+  });
+  return hit ? String(hit.number) : null;
+}
+const short = (s) => String(s ?? "unknown").slice(0, 12);
+
+/** One monitored follow-up. Outcomes: skipped | unrunnable | not-due | due (dry run) | pass | window-ended | fail. */
+export async function monitorOne({ issue, repo, def, defaultSha, cwd, io, now, dryRun, config, env, spawn }) {
+  const row = (outcome, detail = "", extra = {}) => ({ issue, outcome, detail, ...extra });
+  const iss = JSON.parse(String(await io.gh(["issue", "view", String(issue), "-R", repo, "--json", "number,title,body,state,labels,comments"])));
+  if (String(iss.state).toUpperCase() !== "OPEN") return row("skipped", "not open");
+  const recs = (iss.comments ?? []).filter((c) => isTrustedIssueComment(c)).map((c) => monitorRecord(c?.body)).filter(Boolean);
+  const last = recs[recs.length - 1];
+  const finish = async (comment) => {
+    if (comment) await io.gh(["issue", "comment", String(issue), "-R", repo, "--body", comment]);
+    await io.gh(["issue", "close", String(issue), "-R", repo, "--reason", "completed"]);
+    try { await io.gh(["issue", "edit", String(issue), "-R", repo, "--remove-label", MONITORING_LABEL]); } catch { /* label already gone */ }
+  };
+  if (!last || last.result !== "pass" || !last.until) return row("skipped", last?.result === "fail" ? "already failed (FORGE:PROBE)" : "no windowed pass record");
+  if (now >= last.until) {
+    if (!dryRun) await finish(`<!-- FORGE:MONITOR -->\nPost-deploy monitoring window ended ${new Date(last.until).toISOString()} with no failure; closed.`);
+    return row("window-ended", dryRun ? "dry run: not closed" : "closed", { fix: last.fix });
+  }
+  if (now - last.at < last.everyMs && defaultSha && defaultSha === last.defaultSha) return row("not-due", `next run after ${new Date(last.at + last.everyMs).toISOString()} or a new release`);
+  const ids = itemCheckIds(iss.body);
+  const checks = await checksAt(io.git, [`origin/${def}`], cwd);
+  if (!ids.length || ids.some((id) => !id || !checks[id])) return row("unrunnable", `no runnable check on ${def} (${CHECKS_FILE})`);
+  if (dryRun) return row("due", "dry run: not run", { fix: last.fix });
+  const rows = await runNamedChecks({ ids, checks, cwd, config, env, spawn });
+  const ok = rows.every((r) => r.result === "pass");
+  const rec = { fix: last.fix, default_sha: defaultSha, result: ok ? "pass" : "fail", at: new Date(now).toISOString(), ...(ok ? { window_until: new Date(last.until).toISOString(), every_ms: last.everyMs } : {}) };
+  const sum = rows.map((r) => ({ id: r.id, result: r.result, exit: r.exit, ms: r.ms }));
+  if (ok) {
+    await io.gh(["issue", "comment", String(issue), "-R", repo, "--body", `<!-- ${PROBE_MARKER} -->\n## Post-deploy check: pass\n${JSON.stringify({ ...rec, checks: sum })}\n\nRe-run on \`${def}\` at \`${short(defaultSha)}\`.`]);
+    return row("pass", `checks ${[...new Set(ids)].join(", ")} passed at ${short(defaultSha)}`, { fix: last.fix });
+  }
+  const bad = rows.filter((r) => r.result === "fail");
+  const parent = deferredFrom(iss.body);
+  const title = `Regression: post-deploy check ${bad.map((r) => r.id).join(", ")} failed on ${def}${parent ? ` (fix of #${parent})` : ""}`;
+  const body = [
+    `<!-- ${REGRESSION_MARKER} -->`, `Post-deploy check ${bad.map((r) => `\`${r.id}\``).join(", ")} passed after the fix and now fails.`, "",
+    `- Fix commit: \`${last.fix}\``, `- Last passing release (\`${def}\`): \`${last.defaultSha ?? "unknown"}\``, `- Breaking release (\`${def}\`): \`${defaultSha ?? "unknown"}\``,
+    `- Monitored follow-up: #${issue}${parent ? `; parent: #${parent}` : ""}`, "", ...bad.flatMap((r) => [`**${r.id}** (exit ${r.exit ?? "timeout"})`, fenced(r.output), ""]),
+  ].join("\n").slice(0, OUTPUT_TAIL * 4);
+  const existing = await findRegression(io, repo, issue, last.fix);
+  const created = existing ? "" : String(await io.gh(["issue", "create", "-R", repo, "--title", title, "--body", body]));
+  const num = existing ?? /\/issues\/(\d+)/.exec(created)?.[1] ?? null;
+  await finish(`<!-- ${PROBE_MARKER} -->\n## Post-deploy check: fail\n${JSON.stringify({ ...rec, regression: num ? Number(num) : null, checks: sum })}\n\nRegression issue ${num ? `#${num}` : "filed"}; monitoring stops.`);
+  return row("fail", `regression ${num ? `#${num}` : "filed"}: ${bad.map((r) => r.id).join(", ")} broke at ${short(defaultSha)} (fix ${short(last.fix)})`, { fix: last.fix, regression: num });
+}
+
+/** Re-run every due monitored follow-up of the repo. Sequential; an error is an `error` row, never a throw. */
+export async function sweepDue({ repo, cwd = process.cwd(), io, dryRun = false, now = () => Date.now(), config = readProbeConfig(cwd), env = process.env, spawn, onResult = null } = {}) {
+  const def = await resolveDefaultBranch({ repo, io, cwd });
+  if (!def) throw new Error("no default branch");
+  const found = JSON.parse(String(await io.gh(["issue", "list", "-R", repo, "--state", "open", "--label", MONITORING_LABEL, "--limit", "100", "--json", "number"])));
+  const list = (Array.isArray(found) ? found : []).map((i) => i.number).filter(Number.isInteger);
+  if (!list.length) return [];
+  try { await io.git(["fetch", "origin", def], { cwd }); } catch { /* offline: the local ref */ }
+  let defaultSha = null;
+  try { defaultSha = String(await io.git(["rev-parse", `origin/${def}`], { cwd })).trim() || null; } catch { /* unknown release */ }
+  const rows = [];
+  for (const issue of list) {
+    try { rows.push(await monitorOne({ issue, repo, def, defaultSha, cwd, io, now: now(), dryRun, config, env, spawn })); } catch (e) {
+      rows.push({ issue, outcome: "error", detail: String(e?.message ?? e).slice(0, 160) });
+    }
+    try { if (!dryRun) await onResult?.(rows[rows.length - 1]); } catch { /* an event sink never fails the sweep */ }
+  }
+  return rows;
+}
+
+/** One line, e.g. `monitoring: pass #1; regression #2; closed #3`; "" when nothing is monitored. */
+export function formatDueSummary(rows) {
+  const g = [["pass", "pass"], ["fail", "regression"], ["window-ended", "window ended"], ["due", "due"], ["not-due", "not due"], ["unrunnable", "unrunnable"], ["error", "error"]];
+  const parts = g.map(([k, l]) => [l, (rows ?? []).filter((r) => r.outcome === k).map((r) => `#${r.issue}`)]).filter(([, a]) => a.length).map(([l, a]) => `${l} ${a.join(" ")}`);
+  return parts.length ? `monitoring: ${parts.join("; ")}` : "";
+}

package/bin/lean/postdeploy-monitor.test.mjs

@@ -0,0 +1,147 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { sweepDue, formatDueSummary } from "./postdeploy-monitor.mjs";
+import { PROBE_MARKER } from "./probe.mjs";
+import { isTrustedIssueComment } from "./prefilter.mjs";
+
+const T0 = Date.parse("2026-10-26T00:00:00Z");
+const H = 3600e3;
+const CHECKS = JSON.stringify({ soak: { run: "echo ok", for: "7d", every: "6h" } });
+const BODY = "## Acceptance Criteria\n\n- [ ] `R1: health row exists` — post-deploy: [check:soak] soak\n\n<!-- FORGE:DEFERRED_FROM: #40 -->\n";
+const rec = (o) => ({ authorAssociation: "OWNER", author: { login: "bot" }, body: `<!-- ${PROBE_MARKER} -->\n${JSON.stringify({ fix: "fix1234567890", default_sha: "rel1", result: "pass", at: new Date(T0).toISOString(), window_until: new Date(T0 + 7 * 24 * H).toISOString(), every_ms: 6 * H, ...o })}` });
+
+function world({ comments = [rec({})], head = "rel1", file = CHECKS, code = 0, state = "OPEN", regressions = [], failClose = false } = {}) {
+  const w = { gh: [], git: [], spawned: [] };
+  w.io = {
+    gh: async (a) => {
+      w.gh.push(a);
+      const k = a.slice(0, 2).join(" ");
+      if (k === "repo view") return "main\n";
+      if (k === "issue list" && a.includes("--search")) return JSON.stringify(regressions);
+      if (failClose && k === "issue close") throw new Error("rate limit");
+      if (k === "issue list") return JSON.stringify([{ number: 41 }]);
+      if (k === "issue view") return JSON.stringify({ number: 41, state, title: "", body: BODY, labels: [{ name: "post-deploy-monitoring" }], comments });
+      if (k === "issue create") return "https://github.com/o/r/issues/77\n";
+      return "";
+    },
+    git: async (a) => { w.git.push(a); if (a[0] === "show") { if (file == null) throw new Error("missing"); return file; } if (a[0] === "rev-parse") return `${head}\n`; return ""; },
+  };
+  w.spawn = async (o) => { w.spawned.push(o.run); return { code, stdout: code ? "broke" : "ok", stderr: "" }; };
+  return w;
+}
+const sweep = (w, o = {}) => sweepDue({ repo: "o/r", cwd: "/repo", io: w.io, now: () => T0 + H, config: { env: [], timeoutS: 5 }, env: {}, spawn: w.spawn, ...o });
+const calls = (w, k) => w.gh.filter((a) => a.slice(0, 2).join(" ") === k);
+
+const trustedOk = isTrustedIssueComment(rec({}));
+
+describe("post-deploy windowed monitoring (#4627)", () => {
+  it("fixture comments count as trusted", () => assert.ok(trustedOk));
+
+  it("not due: cadence not elapsed and no new release", async () => {
+    const w = world();
+    const rows = await sweep(w);
+    assert.equal(rows[0].outcome, "not-due"); assert.deepEqual(w.spawned, []);
+  });
+
+  it("due by cadence: re-runs the default-branch command and records a pass; the follow-up stays open", async () => {
+    const w = world();
+    const rows = await sweep(w, { now: () => T0 + 7 * H });
+    assert.equal(rows[0].outcome, "pass"); assert.deepEqual(w.spawned, ["echo ok"]);
+    assert.ok(w.git.some((a) => a.join(" ") === "show origin/main:.forge/post-deploy-checks.json"));
+    assert.equal(calls(w, "issue comment").length, 1);
+    assert.equal(calls(w, "issue close").length, 0); assert.equal(calls(w, "issue create").length, 0);
+  });
+
+  it("due by a new release (origin/main moved) well before the cadence", async () => {
+    const w = world({ head: "rel2" });
+    const rows = await sweep(w);
+    assert.equal(rows[0].outcome, "pass"); assert.deepEqual(w.spawned, ["echo ok"]);
+  });
+
+  it("window end closes the follow-up and drops the label without running", async () => {
+    const w = world();
+    const rows = await sweep(w, { now: () => T0 + 8 * 24 * H });
+    assert.equal(rows[0].outcome, "window-ended"); assert.deepEqual(w.spawned, []);
+    assert.equal(calls(w, "issue close").length, 1);
+    assert.ok(calls(w, "issue edit").some((a) => a.includes("--remove-label")));
+  });
+
+  it("pass then fail files one regression naming the fix commit and both releases, then closes the follow-up", async () => {
+    const w = world({ head: "rel2", code: 1 });
+    const rows = await sweep(w);
+    assert.equal(rows[0].outcome, "fail"); assert.equal(rows[0].regression, "77");
+    const [create] = calls(w, "issue create");
+    const text = create[create.indexOf("--body") + 1];
+    assert.match(create[create.indexOf("--title") + 1], /Regression: post-deploy check soak failed on main \(fix of #40\)/);
+    assert.match(text, /FORGE:REGRESSION/); assert.match(text, /fix1234567890/); assert.match(text, /`rel1`/); assert.match(text, /`rel2`/); assert.match(text, /broke/);
+    assert.equal(calls(w, "issue close").length, 1);
+    assert.ok(!w.gh.some((a) => a[1] === "reopen"));
+    const cm = calls(w, "issue comment")[0];
+    assert.match(cm[cm.indexOf("--body") + 1], /"result":"fail".*"regression":77/s);
+  });
+
+  const reg = (n, o = {}) => ({ number: n, body: `<!-- FORGE:REGRESSION -->\n- Fix commit: \`${o.fix ?? "fix1234567890"}\`\n- Monitored follow-up: #${o.issue ?? 41}\n` });
+
+  it("#4653: a regression already filed for this follow-up and fix is reused, not duplicated", async () => {
+    const w = world({ head: "rel2", code: 1, regressions: [reg(90)] });
+    const rows = await sweep(w);
+    assert.equal(rows[0].outcome, "fail"); assert.equal(rows[0].regression, "90");
+    assert.equal(calls(w, "issue create").length, 0); assert.equal(calls(w, "issue close").length, 1);
+    const cm = calls(w, "issue comment")[0];
+    assert.match(cm[cm.indexOf("--body") + 1], /"regression":90/);
+  });
+
+  it("#4653: finish() failing after create does not file a second regression on the next sweep", async () => {
+    const w = world({ head: "rel2", code: 1, failClose: true });
+    assert.equal((await sweep(w))[0].outcome, "error"); assert.equal(calls(w, "issue create").length, 1);
+    const w2 = world({ head: "rel2", code: 1, regressions: [reg(77)] });
+    assert.equal((await sweep(w2))[0].regression, "77"); assert.equal(calls(w2, "issue create").length, 0);
+  });
+
+  it("#4653: a regression for another follow-up, fix commit or issue number prefix is not reused", async () => {
+    const w = world({ head: "rel2", code: 1, regressions: [reg(5, { issue: 4 }), reg(6, { issue: 410 }), reg(7, { fix: "other" })] });
+    const rows = await sweep(w);
+    assert.equal(rows[0].regression, "77"); assert.equal(calls(w, "issue create").length, 1);
+  });
+
+  it("a follow-up whose latest record is a fail is never re-run or re-filed", async () => {
+    const w = world({ head: "rel3", code: 1, comments: [rec({}), rec({ result: "fail", default_sha: "rel2", regression: 77 })] });
+    const rows = await sweep(w);
+    assert.equal(rows[0].outcome, "skipped"); assert.deepEqual(w.spawned, []); assert.equal(calls(w, "issue create").length, 0);
+  });
+
+  it("an untrusted record never counts", async () => {
+    const w = world({ comments: [{ authorAssociation: "NONE", author: { login: "rando" }, body: rec({}).body }] });
+    assert.equal((await sweep(w, { now: () => T0 + 9 * H }))[0].outcome, "skipped");
+    assert.deepEqual(w.spawned, []);
+  });
+
+  it("a check missing on the default branch is unrunnable and never run", async () => {
+    for (const file of [null, JSON.stringify({ other: "echo hi" })]) {
+      const w = world({ file, head: "rel2" });
+      assert.equal((await sweep(w))[0].outcome, "unrunnable"); assert.deepEqual(w.spawned, []);
+    }
+  });
+
+  it("dry run runs nothing and writes nothing", async () => {
+    const w = world({ head: "rel2" });
+    assert.equal((await sweep(w, { dryRun: true }))[0].outcome, "due");
+    assert.deepEqual(w.spawned, []);
+    assert.equal(calls(w, "issue comment").length + calls(w, "issue close").length + calls(w, "issue create").length, 0);
+  });
+
+  it("no monitored follow-up: no git, no run; the summary names outcomes", async () => {
+    const w = world();
+    w.io.gh = async (a) => (a[0] === "repo" ? "main\n" : "[]");
+    assert.deepEqual(await sweep(w), []); assert.equal(w.git.length, 0);
+    assert.equal(formatDueSummary([{ issue: 1, outcome: "pass" }, { issue: 2, outcome: "fail" }]), "monitoring: pass #1; regression #2");
+  });
+
+  it("reports sink rows for pass and fail only when not a dry run", async () => {
+    const seen = [];
+    await sweep(world({ head: "rel2", code: 1 }), { onResult: (r) => seen.push(r.outcome) });
+    assert.deepEqual(seen, ["fail"]);
+  });
+});