forgedock 1.49.0 → 1.51.0

package/bin/lean/batch-health.mjs

@@ -0,0 +1,219 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Early batch health (#5167). A batch that fails in its first minutes (roots engine-error, the rest skipped) or is refused
+ * at launch used to look "launched" until someone read its log. The batch runner (`runBatch`, the schedd per-batch runner)
+ * evaluates the batch's `state.json` at T+15 and T+60 min after `plan.created` and once when the batch ends; the first
+ * unhealthy evaluation emits one `BATCH UNHEALTHY` alert (a runner line, a GitHub comment on `orchestrate.health.issue`
+ * or the batch's `--epic` issues, and `notify.command`) with its causes and recovery commands. `health.json` in the batch
+ * dir records the checkpoints done and the alert, so a restarted daemon re-arms only the remaining checks and never alerts
+ * twice. A refused launch has no batch dir: it is recorded in `<batch root>/refusals.json`. `forgedock status` reads both.
+ */
+import { readFileSync } from "node:fs";
+import { spawn as spawnProc } from "node:child_process";
+import { join } from "node:path";
+import { listBatches, openBatch, REQUEUABLE, writeAtomic } from "./batch.mjs";
+
+export const DEFAULT_HEALTH = Object.freeze({ rootFailed: true, terminalShare: 0.5, stalled: true, checksMin: Object.freeze([15, 60]), issue: null, notifyCommand: null });
+/** Outcomes that end a row well: never counted against the batch. */
+export const GOOD_END = new Set(["merged", "invalid", "decomposed"]);
+export const REFUSALS_FILE = "refusals.json";
+export const REFUSAL_KEEP = 20;
+export const REFUSAL_SHOW_MS = 24 * 3600 * 1000;
+export const NOTIFY_TIMEOUT_MS = 30 * 1000;
+
+const readJson = (file) => { try { return JSON.parse(readFileSync(file, "utf8")); } catch { return null; } };
+const list = (ns) => ns.map((n) => `#${n}`).join(", ");
+
+/** forge.yaml (parsed) → health config. The parser flattens `orchestrate.health.*` to `orchestrate.*`. */
+export function parseHealthConfig(yaml) {
+  const o = yaml?.orchestrate && typeof yaml.orchestrate === "object" ? yaml.orchestrate : {};
+  const bool = (v, d) => (v === undefined || String(v).trim() === "" ? d : String(v).trim().toLowerCase() !== "false");
+  const share = Number(o.terminal_share);
+  const issue = Number(String(o.issue ?? "").replace(/^#/, ""));
+  const checks = String(o.checks_min ?? "").replace(/^\[|\]$/g, "").split(",").map((x) => Number(x.trim())).filter((n) => Number.isFinite(n) && n > 0);
+  const cmd = String(yaml?.notify?.command ?? "").trim();
+  return { rootFailed: bool(o.root_failed, DEFAULT_HEALTH.rootFailed), terminalShare: o.terminal_share !== undefined && Number.isFinite(share) && share >= 0 && share <= 1 ? share : DEFAULT_HEALTH.terminalShare,
+    stalled: bool(o.stalled, DEFAULT_HEALTH.stalled), checksMin: checks.length ? checks : [...DEFAULT_HEALTH.checksMin], issue: Number.isSafeInteger(issue) && issue > 0 ? issue : null, notifyCommand: cmd || null };
+}
+
+/** Issues of the batch with no dependency edge to another issue of the batch. */
+export function rootsOf(state) {
+  const order = (state?.order ?? []).map(Number), inBatch = new Set(order);
+  return order.filter((n) => !(state?.deps?.[n] ?? []).some((d) => inBatch.has(Number(d)) && Number(d) !== n));
+}
+
+/**
+ * `{healthy, causes:[{code, issues, detail}], recovery:[cmd]}` for one batch state. `held`: operator-held issues (not a
+ * stall); `refused`: a refusal record (the launch never started). Pure.
+ */
+export function evaluateHealth(state, { held = [], refused = null, config = DEFAULT_HEALTH, batchId = "<id>", repo = null } = {}) {
+  const causes = [], recovery = [];
+  if (refused) {
+    causes.push({ code: "refused", issues: refused.issues ?? [], detail: refused.message ?? "launch refused" });
+    const relaunch = `forgedock orchestrate ${(refused.issues ?? []).join(" ")}${repo ? ` --repo ${repo}` : ""}`;
+    recovery.push(`merge the unmet dependencies, then: ${relaunch}`, `${relaunch} --force "<why the unmet dependencies are acceptable>"`);
+    return { healthy: false, causes, recovery };
+  }
+  const order = (state?.order ?? []).map(Number);
+  const running = new Set(Object.keys(state?.running ?? {}).map(Number));
+  const pending = new Set((state?.pending ?? []).map(Number));
+  const results = state?.results ?? {};
+  const outcomeOf = (n) => results[n]?.outcome;
+  // Terminal and not good: a final row not running and not waiting again (a held row stays pending but counts).
+  const bad = (n) => { const o = outcomeOf(n); return o !== undefined && !GOOD_END.has(o) && !running.has(n) && (!pending.has(n) || o === "held"); };
+  const merged = order.filter((n) => outcomeOf(n) === "merged").length;
+  const failedRoots = rootsOf(state).filter(bad);
+  if (config.rootFailed && failedRoots.length) causes.push({ code: "root-failed", issues: failedRoots, detail: `root ${failedRoots.map((n) => `#${n} ${outcomeOf(n)}`).join(", ")}` });
+  const badRows = order.filter(bad);
+  if (order.length && merged === 0 && badRows.length && badRows.length / order.length >= config.terminalShare) {
+    const by = {};
+    for (const n of badRows) by[outcomeOf(n)] = (by[outcomeOf(n)] ?? 0) + 1;
+    causes.push({ code: "terminal-share", issues: badRows, detail: `${badRows.length}/${order.length} rows ended non-merged (${Object.entries(by).map(([o, k]) => `${k} ${o}`).join(", ")}), 0 merged` });
+  }
+  const heldSet = new Set(held.map(Number));
+  const waiting = [...pending].filter((n) => !heldSet.has(n));
+  if (config.stalled && !running.size && waiting.length && state?.phase !== "setup" && !state?.cancelled) {
+    causes.push({ code: "stalled", issues: waiting, detail: `0 lanes running while ${waiting.length} issue${waiting.length === 1 ? "" : "s"} pending${state?.breaker ? " (breaker tripped)" : ""}` });
+  }
+  if (!causes.length) return { healthy: true, causes, recovery };
+  const failed = [...new Set(causes.filter((c) => c.code !== "stalled").flatMap((c) => c.issues))];
+  const requeue = failed.filter((n) => REQUEUABLE(outcomeOf(n))), relaunch = failed.filter((n) => !REQUEUABLE(outcomeOf(n)));
+  if (requeue.length) recovery.push(`forgedock orchestrate --requeue ${batchId} ${requeue.join(" ")} --reason "batch unhealthy"`);
+  if (relaunch.length) recovery.push(`forgedock orchestrate ${relaunch.join(" ")}${repo ? ` --repo ${repo}` : ""}   # after fixing the cause (see lanes/<n>.log)`);
+  if (pending.size) recovery.push(`forgedock orchestrate --resume ${batchId}`);
+  recovery.push(`forgedock status --batch ${batchId}`, `forgedock orchestrate --cancel ${batchId}`);
+  return { healthy: false, causes, recovery };
+}
+
+/** The `BATCH UNHEALTHY` text: top causes, then the exact recovery commands. */
+export function formatAlert({ batchId, checkpoint, health }) {
+  return [`BATCH UNHEALTHY ${batchId} (${checkpoint})`, ...health.causes.slice(0, 5).map((c) => `  cause: ${c.code}: ${c.detail}${c.issues?.length && c.code !== "refused" ? ` [${list(c.issues.slice(0, 12))}${c.issues.length > 12 ? ", …" : ""}]` : ""}`),
+    "  recovery:", ...health.recovery.map((r) => `    ${r}`)].join("\n");
+}
+
+export const healthPath = (dir) => join(dir, "health.json");
+export const readHealth = (dir) => { const h = readJson(healthPath(dir)); return h && typeof h === "object" && !Array.isArray(h) ? h : null; };
+export const writeHealth = (dir, h) => writeAtomic(healthPath(dir), JSON.stringify(h) + "\n", { private: true });
+
+/** The checkpoints `[{label, at}]` (`T+<min>`) for a batch created at `created` (ms). */
+export const checkpointsOf = (created, checksMin = DEFAULT_HEALTH.checksMin) => checksMin.map((m) => ({ label: `T+${m}`, at: created + m * 60000 }));
+
+/** `health checks: T+15 14:05Z, T+60 14:50Z; forgedock status --batch <id>` */
+export function scheduleLine(batchId, created, checksMin = DEFAULT_HEALTH.checksMin) {
+  const hhmm = (ms) => `${new Date(ms).toISOString().slice(11, 16)}Z`;
+  return `health checks: ${checkpointsOf(created, checksMin).map((c) => `${c.label} ${hhmm(c.at)}`).join(", ")}, and at batch end; forgedock status --batch ${batchId}`;
+}
+
+/**
+ * One checkpoint: evaluate the batch, persist it, and call `alert(text, health)` on the first unhealthy result only.
+ * @returns the evaluation, plus `alerted` when this call raised the alert
+ */
+export async function runCheck({ batch, checkpoint, config = DEFAULT_HEALTH, repo = null, now = () => Date.now(), alert = async () => {} }) {
+  const state = batch.store.load() ?? {};
+  let held = [];
+  try { held = batch.readHeld?.() ?? []; } catch { held = []; }
+  const health = evaluateHealth(state, { held, config, batchId: batch.id, repo });
+  const prev = readHealth(batch.dir) ?? {};
+  const at = new Date(now()).toISOString();
+  const next = { ...prev, checks: { ...(prev.checks ?? {}), [checkpoint]: at }, last: { at, checkpoint, healthy: health.healthy, causes: health.causes, recovery: health.recovery } };
+  const fire = !health.healthy && !prev.alertedAt;
+  if (fire) Object.assign(next, { alertedAt: at, alert: formatAlert({ batchId: batch.id, checkpoint, health }) });
+  try { writeHealth(batch.dir, next); } catch { /* health is advisory */ }
+  if (fire) { try { await alert(next.alert, health); } catch { /* every channel is best effort */ } }
+  return { ...health, alerted: fire };
+}
+
+/**
+ * Arm the timed checkpoints not yet done (per `health.json`); past-due ones run on the next tick. Timers are unref'd.
+ * @returns `{cancel()}`
+ */
+export function scheduleChecks({ batch, created, config = DEFAULT_HEALTH, now = () => Date.now(), setTimer = setTimeout, clearTimer = clearTimeout, check }) {
+  const done = readHealth(batch.dir)?.checks ?? {};
+  const timers = checkpointsOf(created, config.checksMin).filter((c) => !done[c.label]).map((c) => {
+    const t = setTimer(() => { Promise.resolve().then(() => check(c.label)).catch(() => {}); }, Math.max(0, c.at - now()));
+    t?.unref?.();
+    return t;
+  });
+  return { cancel: () => { for (const t of timers) clearTimer(t); } };
+}
+
+/** Run `notify.command` through the shell: alert JSON on stdin, FORGEDOCK_ALERT_* in env. Never throws. */
+export function runNotify(command, payload, { spawn = spawnProc, timeoutMs = NOTIFY_TIMEOUT_MS, env = process.env } = {}) {
+  return new Promise((resolve) => {
+    let child;
+    try {
+      child = spawn(command, { shell: true, stdio: ["pipe", "ignore", "ignore"], env: { ...env, FORGEDOCK_ALERT: "BATCH_UNHEALTHY", FORGEDOCK_ALERT_BATCH: String(payload.batch ?? ""), FORGEDOCK_ALERT_TEXT: String(payload.text ?? "") } });
+    } catch (e) { return resolve({ ok: false, error: String(e?.message ?? e) }); }
+    const timer = setTimeout(() => { try { child.kill("SIGKILL"); } catch { /* gone */ } resolve({ ok: false, error: `timed out after ${Math.round(timeoutMs / 1000)}s` }); }, timeoutMs);
+    timer.unref?.();
+    child.on("error", (e) => { clearTimeout(timer); resolve({ ok: false, error: String(e?.message ?? e) }); });
+    child.on("close", (code) => { clearTimeout(timer); resolve(code === 0 ? { ok: true } : { ok: false, error: `exit ${code}` }); });
+    try { child.stdin?.on?.("error", () => {}); child.stdin?.end?.(JSON.stringify(payload) + "\n"); } catch { /* the command ignores stdin */ }
+  });
+}
+
+/**
+ * The alert channels: a runner line, a comment on the configured issue (else the batch's epics), and notify.command.
+ * Each is best effort.
+ */
+export async function deliverAlert({ text, batchId, health, sink = { line: () => {} }, gh = null, repo = null, config = DEFAULT_HEALTH, epics = [], notify = runNotify }) {
+  sink.line(text);
+  const targets = config.issue ? [config.issue] : [...new Set(epics)];
+  if (gh && repo) for (const n of targets) {
+    try { await gh(["issue", "comment", String(n), "-R", repo, "--body", `<!-- FORGE:BATCH-HEALTH batch=${batchId} -->\n## BATCH UNHEALTHY\n\`\`\`text\n${text}\n\`\`\``]); } catch (e) { sink.line(`health: comment on #${n} failed (${String(e?.message ?? e).slice(0, 160)})`); }
+  }
+  if (config.notifyCommand) {
+    const r = await notify(config.notifyCommand, { alert: "BATCH_UNHEALTHY", batch: batchId, repo, text, causes: health?.causes ?? [], recovery: health?.recovery ?? [] });
+    if (!r?.ok) sink.line(`health: notify.command failed (${r?.error ?? "unknown"})`);
+  }
+}
+
+/** Record a refused launch under the batch root (no batch dir exists); keeps the newest `REFUSAL_KEEP`. */
+export function recordRefusal(root, rec) {
+  const prev = readRefusals(root, { maxAgeMs: Infinity });
+  writeAtomic(join(root, REFUSALS_FILE), JSON.stringify([...prev, rec].slice(-REFUSAL_KEEP)) + "\n", { private: true });
+}
+
+/** Refusals newer than `maxAgeMs`, oldest first. */
+export function readRefusals(root, { now = Date.now(), maxAgeMs = REFUSAL_SHOW_MS } = {}) {
+  const v = readJson(join(root, REFUSALS_FILE));
+  return (Array.isArray(v) ? v : []).filter((r) => r && typeof r.at === "string" && now - Date.parse(r.at) <= maxAgeMs);
+}
+
+/** Health of one batch for `forgedock status --batch <id>`. */
+export function batchStatusLines(root, id, { now = Date.now() } = {}) {
+  const b = openBatch(root, id);
+  if (!b.exists()) return [`batch ${id}: not found in ${root}`];
+  const h = readHealth(b.dir), plan = b.readPlan() ?? {};
+  const created = Date.parse(plan.created ?? "");
+  const out = [];
+  if (h?.alertedAt) out.push(`batch ${id}: UNHEALTHY since ${h.alertedAt}`, ...String(h.alert ?? "").split("\n").slice(1));
+  else if (h?.last) out.push(`batch ${id}: healthy at ${h.last.checkpoint} (${h.last.at})`);
+  else out.push(`batch ${id}: no health check yet`);
+  if (h?.alertedAt && h.last && h.last.at !== h.alertedAt) out.push(`  last check ${h.last.checkpoint} (${h.last.at}): ${h.last.healthy ? "healthy" : h.last.causes.map((c) => c.code).join(", ")}`);
+  if (Number.isFinite(created)) {
+    const due = checkpointsOf(created, DEFAULT_HEALTH.checksMin).filter((c) => !h?.checks?.[c.label]);
+    if (due.length && !b.readReport()) out.push(`  next checks: ${due.map((c) => `${c.label} ${new Date(c.at).toISOString().slice(11, 16)}Z${c.at < now ? " (due)" : ""}`).join(", ")}`);
+  }
+  return out;
+}
+
+/**
+ * `forgedock status` lines: unhealthy batches still unresolved (no report, or alerted within a day) and recent refusals;
+ * with `batch`, that one batch's health.
+ */
+export function statusLines(root, { now = Date.now(), batch = null } = {}) {
+  if (batch) return batchStatusLines(root, batch, { now });
+  const out = [];
+  for (const id of listBatches(root)) {
+    try {
+      const b = openBatch(root, id), h = readHealth(b.dir);
+      if (!h?.alertedAt || (b.readReport() && now - Date.parse(h.alertedAt) > REFUSAL_SHOW_MS)) continue;
+      const causes = (h.last?.healthy === false ? h.last.causes : null) ?? [];
+      out.push(`  BATCH UNHEALTHY ${id} (since ${h.alertedAt})${causes.length ? `: ${causes.map((c) => c.detail).join("; ")}` : ""} — forgedock status --batch ${id}`);
+    } catch { /* an unreadable batch has no line */ }
+  }
+  for (const r of readRefusals(root, { now })) out.push(`  LAUNCH REFUSED ${r.at}: ${String(r.message ?? "").slice(0, 300)}`);
+  return out;
+}

package/bin/lean/batch-health.test.mjs

@@ -0,0 +1,156 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpDir } from "../tests/_tmp.mjs";
+import { openBatch } from "./batch.mjs";
+import { DEFAULT_HEALTH, parseHealthConfig, rootsOf, evaluateHealth, formatAlert, runCheck, scheduleChecks, scheduleLine, runNotify, deliverAlert,
+  recordRefusal, readRefusals, statusLines, REFUSAL_KEEP } from "./batch-health.mjs";
+
+// #1 and #3 are roots; #2 depends on #1, #4 on #3.
+const base = (results, extra = {}) => ({ order: [1, 2, 3, 4], deps: { 2: [1], 4: [3] }, results, running: {}, pending: [], ...extra });
+
+describe("#5167 evaluateHealth", () => {
+  it("roots are the issues with no in-batch dependency", () => {
+    assert.deepEqual(rootsOf(base({})), [1, 3]);
+    assert.deepEqual(rootsOf({ order: [5, 6], deps: { 6: [99] } }), [5, 6], "an out-of-batch dependency keeps a root");
+  });
+
+  it("all roots engine-error, the rest skipped: root-failed and terminal-share, with recovery commands", () => {
+    const st = base({ 1: { outcome: "engine-error" }, 2: { outcome: "skipped" }, 3: { outcome: "engine-error" }, 4: { outcome: "skipped" } });
+    const h = evaluateHealth(st, { batchId: "b1", repo: "o/r" });
+    assert.equal(h.healthy, false);
+    assert.deepEqual(h.causes.map((c) => c.code), ["root-failed", "terminal-share"]);
+    assert.deepEqual(h.causes[0].issues, [1, 3]);
+    assert.match(h.causes[1].detail, /4\/4 rows ended non-merged \(2 engine-error, 2 skipped\), 0 merged/);
+    assert.ok(h.recovery.includes(`forgedock orchestrate --requeue b1 1 3 2 4 --reason "batch unhealthy"`), h.recovery.join("\n"));
+    assert.ok(h.recovery.includes("forgedock orchestrate --cancel b1") && h.recovery.includes("forgedock status --batch b1"));
+    const text = formatAlert({ batchId: "b1", checkpoint: "T+15", health: h });
+    assert.match(text, /^BATCH UNHEALTHY b1 \(T\+15\)\n {2}cause: root-failed: root #1 engine-error, #3 engine-error \[#1, #3\]/);
+    assert.match(text, /\n {2}recovery:\n {4}forgedock orchestrate --requeue b1 1 3 2 4/);
+  });
+
+  it("a healthy batch (running, merged, or a requeued row waiting again) has no cause", () => {
+    assert.deepEqual(evaluateHealth(base({ 1: { outcome: "merged" } }, { running: { 3: {} }, pending: [2, 4] })), { healthy: true, causes: [], recovery: [] });
+    assert.equal(evaluateHealth(base({ 1: { outcome: "merged" }, 2: { outcome: "merged" }, 3: { outcome: "invalid" }, 4: { outcome: "decomposed" } })).healthy, true);
+    assert.equal(evaluateHealth(base({ 1: { outcome: "engine-error" } }, { running: { 1: {} } })).healthy, true, "a relaunched root is running");
+    assert.equal(evaluateHealth(base({}, { phase: "setup", pending: [1, 2, 3, 4] })).healthy, true, "setup is not a stall");
+  });
+
+  it("terminal-share needs 0 merged and the configured share; root_failed: false turns that cause off", () => {
+    const st = base({ 2: { outcome: "needs-human" }, 4: { outcome: "held" }, 1: { outcome: "merged" } }, { running: { 3: {} }, pending: [4] });
+    assert.equal(evaluateHealth(st).healthy, true, "one merge keeps the share cause off");
+    const st2 = base({ 2: { outcome: "needs-human" }, 4: { outcome: "held" } }, { running: { 1: {}, 3: {} }, pending: [4] });
+    assert.deepEqual(evaluateHealth(st2).causes.map((c) => c.code), ["terminal-share"], "held counts even though it stays pending");
+    assert.equal(evaluateHealth(st2, { config: { ...DEFAULT_HEALTH, terminalShare: 0.75 } }).healthy, true);
+    const st3 = base({ 1: { outcome: "engine-error" } }, { running: { 3: {} }, pending: [2, 4] });
+    assert.equal(evaluateHealth(st3, { config: { ...DEFAULT_HEALTH, rootFailed: false } }).healthy, true);
+  });
+
+  it("stalled: 0 running with pending, operator-held issues excepted", () => {
+    const st = base({ 1: { outcome: "merged" } }, { pending: [2, 3, 4] });
+    const h = evaluateHealth(st, { batchId: "b2" });
+    assert.deepEqual(h.causes.map((c) => [c.code, c.issues]), [["stalled", [2, 3, 4]]]);
+    assert.ok(h.recovery.includes("forgedock orchestrate --resume b2"));
+    assert.equal(evaluateHealth(st, { held: [2, 3, 4] }).healthy, true);
+    assert.equal(evaluateHealth({ ...st, cancelled: true }).healthy, true);
+  });
+
+  it("a refused launch is unhealthy with the merge-or-force relaunch", () => {
+    const h = evaluateHealth(null, { refused: { issues: [7], message: "parallel milestone start refused: #7 depends on #9" }, repo: "o/r" });
+    assert.deepEqual(h.causes.map((c) => c.code), ["refused"]);
+    assert.deepEqual(h.recovery, ["merge the unmet dependencies, then: forgedock orchestrate 7 --repo o/r", `forgedock orchestrate 7 --repo o/r --force "<why the unmet dependencies are acceptable>"`]);
+  });
+});
+
+describe("#5167 config", () => {
+  it("defaults, and orchestrate.health.* (flattened) plus notify.command", () => {
+    assert.deepEqual(parseHealthConfig(null), { ...DEFAULT_HEALTH, checksMin: [15, 60] });
+    const c = parseHealthConfig({ orchestrate: { root_failed: "false", terminal_share: "0.4", stalled: "false", issue: "#12", checks_min: "[10, 30]" }, notify: { command: "curl -d @- x" } });
+    assert.deepEqual(c, { rootFailed: false, terminalShare: 0.4, stalled: false, checksMin: [10, 30], issue: 12, notifyCommand: "curl -d @- x" });
+    assert.equal(parseHealthConfig({ orchestrate: { terminal_share: "2" } }).terminalShare, 0.5, "out of range keeps the default");
+  });
+});
+
+describe("#5167 checkpoints, alert-once and persistence", () => {
+  const mkBatch = (state) => {
+    const root = tmpDir("bh-"), b = openBatch(root, "b20260101-000000-1");
+    mkdirSync(b.dir, { recursive: true });
+    b.writePlan({ v: 1, id: b.id, argv: ["1"], created: "2026-01-01T00:00:00.000Z" });
+    b.store.save(state);
+    return { root, b };
+  };
+  const bad = base({ 1: { outcome: "engine-error" }, 2: { outcome: "skipped" }, 3: { outcome: "engine-error" }, 4: { outcome: "skipped" } });
+
+  it("the first unhealthy check alerts; later checks and a restart do not alert again", async () => {
+    const { root, b } = mkBatch(bad), alerts = [];
+    const alert = async (t) => { alerts.push(t); };
+    const r1 = await runCheck({ batch: b, checkpoint: "T+15", alert, now: () => Date.parse("2026-01-01T00:15:00Z") });
+    const r2 = await runCheck({ batch: openBatch(root, b.id), checkpoint: "T+60", alert });
+    await runCheck({ batch: b, checkpoint: "end", alert });
+    assert.deepEqual([r1.alerted, r2.alerted, alerts.length], [true, false, 1]);
+    const h = JSON.parse(readFileSync(join(b.dir, "health.json"), "utf8"));
+    assert.equal(h.alertedAt, "2026-01-01T00:15:00.000Z");
+    assert.deepEqual(Object.keys(h.checks), ["T+15", "T+60", "end"]);
+    assert.match(statusLines(root, { now: Date.parse("2026-01-01T01:00:00Z") }).join("\n"), /BATCH UNHEALTHY b20260101-000000-1 \(since 2026-01-01T00:15:00.000Z\): root #1 engine-error/);
+    const one = statusLines(root, { batch: b.id }).join("\n");
+    assert.match(one, /UNHEALTHY since/);
+    assert.match(one, /forgedock orchestrate --cancel b20260101-000000-1/);
+  });
+
+  it("a healthy check records no alert, and a throwing channel never escapes", async () => {
+    const { b, root } = mkBatch(base({ 1: { outcome: "merged" } }, { running: { 3: {} }, pending: [2, 4] }));
+    const r = await runCheck({ batch: b, checkpoint: "T+15", alert: async () => { throw new Error("x"); } });
+    assert.equal(r.healthy, true);
+    assert.equal(statusLines(root).length, 0);
+    assert.match(statusLines(root, { batch: b.id })[0], /healthy at T\+15/);
+    b.store.save(bad);
+    const r2 = await runCheck({ batch: b, checkpoint: "T+60", alert: async () => { throw new Error("gh down"); } });
+    assert.equal(r2.alerted, true);
+  });
+
+  it("scheduleChecks arms only the checkpoints health.json has not recorded, relative to created", () => {
+    const { b } = mkBatch(bad);
+    writeFileSync(join(b.dir, "health.json"), JSON.stringify({ checks: { "T+15": "x" } }));
+    const armed = [];
+    const created = Date.parse("2026-01-01T00:00:00Z");
+    const s = scheduleChecks({ batch: b, created, now: () => created + 20 * 60000, setTimer: (fn, ms) => { armed.push(ms); return 1; }, clearTimer: () => armed.push("cleared"), check: () => {} });
+    s.cancel();
+    assert.deepEqual(armed, [40 * 60000, "cleared"]);
+    assert.equal(scheduleLine("b9", created), "health checks: T+15 00:15Z, T+60 01:00Z, and at batch end; forgedock status --batch b9");
+  });
+});
+
+describe("#5167 channels and refusals", () => {
+  it("runNotify pipes the alert JSON and env to the shell command; failures are returned", async () => {
+    const out = join(tmpDir("bh-n-"), "got");
+    const ok = await runNotify(`cat > ${out}; echo "$FORGEDOCK_ALERT $FORGEDOCK_ALERT_BATCH" >> ${out}`, { batch: "b1", text: "t" });
+    assert.deepEqual(ok, { ok: true });
+    const got = readFileSync(out, "utf8").split("\n");
+    assert.equal(JSON.parse(got[0]).batch, "b1");
+    assert.equal(got[1], "BATCH_UNHEALTHY b1");
+    assert.deepEqual(await runNotify("exit 3", { batch: "b1" }), { ok: false, error: "exit 3" });
+    assert.equal((await runNotify("sleep 5", {}, { timeoutMs: 50 })).ok, false);
+  });
+
+  it("deliverAlert comments on the configured issue, else the epics, and reports failed channels", async () => {
+    const calls = [], lines = [];
+    const sink = { line: (l) => lines.push(l) };
+    await deliverAlert({ text: "BATCH UNHEALTHY b1", batchId: "b1", sink, gh: async (a) => { calls.push(a[2]); }, repo: "o/r", epics: [5, 6, 5] });
+    await deliverAlert({ text: "BATCH UNHEALTHY b1", batchId: "b1", sink, gh: async (a) => { calls.push(a[2]); throw new Error("403"); }, repo: "o/r", config: { ...DEFAULT_HEALTH, issue: 9, notifyCommand: "x" }, epics: [5], notify: async () => ({ ok: false, error: "exit 1" }) });
+    assert.deepEqual(calls, ["5", "6", "9"]);
+    assert.deepEqual(lines.filter((l) => /^health:/.test(l)), ["health: comment on #9 failed (403)", "health: notify.command failed (exit 1)"]);
+  });
+
+  it("refusals are kept under the batch root, pruned, and shown by status for a day", () => {
+    const root = tmpDir("bh-r-");
+    for (let i = 0; i < REFUSAL_KEEP + 3; i++) recordRefusal(root, { at: new Date(Date.parse("2026-01-01T00:00:00Z") + i * 1000).toISOString(), issues: [7], message: `refused ${i}` });
+    assert.equal(readRefusals(root, { maxAgeMs: Infinity }).length, REFUSAL_KEEP);
+    const now = Date.parse("2026-01-01T01:00:00Z");
+    assert.match(statusLines(root, { now }).at(-1), /^ {2}LAUNCH REFUSED 2026-01-01T00:00:22.000Z: refused 22$/);
+    assert.equal(statusLines(root, { now: now + 2 * 86400000 }).length, 0);
+    assert.ok(existsSync(join(root, "refusals.json")));
+  });
+});

package/bin/lean/batch.mjs

@@ -19,6 +19,7 @@ import { appendFileSync, chmodSync, closeSync, existsSync, fchmodSync, fsyncSync
 import { spawn } from "node:child_process";
 import { homedir } from "node:os";
 import { dirname, join, resolve, sep } from "node:path";
+import { sanitizeOverride, validateExecutionPatch } from "./live-execution.mjs";
 
 export const RAM_FS = new Set(["tmpfs", "ramfs"]);
 const ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
@@ -85,8 +86,8 @@ const readAdded = (file) => {
   return Array.isArray(v) ? [...new Set(v.filter((n) => Number.isSafeInteger(n) && n > 0))].sort((a, b) => a - b) : [];
 };
 
-/** `outcome`s `--requeue` may move back to pending (#4550). */
-export const REQUEUABLE = (outcome) => ["needs-human", "invalid", "skipped", "ci-pending", "budget-exceeded", "lane-refused", "deferred"].includes(outcome) || /^dependency-/.test(String(outcome));
+/** `outcome`s `--requeue` may move back to pending (#4550); #5168: a final `engine-error` too (the runner clears its engine-error count); #5162: `held` rows too. */
+export const REQUEUABLE = (outcome) => ["needs-human", "engine-error", "held", "invalid", "skipped", "ci-pending", "budget-exceeded", "lane-refused", "deferred"].includes(outcome) || /^dependency-/.test(String(outcome));
 /** Key of one requeue entry; the runner records the applied keys in state.json `requeueApplied`. */
 export const requeueKey = (e) => `${e.issue}@${e.at}`;
 
@@ -145,6 +146,14 @@ export function openBatch(root, id) {
       if (!Number.isInteger(k) || k < 1) throw new Error("parallel must be a positive integer");
       put(f("parallel"), `${k}\n`);
     },
+    /** Live execution override (#5138): the `execution` file `{maxRemote?, mode?, provider?}`, read on every admission; null when missing or invalid. */
+    readExecution: () => sanitizeOverride(readJson(f("execution"))),
+    /** Merge a validated patch into the `execution` file. @returns the stored override. */
+    setExecution: (patch) => {
+      const next = { ...(sanitizeOverride(readJson(f("execution"))) ?? {}), ...validateExecutionPatch(patch) };
+      put(f("execution"), JSON.stringify(next) + "\n");
+      return next;
+    },
     /** Issues added by `--add` (#4408): the `add` file, sorted unique positive integers; [] when missing or invalid. */
     readAdded: () => readAdded(f("add")),
     /**
@@ -168,7 +177,7 @@ export function openBatch(root, id) {
     },
     /**
      * `--requeue` (#4550): record that each issue returns to pending. Refused: not a positive integer, not in the batch's
-     * `order`, no terminal result yet (`not terminal`), or already requeued and not yet applied. The needs-human label and
+     * `order`, no terminal result yet (`not terminal`; a final `engine-error` is terminal, #5168), or already requeued and not yet applied. The needs-human label and
      * closed checks need GitHub and live in the CLI. @returns {{requeued:number[], refused:Array<{issue, reason}>}}
      */
     requeue: (list, { reason = "operator decision", by = "operator" } = {}) => {