forgedock 1.49.0 → 1.51.0

package/bin/engine/outcome-codes.mjs

@@ -36,6 +36,7 @@ export const OUTCOME_CODES = Object.freeze({
   CI_TIMEOUT: "ci.timeout",
   BASE_CONFLICT: "base.conflict",
   RATE_LIMITED: "rate_limited",
+  BACKEND_AUTH: "backend.auth", // forge#5168: the claude CLI login failed ("Failed to authenticate"); deferred under the account hold, never engine-error, never the breaker
   HOST_BUSY: "host.busy",
   GH_UNAVAILABLE: "gh.unavailable", // forge#4157: `gh` unreachable after retries during the --repo identity check; run deferred, nothing posted
   GH_AUTH: "gh.auth", // forge#4662: gh answered HTTP 401 and one token refresh did not fix it; run deferred, a rerun resumes
@@ -149,6 +150,7 @@ export function codeForRunnerError(err) {
   if (isDiskQuotaError(err)) return OUTCOME_CODES.INFRA_DISK_QUOTA;
   if (err.usageLimit === true || (typeof err.resetAtEpochMs === "number" && Number.isFinite(err.resetAtEpochMs)))
     return OUTCOME_CODES.RATE_LIMITED;
+  if (err.authFailure === true) return OUTCOME_CODES.BACKEND_AUTH; // forge#5168
   switch (err.code) {
     case "CLI_SPAWN_TRANSIENT":
       return OUTCOME_CODES.INFRA_CLI_SPAWN_TRANSIENT;

package/bin/engine-cli.mjs

@@ -14,7 +14,7 @@ import { memAvailableMb } from "./lean/heavy.mjs";
 import { join, resolve as resolvePath } from "node:path";
 import { runIssue, DEFAULT_MAX_ATTEMPTS, sanitizeLeaseHost } from "./engine.mjs";
 import { makeProjector } from "./engine/projector.mjs";
-import { isTrustedActor, parseDependsOn } from "./engine/phases.mjs";
+import { decomposedSubissues, isDecomposedRecord, isTrustedActor, parseDependsOn } from "./engine/phases.mjs";
 import { readLog, readLogFile, deriveState, runLogPath, migrateRunLogs } from "./engine/runlog.mjs";
 import { assertOpenCodeNativeRuntime } from "./runner.mjs";
 import { parseForgeYaml } from "./forge-utils.mjs";
@@ -34,6 +34,7 @@ import { parseTurnBudgets, formatTurnSummary, TURN_BUDGET_DEFAULTS } from "./lea
 import { writeAtomic as writeResultAtomic } from "./lean/batch.mjs";
 import { parseLightConfig, LIGHT_DEFAULTS } from "./lean/light.mjs";
 import { OUTCOME_CODES } from "./engine/outcome-codes.mjs";
+import * as batchHealth from "./lean/batch-health.mjs";
 import * as _autoChildren from "./lean/autochildren.mjs";
 
 /**
@@ -1183,6 +1184,7 @@ function flag(argv, name) { const i = argv.indexOf(name); return i >= 0 ? argv[i
  * `--resume <id>` continues after the dispatcher dies, re-adopting live lanes. `deferred` lanes are requeued by code,
  * forge.yaml `orchestration.breaker` ({errors, window_min}, default 3 in 30) stops admissions on an engine-error
  * burst, and `--set-parallel <id> K` changes K of a running batch.
+ * `--set-max-remote <id> N` and `--set-placement <id> remote|hybrid|local` (#5138) change its execution live (live-execution.mjs).
  * Maintenance budget (#4197), only with forge.yaml `maintenance.enabled: true` (or `deps.maintenance`): open issues carrying
  * a `maintenance.labels` label join the batch, each issue is classified from its labels (maintenance; `priority:P<n>`,
  * unlabelled = P2) and admission reserves `maintenance.share` of lane starts for maintenance (`when_idle` drains the rest).
@@ -1248,7 +1250,7 @@ async function envBatch({ argv, root, print = (l) => console.log(l) }) {
   return { ok: true, batch: id };
 }
 
-const ORCH_USAGE = "usage: forgedock orchestrate <issue...> [--parallel K] [--lane L] [--lane-override] [--force <reason>] [--repo owner/repo] [--auto-children off|decomposed|epic|all] [--epic N] | --resume <id> [issue...] | --set-parallel <id> <K> | --clear-breaker <id> | --add <id> <issue...> | --requeue <id> <issue...> [--reason <text>] | --cancel <id> [--stop-remote] | --hold <id> <issue...> | --release <id> <issue...> | --show-env <id> | --set-env <id> KEY=VAL | --unset-env <id> KEY";
+const ORCH_USAGE = "usage: forgedock orchestrate <issue...> [--parallel K] [--lane L] [--lane-override] [--force <reason>] [--repo owner/repo] [--auto-children off|decomposed|epic|all] [--epic N] | --resume <id> [issue...] | --set-parallel <id> <K> | --set-max-remote <id> <N> | --set-placement <id> remote|hybrid|local | --clear-breaker <id> | --add <id> <issue...> | --requeue <id> <issue...> [--reason <text>] | --cancel <id> [--stop-remote] | --hold <id> <issue...> | --release <id> <issue...> | --show-env <id> | --set-env <id> KEY=VAL | --unset-env <id> KEY";
 const ORCH_VALUED = ["--parallel", "--lane", "--repo", "--model", "--prefilter-model", "--reason", "--force", "--auto-children", "--epic"];
 const issuesOf = (argv) => {
   const issues = [];
@@ -1463,6 +1465,17 @@ async function orchestrateClient(argv, deps = {}) {
     console.log(`batch ${id}: parallel set to ${k}`);
     return { ok: true, batch: id, parallel: k };
   }
+  if (argv.includes("--set-max-remote") || argv.includes("--set-placement")) {
+    // #5138: through the daemon when one runs, else the batch's `execution` file directly (nothing hosts the batch).
+    const liveExec = await import("./lean/live-execution.mjs");
+    const { id, patch } = liveExec.parseSetExecution(argv);
+    let sock = null, execution = null;
+    try { sock = await schedd.connectDaemon({ dir, spawnDaemon: null }); } catch { sock = null; }
+    if (sock) execution = (await schedd.request(sock, { op: "set-execution", id, ...patch }, undefined, { timeoutMs: schedd.REQUEST_MS }))?.execution ?? null;
+    else { const batchMod = await import("./lean/batch.mjs"); execution = liveExec.applySetExecution(batchMod.openBatch(batchMod.batchRoot(), id), patch); }
+    console.log(`batch ${id}: ${liveExec.formatOverride(patch)}`);
+    return { ok: true, batch: id, execution };
+  }
   // #4408: --add, and the extra issues of --resume <id> N..., go to the batch (through the daemon when one runs).
   const addVia = (id) => async (issues) => {
     let sock = null;
@@ -1584,6 +1597,14 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
     sink.line(`batch ${b.id}: parallel set to ${k}`);
     return { ok: true, batch: b.id, parallel: k };
   }
+  if (argv.includes("--set-max-remote") || argv.includes("--set-placement")) {
+    // #5138: live max_remote / placement mode of a batch, re-read by its runner on every admission.
+    const liveExec = await import("./lean/live-execution.mjs");
+    const { id, patch } = liveExec.parseSetExecution(argv);
+    const execution = liveExec.applySetExecution(batchMod.openBatch(root, id), patch);
+    sink.line(`batch ${id}: ${liveExec.formatOverride(patch)}`);
+    return { ok: true, batch: id, execution };
+  }
   if (["--show-env", "--set-env", "--unset-env"].some((f) => argv.includes(f))) return envBatch({ argv, root, print: (l) => sink.line(l) });
   if (argv.includes("--cancel")) return cancelBatch({ id: flag(argv, "--cancel"), root, stopRemote: argv.includes("--stop-remote"), print: (l) => sink.line(l) });
   if (argv.includes("--requeue")) {
@@ -1655,7 +1676,11 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
       }
       for (const [n, l] of baseWaits) sink.line(`held: #${n} waits for ${l.map((w) => `#${w.dep} (${w.sha.slice(0, 10)}) on ${w.base}`).join(", ")}; admitted once the commit lands`);
       unmet = unmet.filter((x) => x.kind !== "commit");
-      if (unmet.length) throw new Error(`parallel milestone start refused: ${unmet.map((u) => `#${u.issue} depends on #${u.dep} (${u.why}; base ${u.base})`).join("; ")}. Merge the dependencies first, or pass --force "<reason>".`);
+      if (unmet.length) {
+        const refusal = new Error(`parallel milestone start refused: ${unmet.map((u) => `#${u.issue} depends on #${u.dep} (${u.why}; base ${u.base})`).join("; ")}. Merge the dependencies first, or pass --force "<reason>".`);
+        await alertRefusal({ root, issues, repo, cwd, argv, message: refusal.message, unmet, sink, gh: gateIo.gh, deps }); // #5167: recorded for `forgedock status`
+        throw refusal;
+      }
     }
     const base = deps.batchId ?? batchMod.newBatchId();
     batch = batchMod.openBatch(root, base);
@@ -1673,6 +1698,10 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
   }
   sink.batch?.(batch);
   sink.line(`batch: ${batch.id} (${batch.dir}); resume with: forgedock orchestrate --resume ${batch.id}`);
+  // #5167: early batch health at T+15/T+60 after plan.created and at batch end; the first unhealthy result alerts once
+  const healthConf = deps.healthConfig ?? readHealthConfig(cwd), healthNow = deps.healthNow ?? (() => Date.now());
+  const healthCreated = Number.isFinite(Date.parse(plan?.created ?? "")) ? Date.parse(plan.created) : healthNow();
+  sink.line(batchHealth.scheduleLine(batch.id, healthCreated, healthConf.checksMin));
   const dir = deps.dir ?? runDir();
   migrateRunLogs(dir); // #4432: split legacy per-issue logs by repo before any lane runs (best effort, never throws)
   const { orchestrate, formatTable, formatWaitSummary, formatSandboxSummary, formatOutcomeLists, formatAlertLines, estimateLaneMinutes, laneHistory, formatEstimateSummary } = await import("./lean/orchestrate.mjs");
@@ -1774,13 +1803,14 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
       const relays = Array.isArray(provider.relays) ? provider.relays : [];
       // #5112: the host-busy guard, shared by hybrid placement and the fallback_local capacity fallback.
       const hostHeadroom = deps.hostHeadroom ?? (() => execMod.hostHeadroom({ cfg: readAdmission(cwd), heavyCfg: readHeavy(cwd) }));
-      const place = !hybrid ? (n) => execMod.placeLane({ exec, needs: needsOf.get(n) ?? [], relays }) : await (async () => {
-        const verifyMode = (deps.verify !== undefined ? deps.verify : readVerify(cwd))?.mode ?? "local";
-        let weight = deps.validationWeight ?? null;
-        if (!weight) { try { weight = await execMod.validationWeight({ cwd, precheckCommands: readPrecheckCommands(cwd) ?? [] }); } catch { weight = { heavy: false, signals: [] }; } }
-        const headroom = hostHeadroom;
-        return (n) => execMod.placeLane({ exec, verifyMode, labels: labelsOf.get(n) ?? [], weight, headroom: headroom(), needs: needsOf.get(n) ?? [], relays });
-      })();
+      // #5138: placement and max_remote re-read the batch's live `execution` override (--set-placement / --set-max-remote).
+      const liveExec = await import("./lean/live-execution.mjs");
+      const verifyMode = (deps.verify !== undefined ? deps.verify : readVerify(cwd))?.mode ?? "local";
+      const computeWeight = () => deps.validationWeight ?? execMod.validationWeight({ cwd, precheckCommands: readPrecheckCommands(cwd) ?? [] });
+      const weight = liveExec.lazyWeight({ initial: hybrid ? (await Promise.resolve().then(computeWeight).catch(() => null)) ?? { heavy: false, signals: [] } : null, compute: computeWeight });
+      const place = (e, n) => (e.mode !== "hybrid" ? execMod.placeLane({ exec: e, needs: needsOf.get(n) ?? [], relays })
+        : execMod.placeLane({ exec: e, verifyMode, labels: labelsOf.get(n) ?? [], weight: weight.get(), headroom: hostHeadroom(), needs: needsOf.get(n) ?? [], relays }));
+      if (batch.readExecution()?.provider !== true) batch.setExecution({ provider: true });
       // #4540: provider free slots, refreshed in the background (admission is sync); unknown (null) never blocks.
       let capSample = null, onlineSample = null, capAt = 0, capLast = -Infinity, capFailed = false; // #5048: capFailed = last probe rejected // #4547: online = agents alive, null unknown
       const capacity = () => {
@@ -1792,18 +1822,19 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
         }
         return capSample === null && onlineSample === null ? null : { free: capSample, online: onlineSample, at: capAt };
       };
-      remote = {
-        capacity, max: exec.maxRemote, isRemote: () => true, place, identity: () => holdId, ...(hybrid ? { fallbackLocal: true } : {}), ...(exec.fallbackLocal ? { offlineFallback: true, headroom: hostHeadroom, ...(exec.capacityFallbackAfter ? { capacityFallbackAfter: exec.capacityFallbackAfter } : {}) } : {}), onBudget: bud.onBudget ?? "stop", costPerMinute: bud.costPerMinute ?? 0,
+      remote = liveExec.liveRemote({
+        capacity, isRemote: () => true, identity: () => holdId, ...(exec.fallbackLocal ? { offlineFallback: true, headroom: hostHeadroom, ...(exec.capacityFallbackAfter ? { capacityFallbackAfter: exec.capacityFallbackAfter } : {}) } : {}), onBudget: bud.onBudget ?? "stop", costPerMinute: bud.costPerMinute ?? 0,
         admit: (u) => execBudget.admit(u),
         onUsage: (n, sb, rec) => {
           const startedAt = rec?.handle?.startedAt ?? rec?.startedAt ?? "?";
           try { execBudget.record(`${batch.id}#${sb.id ?? n}#${startedAt}`, sb); } catch { /* the batch total still counts it */ }
           Promise.resolve(emitSbx({ event: "SANDBOX_USAGE", issue: n, provider: sb.provider, sandbox_id: sb.id, minutes: sb.minutes, cost_usd: sb.cost_usd, batch: batch.id })).catch(() => {});
         },
-      };
+      }, { exec, readOverride: batch.readExecution, place, onChange: (e) => sink.line(`execution: live placement ${e.mode}, max_remote ${e.maxRemote}`) });
       sink.line(`execution: ${hybrid ? "hybrid" : "remote"} via ${provider.name}, max_remote ${exec.maxRemote}`);
     } else {
       lanes = withPlacement(batchMod.detachedLanes({ batch, cwd, env: laneEnv, script: laneScript, bin: deps.laneBin, pollMs: deps.pollMs, laneArgs, spawnFn: laneSpawn }));
+      if (batch.readExecution()?.provider === true) batch.setExecution({ provider: false }); // #5138: a resume without a provider refuses live remote/hybrid
     }
     // #4376: a cancel stops detached host lanes by SIGTERM to their group (pid-reuse checked by the result path); sandboxes run on unless `--cancel --stop-remote` (#4612).
     const { killLedgerPid } = await import("./lean/effects.mjs");
@@ -1814,7 +1845,7 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
   // #4965: --auto-children overrides forge.yaml; it is part of plan.argv, so --resume keeps it
   const autoMod = _autoChildren, yamlAuto = readAutoChildren(cwd);
   const autoChildren = { mode: argv.includes("--auto-children") ? autoMod.parseAutoModeFlag(flag(argv, "--auto-children")) : yamlAuto.mode, pollS: yamlAuto.pollS,
-    epics: argv.flatMap((a, i) => (a === "--epic" && /^#?\d+$/.test(argv[i + 1] ?? "") ? [parseInt(argv[i + 1].replace("#", ""), 10)] : [])) };
+    epics: epicsOf(argv) };
   // #4304: recovery lanes run on the core path only (run-issue throws on --recover with full), so a full-path repo gets none.
   const recoveryOn = Boolean(orch.recovery?.maxAttempts) && (deps.enginePath !== undefined ? deps.enginePath : readEnginePath(cwd)) !== "full";
   if (orch.recovery?.maxAttempts && !recoveryOn) sink.line("recovery: off (engine.path is full; recovery lanes run on the core path only)");
@@ -1935,6 +1966,10 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
     preflight = async () => (await laneCheck.runLaneChecks({ exec: execForNeeds, env, io: hostIo })).filter((c) => !c.ok);
   }
   let rows, childFields = null; // #4965: the host gh's accepted `issue list --json` fields, read once from its help
+  const healthCheck = (checkpoint) => batchHealth.runCheck({ batch, checkpoint, config: healthConf, repo, now: healthNow,
+    alert: (text, health) => batchHealth.deliverAlert({ text, batchId: batch.id, health, sink, gh: (a) => leanIo.gh(a), repo, config: healthConf, epics: epicsOf(argv), ...(deps.notify ? { notify: deps.notify } : {}) }) });
+  let healthTimers = { cancel: () => {} };
+  try { healthTimers = batchHealth.scheduleChecks({ batch, created: healthCreated, config: healthConf, now: healthNow, ...(deps.healthTimer ? { setTimer: deps.healthTimer.set, clearTimer: deps.healthTimer.clear } : {}), check: healthCheck }); } catch { /* health is advisory */ }
   try {
     rows = await orchestrate({
       ...(estimate ? { estimate } : {}),
@@ -1945,7 +1980,7 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
       autoChildren,
       ...(autoMod.addsEpicChildren(autoChildren.mode) ? { getChildren: async () => ({ list: await autoMod.readChildList({ gh: (a) => leanIo.gh(a), repo, fields: childFields ??= await autoMod.hostListFields((a) => leanIo.gh(a)) }), elsewhere: autoMod.liveElsewhere({ batchMod, root, self: batch.id, repo }) }) } : {}),
       issues, parallel, prefilter, resolveLane: laneOf, store: batch.store, getParallel: batch.readParallel, getAdded: batch.readAdded, getRequeued: batch.readRequeued, getCancelled: batch.readCancel, getHeld: batch.readHeld, getGates: batch.readGates, baseWait: baseWaits.size ? baseWaitOf(baseWaits, { io, leanIo, cwd }) : null, ...lanes,
-      breaker: orch.breaker, maxDeferrals: orch.maxDeferrals, ...(deps.pollMs ? { pollMs: deps.pollMs } : {}), ...budget, limits, identity, rotator, batchId: batch.id, drift, lifecycle, integration,
+      breaker: orch.breaker, maxDeferrals: orch.maxDeferrals, ...(Number.isInteger(orch.transientRetries) ? { transientRetries: orch.transientRetries } : {}), ...(deps.pollMs ? { pollMs: deps.pollMs } : {}), ...budget, limits, identity, rotator, batchId: batch.id, drift, lifecycle, integration,
       ...(registry ? { sched: { registry, batch: batch.id, repo, cap: liveLaneCap(orch.globalParallel ?? null, { cfg: readAdmission(cwd) }), exclusive: true }, priorityOf: async (n) => schedMod.priorityFromLabels(await labelNames(n)) } : {}),
       ...(remote ? { remote } : {}),
       // #4267: mechanical needs-human outcomes are recovered in the batch; every runner decision is a FORGE:DECISION.
@@ -1962,7 +1997,12 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
       // #4387: an out-of-set dependency labelled needs-human is not pulled into the batch
       issueLabels: labelNames,
       // #4725: a closed (completed) dependency is satisfied even when it still carries a stale needs-human label
-      issueState: async (n) => { const v = JSON.parse(await leanIo.gh(["issue", "view", String(n), "-R", repo, "--json", "state,stateReason"])); return v.state === "CLOSED" && String(v.stateReason ?? "COMPLETED").toUpperCase() !== "COMPLETED" ? "CLOSED_NOT_PLANNED" : v.state; },
+      // #5152: a NOT_PLANNED close labelled workflow:merged, or a decomposition whose sub-issues all closed, is delivered: CLOSED
+      issueState: async (n) => {
+        const v = JSON.parse(await leanIo.gh(["issue", "view", String(n), "-R", repo, "--json", "state,stateReason,labels"]));
+        if (v.state !== "CLOSED" || String(v.stateReason ?? "COMPLETED").toUpperCase() === "COMPLETED") return v.state;
+        return ["merged", "moved"].includes(await closedScope({ gh: (a) => leanIo.gh(a), repo, issue: n, labels: v.labels })) ? "CLOSED" : "CLOSED_NOT_PLANNED";
+      },
       // #4318: a dispatcher signal relabels the queued recoveries before it exits (the daemon resumes its batches instead)
       ...(daemon ? {} : { onExit: (fn) => { exitRelabel = fn; } }),
       // #4454: the daemon's shutdown stop; orchestrate throws batch.stopped, so no report is written and a resume adopts the lanes
@@ -1990,7 +2030,7 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
         : async (n) => { const v = await viewIssue(leanIo.gh, repo, n, "body,comments,author,milestone"); if ("milestone" in v) milestoneOf.set(n, v.milestone?.title ?? null); trustedOf.set(n, bodyTrusted(v)); needsOf.set(n, detectNeeds(placementText(v), execForNeeds)); if (execForNeeds?.relay) prefetchOf.set(n, relayPrefetchCommands(v, execForNeeds.relay)); return v.body; }, // #4785: decisions and the frozen contract's validate commands too
       onUpdate: (r) => sink.row(r),
     });
-  } finally { for (const [sig, f] of Object.entries(handlers)) off(sig, f); }
+  } finally { for (const [sig, f] of Object.entries(handlers)) off(sig, f); healthTimers.cancel(); }
   try { await integration?.settle?.(); } catch { /* best effort: the state block is the record */ }
   const report = ["\n" + formatTable(rows), ...formatOutcomeLists(rows)];
   const pfLine = formatPrefilterSummary(rows);
@@ -2033,9 +2073,34 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
   let unfinished = false;
   try { unfinished = batch.pendingIssues().length > 0 && !batch.readCancel(); } catch { unfinished = false; } // a cancel ends the batch for good
   if (!unfinished) try { batch.writeReport({ rows, crossLane, ok, reclaimed, report: report.join("\n") }); } catch { /* the report is a cache of the rows */ }
+  try { await healthCheck("end"); } catch { /* #5167: health never fails a batch */ }
   return { rows, crossLane, batch: batch.id, reclaimed, ok };
 }
 
+/**
+ * #5152: how a CLOSED issue's scope ended, read from its labels rather than its `stateReason` (a NOT_PLANNED close of
+ * delivered work, legacy or not, is never dropped scope): "merged" (`workflow:merged`), "moved" (`workflow:decomposed`
+ * whose trusted FORGE:DECOMPOSED sub-issues are all closed), "moving" (decomposed, a sub-issue open or unreadable), or null
+ * (no delivery label: `stateReason` decides). `labels` is a `gh --json labels` or REST label list. Never throws.
+ */
+export async function closedScope({ gh, repo, issue, labels }) {
+  const names = (Array.isArray(labels) ? labels : []).map((l) => String(l?.name ?? l));
+  if (names.includes("workflow:merged")) return "merged";
+  if (!isDecomposedRecord({ labels })) return null;
+  // decomposedSubissues reads `repos/{owner}/{repo}/...`: pin the placeholder to this batch's repo
+  const io = { gh: (a) => gh(a.map((x) => String(x).replace("{owner}/{repo}", repo))) };
+  let subs = null;
+  try { subs = await decomposedSubissues(issue, io); } catch { subs = null; }
+  if (!subs) return "moving";
+  for (const s of subs) {
+    try {
+      const v = JSON.parse(String(await gh(["issue", "view", String(s), "-R", repo, "--json", "state"])));
+      if (String(v?.state ?? "").toUpperCase() !== "CLOSED") return "moving";
+    } catch { return "moving"; }
+  }
+  return "moved";
+}
+
 /**
  * #4481: the dispatch-time "already done" check: `{outcome:"merged", pr}` when the issue's own lane PR
  * (`feat/issue-N-lean`) merged, `{outcome:"invalid"}` when the issue is closed without one, else null. Anything
@@ -2053,7 +2118,7 @@ export function makeDispatchCheck({ gh, repo }) {
     } catch { return null; }
   };
   return async (n) => {
-    const iss = await json(["issue", "view", String(n), "-R", repo, "--json", "state,stateReason"]);
+    const iss = await json(["issue", "view", String(n), "-R", repo, "--json", "state,stateReason,labels"]);
     const state = typeof iss?.state === "string" ? iss.state.toUpperCase() : null;
     const notPlanned = String(iss?.stateReason ?? "").toUpperCase().replace(/\s/g, "_") === "NOT_PLANNED";
     const prs = await json(["pr", "list", "-R", repo, "--head", `feat/issue-${n}-lean`, "--state", "merged", "--json", "number,mergedAt", "--limit", "1"]);
@@ -2064,7 +2129,15 @@ export function makeDispatchCheck({ gh, repo }) {
       if (reopened === null || !Number.isFinite(mergedAt) || reopened > mergedAt) return null;
     }
     if (pr !== null) return { outcome: "merged", pr, detail: `PR #${pr} already merged` };
-    if (state === "CLOSED") return notPlanned ? { outcome: "invalid", detail: "issue closed (not planned)", final: true } : { outcome: "invalid", detail: "issue closed" }; // #4687: final = its dependents stay skipped
+    if (state === "CLOSED") {
+      // #5152: workflow:merged / a fully closed decomposition is delivered whatever the stateReason; a decomposition still
+      // moving is a completed close (its dependents' lanes wait on the sub-issues); only an unlabelled NOT_PLANNED is dropped scope
+      const scope = await closedScope({ gh, repo, issue: n, labels: iss?.labels });
+      if (scope === "merged") return { outcome: "merged", detail: "issue closed, workflow:merged" };
+      if (scope === "moved") return { outcome: "merged", detail: "issue decomposed; all sub-issues closed" };
+      if (scope === "moving") return { outcome: "invalid", detail: "issue closed: decomposed, sub-issues still open" };
+      return notPlanned ? { outcome: "invalid", detail: "issue closed (not planned)", final: true } : { outcome: "invalid", detail: "issue closed" }; // #4687: final = its dependents stay skipped
+    }
     // #4920: a live lease held by another lane, read from GitHub (any batch, any host, across restarts): no lane, no sandbox.
     // Release review #4931: never the open feat/issue-N-lean PR here. That branch is this issue's own lane, which a
     // relaunch or recovery lane resumes by design; a lane still running on it holds the lease.
@@ -2408,6 +2481,7 @@ export function readAutoChildren(cwd = process.cwd()) {
  * `mechanical_max_attempts`): the separate budget of env.*, base.conflict and conflicting-PR rows, default 2, never below `max_attempts`.
  * `orchestration.base_sync.enabled` (#4525; default true, false opts out) and `base_sync.union_globs` (comma list of globs of
  * append-only files whose two-sided line additions a parked PR's base sync keeps both of).
+ * `orchestration.transient_retries` (#5161; default 4, 0 off): backoff retries of a transient (gateway 5xx) engine-error.
  * `orchestration.blocked_label` (#4773; default `blocked`, empty turns it off): an open issue carrying it is admitted but not dispatched.
  */
 export function readOrchestration(cwd = process.cwd()) {
@@ -2417,13 +2491,36 @@ export function readOrchestration(cwd = process.cwd()) {
   const errors = int(o.breaker_errors ?? o.errors, 3), windowMin = int(o.breaker_window_min ?? o.window_min, 30);
   const gp = int(o.global_parallel, null);
   const recoveryMax = int(o.recovery_max_attempts ?? o.max_attempts, 1);
-  return { breaker: errors > 0 && windowMin > 0 ? { errors, window_min: windowMin } : null, maxDeferrals: int(o.max_deferrals, 3), blockedLabel: String(o.blocked_label ?? "blocked").trim(), globalParallel: gp >= 1 ? gp : null,
+  return { breaker: errors > 0 && windowMin > 0 ? { errors, window_min: windowMin } : null, maxDeferrals: int(o.max_deferrals, 3), transientRetries: int(o.transient_retries, 4), blockedLabel: String(o.blocked_label ?? "blocked").trim(), globalParallel: gp >= 1 ? gp : null,
     recovery: { maxAttempts: recoveryMax, mechanicalMaxAttempts: Math.max(int(o.recovery_mechanical_max_attempts ?? o.mechanical_max_attempts, 2), recoveryMax) }, // #4725: env/base/sequence codes have their own budget
     // #4525: parked-PR base sync. The parser flattens one level, so `base_sync.enabled|union_globs` arrive as `enabled|union_globs` (or flat `base_sync_*`).
     baseSync: { enabled: String(o.base_sync_enabled ?? o.enabled ?? "true").trim().toLowerCase() !== "false",
       unionGlobs: String(o.base_sync_union_globs ?? o.union_globs ?? "").replace(/^\[|\]$/g, "").split(",").map((x) => x.trim().replace(/^["']|["']$/g, "")).filter(Boolean) } };
 }
 
+/** `--epic N` operands of an orchestrate argv (#4965, #5167). */
+const epicsOf = (argv) => argv.flatMap((a, i) => (a === "--epic" && /^#?\d+$/.test(argv[i + 1] ?? "") ? [parseInt(argv[i + 1].replace("#", ""), 10)] : []));
+
+/**
+ * #5167: a refused milestone start leaves no batch dir: record it under the batch root for `forgedock status` and send
+ * the BATCH UNHEALTHY channels. Best effort: the caller rethrows the refusal unchanged.
+ */
+async function alertRefusal({ root, issues, repo, cwd, argv, message, unmet, sink, gh, deps }) {
+  try {
+    const rec = { at: new Date((deps.healthNow ?? Date.now)()).toISOString(), issues, repo, cwd, message, unmet: unmet.map(({ issue, dep, base }) => ({ issue, dep, base })) };
+    try { batchHealth.recordRefusal(root, rec); } catch { /* the stderr line still reports it */ }
+    const config = deps.healthConfig ?? readHealthConfig(cwd);
+    const health = batchHealth.evaluateHealth(null, { refused: rec, config, repo });
+    const text = batchHealth.formatAlert({ batchId: "(refused launch)", checkpoint: "launch", health });
+    await batchHealth.deliverAlert({ text, batchId: null, health, sink, gh: gh ? (a) => gh(a) : null, repo, config, epics: epicsOf(argv), ...(deps.notify ? { notify: deps.notify } : {}) });
+  } catch { /* never mask the refusal */ }
+}
+
+/** forge.yaml `orchestrate.health.*` (flattened to `orchestrate.*`) and `notify.command` (#5167); defaults when unset. */
+export function readHealthConfig(cwd = process.cwd()) {
+  try { return batchHealth.parseHealthConfig(parseForgeYaml(readFileSync(join(cwd, "forge.yaml"), "utf8"))); } catch { return batchHealth.parseHealthConfig(null); }
+}
+
 /**
  * forge.yaml `orchestrate.drift` (#4161): `{pairGrowth, packageLines, fileLines, blockDispatch, generatedGlobs}` with
  * only the positive-integer thresholds set, or null when no threshold and no `block_dispatch: true` is set (summary
@@ -2558,6 +2655,12 @@ export async function milestoneSyncFromCli(argv, deps = {}) {
   return run(argv, deps);
 }
 
+/** `forgedock fix-close-reasons <owner/repo> [--milestone m] [--apply]` (#5177): re-close delivered `not planned` issues as completed (lean/fixclosereasons.mjs). Resolves the exit code. */
+export async function fixCloseReasonsFromCli(argv, deps = {}) {
+  const { runFixCloseReasonsCli: run } = await import("./lean/fixclosereasons.mjs");
+  return run(argv, deps);
+}
+
 /**
  * `forgedock milestone train` (#4658): lean/train.mjs over `orchestrateFromCli` (one batch per milestone) and the #4657 lifecycle.
  * The train reads forge.yaml (`orchestration.global_parallel`, `milestones.close_on`) and never writes it.

package/bin/forgedock.mjs

@@ -977,6 +977,18 @@ async function gatherDashboardData(cwd) {
   return result;
 }
 
+/**
+ * #5167: unhealthy batches and recent launch refusals (or one batch's health with `--batch <id>`) under
+ * `forgedock status`, read from the batch root. Any read failure prints nothing.
+ */
+async function batchHealthSection(c, batchId) {
+  try {
+    const [{ batchRoot }, { statusLines }] = await Promise.all([import("./lean/batch.mjs"), import("./lean/batch-health.mjs")]);
+    const lines = statusLines(batchRoot(), batchId ? { batch: batchId } : {});
+    if (lines.length) c.stdout.write(`  batch health\n${lines.map((l) => (batchId ? `  ${l}` : l)).join("\n")}\n\n`);
+  } catch { /* status never fails on batch health */ }
+}
+
 /** Compact status screen for configured/managed directories. */
 async function statusScreen(c) {
   const state = resolveState(c.cwd);
@@ -3375,13 +3387,14 @@ function help() {
     ["npx forgedock init --minimal", "Generate a minimal forge.yaml (required sections only)"],
     ["npx forgedock enable [dir]", "Activate ForgeDock in a directory"],
     ["npx forgedock disable [dir]", "Opt a directory out of ForgeDock"],
+    ["npx forgedock fix-close-reasons <owner/repo> [--milestone <m>] [--apply]", "Re-close issues that were closed not planned but carry workflow:merged or workflow:decomposed (delivered, not dropped) as completed, with a comment; dry run unless --apply; workflow:invalid issues are never touched"],
     ["npx forgedock status [dir]", "Show ForgeDock state for a directory"],
     ["npx forgedock run <cmd> [args]", "Run a command headlessly (local claude CLI or Anthropic API)"],
     ["npx forgedock bootstrap <idea|docs> [--dry-run] [--spike <path>] | --plan <plan.json> [--dry-run]", "Headless /bootstrap: idea to orchestrate-ready backlog (--dry-run previews milestones, titles, DAG and estimated agent cost without GitHub writes); --plan validates and creates (or previews) a plan, and re-running resumes from GitHub"],
     ["npx forgedock run-issue <issue> [--lane <lane>] [--lane-override] [--path core|full] [--retry] [--wait-lease]", "Run the lean core work-on on one issue (default; --path full = legacy multi-phase pipeline; the lane comes from the issue: milestone → milestone/<slug>, else staging; --lane is an assertion, never the default branch) (--retry reopens an engine-error run, or a needs-human run once the label is cleared; --wait-lease waits for a held lease; exit 75 = deferred, lease held)"],
     ["npx forgedock plan validate --live [--repo owner/repo] [--json]", "Re-run the bootstrap dependency-order rules on live GitHub state: each cross-milestone Depends-on edge is ok, held or invalid (exit 1 on invalid)"],
     ["npx forgedock orchestrate <issue...> [--parallel K] [--lane L]", "Run core work-on on several issues with bounded concurrency and Depends-on ordering; each issue on its own lane, cross-lane dependencies reported"],
-    ["npx forgedock orchestrate --resume <id> [N...] | --set-parallel <id> K | --add <id> N... | --requeue <id> N... [--reason \"...\"] | --cancel <id> | --hold <id> N... | --release <id> N... | --show-env <id> | --set-env <id> KEY=VAL | --unset-env <id> KEY", "Reattach to a batch hosted by the schedd daemon (a finished one replays its report, an orphaned one is resumed with live lanes re-adopted), change its concurrency live, add issues to a running (or held) batch under its K and DAG, requeue a terminal issue (and the dependents skipped behind it) after an operator decision (refused while it is labelled needs-human or closed; a finished batch is reopened), or stop one batch without stopping schedd (running lanes torn down with their work saved, the rest held for --resume), or hold/release single issues of a batch (a held issue is not admitted until released or named by --resume), or print (secrets redacted), set or unset a batch's pinned lane env (applies to newly launched lanes)"],
+    ["npx forgedock orchestrate --resume <id> [N...] | --set-parallel <id> K | --set-max-remote <id> N | --set-placement <id> remote|hybrid|local | --add <id> N... | --requeue <id> N... [--reason \"...\"] | --cancel <id> | --hold <id> N... | --release <id> N... | --show-env <id> | --set-env <id> KEY=VAL | --unset-env <id> KEY", "Reattach to a batch hosted by the schedd daemon (a finished one replays its report, an orphaned one is resumed with live lanes re-adopted), change its concurrency live, change its max_remote or placement mode live (remote/hybrid need a batch started with a remote provider; running lanes are not stopped), add issues to a running (or held) batch under its K and DAG, requeue a terminal issue (and the dependents skipped behind it) after an operator decision (refused while it is labelled needs-human or closed; a finished batch is reopened), or stop one batch without stopping schedd (running lanes torn down with their work saved, the rest held for --resume), or hold/release single issues of a batch (a held issue is not admitted until released or named by --resume), or print (secrets redacted), set or unset a batch's pinned lane env (applies to newly launched lanes)"],
     ["npx forgedock triage <issue...> [--lane L] [--dry-run]", "Already-fixed pre-filter: close issues a merged PR or green CI already resolved; print the rest"],
     ["npx forgedock probe [issue...] [--dry-run]", "Run declared acceptance probes for closed issues whose fix is on the default branch; a fail reopens the issue"],
     ["npx forgedock check [--base REF] [--json]", "Run the lane's frozen validate commands and change-scoped pre-PR checks locally in this worktree, as the engine will (no GitHub, no Actions)"],
@@ -3980,7 +3993,7 @@ async function watch() {
 const SPLASH_COMMANDS = new Set(["run", "bootstrap", "run-issue", "orchestrate", "triage", "probe", "gc", "resume-stalled", "demo", "doctor", "watch", "help", "--help", "-h"]);
 const KNOWN_COMMANDS = new Set([
   "install", "init", "enable", "disable", "status", "uninstall", "update",
-  "run", "bootstrap", "backend-check", "run-issue", "orchestrate", "milestone", "milestone-sync", "guards", "triage", "probe", "review-staging", "gc", "resume-stalled", "demo", "doctor", "watch", "issue-watch", "prod-watch", "selftest", "labels", "config", "stats", "limits", "decisions", "sched", "schedd", "pool", "check", "guide", "help", "--help", "-h",
+  "run", "bootstrap", "backend-check", "run-issue", "orchestrate", "milestone", "milestone-sync", "fix-close-reasons", "guards", "triage", "probe", "review-staging", "gc", "resume-stalled", "demo", "doctor", "watch", "issue-watch", "prod-watch", "selftest", "labels", "config", "stats", "limits", "decisions", "sched", "schedd", "pool", "check", "guide", "help", "--help", "-h",
   "version", "--version", "-v",
 ]);
 if (SPLASH_COMMANDS.has(command) || !KNOWN_COMMANDS.has(command)) splash(command);
@@ -4062,8 +4075,12 @@ switch (command) {
   }
   case "status": {
     const c = ctx();
-    if (positional[1]) c.cwd = resolve(positional[1]);
+    // #5167: `--batch <id>` is not the status directory
+    const statusArgs = positional.slice(1), bi = statusArgs.indexOf("--batch");
+    const statusDir = statusArgs.find((a, i) => !(bi !== -1 && (i === bi || i === bi + 1)));
+    if (statusDir) c.cwd = resolve(statusDir);
     await statusScreen(c);
+    await batchHealthSection(c, bi === -1 ? null : statusArgs[bi + 1] ?? null);
     break;
   }
   case "uninstall":
@@ -4334,6 +4351,17 @@ switch (command) {
     }
     break;
   }
+  case "fix-close-reasons": {
+    // Migration helper (#5177): delivered issues closed `not planned` are re-closed `completed`; dry run unless --apply.
+    const { fixCloseReasonsFromCli } = await import("./engine-cli.mjs");
+    try {
+      exitCode = await fixCloseReasonsFromCli(restArgs);
+    } catch (err) {
+      process.stderr.write(`${RED}${err.message}${RESET}\n`);
+      exitCode = 1;
+    }
+    break;
+  }
   case "milestone": {
     // Native unattended milestone train (#4658): `forgedock milestone train <slug...>`.
     const { trainFromCli } = await import("./engine-cli.mjs");