forgedock 1.28.0 → 1.29.0

package/bin/lean/orchestrate.mjs

@@ -50,8 +50,18 @@
  * With `remote.fallbackLocal`, a lane placed remote runs on the host path instead of waiting when `max` remote lanes run,
  * and instead of stopping when the budget refuses. The final `{where, reason}` is kept on the run record and the row as
  * `placement` and passed to `runOne` as `ctx.placement` (the lane records it in its FORGE:STATE).
+ *
+ * In-batch recovery (#4267), off unless `recovery: {maxAttempts}` is passed: a row `classifyNeedsHuman` calls
+ * recoverable (needs-human `ci.red` / `base.conflict` / `builder.block_untyped`, `contract.invalid` after a mid-run issue
+ * edit, `ci-pending` whose PR `conflicting(issue, pr)` says conflicts) is not final while its persisted `recoveries` stay
+ * under `maxAttempts`: it is requeued as a recovery lane (`ctx.recover = {code, attempt}`) under the same admission as any
+ * lane, so its dependents stay queued. A merged recovery row carries `recovered: <code>`; any other end is final with
+ * `recovery` evidence. Human codes (`builder.blocked`) are never recovered. Every operational decision the runner takes
+ * (recovery, engine-error relaunch, dependency requeue, sub-issues added) goes to `decide({issue, what, why, alternative})`,
+ * fire-and-forget: the runner never waits on it.
  */
 import { parseDependsOn } from "../engine/phases.mjs";
+import { classifyNeedsHuman, recoveryCodeOf } from "../engine/outcome-codes.mjs";
 
 // A dependency that finished like this no longer blocks its dependents.
 const SATISFIED = new Set(["merged", "invalid"]);
@@ -144,7 +154,8 @@ const toMap = (o) => new Map(Object.entries(o ?? {}).map(([k, v]) => [Number(k),
  */
 export async function orchestrate({ issues, parallel = 3, runOne, readBody, prefilter = null, resolveLane = null, now = () => Date.now(), onUpdate = () => {}, onCrossLane = () => {},
   store = null, adopt = null, getParallel = null, breaker = null, maxDeferrals = DEFAULT_MAX_DEFERRALS, sleep = cancellableSleep, pollMs = 5000,
-  classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null }) {
+  classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null,
+  recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null }) {
   const saved = store?.load?.() ?? null;
   const order = saved ? [...saved.order] : [];
   const set = new Set(order);
@@ -193,13 +204,20 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   const sandboxUse = toMap(saved?.sandbox); // issue -> {minutes, cost_usd, attempts} (#4255)
   const remoteCpm = Number(remote?.costPerMinute) > 0 ? Number(remote.costPerMinute) : 0;
   let budgetStop = null, budgetHit = null; // this pass's refusal under on_budget stop; the last refusal reason
+  const recoveryMax = Number.isInteger(recovery?.maxAttempts) && recovery.maxAttempts > 0 ? recovery.maxAttempts : 0;
+  const recoveries = toMap(saved?.recoveries); // issue -> recovery lanes launched (#4267)
+  const recovering = toMap(saved?.recovering); // issue -> {code, attempt} of the recovery lane queued or running
+  // #4267: decisions are recorded, never awaited: a failing or slow recorder cannot block the batch.
+  const decided = (d) => { try { Promise.resolve(decide?.(d)).catch(() => {}); } catch { /* best effort */ } };
+  const firstLine = (x) => String(x ?? "").split("\n")[0].slice(0, 300);
 
   const save = () => {
     if (!store) return;
     store.save({ v: 1, order, deps: toObj(deps), lanes: toObj(lanes), crossSeen: [...crossSeen], results: toObj(results), running: toObj(runRecs),
       pending, notBefore: toObj(notBefore), requeues: toObj(requeues), deferrals: toObj(deferrals), engineErrors: toObj(engineErrors),
       prefilter: toObj(checked), errorTimes, breaker: tripped, ...(budget ? { classes: toObj(classes), dispatch: { ...dispatch } } : {}),
-      ...(reg ? { priorities: toObj(priorities) } : {}), ...(sandboxUse.size ? { sandbox: toObj(sandboxUse) } : {}) });
+      ...(reg ? { priorities: toObj(priorities) } : {}), ...(sandboxUse.size ? { sandbox: toObj(sandboxUse) } : {}),
+      ...(recoveries.size || recovering.size ? { recoveries: toObj(recoveries), recovering: toObj(recovering) } : {}) });
   };
   const update = (row) => { store?.log?.(row); onUpdate(row); };
   const classifyOne = async (n) => {
@@ -341,6 +359,8 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       // Dependencies already done but still open on GitHub: a delay instead of a hot relaunch loop.
       const allDone = depList.every((d) => results.has(d) && SATISFIED.has(results.get(d).outcome));
       requeue(n, allDone ? DEP_REQUEUE_DELAY_S * 1000 : 0); done(); save();
+      decided({ issue: n, what: `requeued behind ${depList.map((d) => `#${d}`).join(", ")} (Depends on edge added)`, why: firstLine(row.detail) || "the lane reported an open dependency",
+        alternative: "skip the issue and its dependents" });
       update({ ...row, detail: `requeued behind ${depList.map((d) => `#${d}`).join(", ")}` });
       return;
     }
@@ -365,15 +385,50 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       if ((engineErrors.get(n) ?? 0) < ENGINE_ERROR_RELAUNCHES) {
         engineErrors.set(n, (engineErrors.get(n) ?? 0) + 1);
         requeue(n, 0); done(); save();
+        decided({ issue: n, what: "relaunch once after engine-error", why: firstLine(row.detail) || "engine-error", alternative: "end the issue engine-error and skip its dependents" });
         update({ ...row, detail: `relaunching once: ${row.detail ?? ""}` });
         return;
       }
     }
     if (row.outcome === "decomposed" && subs.length) {
       await addIssues(subs);
+      decided({ issue: n, what: `added sub-issues ${subs.map((d) => `#${d}`).join(", ")} to the batch; dependents wait on them`, why: "the builder decomposed the issue",
+        alternative: "end the batch row decomposed and let dependents wait for a later batch" });
       for (const [m, ds] of deps) if (ds.includes(n)) deps.set(m, [...new Set(ds.flatMap((d) => (d === n ? subs : [d])))]);
       noteCrossLane();
     }
+    // #4267: a recovery pass ends here; a recoverable row under the attempt cap is requeued as a recovery lane.
+    const prior = recovering.get(n) ?? null;
+    if (prior) {
+      recovering.delete(n);
+      // #4298: the row reports the whole issue: earlier attempts' cost and minutes are summed into the recovery's.
+      row.cost_usd = Math.round(((Number(row.cost_usd) || 0) + (Number(prior.cost_usd) || 0)) * 1e6) / 1e6;
+      row.minutes = Math.round(((Number(row.minutes) || 0) + (Number(prior.minutes) || 0)) * 10) / 10;
+      if (row.outcome === "merged") row.recovered = prior.code;
+      else row.recovery = { code: prior.code, attempts: recoveries.get(n) ?? prior.attempt, outcome: row.outcome, ...(row.code ? { result_code: row.code } : {}), detail: firstLine(row.detail) };
+      // A recovery that failed for good keeps needs-human. The lane relabels needs-human/ci-pending itself; engine-error (after its
+      // relaunch) and deferred (no longer requeued) end here, so the runner does it, once. invalid/decomposed/dependency-*/merged never.
+      if (["engine-error", "deferred"].includes(row.outcome) && typeof relabelNeedsHuman === "function") {
+        try { await relabelNeedsHuman({ issue: n, evidence: `Recovery pass for \`${prior.code}\` (attempt ${prior.attempt}) did not merge: it ended \`${row.outcome}\`${row.code ? ` (\`${row.code}\`)` : ""}${firstLine(row.detail) ? `: ${firstLine(row.detail)}` : ""}. The issue stays \`needs-human\`.` }); } catch { /* best effort */ }
+      }
+    }
+    if (recoveryMax && (row.outcome === "needs-human" || row.outcome === "ci-pending")) {
+      let isConflicting = false;
+      if (row.outcome === "ci-pending" && typeof conflicting === "function" && Number.isInteger(row.pr)) {
+        try { isConflicting = (await conflicting(n, row.pr)) === true; } catch { isConflicting = false; }
+      }
+      const facts = { ...row, issueEdited: r?.issueEdited === true, conflicting: isConflicting };
+      const used = recoveries.get(n) ?? 0;
+      if (classifyNeedsHuman(facts) === "recoverable" && used < recoveryMax) {
+        const code = recoveryCodeOf(facts), attempt = used + 1;
+        recoveries.set(n, attempt); recovering.set(n, { code, attempt, cost_usd: Number(row.cost_usd) || 0, minutes: Number(row.minutes) || 0 });
+        requeue(n, 0); done(); save();
+        decided({ issue: n, what: `recovery lane for ${code} (attempt ${attempt}/${recoveryMax}); dependents stay queued`, why: firstLine(row.detail) || code,
+          alternative: "leave needs-human for the operator and skip the dependents" });
+        update({ ...row, outcome: "recovering", detail: `recovering (${code}), attempt ${attempt}/${recoveryMax}: ${firstLine(row.detail)}` });
+        return;
+      }
+    }
     results.set(n, row);
     if (row.outcome === "merged" && drift?.onMerged && r?.structure && String(rec?.lane ?? r?.lane ?? "").startsWith("milestone/")) {
       save(); update(row);
@@ -406,7 +461,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
       if (pf?.closed) r = { terminalReason: "invalid", cost_usd: 0, detail: `pre-filter: ${pf.detail}` };
       else {
         try {
-          r = await runOne(n, lane, { remote: !!rec.remote, ...(rec.placement ? { placement: rec.placement } : {}), onStart: (info) => {
+          r = await runOne(n, lane, { remote: !!rec.remote, ...(rec.placement ? { placement: rec.placement } : {}), ...(recovering.has(n) ? { recover: { code: recovering.get(n).code, attempt: recovering.get(n).attempt } } : {}), onStart: (info) => {
             Object.assign(rec, info ?? {}); save();
             if (reg && slotted.has(n) && info?.pid) reg.setLanePid(keyOf(n), info.pid);
           } });
@@ -521,6 +576,28 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
   return order.map((n) => results.get(n) ?? { issue: n, outcome: "held", pr: null, cost_usd: 0, minutes: 0, detail: tripped?.detail ?? (driftHeld.has(n) ? `drift: ${driftHeld.get(n)} flagged` : budgetStop ? `execution budget: ${budgetStop}` : budgetHeld ? "maintenance share" : "not run") });
 }
 
+/** Max chars of one row's detail in the batch-report outcome lists (#4298). */
+export const OUTCOME_LINE_MAX = 200;
+/**
+ * #4267 batch-report lines after the table: `recovered: #A (ci.red) #B (base.conflict)`, one `unrecovered:` line per
+ * mechanical needs-human row (failed recovery or a terminal code), then one escalation list holding only human codes,
+ * asked once at the end of the batch. Empty when there is nothing to list.
+ */
+export function formatOutcomeLists(rows) {
+  const out = [];
+  const one = (x) => String(x ?? "").split("\n")[0].slice(0, OUTCOME_LINE_MAX);
+  const rec = (rows ?? []).filter((r) => r?.recovered);
+  if (rec.length) out.push(`recovered: ${rec.map((r) => `#${r.issue} (${r.recovered})`).join(" ")}`);
+  for (const r of rows ?? []) {
+    if (!r || r.recovered) continue;
+    const human = r.outcome === "needs-human" && classifyNeedsHuman(r) === "human";
+    if (!human && (r.outcome === "needs-human" || r.recovery)) out.push(`unrecovered: #${r.issue} (${r.code ?? r.recovery?.code ?? "unclassified"}: ${one(r.recovery ? `recovery ${r.recovery.code} ended ${r.recovery.outcome}: ${r.recovery.detail}` : r.detail)})`);
+  }
+  const esc = (rows ?? []).filter((r) => r?.outcome === "needs-human" && classifyNeedsHuman(r) === "human");
+  if (esc.length) out.push(["escalations (human decisions, asked once):", ...esc.map((r) => `- #${r.issue} (${r.code}): ${one(r.detail)}`)].join("\n"));
+  return out;
+}
+
 export function formatTable(rows) {
   // #4255: sandbox minutes and cost next to the token cost, only when some lane ran in a sandbox.
   const sbx = rows.some((r) => r?.sandbox);

package/bin/lean/prefilter.mjs

@@ -10,7 +10,7 @@
  * ALREADY-FIXED/INVALID verdict with evidence closes the issue; everything else, and every error, goes to the normal lane.
  * All effects are injected (gh, git, runner, transitionLabel, projector?) so it is unit-testable.
  */
-import { capComment, sanitizeMd } from "./render.mjs";
+import { capComment, sanitizeMd, noComments } from "./render.mjs";
 import { PREFILTER_SYSTEM, prefilterPrompt, parsePrefilterVerdict } from "./prompts.mjs";
 
 export const DEFAULT_PREFILTER_MODEL = "claude-haiku-5-5";
@@ -38,7 +38,7 @@ export function ciFailureRun(iss) {
 const PATH_RE = /(?:^|[\s`'"(\[])((?:\.{0,2}\/)?(?:[\w@.-]+\/)+[\w@.-]+\.[A-Za-z0-9]{1,8})(?::\d+(?:-\d+)?)?(?=$|[\s`'"),.:;\]])/gm;
 /** Repo-relative file paths the issue names (`dir/file.ext`, optionally `:line`); URLs and HTML comments are ignored. */
 export function namedPaths(text) {
-  const s = sanitizeMd(text).replace(/<!--[\s\S]*?-->/g, " ").replace(/\b[a-z]+:\/\/\S+/gi, " ");
+  const s = noComments(sanitizeMd(text), " ").replace(/\b[a-z]+:\/\/\S+/gi, " ");
   return [...new Set([...s.matchAll(PATH_RE)].map((m) => m[1].replace(/^\.\//, "")).filter((p) => !p.startsWith("/") && !p.startsWith("../")))];
 }
 

package/bin/lean/prefilter.test.mjs

@@ -112,6 +112,9 @@ describe("pre-filter (#4123)", () => {
 
   it("every named path missing on the lane is a candidate; any surviving path is not", async () => {
     assert.deepEqual(namedPaths("see `src/old/foo.js:12` and lib/bar.py, not https://x.io/a/b.js <!-- FORGE:STATE {\"a\":\"c/d.js\"} -->"), ["src/old/foo.js", "lib/bar.py"]);
+    // #4298: the scan-based noComments: a `--!>` closer ends a comment and an unclosed one runs to the end (the old
+    // one-pass regex kept both hidden paths).
+    assert.deepEqual(namedPaths("fix lib/a.js <!-- x/hidden.js --!> visible/e.js and <!-- unclosed c/d.js"), ["lib/a.js", "visible/e.js"]);
     const gone = world({ issues: { 5: { body: "crash in src/old/foo.js:12" } }, verdict: { verdict: "INVALID", evidence: ["src/old/foo.js removed in abc123"] } });
     assert.equal((await pf(gone, 5)).closed, true);
     assert.match(gone.runner[0].prompt, /do not exist on origin\/staging: src\/old\/foo\.js/);

package/bin/lean/prompts.mjs

@@ -50,14 +50,17 @@ export function ciUnavailablePrompt() {
 /** #4163: the extra host-rules line when the engine pinned the repo's toolchain on PATH; "" otherwise (prompt unchanged). */
 const toolchainRule = (t) => (t ? `\nThe engine pinned this repo's toolchain on PATH (${t}). Use it as is: do not install global shims or other versions and do not edit PATH; if the pinned tool itself fails, report it as a toolchain/environment problem instead of working around it.` : "");
 
+/** #4272: the extra line citing the repo-facts cache by path (never its content) when it exists; "" otherwise (prompt unchanged). */
+const factsRule = (f) => (f ? `\nRepo facts (toolchain pins, slow suites, layer rules, hotspots) are cached in ${f}: read it before choosing checks or placing code.` : "");
+
 /** #4221: the extra host-rules line when the lane worktree is sparse; "" otherwise (prompt unchanged). */
 const sparseRule = (s) => (s?.excluded ? `\nThis worktree is a sparse checkout: ${s.excluded} tracked file(s) of ${s.maxKb} KB or more are not on disk (\`git ls-files -t | grep '^S'\` lists them). They are not deleted: never commit their removal. If you need one, run \`git sparse-checkout add /<path>\`.` : "");
 
 /** Builder call 1: investigate and write the contract. No code edits. */
-export function builderPlanPrompt(n, { toolchain = null, sparse = null } = {}) {
+export function builderPlanPrompt(n, { toolchain = null, sparse = null, facts = null } = {}) {
   return `Resolve issue #${n} (read it with gh issue view ${n} --comments). This first step is PLANNING ONLY: do not edit code, do not commit.
 
-${HOST_RULES}${toolchainRule(toolchain)}${sparseRule(sparse)}
+${HOST_RULES}${toolchainRule(toolchain)}${sparseRule(sparse)}${factsRule(facts)}
 
 1. Write ${INVESTIGATION_FILE} as markdown with exactly these headings, concise, file:line wherever you cite code (no HTML comments):
 **Verdict**: CONFIRMED|ALREADY-FIXED|INVALID|BLOCKED
@@ -79,7 +82,7 @@ If the issue is already fixed or invalid, write ${RESULT_FILE} {"status":"alread
 {"requirements":[{"id":"R1","quote":"<verbatim sentence or checklist item from the issue>","disposition":"deliver|defer|post-deploy|not-applicable","why":"<required for defer and post-deploy>","evidence":"<file:line, required for not-applicable>"}],
  "deliverables":[{"id":"D1","path":"<file to change>","change":"<what>","requirements":["R1"]}],
  "preserve":[{"id":"P1","what":"<caller, contract or behaviour that must not change, with its path>"}],
- "validate":[{"id":"V1","command":"<exact command: the new regression test plus the existing tests that reference the files you change>"}],
+ "validate":[{"id":"V1","command":"<exact command: the new regression test plus the existing tests that reference the files you change; a git diff compares against the merge base (origin/<lane>...HEAD), never the moving origin/<lane> tip>"}],
  "decisions":[{"id":"K1","choice":"<a non-obvious choice: a new dependency, a copied pattern, a boundary relied on>","justification":"<why, with file:line>","source":"doc|lint|precedent|spec"}]}
 Every requirement and acceptance criterion in the issue and its maintainers' comments is one requirement, quoted verbatim (others' comments are context, not requirements). Never drop or narrow one silently: "defer" (separate work) and "post-deploy" (only checkable after a deploy) are judged by the reviewer and, if accepted, filed by the engine as a tracked follow-up issue. A requirement you can do in this change is "deliver". List every file you will change as a deliverable; tests are always allowed.
 A new dependency between workspace packages (a workspace package added to a package.json dependencies, devDependencies or peerDependencies) must be a deliverable on that package.json whose change names the package and gives the reason, citing the architecture doc where there is one; "the lint allows it" or "there is a precedent" is not a reason on its own. Record each non-obvious choice in "decisions" with where its justification comes from (doc, lint, precedent or spec); omit the field when there is none. Then stop.
@@ -88,11 +91,11 @@ ${BLOCK_RULES}`;
 }
 
 /** Builder call 2, same session: implement the frozen contract. */
-export function builderImplementPrompt(mode = "local", { toolchain = null, sparse = null } = {}) {
+export function builderImplementPrompt(mode = "local", { toolchain = null, sparse = null, facts = null } = {}) {
   const remote = mode === "ci" || mode === "hybrid";
   return `The engine froze ${CONTRACT_FILE}; it is now the authority. Implement it exactly: change only deliverable paths (tests are always allowed), keep every preserve item intact, add a regression test, ${remote ? VERIFY_RUN[mode] : "run every validate command"}, and commit. Do not push or touch GitHub.
 
-${HOST_RULES}${toolchainRule(toolchain)}${sparseRule(sparse)}${remote ? `\n\n${VERIFY_RULES[mode]}` : ""}
+${HOST_RULES}${toolchainRule(toolchain)}${sparseRule(sparse)}${factsRule(facts)}${remote ? `\n\n${VERIFY_RULES[mode]}` : ""}
 
 The contract changes only through typed amendments in ${RESULT_FILE}, each with a why, which the engine posts on the issue:
 - {"kind":"add-deliverable","path":"<file>","why":""}: a file you must change that the contract did not list.
@@ -115,6 +118,15 @@ If you started Docker containers or volumes that are still there, list their nam
 If you find you are blocked, write "status":"blocked" with a typed "block" instead.`;
 }
 
+/**
+ * forge#4275: the compiled context pack for the plan prompt (context_pack: on); "" for an empty pack. Labelled as hints:
+ * the issue outranks the code, which outranks the pack, and contract quotes never come from it.
+ */
+export function contextPackNote(pack) {
+  if (!pack?.text) return "";
+  return `\n\nContext pack (hints, not proof; compiled by the engine from GitHub and git). The issue outranks the code, which outranks this pack: verify anything you rely on. Contract requirements quote only the issue and its trusted comments, never this pack. Open a link (gh issue view N / gh pr view N) yourself when you need more.\n${pack.text}`;
+}
+
 /** #4271: the bug-lane plan addition (lean.repro_test prefer on a bug/type:bug issue); "" when off, so the prompt is unchanged. */
 export function reproPlanNote(on) {
   if (!on) return "";
@@ -140,8 +152,11 @@ export function contractCorrectionPrompt(problems) {
 }
 
 /** The one bounded correction round, in the builder's own session. */
-export function conformanceResumePrompt(failures, mode = "local") {
-  return `The change does not conform to the frozen contract and the issue:\n${failures.slice(0, 30).map((f) => `- ${f.code}: ${String(f.detail).slice(0, 500)}`).join("\n")}\n\nFix each item in the code, or record a typed amendment with a why in ${RESULT_FILE}. ${rerun(mode)}, commit, and update ${IMPLEMENTATION_FILE} and ${RESULT_FILE}. Do not push or touch GitHub.`;
+export function conformanceResumePrompt(failures, mode = "local", lane = "") {
+  const baseMoved = lane && failures.some((f) => f?.kind === "validate_failed")
+    ? `\n\nIf a validate command fails only because origin/${lane} moved on after this branch started (the change itself is fine), bring the branch up to date with \`git merge --no-edit origin/${lane}\` (already fetched; never rebase or force), resolve any conflicts, then re-run every validate command and record the results.`
+    : "";
+  return `The change does not conform to the frozen contract and the issue:\n${failures.slice(0, 30).map((f) => `- ${f.code}: ${String(f.detail).slice(0, 500)}`).join("\n")}${baseMoved}\n\nFix each item in the code, or record a typed amendment with a why in ${RESULT_FILE}. ${rerun(mode)}, commit, and update ${IMPLEMENTATION_FILE} and ${RESULT_FILE}. Do not push or touch GitHub.`;
 }
 
 /** forge#4159: the one resume for package.json edges between workspace packages the contract does not name. */

package/bin/lean/recovery.test.mjs

@@ -0,0 +1,230 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// #4267: in-batch recovery of mechanical needs-human (orchestrate, run-issue/orchestrate CLI, /orchestrate spec).
+// The runCore side (--recover label/decision/prompt, issueEdited) is in lean.test.mjs next to its harness.
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { tmpDir } from "../tests/_tmp.mjs";
+import { orchestrate, formatOutcomeLists, OUTCOME_LINE_MAX } from "./orchestrate.mjs";
+import { RECOVER_HOW } from "./lean.mjs";
+import { RECOVERABLE_CODES } from "../engine/outcome-codes.mjs";
+import { runFromCli, orchestrateFromCli } from "../engine-cli.mjs";
+// orchestrateFromCli opens the host lane registry (#4191): keep it in a temp dir.
+process.env.FORGEDOCK_SCHED_DIR = tmpDir("recovery-sched-");
+
+describe("in-batch recovery of mechanical needs-human (#4267)", () => {
+  // runOne: `script[n]` is a list of results, one per launch of issue n (the last repeats); ctx is recorded.
+  const batch = (script, bodies = {}, extra = {}) => {
+    const calls = [], decisions = [];
+    const p = orchestrate({ issues: Object.keys(script).map(Number), parallel: 3, recovery: { maxAttempts: 1 },
+      readBody: async (n) => bodies[n] ?? "",
+      runOne: async (n, _lane, ctx = {}) => {
+        calls.push({ n, recover: ctx.recover ?? null });
+        await new Promise((r) => setImmediate(r));
+        const list = script[n] ?? [{ terminalReason: "merged" }];
+        const k = calls.filter((c) => c.n === n).length - 1;
+        return list[Math.min(k, list.length - 1)];
+      },
+      decide: (d) => { decisions.push(d); }, ...extra });
+    return p.then((rows) => ({ rows, calls, decisions }));
+  };
+  const NH = (code, detail = `ended ${code}`, more = {}) => ({ terminalReason: "needs-human", code, detail, ...more });
+  const byIssue = (rows) => Object.fromEntries(rows.map((r) => [r.issue, r]));
+
+  it("ci.red: one recovery lane on the PR; green re-queues the issue and its dependent in the same batch, no operator action", async () => {
+    const { rows, calls, decisions } = await batch({ 1: [NH("ci.red", "Required check(s) failed on PR #101: test"), { terminalReason: "merged", pr: 101 }], 2: [{ terminalReason: "merged", pr: 102 }] }, { 2: "**Depends on**: #1" });
+    const r = byIssue(rows);
+    assert.equal(r[1].outcome, "merged"); assert.equal(r[1].recovered, "ci.red");
+    assert.equal(r[2].outcome, "merged", "the dependent was not skipped");
+    assert.deepEqual(calls.filter((c) => c.n === 1).map((c) => c.recover), [null, { code: "ci.red", attempt: 1 }]);
+    assert.ok(calls.findIndex((c) => c.n === 2) > calls.findLastIndex((c) => c.n === 1), "the dependent ran after the recovered merge");
+    const d = decisions.find((x) => x.issue === 1);
+    assert.match(d.what, /recovery lane for ci\.red/); assert.match(d.why, /Required check/); assert.ok(d.alternative);
+    assert.deepEqual(formatOutcomeLists(rows), ["recovered: #1 (ci.red)"]);
+  });
+
+  it("a recovery that stays red is final with its evidence; dependents stay skipped; no attempt beyond max_attempts", async () => {
+    const { rows, calls } = await batch({ 1: [NH("ci.red"), NH("ci.red", "CI verification failed again", { recovery: true })], 2: [] }, { 2: "**Depends on**: #1" });
+    const r = byIssue(rows);
+    assert.equal(calls.filter((c) => c.n === 1).length, 2, "the first run plus exactly one recovery");
+    assert.equal(r[1].outcome, "needs-human");
+    assert.deepEqual(r[1].recovery, { code: "ci.red", attempts: 1, outcome: "needs-human", result_code: "ci.red", detail: "CI verification failed again" });
+    assert.equal(r[2].outcome, "skipped"); assert.ok(!calls.some((c) => c.n === 2));
+    const lines = formatOutcomeLists(rows);
+    assert.equal(lines.length, 1); assert.match(lines[0], /^unrecovered: #1 \(ci\.red: recovery ci\.red ended needs-human: CI verification failed again\)$/);
+  });
+
+  it("human codes (product-decision, spend) are never recovered, still skip dependents and are the only escalations, asked once while other lanes keep running", async () => {
+    const { rows, calls, decisions } = await batch({
+      1: [NH("builder.blocked", "Builder is blocked (product-decision): pick a pricing model")],
+      2: [],
+      3: [NH("builder.blocked", "Builder is blocked (spend): needs a paid plan")],
+      4: [NH("conformance.omission", "R2 not covered")],
+      5: [{ terminalReason: "merged" }] }, { 2: "**Depends on**: #1" });
+    const r = byIssue(rows);
+    assert.equal(calls.filter((c) => c.n === 1).length, 1); assert.equal(calls.filter((c) => c.n === 3).length, 1);
+    assert.equal(calls.filter((c) => c.n === 4).length, 1, "terminal mechanical codes are not recovered either");
+    assert.ok(!decisions.some((d) => [1, 3].includes(d.issue)));
+    assert.equal(r[2].outcome, "skipped"); assert.equal(r[5].outcome, "merged");
+    const lines = formatOutcomeLists(rows);
+    assert.equal(lines.filter((l) => l.startsWith("escalations")).length, 1, "one escalation list");
+    const esc = lines.find((l) => l.startsWith("escalations"));
+    assert.match(esc, /#1 \(builder\.blocked\): Builder is blocked \(product-decision\)/);
+    assert.match(esc, /#3 \(builder\.blocked\): Builder is blocked \(spend\)/);
+    assert.doesNotMatch(esc, /#4/, "the needs-human list holds only human codes");
+    assert.ok(lines.some((l) => /^unrecovered: #4 \(conformance\.omission: R2 not covered\)$/.test(l)));
+  });
+
+  it("base.conflict: a recovery base-sync with a clean result re-queues the issue; the report lists recovered issues separately", async () => {
+    const { rows, calls } = await batch({ 1: [NH("ci.red"), { terminalReason: "merged" }], 2: [NH("base.conflict", "PR #102 still conflicts with staging after 2 base-syncs"), { terminalReason: "merged" }] });
+    assert.deepEqual(calls.filter((c) => c.n === 2).map((c) => c.recover?.code ?? null), [null, "base.conflict"]);
+    assert.equal(byIssue(rows)[2].recovered, "base.conflict");
+    assert.deepEqual(formatOutcomeLists(rows), ["recovered: #1 (ci.red) #2 (base.conflict)"]);
+  });
+
+  it("ci-pending recovers only when its PR conflicts (as base.conflict); a slow-CI ci-pending is left as is", async () => {
+    const seen = [];
+    const conflicting = async (n, pr) => { seen.push([n, pr]); return n === 1; };
+    const { rows, calls } = await batch({ 1: [{ terminalReason: "ci-pending", pr: 101 }, { terminalReason: "merged", pr: 101 }], 2: [{ terminalReason: "ci-pending", pr: 102 }] }, {}, { conflicting });
+    const r = byIssue(rows);
+    assert.equal(r[1].recovered, "base.conflict");
+    assert.equal(r[2].outcome, "ci-pending"); assert.equal(calls.filter((c) => c.n === 2).length, 1);
+    assert.deepEqual(seen.sort(), [[1, 101], [2, 102]]);
+  });
+
+  it("contract.invalid recovers only when the issue was edited mid-run", async () => {
+    const { rows, calls } = await batch({ 1: [NH("contract.invalid", "contract invalid", { issueEdited: true }), { terminalReason: "merged" }], 2: [NH("contract.invalid")] });
+    assert.equal(byIssue(rows)[1].recovered, "contract.invalid");
+    assert.equal(byIssue(rows)[2].outcome, "needs-human"); assert.equal(calls.filter((c) => c.n === 2).length, 1);
+  });
+
+  it("relabelNeedsHuman (SPEC-7/8): only a recovery that ends final engine-error/deferred relabels; invalid/merged never; a requeued deferred waits", async () => {
+    const relabels = [];
+    const rl = { relabelNeedsHuman: async (x) => { relabels.push(x); }, pollMs: 5 };
+    // engine-error after the recovery lane's own relaunch: final -> exactly one relabel with the recovery evidence
+    const a = await batch({ 1: [NH("ci.red"), { terminalReason: "engine-error", detail: "boom" }] }, {}, rl);
+    assert.equal(a.rows[0].outcome, "engine-error"); assert.equal(relabels.length, 1);
+    assert.equal(relabels[0].issue, 1); assert.match(relabels[0].evidence, /Recovery pass for `ci\.red` \(attempt 1\) did not merge: it ended `engine-error`/);
+    // invalid / merged / decomposed: never
+    relabels.length = 0;
+    await batch({ 1: [NH("ci.red"), { terminalReason: "invalid", detail: "not a bug" }], 2: [NH("ci.red"), { terminalReason: "merged" }],
+      3: [NH("ci.red"), { terminalReason: "decomposed", subissues: [] }] }, {}, rl);
+    assert.deepEqual(relabels, []);
+    // a deferred recovery that is requeued is not relabelled until it is final (capped)
+    const defer = { terminalReason: "deferred", code: "gh.unavailable", retryInSeconds: 0.01, detail: "gh down" };
+    const seen = [];
+    const d = await batch({ 1: [NH("ci.red"), defer] }, {}, { ...rl, maxDeferrals: 2, onUpdate: (r) => seen.push([r.outcome, relabels.length, String(r.detail)]) });
+    assert.equal(d.rows[0].outcome, "deferred");
+    const rq = seen.filter(([o, , det]) => o === "deferred" && /^requeued in/.test(det));
+    assert.ok(rq.length >= 1 && rq.every(([, n]) => n === 0), "a requeued deferral is not relabelled");
+    assert.equal(relabels.filter((x) => x.issue === 1).length, 1, "once, only when final");
+    // without a recovery pass nothing is relabelled
+    relabels.length = 0;
+    await batch({ 1: [{ terminalReason: "engine-error" }] }, {}, rl);
+    assert.deepEqual(relabels, []);
+  });
+
+  it("without the recovery option the runner behaves as before: needs-human is final and dependents skip", async () => {
+    const calls = [];
+    const rows = await orchestrate({ issues: [1, 2], readBody: async (n) => (n === 2 ? "**Depends on**: #1" : ""),
+      runOne: async (n) => { calls.push(n); return NH("ci.red"); } });
+    assert.deepEqual(calls, [1]); assert.deepEqual(rows.map((r) => r.outcome), ["needs-human", "skipped"]);
+  });
+
+  it("the attempt count survives a resume: a persisted recovery lane is launched with ctx.recover and is not retried again", async () => {
+    let state = { v: 1, order: [1], deps: { 1: [] }, pending: [1], results: {}, running: {}, recoveries: { 1: 1 }, recovering: { 1: { code: "ci.red", attempt: 1 } } };
+    const store = { load: () => state, save: (s) => { state = s; }, log: () => {} };
+    const ctxs = [];
+    const rows = await orchestrate({ issues: [1], store, recovery: { maxAttempts: 1 }, readBody: async () => "",
+      runOne: async (n, l, ctx) => { ctxs.push(ctx.recover); return NH("ci.red"); } });
+    assert.deepEqual(ctxs, [{ code: "ci.red", attempt: 1 }]);
+    assert.equal(rows[0].outcome, "needs-human"); assert.equal(rows[0].recovery.code, "ci.red");
+    assert.deepEqual(state.recovering, {});
+  });
+
+  it("other runner decisions are recorded too (engine-error relaunch, dependency requeue, sub-issues), and a throwing recorder never blocks the run", async () => {
+    const { decisions } = await batch({ 1: [{ terminalReason: "engine-error", detail: "boom" }, { terminalReason: "merged" }], 2: [{ terminalReason: "decomposed", subissues: [8] }] });
+    assert.ok(decisions.some((d) => d.issue === 1 && /relaunch/.test(d.what) && d.why === "boom" && d.alternative));
+    assert.ok(decisions.some((d) => d.issue === 2 && /#8/.test(d.what)));
+    const rows = await orchestrate({ issues: [1], recovery: { maxAttempts: 1 }, readBody: async () => "", decide: () => { throw new Error("gh down"); },
+      runOne: async (n, l, ctx) => (ctx.recover ? { terminalReason: "merged" } : NH("ci.red")) });
+    assert.equal(rows[0].recovered, "ci.red");
+  });
+
+
+  it("run-issue --recover reaches runCore; a batch relaunches a ci.red lane with recover and reports it", async () => {
+    const got = [];
+    const l = console.log; console.log = () => {};
+    try {
+      await runFromCli(["7", "--lane", "staging", "--repo", "o/r", "--recover", "base.conflict", "--recover-attempt", "1"], { dir: tmpDir("core-rec-"), emitTelemetry: async () => {}, leanIo: {}, io: { gh: async () => "o/r\n" },
+        leanModule: { runCore: async (o) => { got.push(o.recover); return { terminalReason: "merged" }; } } });
+    } finally { console.log = l; }
+    assert.deepEqual(got, [{ code: "base.conflict", attempt: 1 }]);
+    // The batch: a ci.red lane is relaunched in-process with recover, the decision is recorded and the report lists it.
+    const leanIo = {
+      gh: async (a) => (a.includes("milestone") ? JSON.stringify({ milestone: null }) : JSON.stringify({ body: "" })),
+      git: async (a) => (a[0] === "ls-remote" ? "abc\trefs/heads/staging\n" : ""),
+    };
+    const runs = [], decisions = [], logs = [];
+    console.log = (x) => logs.push(String(x));
+    let res;
+    try {
+      res = await orchestrateFromCli(["5", "--repo", "o/r", "--no-prefilter"], { dir: tmpDir("core-rec-"), batchRoot: tmpDir("core-rec-batch-"), allowTmpfs: true, emitTelemetry: async () => {},
+        io: { gh: async () => "o/r\n" }, leanIo, sched: null, orchestration: { breaker: null, maxDeferrals: 3, globalParallel: null, recovery: { maxAttempts: 1 } }, decide: (d) => decisions.push(d),
+        leanModule: { runCore: async (o) => { runs.push(o.recover ?? null); return o.recover ? { terminalReason: "merged", pr: 55 } : { terminalReason: "needs-human", code: "ci.red", detail: "Required check(s) failed on PR #55: test" }; } } });
+    } finally { console.log = l; }
+    assert.deepEqual(runs, [null, { code: "ci.red", attempt: 1 }]);
+    assert.equal(res.ok, true);
+    assert.ok(decisions.some((d) => d.issue === 5 && /recovery lane for ci\.red/.test(d.what)));
+    assert.ok(logs.includes("recovered: #5 (ci.red)"), logs.join("\n"));
+  });
+
+  it("/orchestrate spec step 3 lists the recoverable codes, asks no confirmation on them, and escalates once at the end", async () => {
+    const spec = readFileSync(new URL("../../commands/orchestrate.md", import.meta.url), "utf8");
+    const step3 = spec.slice(spec.indexOf("## 3. Launch lanes"), spec.indexOf("## 4. Report"));
+    for (const c of ["`ci.red`", "`base.conflict`", "`ci-pending` on a conflicting", "`contract.invalid` caused by an issue edited mid-run"]) assert.ok(step3.includes(c), c);
+    assert.match(step3, /\*\*Do not ask for confirmation on them\.\*\*/);
+    assert.match(step3, /FORGE:DECISION/);
+    assert.match(step3, /one question list[\s\S]*once, at the end of the batch/);
+    assert.match(spec, /recovered: #A \(ci\.red\) #B \(base\.conflict\)/);
+  });
+});
+
+describe("#4298 staging-review follow-ups: recovery", () => {
+  it("RECOVER_HOW has a prompt for exactly the RECOVERABLE_CODES", () => {
+    assert.deepEqual(Object.keys(RECOVER_HOW).sort(), [...RECOVERABLE_CODES].sort());
+    for (const c of RECOVERABLE_CODES) assert.ok(String(RECOVER_HOW[c]).length > 20, c);
+  });
+
+  it("a recovered row reports the first attempt's cost and minutes summed with the recovery's; the lane sees only {code, attempt}", async () => {
+    let t = 0; const seen = [];
+    const script = { 1: [{ terminalReason: "needs-human", code: "ci.red", detail: "red", cost_usd: 0.5 }, { terminalReason: "merged", pr: 9, cost_usd: 0.25 }],
+      2: [{ terminalReason: "needs-human", code: "ci.red", detail: "red", cost_usd: 1 }, { terminalReason: "needs-human", code: "ci.red", detail: "red again", cost_usd: 2 }] };
+    const k = { 1: 0, 2: 0 };
+    const rows = await orchestrate({ issues: [1, 2], parallel: 1, recovery: { maxAttempts: 1 }, readBody: async () => "", now: () => t,
+      runOne: async (n, _l, ctx = {}) => { seen.push(ctx.recover ?? null); t += 60000 * (n === 1 ? 3 : 1); return script[n][k[n]++]; } });
+    const r = Object.fromEntries(rows.map((x) => [x.issue, x]));
+    assert.equal(r[1].outcome, "merged"); assert.equal(r[1].cost_usd, 0.75); assert.equal(r[1].minutes, 6);
+    assert.equal(r[2].outcome, "needs-human"); assert.equal(r[2].cost_usd, 3); assert.equal(r[2].minutes, 2);
+    assert.deepEqual(seen.filter(Boolean), [{ code: "ci.red", attempt: 1 }, { code: "ci.red", attempt: 1 }]);
+  });
+
+  it("formatOutcomeLists caps a detail at OUTCOME_LINE_MAX", () => {
+    const [l] = formatOutcomeLists([{ issue: 3, outcome: "needs-human", code: "conformance.omission", detail: "x".repeat(500) }]);
+    assert.ok(l.includes("x".repeat(OUTCOME_LINE_MAX)) && !l.includes("x".repeat(OUTCOME_LINE_MAX + 1)));
+  });
+
+  it("run-issue rejects --recover on the legacy --path full", async () => {
+    await assert.rejects(runFromCli(["7", "--lane", "staging", "--path", "full", "--recover", "ci.red"], { dir: tmpDir("core-rec-"), emitTelemetry: async () => {}, io: { gh: async () => "o/r\n" } }),
+      /--recover is not supported with --path full/);
+  });
+
+  it("/orchestrate spec describes terminal non-human needs-human rows, --recover scope and the schedd resume scope", () => {
+    const spec = readFileSync(new URL("../../commands/orchestrate.md", import.meta.url), "utf8");
+    assert.match(spec, /\*\*terminal code\*\* that is neither recoverable nor human/);
+    assert.match(spec, /`unrecovered: #N/);
+    assert.match(spec, /core path only:\s+`--path full` rejects `--recover`/);
+    assert.match(spec, /resumes every daemon-hosted batch that has no report yet/);
+  });
+});

package/bin/lean/render.mjs

@@ -20,6 +20,25 @@ const bullets = (v) => {
 const objs = (v, key) => arr(v).map((x) => (x && typeof x === "object" ? x : { [key]: x }));
 const norm = (r) => r ?? {};
 
+/**
+ * Untrusted issue/comment text with HTML comments removed and no comment opener left: comments are cut out by plain
+ * scanning (closer `-->` or `--!>`; an unclosed one runs to the end), then every `<` that starts `<!` becomes `‹`.
+ * With no `<!` in the output, no HTML comment and no FORGE marker can open, whatever the input nesting. Pack text is
+ * posted back in a trusted bot comment, so this is the guarantee that matters.
+ */
+export function noComments(s, repl = "") {
+  const t = String(s ?? "");
+  let out = "", i = 0;
+  for (;;) {
+    const a = t.indexOf("<!--", i);
+    if (a < 0) { out += t.slice(i); break; }
+    out += t.slice(i, a) + repl;
+    const ends = [[t.indexOf("-->", a + 4), 3], [t.indexOf("--!>", a + 4), 4]].filter(([k]) => k >= 0).sort((x, y) => x[0] - y[0]);
+    if (!ends.length) break;
+    i = ends[0][0] + ends[0][1];
+  }
+  return out.replace(/<(?=!)/g, "\u2039");
+}
 /** Builder-written markdown is untrusted: drop any FORGE marker comment (even unterminated ones) and normalise newlines. */
 export function sanitizeMd(md) {
   return String(md ?? "").replace(/\r\n/g, "\n").replace(/<!--\s*FORGE:[\s\S]*?(?:-->|$)/gi, "").trim();
@@ -86,11 +105,12 @@ const esc = (v) => cell(v).replace(/</g, "&lt;");
 const escl = (v) => line(v).replace(/</g, "&lt;");
 /** Canonical contract JSON: stable, and `<` escaped so builder text can never open an HTML/FORGE comment. */
 export const contractJson = (c) => JSON.stringify(c, null, 1).replace(/</g, "\\u003c");
-const CONTRACT_BLOCK_RE = /<summary>contract\.json \(sha256 ([0-9a-f]{12})\)<\/summary>\s*```json\n([\s\S]*?)\n```/;
+// #4265: an optional ", issue S" records the hash of the issue scope (conformance.mjs scopeText) the contract was planned from.
+const CONTRACT_BLOCK_RE = /<summary>contract\.json \(sha256 ([0-9a-f]{12})(?:, issue ([0-9a-f]{12}))?\)<\/summary>\s*```json\n([\s\S]*?)\n```/;
 
 /** FORGE:CONTRACT body: readable tables plus the frozen JSON and its hash (read back on a rerun). */
-export function renderContract(c, hash, { compact = false } = {}) {
-  const json = `<details><summary>contract.json (sha256 ${hash})</summary>\n\n\`\`\`json\n${contractJson(c)}\n\`\`\`\n</details>`;
+export function renderContract(c, hash, { compact = false, scope = null } = {}) {
+  const json = `<details><summary>contract.json (sha256 ${hash}${scope ? `, issue ${scope}` : ""})</summary>\n\n\`\`\`json\n${contractJson(c)}\n\`\`\`\n</details>`;
   if (compact) return `## Builder Contract (frozen \`${hash}\`)\n\n${c.requirements.length} requirement(s), ${(c.deliverables ?? []).length} deliverable(s); the tables are omitted because the contract is large.\n\n${json}`;
   const disp = (r) => `${r.disposition}${r.tracked ? ` → ${r.tracked}` : ["defer", "post-deploy"].includes(r.disposition) ? " (filed after review accepts it)" : ""}${r.why ? ` (${escl(r.why)})` : ""}`;
   const req = c.requirements.map((r) => `| ${esc(r.id)} | ${esc(r.quote)} | ${esc(disp(r))} |`).join("\n");
@@ -99,11 +119,11 @@ export function renderContract(c, hash, { compact = false } = {}) {
     ? `\n\n### Decisions\n| Id | Choice | Justification | Source |\n| --- | --- | --- | --- |\n${c.decisions.map((k) => `| ${esc(k?.id)} | ${esc(k?.choice)} | ${esc(k?.justification)} | ${esc(k?.source)} |`).join("\n")}` : "";
   return `## Builder Contract (frozen \`${hash}\`)\n\n### Requirements\n| Id | From the issue | Disposition |\n| --- | --- | --- |\n${req}\n\n### Deliverables\n${del ? `| Id | Path | Change | Requirements |\n| --- | --- | --- | --- |\n${del}` : "None."}\n\n### Preserve\n${(c.preserve ?? []).map((p) => `- ${escl(p.id)}: ${escl(p.what)}`).join("\n") || "None."}\n\n### Validate\n${(c.validate ?? []).map((v) => `- ${escl(v.id)}: \`${escl(v.command).replace(/`/g, "'")}\``).join("\n") || NR}${dec}\n\n${json}`;
 }
-/** {contract, hash, text} from a FORGE:CONTRACT comment body, or null. */
+/** {contract, hash, scope, text} from a FORGE:CONTRACT comment body (scope null for a pre-#4265 comment), or null. */
 export function parseContractComment(body) {
   const m = CONTRACT_BLOCK_RE.exec(String(body ?? ""));
   if (!m) return null;
-  try { return { hash: m[1], text: m[2], contract: JSON.parse(m[2]) }; } catch { return null; }
+  try { return { hash: m[1], scope: m[2] ?? null, text: m[3], contract: JSON.parse(m[3]) }; } catch { return null; }
 }
 
 /** Typed amendments, posted visibly before review. */