forgedock 1.28.0 → 1.29.1

package/bin/lean/recovery.test.mjs

@@ -0,0 +1,419 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// #4267: in-batch recovery of mechanical needs-human (orchestrate, run-issue/orchestrate CLI, /orchestrate spec).
+// The runCore side (--recover label/decision/prompt, issueEdited) is in lean.test.mjs next to its harness.
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { tmpDir } from "../tests/_tmp.mjs";
+import { orchestrate, formatOutcomeLists, OUTCOME_LINE_MAX } from "./orchestrate.mjs";
+import { RECOVER_HOW } from "./lean.mjs";
+import { recoveryEvidence } from "./render.mjs";
+import { RECOVERABLE_CODES } from "../engine/outcome-codes.mjs";
+import { runFromCli, orchestrateFromCli } from "../engine-cli.mjs";
+// orchestrateFromCli opens the host lane registry (#4191): keep it in a temp dir.
+process.env.FORGEDOCK_SCHED_DIR = tmpDir("recovery-sched-");
+
+describe("in-batch recovery of mechanical needs-human (#4267)", () => {
+  // runOne: `script[n]` is a list of results, one per launch of issue n (the last repeats); ctx is recorded.
+  const batch = (script, bodies = {}, extra = {}) => {
+    const calls = [], decisions = [];
+    const p = orchestrate({ issues: Object.keys(script).map(Number), parallel: 3, recovery: { maxAttempts: extra.maxAttempts ?? 1 },
+      readBody: async (n) => bodies[n] ?? "",
+      runOne: async (n, _lane, ctx = {}) => {
+        calls.push({ n, recover: ctx.recover ?? null });
+        await new Promise((r) => setImmediate(r));
+        const list = script[n] ?? [{ terminalReason: "merged" }];
+        const k = calls.filter((c) => c.n === n).length - 1;
+        return list[Math.min(k, list.length - 1)];
+      },
+      decide: (d) => { decisions.push(d); }, ...extra });
+    return p.then((rows) => ({ rows, calls, decisions }));
+  };
+  const NH = (code, detail = `ended ${code}`, more = {}) => ({ terminalReason: "needs-human", code, detail, ...more });
+  const byIssue = (rows) => Object.fromEntries(rows.map((r) => [r.issue, r]));
+
+  it("ci.red: one recovery lane on the PR; green re-queues the issue and its dependent in the same batch, no operator action", async () => {
+    const { rows, calls, decisions } = await batch({ 1: [NH("ci.red", "Required check(s) failed on PR #101: test"), { terminalReason: "merged", pr: 101 }], 2: [{ terminalReason: "merged", pr: 102 }] }, { 2: "**Depends on**: #1" });
+    const r = byIssue(rows);
+    assert.equal(r[1].outcome, "merged"); assert.equal(r[1].recovered, "ci.red");
+    assert.equal(r[2].outcome, "merged", "the dependent was not skipped");
+    assert.deepEqual(calls.filter((c) => c.n === 1).map((c) => c.recover), [null, { code: "ci.red", attempt: 1, final: true }]);
+    assert.ok(calls.findIndex((c) => c.n === 2) > calls.findLastIndex((c) => c.n === 1), "the dependent ran after the recovered merge");
+    const d = decisions.find((x) => x.issue === 1);
+    assert.match(d.what, /recovery lane for ci\.red/); assert.match(d.why, /Required check/); assert.ok(d.alternative);
+    assert.deepEqual(formatOutcomeLists(rows), ["recovered: #1 (ci.red)"]);
+  });
+
+  it("a recovery that stays red is final with its evidence; dependents stay skipped; no attempt beyond max_attempts", async () => {
+    const { rows, calls } = await batch({ 1: [NH("ci.red"), NH("ci.red", "CI verification failed again", { recovery: true })], 2: [] }, { 2: "**Depends on**: #1" });
+    const r = byIssue(rows);
+    assert.equal(calls.filter((c) => c.n === 1).length, 2, "the first run plus exactly one recovery");
+    assert.equal(r[1].outcome, "needs-human");
+    assert.deepEqual(r[1].recovery, { code: "ci.red", attempts: 1, outcome: "needs-human", result_code: "ci.red", detail: "CI verification failed again" });
+    assert.equal(r[2].outcome, "skipped"); assert.ok(!calls.some((c) => c.n === 2));
+    const lines = formatOutcomeLists(rows);
+    assert.equal(lines.length, 1); assert.match(lines[0], /^unrecovered: #1 \(ci\.red: recovery ci\.red ended needs-human: CI verification failed again\)$/);
+  });
+
+  it("human codes (product-decision, spend) are never recovered, still skip dependents and are the only escalations, asked once while other lanes keep running", async () => {
+    const { rows, calls, decisions } = await batch({
+      1: [NH("builder.blocked", "Builder is blocked (product-decision): pick a pricing model")],
+      2: [],
+      3: [NH("builder.blocked", "Builder is blocked (spend): needs a paid plan")],
+      4: [NH("conformance.omission", "R2 not covered")],
+      5: [{ terminalReason: "merged" }] }, { 2: "**Depends on**: #1" });
+    const r = byIssue(rows);
+    assert.equal(calls.filter((c) => c.n === 1).length, 1); assert.equal(calls.filter((c) => c.n === 3).length, 1);
+    assert.equal(calls.filter((c) => c.n === 4).length, 1, "terminal mechanical codes are not recovered either");
+    assert.ok(!decisions.some((d) => [1, 3].includes(d.issue)));
+    assert.equal(r[2].outcome, "skipped"); assert.equal(r[5].outcome, "merged");
+    const lines = formatOutcomeLists(rows);
+    assert.equal(lines.filter((l) => l.startsWith("escalations")).length, 1, "one escalation list");
+    const esc = lines.find((l) => l.startsWith("escalations"));
+    assert.match(esc, /#1 \(builder\.blocked\): Builder is blocked \(product-decision\)/);
+    assert.match(esc, /#3 \(builder\.blocked\): Builder is blocked \(spend\)/);
+    assert.doesNotMatch(esc, /#4/, "the needs-human list holds only human codes");
+    assert.ok(lines.some((l) => /^unrecovered: #4 \(conformance\.omission: R2 not covered\)$/.test(l)));
+  });
+
+  it("base.conflict: a recovery base-sync with a clean result re-queues the issue; the report lists recovered issues separately", async () => {
+    const { rows, calls } = await batch({ 1: [NH("ci.red"), { terminalReason: "merged" }], 2: [NH("base.conflict", "PR #102 still conflicts with staging after 2 base-syncs"), { terminalReason: "merged" }] });
+    assert.deepEqual(calls.filter((c) => c.n === 2).map((c) => c.recover?.code ?? null), [null, "base.conflict"]);
+    assert.equal(byIssue(rows)[2].recovered, "base.conflict");
+    assert.deepEqual(formatOutcomeLists(rows), ["recovered: #1 (ci.red) #2 (base.conflict)"]);
+  });
+
+  it("ci-pending recovers only when its PR conflicts (as base.conflict); a slow-CI ci-pending is left as is", async () => {
+    const seen = [];
+    const conflicting = async (n, pr) => { seen.push([n, pr]); return n === 1; };
+    const { rows, calls } = await batch({ 1: [{ terminalReason: "ci-pending", pr: 101 }, { terminalReason: "merged", pr: 101 }], 2: [{ terminalReason: "ci-pending", pr: 102 }] }, {}, { conflicting });
+    const r = byIssue(rows);
+    assert.equal(r[1].recovered, "base.conflict");
+    assert.equal(r[2].outcome, "ci-pending"); assert.equal(calls.filter((c) => c.n === 2).length, 1);
+    assert.deepEqual(seen.sort(), [[1, 101], [2, 102]]);
+  });
+
+  it("contract.invalid recovers only when the issue was edited mid-run", async () => {
+    const { rows, calls } = await batch({ 1: [NH("contract.invalid", "contract invalid", { issueEdited: true }), { terminalReason: "merged" }], 2: [NH("contract.invalid")] });
+    assert.equal(byIssue(rows)[1].recovered, "contract.invalid");
+    assert.equal(byIssue(rows)[2].outcome, "needs-human"); assert.equal(calls.filter((c) => c.n === 2).length, 1);
+  });
+
+  it("relabelNeedsHuman (SPEC-7/8): only a recovery that ends final engine-error/deferred relabels; invalid/merged never; a requeued deferred waits", async () => {
+    const relabels = [];
+    const rl = { relabelNeedsHuman: async (x) => { relabels.push(x); }, pollMs: 5 };
+    // engine-error after the recovery lane's own relaunch: final -> exactly one relabel with the recovery evidence
+    const a = await batch({ 1: [NH("ci.red"), { terminalReason: "engine-error", detail: "boom" }] }, {}, rl);
+    assert.equal(a.rows[0].outcome, "engine-error"); assert.equal(relabels.length, 1);
+    assert.equal(relabels[0].issue, 1); assert.match(relabels[0].evidence, /Recovery pass for `ci\.red` \(attempt 1\) did not merge: it ended `engine-error`/);
+    // invalid / merged / decomposed: never
+    relabels.length = 0;
+    await batch({ 1: [NH("ci.red"), { terminalReason: "invalid", detail: "not a bug" }], 2: [NH("ci.red"), { terminalReason: "merged" }],
+      3: [NH("ci.red"), { terminalReason: "decomposed", subissues: [] }] }, {}, rl);
+    assert.deepEqual(relabels, []);
+    // a deferred recovery that is requeued is not relabelled until it is final (capped)
+    const defer = { terminalReason: "deferred", code: "gh.unavailable", retryInSeconds: 0.01, detail: "gh down" };
+    const seen = [];
+    const d = await batch({ 1: [NH("ci.red"), defer] }, {}, { ...rl, maxDeferrals: 2, onUpdate: (r) => seen.push([r.outcome, relabels.length, String(r.detail)]) });
+    assert.equal(d.rows[0].outcome, "deferred");
+    const rq = seen.filter(([o, , det]) => o === "deferred" && /^requeued in/.test(det));
+    assert.ok(rq.length >= 1 && rq.every(([, n]) => n === 0), "a requeued deferral is not relabelled");
+    assert.equal(relabels.filter((x) => x.issue === 1).length, 1, "once, only when final");
+    // without a recovery pass nothing is relabelled
+    relabels.length = 0;
+    await batch({ 1: [{ terminalReason: "engine-error" }] }, {}, rl);
+    assert.deepEqual(relabels, []);
+  });
+
+  it("without the recovery option the runner behaves as before: needs-human is final and dependents skip", async () => {
+    const calls = [];
+    const rows = await orchestrate({ issues: [1, 2], readBody: async (n) => (n === 2 ? "**Depends on**: #1" : ""),
+      runOne: async (n) => { calls.push(n); return NH("ci.red"); } });
+    assert.deepEqual(calls, [1]); assert.deepEqual(rows.map((r) => r.outcome), ["needs-human", "skipped"]);
+  });
+
+  it("the attempt count survives a resume: a persisted recovery lane is launched with ctx.recover and is not retried again", async () => {
+    let state = { v: 1, order: [1], deps: { 1: [] }, pending: [1], results: {}, running: {}, recoveries: { 1: 1 }, recovering: { 1: { code: "ci.red", attempt: 1 } } };
+    const store = { load: () => state, save: (s) => { state = s; }, log: () => {} };
+    const ctxs = [];
+    const rows = await orchestrate({ issues: [1], store, recovery: { maxAttempts: 1 }, readBody: async () => "",
+      runOne: async (n, l, ctx) => { ctxs.push(ctx.recover); return NH("ci.red"); } });
+    assert.deepEqual(ctxs, [{ code: "ci.red", attempt: 1, final: true }]);
+    assert.equal(rows[0].outcome, "needs-human"); assert.equal(rows[0].recovery.code, "ci.red");
+    assert.deepEqual(state.recovering, {});
+  });
+
+  it("other runner decisions are recorded too (engine-error relaunch, dependency requeue, sub-issues), and a throwing recorder never blocks the run", async () => {
+    const { decisions } = await batch({ 1: [{ terminalReason: "engine-error", detail: "boom" }, { terminalReason: "merged" }], 2: [{ terminalReason: "decomposed", subissues: [8] }] });
+    assert.ok(decisions.some((d) => d.issue === 1 && /relaunch/.test(d.what) && d.why === "boom" && d.alternative));
+    assert.ok(decisions.some((d) => d.issue === 2 && /#8/.test(d.what)));
+    const rows = await orchestrate({ issues: [1], recovery: { maxAttempts: 1 }, readBody: async () => "", decide: () => { throw new Error("gh down"); },
+      runOne: async (n, l, ctx) => (ctx.recover ? { terminalReason: "merged" } : NH("ci.red")) });
+    assert.equal(rows[0].recovered, "ci.red");
+  });
+
+
+  it("run-issue --recover reaches runCore; a batch relaunches a ci.red lane with recover and reports it", async () => {
+    const got = [];
+    const l = console.log; console.log = () => {};
+    try {
+      await runFromCli(["7", "--lane", "staging", "--repo", "o/r", "--recover", "base.conflict", "--recover-attempt", "1"], { dir: tmpDir("core-rec-"), emitTelemetry: async () => {}, leanIo: {}, io: { gh: async () => "o/r\n" },
+        leanModule: { runCore: async (o) => { got.push(o.recover); return { terminalReason: "merged" }; } } });
+    } finally { console.log = l; }
+    assert.deepEqual(got, [{ code: "base.conflict", attempt: 1, final: true }]);
+    // The batch: a ci.red lane is relaunched in-process with recover, the decision is recorded and the report lists it.
+    const leanIo = {
+      gh: async (a) => (a.includes("milestone") ? JSON.stringify({ milestone: null }) : JSON.stringify({ body: "" })),
+      git: async (a) => (a[0] === "ls-remote" ? "abc\trefs/heads/staging\n" : ""),
+    };
+    const runs = [], decisions = [], logs = [];
+    console.log = (x) => logs.push(String(x));
+    let res;
+    try {
+      res = await orchestrateFromCli(["5", "--repo", "o/r", "--no-prefilter"], { dir: tmpDir("core-rec-"), batchRoot: tmpDir("core-rec-batch-"), allowTmpfs: true, emitTelemetry: async () => {},
+        io: { gh: async () => "o/r\n" }, leanIo, sched: null, orchestration: { breaker: null, maxDeferrals: 3, globalParallel: null, recovery: { maxAttempts: 1 } }, decide: (d) => decisions.push(d),
+        leanModule: { runCore: async (o) => { runs.push(o.recover ?? null); return o.recover ? { terminalReason: "merged", pr: 55 } : { terminalReason: "needs-human", code: "ci.red", detail: "Required check(s) failed on PR #55: test" }; } } });
+    } finally { console.log = l; }
+    assert.deepEqual(runs, [null, { code: "ci.red", attempt: 1, final: true }]);
+    assert.equal(res.ok, true);
+    assert.ok(decisions.some((d) => d.issue === 5 && /recovery lane for ci\.red/.test(d.what)));
+    assert.ok(logs.includes("recovered: #5 (ci.red)"), logs.join("\n"));
+  });
+
+  it("/orchestrate spec step 3 lists the recoverable codes, asks no confirmation on them, and escalates once at the end", async () => {
+    const spec = readFileSync(new URL("../../commands/orchestrate.md", import.meta.url), "utf8");
+    const step3 = spec.slice(spec.indexOf("## 3. Launch lanes"), spec.indexOf("## 4. Report"));
+    for (const c of ["`ci.red`", "`base.conflict`", "`ci-pending` on a conflicting", "`contract.invalid` caused by an issue edited mid-run"]) assert.ok(step3.includes(c), c);
+    assert.match(step3, /\*\*Do not ask for confirmation on them\.\*\*/);
+    assert.match(step3, /FORGE:DECISION/);
+    assert.match(step3, /one question list[\s\S]*once, at the end of the batch/);
+    assert.match(spec, /recovered: #A \(ci\.red\) #B \(base\.conflict\)/);
+  });
+
+  it("#4302: with maxAttempts 2 only the last attempt is final; a non-final attempt is not relabelled while its successor runs", async () => {
+    const relabels = [];
+    const conf = { maxAttempts: 2, conflicting: async () => true, relabelNeedsHuman: async (x) => { relabels.push(x); } };
+    const pend = { terminalReason: "ci-pending", pr: 9, detail: "CI still running" };
+    const seen = [];
+    const { rows, calls } = await batch({ 1: [NH("base.conflict"), pend, pend] }, {}, { ...conf, onUpdate: (r) => seen.push([r.outcome, relabels.length]) });
+    assert.deepEqual(calls.map((c) => c.recover), [null, { code: "base.conflict", attempt: 1, final: false }, { code: "base.conflict", attempt: 2, final: true }]);
+    assert.equal(rows[0].outcome, "ci-pending");
+    assert.ok(seen.filter(([o]) => o === "recovering").every(([, k]) => k === 0), "no relabel between attempts");
+    assert.deepEqual(relabels, [], "the final lane relabels ci-pending itself; the runner does not");
+  });
+
+  it("#4302: a non-final attempt that ends needs-human/ci-pending without a successor is relabelled by the runner, once", async () => {
+    const relabels = [];
+    const rl = { maxAttempts: 2, relabelNeedsHuman: async (x) => { relabels.push(x); } };
+    // ci-pending on a PR that does not conflict is not recoverable: no attempt 2.
+    const a = await batch({ 1: [NH("ci.red"), { terminalReason: "ci-pending", pr: 9, detail: "CI still running" }] }, {}, { ...rl, conflicting: async () => false });
+    assert.equal(a.calls.length, 2); assert.equal(a.rows[0].outcome, "ci-pending");
+    assert.equal(relabels.length, 1); assert.match(relabels[0].evidence, /Recovery pass for `ci\.red` \(attempt 1\) did not merge: it ended `ci-pending`/);
+    // needs-human with a human code is never recovered: no attempt 2.
+    relabels.length = 0;
+    const b = await batch({ 1: [NH("ci.red"), NH("builder.blocked", "needs credentials")] }, {}, rl);
+    assert.equal(b.calls.length, 2); assert.equal(b.rows[0].outcome, "needs-human");
+    assert.equal(relabels.length, 1); assert.match(relabels[0].evidence, /ended `needs-human` \(`builder\.blocked`\)/);
+    // maxAttempts 1: the only attempt is final, so the lane relabels and the runner does not (unchanged).
+    relabels.length = 0;
+    const c = await batch({ 1: [NH("ci.red"), NH("builder.blocked")] }, {}, { ...rl, maxAttempts: 1 });
+    assert.deepEqual(c.calls.map((x) => x.recover), [null, { code: "ci.red", attempt: 1, final: true }]);
+    assert.deepEqual(relabels, []);
+  });
+
+  describe("#4315: a recovery queued when the batch stops is relabelled needs-human with the full reason", () => {
+    const LONG = `ci.red: ${"x".repeat(400)} END-OF-REASON`;
+    // issue 1 ends ci.red twice (original lane + non-final attempt 1/2); `stop` ends the batch before attempt 2 launches
+    const stopped = async (stop) => {
+      const relabels = [], calls = [];
+      let tick; const secondRun = new Promise((r) => { tick = r; });
+      const rows = await orchestrate({ issues: stop.issues ?? [1], parallel: 2, recovery: { maxAttempts: 2 }, readBody: async () => "",
+        relabelNeedsHuman: async (x) => { relabels.push(x); },
+        runOne: async (n, _l, ctx = {}) => {
+          calls.push({ n, recover: ctx.recover ?? null });
+          if (n === 2) { await secondRun; return { terminalReason: "engine-error", detail: "boom" }; }
+          if (calls.filter((c) => c.n === 1).length === 2) tick();
+          await new Promise((r) => setImmediate(r));
+          return NH("ci.red", LONG);
+        }, ...stop.opts(calls) });
+      return { rows, relabels, calls: calls.filter((c) => c.n === 1) };
+    };
+    const check = ({ rows, relabels, calls }, cause) => {
+      assert.deepEqual(calls.map((c) => c.recover), [null, { code: "ci.red", attempt: 1, final: false }], "attempt 2 never launched");
+      assert.equal(relabels.length, 1); assert.equal(relabels[0].issue, 1);
+      assert.match(relabels[0].evidence, /Recovery pass for `ci\.red` \(attempt 1\) did not merge: it ended `needs-human` \(`ci\.red`\)/);
+      assert.ok(relabels[0].evidence.includes(LONG), "full reason, not 300 chars");
+      assert.match(relabels[0].evidence, /Recovery attempt 2\/2 for `ci\.red` was queued, but the batch stopped \(/);
+      assert.match(relabels[0].evidence, cause);
+      const r1 = byIssue(rows)[1];
+      assert.equal(r1.outcome, "held"); assert.equal(r1.recovery.code, "ci.red"); assert.equal(r1.recovery.attempts, 2);
+      assert.match(formatOutcomeLists(rows).join("\n"), /unrecovered: #1/);
+    };
+
+    it("breaker", async () => {
+      check(await stopped({ issues: [1, 2], opts: () => ({ breaker: { errors: 1, window_min: 30 } }) }), /circuit breaker/);
+    });
+    it("execution budget (on_budget stop)", async () => {
+      check(await stopped({ opts: (calls) => ({ remote: { max: 3, isRemote: () => true, admit: () => (calls.length >= 2 ? { ok: false, reason: "cap $5 reached" } : { ok: true }) } }) }),
+        /execution budget: cap \$5 reached/);
+    });
+    it("drift hold", async () => {
+      check(await stopped({ opts: (calls) => ({ drift: { holds: () => (calls.length >= 2 ? "packages/ui" : null) } }) }), /drift: packages\/ui flagged/);
+    });
+
+    it("a resume of a lost run that does not relaunch it relabels once; a second resume does not repeat it", async () => {
+      let state = { v: 1, order: [1], deps: { 1: [] }, pending: [], results: {}, running: { 1: { startedAt: 1, pid: 999 } }, recoveries: { 1: 2 },
+        recovering: { 1: { code: "ci.red", attempt: 2, detail: LONG, outcome: "needs-human", result_code: "ci.red", prev_attempt: 1 } },
+        breaker: { at: 1, detail: "circuit breaker: 3 engine-errors within 30 min; admissions stopped" } };
+      const store = { load: () => state, save: (s) => { state = s; }, log: () => {} };
+      const relabels = [], ran = [];
+      const go = () => orchestrate({ issues: [1], store, recovery: { maxAttempts: 2 }, readBody: async () => "", adopt: async () => null,
+        relabelNeedsHuman: async (x) => { relabels.push(x); }, runOne: async (n) => { ran.push(n); return { terminalReason: "merged" }; } });
+      const rows = await go();
+      assert.deepEqual(ran, []); assert.equal(rows[0].outcome, "held");
+      assert.equal(relabels.length, 1); assert.ok(relabels[0].evidence.includes(LONG));
+      assert.match(relabels[0].evidence, /attempt 1\) did not merge.*[\s\S]*circuit breaker/);
+      assert.deepEqual(state.relabelled, [1]);
+      await go();
+      assert.equal(relabels.length, 1, "relabelled once across resumes");
+    });
+
+    it("a resume that relaunches the lost recovery lane does not relabel; the lane gets ctx.recover", async () => {
+      let state = { v: 1, order: [1], deps: { 1: [] }, pending: [], results: {}, running: { 1: { startedAt: 1, pid: 999 } }, recoveries: { 1: 2 },
+        recovering: { 1: { code: "ci.red", attempt: 2, detail: LONG, outcome: "needs-human", prev_attempt: 1 } }, relabelled: [1] };
+      const store = { load: () => state, save: (s) => { state = s; }, log: () => {} };
+      const relabels = [], ctxs = [];
+      const rows = await orchestrate({ issues: [1], store, recovery: { maxAttempts: 2 }, readBody: async () => "", adopt: async () => null,
+        relabelNeedsHuman: async (x) => { relabels.push(x); }, runOne: async (n, l, ctx) => { ctxs.push(ctx.recover); return { terminalReason: "merged" }; } });
+      assert.deepEqual(ctxs, [{ code: "ci.red", attempt: 2, final: true }]);
+      assert.equal(rows[0].outcome, "merged"); assert.deepEqual(relabels, []);
+      assert.equal(state.relabelled, undefined);
+    });
+
+    it("a store from before #4315 (no detail, no relabelled) still resumes and relabels with the code", async () => {
+      let state = { v: 1, order: [1], deps: { 1: [] }, pending: [1], results: {}, running: {}, recoveries: { 1: 1 }, recovering: { 1: { code: "ci.red", attempt: 1 } },
+        breaker: { at: 1, detail: "circuit breaker: tripped" } };
+      const store = { load: () => state, save: (s) => { state = s; }, log: () => {} };
+      const relabels = [];
+      const rows = await orchestrate({ issues: [1], store, recovery: { maxAttempts: 2 }, readBody: async () => "",
+        relabelNeedsHuman: async (x) => { relabels.push(x); }, runOne: async () => ({ terminalReason: "merged" }) });
+      assert.equal(rows[0].outcome, "held"); assert.equal(rows[0].recovery.detail, "ci.red");
+      assert.equal(relabels.length, 1); assert.match(relabels[0].evidence, /The lane did not merge: it ended `needs-human`/);
+    });
+  });
+
+  it("#4302: --recover-nonfinal reaches runCore as final: false; without it a recovery lane is final", async () => {
+    const got = [];
+    const l = console.log; console.log = () => {};
+    try {
+      for (const extra of [["--recover-nonfinal"], []]) {
+        await runFromCli(["7", "--lane", "staging", "--repo", "o/r", "--recover", "ci.red", "--recover-attempt", "1", ...extra], { dir: tmpDir("core-rec-"), emitTelemetry: async () => {}, leanIo: {}, io: { gh: async () => "o/r\n" },
+          leanModule: { runCore: async (o) => { got.push(o.recover); return { terminalReason: "merged" }; } } });
+      }
+    } finally { console.log = l; }
+    assert.deepEqual(got, [{ code: "ci.red", attempt: 1, final: false }, { code: "ci.red", attempt: 1, final: true }]);
+  });
+});
+
+describe("#4298 staging-review follow-ups: recovery", () => {
+  it("RECOVER_HOW has a prompt for exactly the RECOVERABLE_CODES", () => {
+    assert.deepEqual(Object.keys(RECOVER_HOW).sort(), [...RECOVERABLE_CODES].sort());
+    for (const c of RECOVERABLE_CODES) assert.ok(String(RECOVER_HOW[c]).length > 20, c);
+  });
+
+  it("a recovered row reports the first attempt's cost and minutes summed with the recovery's; the lane sees only {code, attempt, final}", async () => {
+    let t = 0; const seen = [];
+    const script = { 1: [{ terminalReason: "needs-human", code: "ci.red", detail: "red", cost_usd: 0.5 }, { terminalReason: "merged", pr: 9, cost_usd: 0.25 }],
+      2: [{ terminalReason: "needs-human", code: "ci.red", detail: "red", cost_usd: 1 }, { terminalReason: "needs-human", code: "ci.red", detail: "red again", cost_usd: 2 }] };
+    const k = { 1: 0, 2: 0 };
+    const rows = await orchestrate({ issues: [1, 2], parallel: 1, recovery: { maxAttempts: 1 }, readBody: async () => "", now: () => t,
+      runOne: async (n, _l, ctx = {}) => { seen.push(ctx.recover ?? null); t += 60000 * (n === 1 ? 3 : 1); return script[n][k[n]++]; } });
+    const r = Object.fromEntries(rows.map((x) => [x.issue, x]));
+    assert.equal(r[1].outcome, "merged"); assert.equal(r[1].cost_usd, 0.75); assert.equal(r[1].minutes, 6);
+    assert.equal(r[2].outcome, "needs-human"); assert.equal(r[2].cost_usd, 3); assert.equal(r[2].minutes, 2);
+    assert.deepEqual(seen.filter(Boolean), [{ code: "ci.red", attempt: 1, final: true }, { code: "ci.red", attempt: 1, final: true }]);
+  });
+
+  it("formatOutcomeLists caps a detail at OUTCOME_LINE_MAX", () => {
+    const [l] = formatOutcomeLists([{ issue: 3, outcome: "needs-human", code: "conformance.omission", detail: "x".repeat(500) }]);
+    assert.ok(l.includes("x".repeat(OUTCOME_LINE_MAX)) && !l.includes("x".repeat(OUTCOME_LINE_MAX + 1)));
+  });
+
+  it("run-issue rejects --recover on the legacy --path full", async () => {
+    await assert.rejects(runFromCli(["7", "--lane", "staging", "--path", "full", "--recover", "ci.red"], { dir: tmpDir("core-rec-"), emitTelemetry: async () => {}, io: { gh: async () => "o/r\n" } }),
+      /--recover is not supported with --path full/);
+  });
+
+  it("/orchestrate spec describes terminal non-human needs-human rows, --recover scope and the schedd resume scope", () => {
+    const spec = readFileSync(new URL("../../commands/orchestrate.md", import.meta.url), "utf8");
+    assert.match(spec, /\*\*terminal code\*\* that is neither recoverable nor human/);
+    assert.match(spec, /`unrecovered: #N/);
+    assert.match(spec, /core path only:\s+`--path full` rejects `--recover`/);
+    assert.match(spec, /resumes every daemon-hosted batch that has no report yet/);
+  });
+});
+
+describe("#4304 staging-review follow-ups: recovery", () => {
+  it("a recovery row counts the pre-filter cost once; a single pass still includes it", async () => {
+    const script = { 1: [{ terminalReason: "needs-human", code: "ci.red", detail: "red", cost_usd: 1 }, { terminalReason: "needs-human", code: "ci.red", detail: "red again", cost_usd: 1 }],
+      2: [{ terminalReason: "merged", pr: 4, cost_usd: 1 }] };
+    const k = { 1: 0, 2: 0 };
+    const rows = await orchestrate({ issues: [1, 2], parallel: 1, recovery: { maxAttempts: 1 }, readBody: async () => "", now: () => 0,
+      prefilter: async () => ({ closed: false, cost_usd: 0.1 }), runOne: async (n) => script[n][k[n]++] });
+    const r = Object.fromEntries(rows.map((x) => [x.issue, x]));
+    assert.equal(k[1], 2, "issue 1 ran a recovery lane");
+    assert.equal(r[1].cost_usd, 2.1);
+    assert.equal(r[2].cost_usd, 1.1);
+  });
+
+  const runBatchWith = async (enginePath) => {
+    const leanIo = {
+      gh: async (a) => (a.includes("milestone") ? JSON.stringify({ milestone: null }) : JSON.stringify({ body: "" })),
+      git: async (a) => (a[0] === "ls-remote" ? "abc\trefs/heads/staging\n" : ""),
+    };
+    const runs = [], logs = [], l = console.log;
+    console.log = (x) => logs.push(String(x));
+    try {
+      await orchestrateFromCli(["5", "--repo", "o/r", "--no-prefilter"], { dir: tmpDir("core-rec-"), batchRoot: tmpDir("core-rec-batch-"), allowTmpfs: true, emitTelemetry: async () => {},
+        io: { gh: async () => "o/r\n" }, leanIo, sched: null, enginePath, orchestration: { breaker: null, maxDeferrals: 3, globalParallel: null, recovery: { maxAttempts: 1 } }, decide: () => {},
+        relabelNeedsHuman: async () => {},
+        leanModule: { runCore: async (o) => { runs.push(o.recover ?? null); return { terminalReason: "needs-human", code: "ci.red", detail: "Required check(s) failed on PR #55: test" }; } } });
+    } finally { console.log = l; }
+    return { runs, logs };
+  };
+
+  it("engine.path full turns batch recovery off (run-issue rejects --recover there); core keeps it", async () => {
+    const full = await runBatchWith("full");
+    assert.deepEqual(full.runs, [null]);
+    assert.ok(full.logs.some((x) => /recovery: off \(engine\.path is full/.test(x)), full.logs.join("\n"));
+    const core = await runBatchWith("core");
+    assert.deepEqual(core.runs, [null, { code: "ci.red", attempt: 1, final: true }]);
+  });
+});
+
+describe("#4316 one shared recovery evidence text", () => {
+  it("lane, settle and batch-exit relabels render the same evidence shape", () => {
+    const long = `${"x".repeat(320)}\nsecond line`;
+    assert.equal(recoveryEvidence({ code: "ci.red", attempt: 1, outcome: "needs-human", resultCode: "ci.red", detail: "  red check\nmore  " }),
+      "Recovery pass for `ci.red` (attempt 1) did not merge: it ended `needs-human` (`ci.red`): red check. The issue stays `needs-human`.");
+    assert.equal(recoveryEvidence({ code: "ci.red", attempt: 2, outcome: "ci-pending", detail: "" }),
+      "Recovery pass for `ci.red` (attempt 2) did not merge: it ended `ci-pending`. The issue stays `needs-human`.");
+    assert.equal(recoveryEvidence({ code: "c", attempt: 1, outcome: "o", detail: long }).includes(`: ${"x".repeat(300)}. The issue`), true, "first line, 300 chars");
+    const queued = recoveryEvidence({ code: "base.conflict", attempt: 0, outcome: undefined, resultCode: "base.conflict", detail: long, full: true, queued: 1, cause: "breaker", recoveryMax: 2 });
+    assert.equal(queued, `The lane did not merge: it ended \`needs-human\` (\`base.conflict\`): ${long}\n\n`
+      + "Recovery attempt 1/2 for `base.conflict` was queued, but the batch stopped (breaker) before it launched. The issue stays `needs-human`.");
+    assert.match(recoveryEvidence({ code: "c", attempt: 1, outcome: "o", queued: 2, cause: "batch stopped" }), /^Recovery pass for `c` \(attempt 1\) did not merge: it ended `o`\n\nRecovery attempt 2\/2 /);
+  });
+  it("/orchestrate spec names the pre-filter's flaky exceptions and recovery being off on engine.path full", () => {
+    const spec = readFileSync(new URL("../../commands/orchestrate.md", import.meta.url), "utf8");
+    assert.match(spec, /unless it is flaky \(wording or `flaky` label\), was reopened after a pre-filter close, or\s+the workflow failed and passed at the same SHA/);
+    assert.match(spec, /recovery is off when `engine\.path: full`/);
+  });
+  it("orchestrate.mjs and lean.mjs build no copy of their own", () => {
+    for (const f of ["orchestrate.mjs", "lean.mjs"]) {
+      const src = readFileSync(new URL(`./${f}`, import.meta.url), "utf8");
+      assert.equal((src.match(/did not merge/g) ?? []).length, 0, f);
+    }
+  });
+});

package/bin/lean/render.mjs

@@ -20,6 +20,25 @@ const bullets = (v) => {
 const objs = (v, key) => arr(v).map((x) => (x && typeof x === "object" ? x : { [key]: x }));
 const norm = (r) => r ?? {};
 
+/**
+ * Untrusted issue/comment text with HTML comments removed and no comment opener left: comments are cut out by plain
+ * scanning (closer `-->` or `--!>`; an unclosed one runs to the end), then every `<` that starts `<!` becomes `‹`.
+ * With no `<!` in the output, no HTML comment and no FORGE marker can open, whatever the input nesting. Pack text is
+ * posted back in a trusted bot comment, so this is the guarantee that matters.
+ */
+export function noComments(s, repl = "") {
+  const t = String(s ?? "");
+  let out = "", i = 0;
+  for (;;) {
+    const a = t.indexOf("<!--", i);
+    if (a < 0) { out += t.slice(i); break; }
+    out += t.slice(i, a) + repl;
+    const ends = [[t.indexOf("-->", a + 4), 3], [t.indexOf("--!>", a + 4), 4]].filter(([k]) => k >= 0).sort((x, y) => x[0] - y[0]);
+    if (!ends.length) break;
+    i = ends[0][0] + ends[0][1];
+  }
+  return out.replace(/<(?=!)/g, "\u2039");
+}
 /** Builder-written markdown is untrusted: drop any FORGE marker comment (even unterminated ones) and normalise newlines. */
 export function sanitizeMd(md) {
   return String(md ?? "").replace(/\r\n/g, "\n").replace(/<!--\s*FORGE:[\s\S]*?(?:-->|$)/gi, "").trim();
@@ -31,6 +50,19 @@ export function capComment(body, max = MAX_COMMENT_CHARS) {
   const note = "\n\n_[truncated: comment exceeded the size cap]_";
   return s.slice(0, max - note.length) + note;
 }
+/**
+ * #4297/#4316: the one evidence text for a recovery that did not merge (lane, settle and batch-exit relabel). `attempt` 0 is
+ * the first lane; the detail is its first line capped at 300 chars unless `full`. `queued` (the attempt that never launched)
+ * adds why the batch stopped before it.
+ */
+export function recoveryEvidence({ code, attempt, outcome, resultCode, detail, full = false, queued = 0, cause = "", recoveryMax = 0 }) {
+  const how = attempt ? `Recovery pass for \`${code}\` (attempt ${attempt}) did not merge` : "The lane did not merge";
+  const d = String(detail ?? "").trim();
+  const shown = full ? d : d.split("\n")[0].slice(0, 300);
+  const head = `${how}: it ended \`${outcome ?? "needs-human"}\`${resultCode ? ` (\`${resultCode}\`)` : ""}${shown ? `: ${shown}` : ""}`;
+  return queued ? `${head}\n\nRecovery attempt ${queued}/${recoveryMax || queued} for \`${code}\` was queued, but the batch stopped (${cause}) before it launched. The issue stays \`needs-human\`.`
+    : `${head}. The issue stays \`needs-human\`.`;
+}
 const usable = (md) => sanitizeMd(md).replace(/\s+/g, " ").length >= 20;
 
 /** Text of the `### heading` section (up to the next heading), or "". */
@@ -86,11 +118,12 @@ const esc = (v) => cell(v).replace(/</g, "&lt;");
 const escl = (v) => line(v).replace(/</g, "&lt;");
 /** Canonical contract JSON: stable, and `<` escaped so builder text can never open an HTML/FORGE comment. */
 export const contractJson = (c) => JSON.stringify(c, null, 1).replace(/</g, "\\u003c");
-const CONTRACT_BLOCK_RE = /<summary>contract\.json \(sha256 ([0-9a-f]{12})\)<\/summary>\s*```json\n([\s\S]*?)\n```/;
+// #4265: an optional ", issue S" records the hash of the issue scope (conformance.mjs scopeText) the contract was planned from.
+const CONTRACT_BLOCK_RE = /<summary>contract\.json \(sha256 ([0-9a-f]{12})(?:, issue ([0-9a-f]{12}))?\)<\/summary>\s*```json\n([\s\S]*?)\n```/;
 
 /** FORGE:CONTRACT body: readable tables plus the frozen JSON and its hash (read back on a rerun). */
-export function renderContract(c, hash, { compact = false } = {}) {
-  const json = `<details><summary>contract.json (sha256 ${hash})</summary>\n\n\`\`\`json\n${contractJson(c)}\n\`\`\`\n</details>`;
+export function renderContract(c, hash, { compact = false, scope = null } = {}) {
+  const json = `<details><summary>contract.json (sha256 ${hash}${scope ? `, issue ${scope}` : ""})</summary>\n\n\`\`\`json\n${contractJson(c)}\n\`\`\`\n</details>`;
   if (compact) return `## Builder Contract (frozen \`${hash}\`)\n\n${c.requirements.length} requirement(s), ${(c.deliverables ?? []).length} deliverable(s); the tables are omitted because the contract is large.\n\n${json}`;
   const disp = (r) => `${r.disposition}${r.tracked ? ` → ${r.tracked}` : ["defer", "post-deploy"].includes(r.disposition) ? " (filed after review accepts it)" : ""}${r.why ? ` (${escl(r.why)})` : ""}`;
   const req = c.requirements.map((r) => `| ${esc(r.id)} | ${esc(r.quote)} | ${esc(disp(r))} |`).join("\n");
@@ -99,11 +132,11 @@ export function renderContract(c, hash, { compact = false } = {}) {
     ? `\n\n### Decisions\n| Id | Choice | Justification | Source |\n| --- | --- | --- | --- |\n${c.decisions.map((k) => `| ${esc(k?.id)} | ${esc(k?.choice)} | ${esc(k?.justification)} | ${esc(k?.source)} |`).join("\n")}` : "";
   return `## Builder Contract (frozen \`${hash}\`)\n\n### Requirements\n| Id | From the issue | Disposition |\n| --- | --- | --- |\n${req}\n\n### Deliverables\n${del ? `| Id | Path | Change | Requirements |\n| --- | --- | --- | --- |\n${del}` : "None."}\n\n### Preserve\n${(c.preserve ?? []).map((p) => `- ${escl(p.id)}: ${escl(p.what)}`).join("\n") || "None."}\n\n### Validate\n${(c.validate ?? []).map((v) => `- ${escl(v.id)}: \`${escl(v.command).replace(/`/g, "'")}\``).join("\n") || NR}${dec}\n\n${json}`;
 }
-/** {contract, hash, text} from a FORGE:CONTRACT comment body, or null. */
+/** {contract, hash, scope, text} from a FORGE:CONTRACT comment body (scope null for a pre-#4265 comment), or null. */
 export function parseContractComment(body) {
   const m = CONTRACT_BLOCK_RE.exec(String(body ?? ""));
   if (!m) return null;
-  try { return { hash: m[1], text: m[2], contract: JSON.parse(m[2]) }; } catch { return null; }
+  try { return { hash: m[1], scope: m[2] ?? null, text: m[3], contract: JSON.parse(m[3]) }; } catch { return null; }
 }
 
 /** Typed amendments, posted visibly before review. */

package/bin/lean/testmap.mjs

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Optional repo caches for the pre-PR step (#4272): `.forge/test-map.json` (scripts/test-map.sh, file glob → check
+ * command) picks the checks for the lane's changed files; `.forge/repo-facts.json` (scripts/repo-facts.sh) is cited to
+ * the builder by path. The test map's commands are run, so it is read from the base ref, then the main checkout, never
+ * the builder-writable lane worktree (#4297); repo facts are looked up in the worktree, then the main checkout. A missing
+ * or invalid file is absent: the lane keeps forge.yaml `learned.test_commands` and today's prompts.
+ *
+ *   node bin/lean/testmap.mjs --map FILE --select FILE...   print the selected commands, one per line
+ */
+import fs from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { globToRegex, shellQuote } from "../../scripts/ci-local/lib.mjs";
+
+export const TEST_MAP_FILE = ".forge/test-map.json";
+export const REPO_FACTS_FILE = ".forge/repo-facts.json";
+
+/** A parsed test map when valid ({version:1, entries:[{command, globs:string[]}]}), else null. */
+export function parseTestMap(text) {
+  let j;
+  try { j = JSON.parse(text); } catch { return null; }
+  if (!j || j.version !== 1 || !Array.isArray(j.entries)) return null;
+  const ok = j.entries.every((e) => e && typeof e.command === "string" && e.command.trim() && Array.isArray(e.globs) && e.globs.length && e.globs.every((g) => typeof g === "string" && g));
+  return ok ? j : null;
+}
+
+/** The first valid test map under `dirs` (in order), or null; never throws. */
+export async function readTestMap(dirs, read) {
+  if (typeof read !== "function") return null;
+  for (const d of dirs.filter(Boolean)) {
+    let text;
+    try { text = await read(join(d, TEST_MAP_FILE)); } catch { continue; }
+    const map = parseTestMap(String(text ?? ""));
+    if (map) return map;
+  }
+  return null;
+}
+
+/** #4297: the valid test map committed at `ref` (`git show <ref>:.forge/test-map.json`), or null; never throws. */
+export async function readTestMapAtRef(git, ref) {
+  if (typeof git !== "function" || !ref) return null;
+  try { return parseTestMap(String((await git(["show", `${ref}:${TEST_MAP_FILE}`])) ?? "")); } catch { return null; }
+}
+
+/** Commands of entries with a glob matching a changed file: map order, deduped; a non-root cwd runs as `cd <cwd> && cmd`. */
+export function selectCommands(map, files) {
+  const out = [];
+  const changed = (Array.isArray(files) ? files : []).map((f) => String(f).replace(/^\.\//, ""));
+  for (const e of map?.entries ?? []) {
+    const res = e.globs.map((g) => globToRegex(g.replace(/^\.\//, "")));
+    if (!changed.some((f) => res.some((re) => re.test(f)))) continue;
+    const cwd = typeof e.cwd === "string" && e.cwd && e.cwd !== "." ? e.cwd : "";
+    const cmd = cwd ? `cd ${shellQuote(cwd)} && ${e.command.trim()}` : e.command.trim();
+    if (!out.includes(cmd)) out.push(cmd);
+  }
+  return out;
+}
+
+/**
+ * #4315: the pre-PR checks are the learned commands (in order) plus each map-selected command they lack; compared
+ * whitespace-collapsed, so the map can only add checks, never remove one. `source`: "learned" when the map added
+ * nothing, "test-map" when there were no learned commands, else "test-map+learned".
+ * @returns {{commands:string[], source:string, added:number}}
+ */
+export function mergeChecks(selected, learned) {
+  const norm = (c) => String(c ?? "").trim().replace(/\s+/g, " ");
+  const commands = [], seen = new Set();
+  const add = (c) => { const t = String(c ?? "").trim(), k = norm(t); if (!k || seen.has(k)) return false; seen.add(k); commands.push(t); return true; };
+  for (const c of Array.isArray(learned) ? learned : []) add(c);
+  const base = commands.length;
+  let added = 0;
+  for (const c of Array.isArray(selected) ? selected : []) if (add(c)) added++;
+  return { commands, added, source: !added ? "learned" : base ? "test-map+learned" : "test-map" };
+}
+
+/** The path of the first repo-facts file under `dirs` that exists, or null; never throws. */
+export async function factsPath(dirs, exists) {
+  if (typeof exists !== "function") return null;
+  for (const d of dirs.filter(Boolean)) {
+    const p = join(d, REPO_FACTS_FILE);
+    try { if (await exists(p)) return p; } catch { /* absent */ }
+  }
+  return null;
+}
+
+function main(argv) {
+  const i = argv.indexOf("--map"), j = argv.indexOf("--select");
+  if (i < 0 || !argv[i + 1] || j !== i + 2) { console.error("usage: testmap.mjs --map FILE --select FILE..."); return 2; }
+  let text;
+  try { text = fs.readFileSync(argv[i + 1], "utf8"); } catch { console.error(`testmap: cannot read ${argv[i + 1]}`); return 2; }
+  const map = parseTestMap(text);
+  if (!map) { console.error("testmap: invalid test map"); return 2; }
+  const files = argv.slice(j + 1);
+  for (const c of selectCommands(map, files)) console.log(c);
+  return 0;
+}
+
+if (process.argv[1] && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) process.exitCode = main(process.argv.slice(2));