forgedock 1.29.0 → 1.30.0

package/bin/lean/prompts.mjs

@@ -85,6 +85,7 @@ If the issue is already fixed or invalid, write ${RESULT_FILE} {"status":"alread
  "validate":[{"id":"V1","command":"<exact command: the new regression test plus the existing tests that reference the files you change; a git diff compares against the merge base (origin/<lane>...HEAD), never the moving origin/<lane> tip>"}],
  "decisions":[{"id":"K1","choice":"<a non-obvious choice: a new dependency, a copied pattern, a boundary relied on>","justification":"<why, with file:line>","source":"doc|lint|precedent|spec"}]}
 Every requirement and acceptance criterion in the issue and its maintainers' comments is one requirement, quoted verbatim (others' comments are context, not requirements). Never drop or narrow one silently: "defer" (separate work) and "post-deploy" (only checkable after a deploy) are judged by the reviewer and, if accepted, filed by the engine as a tracked follow-up issue. A requirement you can do in this change is "deliver". List every file you will change as a deliverable; tests are always allowed.
+A validate or repro command must not read a $TMPDIR file it does not create itself: each run gets a fresh TMPDIR.
 A new dependency between workspace packages (a workspace package added to a package.json dependencies, devDependencies or peerDependencies) must be a deliverable on that package.json whose change names the package and gives the reason, citing the architecture doc where there is one; "the lint allows it" or "there is a precedent" is not a reason on its own. Record each non-obvious choice in "decisions" with where its justification comes from (doc, lint, precedent or spec); omit the field when there is none. Then stop.
 
 ${BLOCK_RULES}`;
@@ -93,7 +94,7 @@ ${BLOCK_RULES}`;
 /** Builder call 2, same session: implement the frozen contract. */
 export function builderImplementPrompt(mode = "local", { toolchain = null, sparse = null, facts = null } = {}) {
   const remote = mode === "ci" || mode === "hybrid";
-  return `The engine froze ${CONTRACT_FILE}; it is now the authority. Implement it exactly: change only deliverable paths (tests are always allowed), keep every preserve item intact, add a regression test, ${remote ? VERIFY_RUN[mode] : "run every validate command"}, and commit. Do not push or touch GitHub.
+  return `The engine froze ${CONTRACT_FILE}; it is now the authority. Implement it exactly: change only deliverable paths (tests are always allowed), keep every preserve item intact, add a regression test, ${remote ? VERIFY_RUN[mode] : "run every validate command"}, run the repo's formatter on the files you changed, and commit. Do not push or touch GitHub.
 
 ${HOST_RULES}${toolchainRule(toolchain)}${sparseRule(sparse)}${factsRule(facts)}${remote ? `\n\n${VERIFY_RULES[mode]}` : ""}
 
@@ -147,6 +148,39 @@ export function hotspotBuildRule(facts) {
   return `\n\nDon't grow a file over ${facts.file_lines} lines by more than ${facts.growth_lines} lines. Put new code next to it.`;
 }
 
+/** #4322: tests that enumerate the siblings of a module the issue names (bin/lean/enumerating.mjs), for the plan; "" when none. */
+export function enumeratingPlanNote(list) {
+  if (!Array.isArray(list) || !list.length) return "";
+  return `\n\nEnumerating tests (found by the engine): these tests list every member of a module or directory this issue names, so a new export, table, job, action, namespace or file there must join their list or CI goes red. List each one you will touch under Affected Files in ${INVESTIGATION_FILE}, and add a validate command that runs only those tests.\n${list.slice(0, 40).map((e) => `- ${e.module} -> ${e.test}:${e.line} (${e.why})`).join("\n")}`;
+}
+
+const NOT_MEMBER = `or, when the symbol is not a member of that list, record {"symbol":"<name>","test":"<test path>","why":"<why it is not a member>"} in ${RESULT_FILE} "not_members"`;
+
+/** #4322: the builder rule citing the plan's enumerating tests; "" when none. */
+export function enumeratingBuildRule(list) {
+  if (!Array.isArray(list) || !list.length) return "";
+  return `\n\nEnumerating tests: add every new exported symbol, table, job, action, namespace or file to each test listed below whose list enumerates its siblings, one test-list edit per new symbol, ${NOT_MEMBER}.\n${list.slice(0, 40).map((e) => `- ${e.module} -> ${e.test}:${e.line}`).join("\n")}`;
+}
+
+/** #4322: after implement, new symbols that tests enumerating their siblings do not name. */
+export function missingEntriesPrompt(missing, mode = "local") {
+  return `These tests enumerate the siblings of symbols or files this change adds, but do not name them (CI will fail on the stale list):\n${missing.slice(0, 30).map((m) => `- ${m.symbol} (${m.module}) -> ${m.test}:${m.line}`).join("\n")}\n\nFor each, add the symbol to that test's list, ${NOT_MEMBER}. ${rerun(mode)}, commit, and update ${IMPLEMENTATION_FILE} and ${RESULT_FILE}. Do not push or touch GitHub.`;
+}
+
+const RENUMBER_RULE = "Renumber the lane's files, never the base's: rename each with git mv to its new number, keep the SQL (or other migration source) byte-identical, regenerate the snapshot and journal with the repo's own migration generator (never hand- or text-merge the journal), keep journal entries strictly ordered, and update every reference to the old name.";
+
+/** #4322: lane-added sequence-numbered files whose number the base already uses, at hand-off or after a base-sync. */
+export function sequencePrompt(collisions, lane, mode = "local") {
+  const rows = collisions.slice(0, 30).map((c) => `- ${c.lane} -> number ${c.next}${c.base.length ? ` (base has ${c.base.join(", ")})` : " (moves to keep the order)"}`).join("\n");
+  return `origin/${lane} already uses sequence numbers this change adds:\n${rows}\n\nFirst bring the branch up to date with \`git merge --no-edit origin/${lane}\` if it is not (already fetched; never rebase or force). ${RENUMBER_RULE} ${rerun(mode)}, commit, and update ${IMPLEMENTATION_FILE} and ${RESULT_FILE}. Do not push or touch GitHub.`;
+}
+
+/** #4322: the base-sync conflict addition when a conflicted file is a numbered migration or a journal; "" otherwise. */
+export function sequenceSyncNote(files) {
+  const hit = (files ?? []).some((f) => /(?:^|\/)\d{3,}[_-][^/]*$|(?:^|\/)_journal\.json$/.test(String(f)));
+  return hit ? ` A numbered migration or its journal conflicts: do not text-merge it. ${RENUMBER_RULE}` : "";
+}
+
 export function contractCorrectionPrompt(problems) {
   return `Your ${CONTRACT_FILE} is not valid:\n${problems.slice(0, 20).map((p) => `- ${p}`).join("\n")}\n\nFix the contract (planning only, no code, no commit) and stop. Do not push or touch GitHub.`;
 }
@@ -207,7 +241,7 @@ export const REVIEWER_SYSTEM = `You are an independent conformance reviewer of o
 
 You receive the GitHub issue with its comments, the frozen contract (requirements quoted from the issue, deliverables, preserve constraints, validate commands) with typed amendments, the builder's result with validation output, the changed files, the diff, and a row skeleton with one row per contract id and changed file. The engine computes the verdict from your rows; a reply without rows is rejected, so fill the skeleton rather than composing rows: fill EVERY row (set each empty status, evidence and validated_by, and each null deferral_ok; keep the ids and files as given):
 - issue_coverage: one row per requirement or acceptance criterion in the issue, quoted verbatim, saying whether the contract covers it, narrows it, or misses it. The skeleton lists the contract's quotes; add a row for any issue requirement the contract does not quote.
-- requirements: one row per contract requirement id: implemented|partial|missing|deferred|post-deploy|not-applicable, with repo-relative path:line evidence (e.g. src/app/route.ts:12, not route.ts:12) and the validate id or test that proves it (validated_by). For a defer or post-deploy requirement add deferral_ok: true only if it is genuinely separate work or only checkable after a deploy, false if it belongs in this change.
+- requirements: one row per contract requirement id: implemented|partial|missing|deferred|post-deploy|not-applicable, with evidence as a backtick-quoted repo-relative path:line copied verbatim from the changed files, including (group), [slug] and [...slug] folders (e.g. \`apps/web/app/(console)/layout.tsx:12\`, not route.ts:12) and the validate id or test that proves it (validated_by). For a defer or post-deploy requirement add deferral_ok: true only if it is genuinely separate work or only checkable after a deploy, false if it belongs in this change.
 - scope: one row per changed file, pre-filled with the engine's authorization: the deliverable id that authorizes it (deliverables may be directories or globs), "test", "incidental" (changelog, lockfile), or "none" with what it changes. Change it only if the diff shows the file does something its deliverable does not cover.
 - preserve: one row per preserve id: intact|violated|unverified, with path:line. The skeleton starts each at "unverified"; set intact only after checking it.
 - regressions: concrete breakages only, each with path:line.
@@ -257,7 +291,7 @@ export function reviewerPrompt({ issue, comments = [], contract, result, files =
  * for, with the exact problems and the reviewer's previous reply; no diff is re-sent (the worktree still holds the change).
  */
 export function reviewerRepairPrompt({ missing = [], problems = [], rows = null }) {
-  return `Your conformance review is incomplete. The engine keeps your other rows; answer ONLY for these ids or sections: ${missing.join(", ") || "(see the problems)"}.\n\n<previous_reply_problems>\nYour previous reply was rejected as incomplete. Fix exactly these:\n${problems.slice(0, 60).map((p) => `- ${String(p).slice(0, 300)}`).join("\n")}\n</previous_reply_problems>\n\n<previous_reply>\n${cap(JSON.stringify(rows ?? {}), 20000)}\n</previous_reply>\n\nRead the changed files in your worktree as needed. Output ONLY a JSON object in a json code fence with just the rows for those ids (requirements rows by id, preserve rows by id, scope rows by file, issue_coverage in full if it is listed) plus "verdict":"CONFORMS|NONCONFORMING" consistent with all rows. Evidence is repo-relative path:line.`;
+  return `Your conformance review is incomplete. The engine keeps your other rows; answer ONLY for these ids or sections: ${missing.join(", ") || "(see the problems)"}.\n\n<previous_reply_problems>\nYour previous reply was rejected as incomplete. Fix exactly these:\n${problems.slice(0, 60).map((p) => `- ${String(p).slice(0, 300)}`).join("\n")}\n</previous_reply_problems>\n\n<previous_reply>\n${cap(JSON.stringify(rows ?? {}), 20000)}\n</previous_reply>\n\nRead the changed files in your worktree as needed. Output ONLY a JSON object in a json code fence with just the rows for those ids (requirements rows by id, preserve rows by id, scope rows by file, issue_coverage in full if it is listed) plus "verdict":"CONFORMS|NONCONFORMING" consistent with all rows. Evidence is a backtick-quoted repo-relative path:line copied verbatim (e.g. \`apps/web/app/(console)/layout.tsx:12\`); for an evidence path reported as not existing, give the full repo-relative path of that file from the changed files or the repository.`;
 }
 
 /** Pre-filter (#4123): one read-only, single-turn check of an issue a cheap signal flagged as possibly already fixed. */

package/bin/lean/recovery.test.mjs

@@ -8,6 +8,7 @@ import { readFileSync } from "node:fs";
 import { tmpDir } from "../tests/_tmp.mjs";
 import { orchestrate, formatOutcomeLists, OUTCOME_LINE_MAX } from "./orchestrate.mjs";
 import { RECOVER_HOW } from "./lean.mjs";
+import { recoveryEvidence } from "./render.mjs";
 import { RECOVERABLE_CODES } from "../engine/outcome-codes.mjs";
 import { runFromCli, orchestrateFromCli } from "../engine-cli.mjs";
 // orchestrateFromCli opens the host lane registry (#4191): keep it in a temp dir.
@@ -17,7 +18,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
   // runOne: `script[n]` is a list of results, one per launch of issue n (the last repeats); ctx is recorded.
   const batch = (script, bodies = {}, extra = {}) => {
     const calls = [], decisions = [];
-    const p = orchestrate({ issues: Object.keys(script).map(Number), parallel: 3, recovery: { maxAttempts: 1 },
+    const p = orchestrate({ issues: Object.keys(script).map(Number), parallel: 3, recovery: { maxAttempts: extra.maxAttempts ?? 1 },
       readBody: async (n) => bodies[n] ?? "",
       runOne: async (n, _lane, ctx = {}) => {
         calls.push({ n, recover: ctx.recover ?? null });
@@ -37,7 +38,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
     const r = byIssue(rows);
     assert.equal(r[1].outcome, "merged"); assert.equal(r[1].recovered, "ci.red");
     assert.equal(r[2].outcome, "merged", "the dependent was not skipped");
-    assert.deepEqual(calls.filter((c) => c.n === 1).map((c) => c.recover), [null, { code: "ci.red", attempt: 1 }]);
+    assert.deepEqual(calls.filter((c) => c.n === 1).map((c) => c.recover), [null, { code: "ci.red", attempt: 1, final: true }]);
     assert.ok(calls.findIndex((c) => c.n === 2) > calls.findLastIndex((c) => c.n === 1), "the dependent ran after the recovered merge");
     const d = decisions.find((x) => x.issue === 1);
     assert.match(d.what, /recovery lane for ci\.red/); assert.match(d.why, /Required check/); assert.ok(d.alternative);
@@ -138,7 +139,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
     const ctxs = [];
     const rows = await orchestrate({ issues: [1], store, recovery: { maxAttempts: 1 }, readBody: async () => "",
       runOne: async (n, l, ctx) => { ctxs.push(ctx.recover); return NH("ci.red"); } });
-    assert.deepEqual(ctxs, [{ code: "ci.red", attempt: 1 }]);
+    assert.deepEqual(ctxs, [{ code: "ci.red", attempt: 1, final: true }]);
     assert.equal(rows[0].outcome, "needs-human"); assert.equal(rows[0].recovery.code, "ci.red");
     assert.deepEqual(state.recovering, {});
   });
@@ -160,7 +161,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
       await runFromCli(["7", "--lane", "staging", "--repo", "o/r", "--recover", "base.conflict", "--recover-attempt", "1"], { dir: tmpDir("core-rec-"), emitTelemetry: async () => {}, leanIo: {}, io: { gh: async () => "o/r\n" },
         leanModule: { runCore: async (o) => { got.push(o.recover); return { terminalReason: "merged" }; } } });
     } finally { console.log = l; }
-    assert.deepEqual(got, [{ code: "base.conflict", attempt: 1 }]);
+    assert.deepEqual(got, [{ code: "base.conflict", attempt: 1, final: true }]);
     // The batch: a ci.red lane is relaunched in-process with recover, the decision is recorded and the report lists it.
     const leanIo = {
       gh: async (a) => (a.includes("milestone") ? JSON.stringify({ milestone: null }) : JSON.stringify({ body: "" })),
@@ -174,7 +175,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
         io: { gh: async () => "o/r\n" }, leanIo, sched: null, orchestration: { breaker: null, maxDeferrals: 3, globalParallel: null, recovery: { maxAttempts: 1 } }, decide: (d) => decisions.push(d),
         leanModule: { runCore: async (o) => { runs.push(o.recover ?? null); return o.recover ? { terminalReason: "merged", pr: 55 } : { terminalReason: "needs-human", code: "ci.red", detail: "Required check(s) failed on PR #55: test" }; } } });
     } finally { console.log = l; }
-    assert.deepEqual(runs, [null, { code: "ci.red", attempt: 1 }]);
+    assert.deepEqual(runs, [null, { code: "ci.red", attempt: 1, final: true }]);
     assert.equal(res.ok, true);
     assert.ok(decisions.some((d) => d.issue === 5 && /recovery lane for ci\.red/.test(d.what)));
     assert.ok(logs.includes("recovered: #5 (ci.red)"), logs.join("\n"));
@@ -189,6 +190,130 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
     assert.match(step3, /one question list[\s\S]*once, at the end of the batch/);
     assert.match(spec, /recovered: #A \(ci\.red\) #B \(base\.conflict\)/);
   });
+
+  it("#4302: with maxAttempts 2 only the last attempt is final; a non-final attempt is not relabelled while its successor runs", async () => {
+    const relabels = [];
+    const conf = { maxAttempts: 2, conflicting: async () => true, relabelNeedsHuman: async (x) => { relabels.push(x); } };
+    const pend = { terminalReason: "ci-pending", pr: 9, detail: "CI still running" };
+    const seen = [];
+    const { rows, calls } = await batch({ 1: [NH("base.conflict"), pend, pend] }, {}, { ...conf, onUpdate: (r) => seen.push([r.outcome, relabels.length]) });
+    assert.deepEqual(calls.map((c) => c.recover), [null, { code: "base.conflict", attempt: 1, final: false }, { code: "base.conflict", attempt: 2, final: true }]);
+    assert.equal(rows[0].outcome, "ci-pending");
+    assert.ok(seen.filter(([o]) => o === "recovering").every(([, k]) => k === 0), "no relabel between attempts");
+    assert.deepEqual(relabels, [], "the final lane relabels ci-pending itself; the runner does not");
+  });
+
+  it("#4302: a non-final attempt that ends needs-human/ci-pending without a successor is relabelled by the runner, once", async () => {
+    const relabels = [];
+    const rl = { maxAttempts: 2, relabelNeedsHuman: async (x) => { relabels.push(x); } };
+    // ci-pending on a PR that does not conflict is not recoverable: no attempt 2.
+    const a = await batch({ 1: [NH("ci.red"), { terminalReason: "ci-pending", pr: 9, detail: "CI still running" }] }, {}, { ...rl, conflicting: async () => false });
+    assert.equal(a.calls.length, 2); assert.equal(a.rows[0].outcome, "ci-pending");
+    assert.equal(relabels.length, 1); assert.match(relabels[0].evidence, /Recovery pass for `ci\.red` \(attempt 1\) did not merge: it ended `ci-pending`/);
+    // needs-human with a human code is never recovered: no attempt 2.
+    relabels.length = 0;
+    const b = await batch({ 1: [NH("ci.red"), NH("builder.blocked", "needs credentials")] }, {}, rl);
+    assert.equal(b.calls.length, 2); assert.equal(b.rows[0].outcome, "needs-human");
+    assert.equal(relabels.length, 1); assert.match(relabels[0].evidence, /ended `needs-human` \(`builder\.blocked`\)/);
+    // maxAttempts 1: the only attempt is final, so the lane relabels and the runner does not (unchanged).
+    relabels.length = 0;
+    const c = await batch({ 1: [NH("ci.red"), NH("builder.blocked")] }, {}, { ...rl, maxAttempts: 1 });
+    assert.deepEqual(c.calls.map((x) => x.recover), [null, { code: "ci.red", attempt: 1, final: true }]);
+    assert.deepEqual(relabels, []);
+  });
+
+  describe("#4315: a recovery queued when the batch stops is relabelled needs-human with the full reason", () => {
+    const LONG = `ci.red: ${"x".repeat(400)} END-OF-REASON`;
+    // issue 1 ends ci.red twice (original lane + non-final attempt 1/2); `stop` ends the batch before attempt 2 launches
+    const stopped = async (stop) => {
+      const relabels = [], calls = [];
+      let tick; const secondRun = new Promise((r) => { tick = r; });
+      const rows = await orchestrate({ issues: stop.issues ?? [1], parallel: 2, recovery: { maxAttempts: 2 }, readBody: async () => "",
+        relabelNeedsHuman: async (x) => { relabels.push(x); },
+        runOne: async (n, _l, ctx = {}) => {
+          calls.push({ n, recover: ctx.recover ?? null });
+          if (n === 2) { await secondRun; return { terminalReason: "engine-error", detail: "boom" }; }
+          if (calls.filter((c) => c.n === 1).length === 2) tick();
+          await new Promise((r) => setImmediate(r));
+          return NH("ci.red", LONG);
+        }, ...stop.opts(calls) });
+      return { rows, relabels, calls: calls.filter((c) => c.n === 1) };
+    };
+    const check = ({ rows, relabels, calls }, cause) => {
+      assert.deepEqual(calls.map((c) => c.recover), [null, { code: "ci.red", attempt: 1, final: false }], "attempt 2 never launched");
+      assert.equal(relabels.length, 1); assert.equal(relabels[0].issue, 1);
+      assert.match(relabels[0].evidence, /Recovery pass for `ci\.red` \(attempt 1\) did not merge: it ended `needs-human` \(`ci\.red`\)/);
+      assert.ok(relabels[0].evidence.includes(LONG), "full reason, not 300 chars");
+      assert.match(relabels[0].evidence, /Recovery attempt 2\/2 for `ci\.red` was queued, but the batch stopped \(/);
+      assert.match(relabels[0].evidence, cause);
+      const r1 = byIssue(rows)[1];
+      assert.equal(r1.outcome, "held"); assert.equal(r1.recovery.code, "ci.red"); assert.equal(r1.recovery.attempts, 2);
+      assert.match(formatOutcomeLists(rows).join("\n"), /unrecovered: #1/);
+    };
+
+    it("breaker", async () => {
+      check(await stopped({ issues: [1, 2], opts: () => ({ breaker: { errors: 1, window_min: 30 } }) }), /circuit breaker/);
+    });
+    it("execution budget (on_budget stop)", async () => {
+      check(await stopped({ opts: (calls) => ({ remote: { max: 3, isRemote: () => true, admit: () => (calls.length >= 2 ? { ok: false, reason: "cap $5 reached" } : { ok: true }) } }) }),
+        /execution budget: cap \$5 reached/);
+    });
+    it("drift hold", async () => {
+      check(await stopped({ opts: (calls) => ({ drift: { holds: () => (calls.length >= 2 ? "packages/ui" : null) } }) }), /drift: packages\/ui flagged/);
+    });
+
+    it("a resume of a lost run that does not relaunch it relabels once; a second resume does not repeat it", async () => {
+      let state = { v: 1, order: [1], deps: { 1: [] }, pending: [], results: {}, running: { 1: { startedAt: 1, pid: 999 } }, recoveries: { 1: 2 },
+        recovering: { 1: { code: "ci.red", attempt: 2, detail: LONG, outcome: "needs-human", result_code: "ci.red", prev_attempt: 1 } },
+        breaker: { at: 1, detail: "circuit breaker: 3 engine-errors within 30 min; admissions stopped" } };
+      const store = { load: () => state, save: (s) => { state = s; }, log: () => {} };
+      const relabels = [], ran = [];
+      const go = () => orchestrate({ issues: [1], store, recovery: { maxAttempts: 2 }, readBody: async () => "", adopt: async () => null,
+        relabelNeedsHuman: async (x) => { relabels.push(x); }, runOne: async (n) => { ran.push(n); return { terminalReason: "merged" }; } });
+      const rows = await go();
+      assert.deepEqual(ran, []); assert.equal(rows[0].outcome, "held");
+      assert.equal(relabels.length, 1); assert.ok(relabels[0].evidence.includes(LONG));
+      assert.match(relabels[0].evidence, /attempt 1\) did not merge.*[\s\S]*circuit breaker/);
+      assert.deepEqual(state.relabelled, [1]);
+      await go();
+      assert.equal(relabels.length, 1, "relabelled once across resumes");
+    });
+
+    it("a resume that relaunches the lost recovery lane does not relabel; the lane gets ctx.recover", async () => {
+      let state = { v: 1, order: [1], deps: { 1: [] }, pending: [], results: {}, running: { 1: { startedAt: 1, pid: 999 } }, recoveries: { 1: 2 },
+        recovering: { 1: { code: "ci.red", attempt: 2, detail: LONG, outcome: "needs-human", prev_attempt: 1 } }, relabelled: [1] };
+      const store = { load: () => state, save: (s) => { state = s; }, log: () => {} };
+      const relabels = [], ctxs = [];
+      const rows = await orchestrate({ issues: [1], store, recovery: { maxAttempts: 2 }, readBody: async () => "", adopt: async () => null,
+        relabelNeedsHuman: async (x) => { relabels.push(x); }, runOne: async (n, l, ctx) => { ctxs.push(ctx.recover); return { terminalReason: "merged" }; } });
+      assert.deepEqual(ctxs, [{ code: "ci.red", attempt: 2, final: true }]);
+      assert.equal(rows[0].outcome, "merged"); assert.deepEqual(relabels, []);
+      assert.equal(state.relabelled, undefined);
+    });
+
+    it("a store from before #4315 (no detail, no relabelled) still resumes and relabels with the code", async () => {
+      let state = { v: 1, order: [1], deps: { 1: [] }, pending: [1], results: {}, running: {}, recoveries: { 1: 1 }, recovering: { 1: { code: "ci.red", attempt: 1 } },
+        breaker: { at: 1, detail: "circuit breaker: tripped" } };
+      const store = { load: () => state, save: (s) => { state = s; }, log: () => {} };
+      const relabels = [];
+      const rows = await orchestrate({ issues: [1], store, recovery: { maxAttempts: 2 }, readBody: async () => "",
+        relabelNeedsHuman: async (x) => { relabels.push(x); }, runOne: async () => ({ terminalReason: "merged" }) });
+      assert.equal(rows[0].outcome, "held"); assert.equal(rows[0].recovery.detail, "ci.red");
+      assert.equal(relabels.length, 1); assert.match(relabels[0].evidence, /The lane did not merge: it ended `needs-human`/);
+    });
+  });
+
+  it("#4302: --recover-nonfinal reaches runCore as final: false; without it a recovery lane is final", async () => {
+    const got = [];
+    const l = console.log; console.log = () => {};
+    try {
+      for (const extra of [["--recover-nonfinal"], []]) {
+        await runFromCli(["7", "--lane", "staging", "--repo", "o/r", "--recover", "ci.red", "--recover-attempt", "1", ...extra], { dir: tmpDir("core-rec-"), emitTelemetry: async () => {}, leanIo: {}, io: { gh: async () => "o/r\n" },
+          leanModule: { runCore: async (o) => { got.push(o.recover); return { terminalReason: "merged" }; } } });
+      }
+    } finally { console.log = l; }
+    assert.deepEqual(got, [{ code: "ci.red", attempt: 1, final: false }, { code: "ci.red", attempt: 1, final: true }]);
+  });
 });
 
 describe("#4298 staging-review follow-ups: recovery", () => {
@@ -197,7 +322,7 @@ describe("#4298 staging-review follow-ups: recovery", () => {
     for (const c of RECOVERABLE_CODES) assert.ok(String(RECOVER_HOW[c]).length > 20, c);
   });
 
-  it("a recovered row reports the first attempt's cost and minutes summed with the recovery's; the lane sees only {code, attempt}", async () => {
+  it("a recovered row reports the first attempt's cost and minutes summed with the recovery's; the lane sees only {code, attempt, final}", async () => {
     let t = 0; const seen = [];
     const script = { 1: [{ terminalReason: "needs-human", code: "ci.red", detail: "red", cost_usd: 0.5 }, { terminalReason: "merged", pr: 9, cost_usd: 0.25 }],
       2: [{ terminalReason: "needs-human", code: "ci.red", detail: "red", cost_usd: 1 }, { terminalReason: "needs-human", code: "ci.red", detail: "red again", cost_usd: 2 }] };
@@ -207,7 +332,7 @@ describe("#4298 staging-review follow-ups: recovery", () => {
     const r = Object.fromEntries(rows.map((x) => [x.issue, x]));
     assert.equal(r[1].outcome, "merged"); assert.equal(r[1].cost_usd, 0.75); assert.equal(r[1].minutes, 6);
     assert.equal(r[2].outcome, "needs-human"); assert.equal(r[2].cost_usd, 3); assert.equal(r[2].minutes, 2);
-    assert.deepEqual(seen.filter(Boolean), [{ code: "ci.red", attempt: 1 }, { code: "ci.red", attempt: 1 }]);
+    assert.deepEqual(seen.filter(Boolean), [{ code: "ci.red", attempt: 1, final: true }, { code: "ci.red", attempt: 1, final: true }]);
   });
 
   it("formatOutcomeLists caps a detail at OUTCOME_LINE_MAX", () => {
@@ -228,3 +353,142 @@ describe("#4298 staging-review follow-ups: recovery", () => {
     assert.match(spec, /resumes every daemon-hosted batch that has no report yet/);
   });
 });
+
+describe("#4304 staging-review follow-ups: recovery", () => {
+  it("a recovery row counts the pre-filter cost once; a single pass still includes it", async () => {
+    const script = { 1: [{ terminalReason: "needs-human", code: "ci.red", detail: "red", cost_usd: 1 }, { terminalReason: "needs-human", code: "ci.red", detail: "red again", cost_usd: 1 }],
+      2: [{ terminalReason: "merged", pr: 4, cost_usd: 1 }] };
+    const k = { 1: 0, 2: 0 };
+    const rows = await orchestrate({ issues: [1, 2], parallel: 1, recovery: { maxAttempts: 1 }, readBody: async () => "", now: () => 0,
+      prefilter: async () => ({ closed: false, cost_usd: 0.1 }), runOne: async (n) => script[n][k[n]++] });
+    const r = Object.fromEntries(rows.map((x) => [x.issue, x]));
+    assert.equal(k[1], 2, "issue 1 ran a recovery lane");
+    assert.equal(r[1].cost_usd, 2.1);
+    assert.equal(r[2].cost_usd, 1.1);
+  });
+
+  const runBatchWith = async (enginePath) => {
+    const leanIo = {
+      gh: async (a) => (a.includes("milestone") ? JSON.stringify({ milestone: null }) : JSON.stringify({ body: "" })),
+      git: async (a) => (a[0] === "ls-remote" ? "abc\trefs/heads/staging\n" : ""),
+    };
+    const runs = [], logs = [], l = console.log;
+    console.log = (x) => logs.push(String(x));
+    try {
+      await orchestrateFromCli(["5", "--repo", "o/r", "--no-prefilter"], { dir: tmpDir("core-rec-"), batchRoot: tmpDir("core-rec-batch-"), allowTmpfs: true, emitTelemetry: async () => {},
+        io: { gh: async () => "o/r\n" }, leanIo, sched: null, enginePath, orchestration: { breaker: null, maxDeferrals: 3, globalParallel: null, recovery: { maxAttempts: 1 } }, decide: () => {},
+        relabelNeedsHuman: async () => {},
+        leanModule: { runCore: async (o) => { runs.push(o.recover ?? null); return { terminalReason: "needs-human", code: "ci.red", detail: "Required check(s) failed on PR #55: test" }; } } });
+    } finally { console.log = l; }
+    return { runs, logs };
+  };
+
+  it("engine.path full turns batch recovery off (run-issue rejects --recover there); core keeps it", async () => {
+    const full = await runBatchWith("full");
+    assert.deepEqual(full.runs, [null]);
+    assert.ok(full.logs.some((x) => /recovery: off \(engine\.path is full/.test(x)), full.logs.join("\n"));
+    const core = await runBatchWith("core");
+    assert.deepEqual(core.runs, [null, { code: "ci.red", attempt: 1, final: true }]);
+  });
+});
+
+describe("#4316 one shared recovery evidence text", () => {
+  it("lane, settle and batch-exit relabels render the same evidence shape", () => {
+    const long = `${"x".repeat(320)}\nsecond line`;
+    assert.equal(recoveryEvidence({ code: "ci.red", attempt: 1, outcome: "needs-human", resultCode: "ci.red", detail: "  red check\nmore  " }),
+      "Recovery pass for `ci.red` (attempt 1) did not merge: it ended `needs-human` (`ci.red`): red check. The issue stays `needs-human`.");
+    assert.equal(recoveryEvidence({ code: "ci.red", attempt: 2, outcome: "ci-pending", detail: "" }),
+      "Recovery pass for `ci.red` (attempt 2) did not merge: it ended `ci-pending`. The issue stays `needs-human`.");
+    assert.equal(recoveryEvidence({ code: "c", attempt: 1, outcome: "o", detail: long }).includes(`: ${"x".repeat(300)}. The issue`), true, "first line, 300 chars");
+    const queued = recoveryEvidence({ code: "base.conflict", attempt: 0, outcome: undefined, resultCode: "base.conflict", detail: long, full: true, queued: 1, cause: "breaker", recoveryMax: 2 });
+    assert.equal(queued, `The lane did not merge: it ended \`needs-human\` (\`base.conflict\`): ${long}\n\n`
+      + "Recovery attempt 1/2 for `base.conflict` was queued, but the batch stopped (breaker) before it launched. The issue stays `needs-human`.");
+    assert.match(recoveryEvidence({ code: "c", attempt: 1, outcome: "o", queued: 2, cause: "batch stopped" }), /^Recovery pass for `c` \(attempt 1\) did not merge: it ended `o`\n\nRecovery attempt 2\/2 /);
+  });
+  it("/orchestrate spec names the pre-filter's flaky exceptions and recovery being off on engine.path full", () => {
+    const spec = readFileSync(new URL("../../commands/orchestrate.md", import.meta.url), "utf8");
+    assert.match(spec, /unless it is flaky \(wording or `flaky` label\), was reopened after a pre-filter close, or\s+the workflow failed and passed at the same SHA/);
+    assert.match(spec, /recovery is off when `engine\.path: full`/);
+  });
+  it("orchestrate.mjs and lean.mjs build no copy of their own", () => {
+    for (const f of ["orchestrate.mjs", "lean.mjs"]) {
+      const src = readFileSync(new URL(`./${f}`, import.meta.url), "utf8");
+      assert.equal((src.match(/did not merge/g) ?? []).length, 0, f);
+    }
+  });
+});
+
+describe("#4318: the batch-exit relabel survives a throw and a signal; relabel results; full non-final reason", () => {
+  const LONG = `ci.red: line one ${"x".repeat(400)}\nsecond line END-OF-REASON`;
+  const NH = (code, detail) => ({ terminalReason: "needs-human", code, detail });
+  // issue 1 ends ci.red; attempt 1/2 is queued, and `onRecovering` runs when the runner reports it
+  const queued = (opts) => {
+    const relabels = [], ran = [];
+    const p = orchestrate({ issues: [1], parallel: 1, recovery: { maxAttempts: 2 }, readBody: async () => "",
+      relabelNeedsHuman: opts.relabel ?? (async (x) => { relabels.push(x); }),
+      runOne: async (n, _l, ctx = {}) => { ran.push(ctx.recover ?? null); await new Promise((r) => setImmediate(r)); return NH("ci.red", LONG); },
+      ...opts.extra(relabels) });
+    return { p, relabels, ran };
+  };
+
+  it("an exception out of the loop relabels the queued recovery once, then propagates", async () => {
+    let boom = false;
+    const { p, relabels, ran } = queued({ extra: () => ({
+      now: () => { if (boom) throw new Error("clock broke"); return 1000; },
+      onUpdate: (r) => { if (r.outcome === "recovering") boom = true; } }) });
+    await assert.rejects(p, /clock broke/);
+    assert.deepEqual(ran, [null], "attempt 1 never launched");
+    assert.equal(relabels.length, 1); assert.equal(relabels[0].issue, 1);
+    assert.ok(relabels[0].evidence.includes(LONG), "full persisted reason");
+    assert.match(relabels[0].evidence, /the batch stopped \(batch failed: clock broke\)/);
+  });
+
+  it("onExit hands the caller the relabel: a signal relabels with its cause, once; the normal exit does not repeat it", async () => {
+    let exitFn = null;
+    const { p, relabels } = queued({ extra: () => ({
+      onExit: (fn) => { exitFn = fn; },
+      onUpdate: async (r) => { if (r.outcome === "recovering" && exitFn) { const f = exitFn; exitFn = null; await f("interrupted by SIGINT"); await f("again"); } } }) });
+    await p;
+    assert.equal(relabels.length, 1, "idempotent");
+    assert.match(relabels[0].evidence, /the batch stopped \(interrupted by SIGINT\)/);
+  });
+
+  it("a non-final attempt's relabel carries the full multi-line reason, not 300 chars", async () => {
+    const relabels = [];
+    await orchestrate({ issues: [1], parallel: 1, recovery: { maxAttempts: 2 }, readBody: async () => "", relabelNeedsHuman: async (x) => { relabels.push(x); },
+      runOne: async (_n, _l, ctx = {}) => (ctx.recover ? NH("builder.blocked", LONG) : NH("ci.red", "red")) });
+    assert.equal(relabels.length, 1);
+    assert.ok(relabels[0].evidence.includes(LONG));
+    assert.ok(relabels[0].evidence.includes("END-OF-REASON"));
+  });
+
+  describe("relabel results on the batch-exit path", () => {
+    const lost = (relabel) => {
+      let state = { v: 1, order: [1], deps: { 1: [] }, pending: [], results: {}, running: { 1: { startedAt: 1, pid: 999 } }, recoveries: { 1: 2 },
+        recovering: { 1: { code: "ci.red", attempt: 2, detail: LONG, outcome: "needs-human", result_code: "ci.red", prev_attempt: 1 } },
+        breaker: { at: 1, detail: "circuit breaker: tripped" } };
+      const store = { load: () => state, save: (s) => { state = s; }, log: () => {} };
+      const calls = [];
+      const go = () => orchestrate({ issues: [1], store, recovery: { maxAttempts: 2 }, readBody: async () => "", adopt: async () => null,
+        relabelNeedsHuman: async (x) => { calls.push(x); return relabel(); }, runOne: async () => ({ terminalReason: "merged" }) });
+      return { go, calls, state: () => state };
+    };
+    it("a failed label add (false / \"failed\" / a throw) is not recorded: the next resume retries it", async () => {
+      for (const relabel of [() => false, () => "failed", () => { throw new Error("gh down"); }]) {
+        const b = lost(relabel);
+        await b.go();
+        assert.equal(b.state().relabelled, undefined);
+        await b.go();
+        assert.equal(b.calls.length, 2, "retried on resume");
+      }
+    });
+    it("\"skipped\" (closed or leased issue) and true are recorded: no retry", async () => {
+      for (const relabel of [() => "skipped", () => true]) {
+        const b = lost(relabel);
+        await b.go(); await b.go();
+        assert.deepEqual(b.state().relabelled, [1]);
+        assert.equal(b.calls.length, 1);
+      }
+    });
+  });
+});

package/bin/lean/render.mjs

@@ -50,6 +50,19 @@ export function capComment(body, max = MAX_COMMENT_CHARS) {
   const note = "\n\n_[truncated: comment exceeded the size cap]_";
   return s.slice(0, max - note.length) + note;
 }
+/**
+ * #4297/#4316: the one evidence text for a recovery that did not merge (lane, settle and batch-exit relabel). `attempt` 0 is
+ * the first lane; the detail is its first line capped at 300 chars unless `full`. `queued` (the attempt that never launched)
+ * adds why the batch stopped before it.
+ */
+export function recoveryEvidence({ code, attempt, outcome, resultCode, detail, full = false, queued = 0, cause = "", recoveryMax = 0 }) {
+  const how = attempt ? `Recovery pass for \`${code}\` (attempt ${attempt}) did not merge` : "The lane did not merge";
+  const d = String(detail ?? "").trim();
+  const shown = full ? d : d.split("\n")[0].slice(0, 300);
+  const head = `${how}: it ended \`${outcome ?? "needs-human"}\`${resultCode ? ` (\`${resultCode}\`)` : ""}${shown ? `: ${shown}` : ""}`;
+  return queued ? `${head}\n\nRecovery attempt ${queued}/${recoveryMax || queued} for \`${code}\` was queued, but the batch stopped (${cause}) before it launched. The issue stays \`needs-human\`.`
+    : `${head}. The issue stays \`needs-human\`.`;
+}
 const usable = (md) => sanitizeMd(md).replace(/\s+/g, " ").length >= 20;
 
 /** Text of the `### heading` section (up to the next heading), or "". */

package/bin/lean/sequence.mjs

@@ -0,0 +1,102 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Sequence-numbered artifacts at hand-off (#4322): migrations named `<number>_<name>` (0007_usage.sql, 0007_x.py,
+ * meta/0007_snapshot.json) that the lane added with a number the base already uses. Union merge (#3969) cannot help: a
+ * duplicate number is a semantic collision and a drizzle-style journal is structured JSON. The engine only detects and
+ * verifies; the builder renumbers with the repo's own migration generator. Deterministic (git only), never throws.
+ */
+import { basename, dirname, posix } from "node:path";
+
+const NUM_RE = /^(\d{3,})[_-]/;
+const SKIP_RE = /(?:^|\/)(?:node_modules|vendor|dist|build)\//;
+const lines = (s) => String(s ?? "").split("\n").map((x) => x.trim()).filter(Boolean);
+const numOf = (p) => NUM_RE.exec(basename(p))?.[1] ?? null;
+const inDir = (dir) => (dir === "." ? [] : [`${dir}/`]);
+const out = (git) => async (args) => { try { return String(await git(args)); } catch { return ""; } };
+
+/** Lane-added numbered files (base...HEAD), with their blob ids. */
+async function laneNumbered(git, base) {
+  const g = out(git);
+  const added = lines(await g(["-c", "core.quotepath=off", "diff", "--name-only", "--diff-filter=A", `${base}...HEAD`])).filter((f) => numOf(f) && !SKIP_RE.test(f));
+  const res = [];
+  for (const path of added) res.push({ path, blob: (await g(["rev-parse", `HEAD:${path}`])).trim() });
+  return res;
+}
+
+/**
+ * Collisions: in each directory where a lane-added numbered file shares its number with a different base file (another
+ * name, or the same path with other content), every lane-added numbered file there must move, in order, above the base's
+ * highest number. @returns {Promise<Array<{dir, number, lane, base:string[], next}>>} (base [] = moves only to keep order)
+ */
+export async function sequenceCollisions({ git, base }) {
+  try {
+    const g = out(git);
+    const lane = await laneNumbered(git, base);
+    const res = [];
+    for (const dir of [...new Set(lane.map((f) => dirname(f.path)))]) {
+      const mine = lane.filter((f) => dirname(f.path) === dir).sort((a, b) => Number(numOf(a.path)) - Number(numOf(b.path)) || a.path.localeCompare(b.path));
+      const theirs = [];
+      for (const l of lines(await g(["-c", "core.quotepath=off", "ls-tree", base, ...inDir(dir)]))) {
+        const m = /^\d+\s+blob\s+(\S+)\t(.+)$/.exec(l);
+        if (m && dirname(m[2]) === dir && numOf(m[2])) theirs.push({ path: m[2], blob: m[1] });
+      }
+      const clash = (f) => theirs.filter((t) => numOf(t.path) === numOf(f.path) && (t.path !== f.path || t.blob !== f.blob)).map((t) => t.path);
+      if (!mine.some((f) => clash(f).length)) continue;
+      const width = Math.max(...mine.map((f) => numOf(f.path).length));
+      let next = Math.max(0, ...theirs.map((t) => Number(numOf(t.path))));
+      for (const f of mine) res.push({ dir, number: numOf(f.path), lane: f.path, base: clash(f), next: String(++next).padStart(width, "0") });
+    }
+    return res;
+  } catch { return []; }
+}
+
+/**
+ * Drizzle-style journal checks for `dirs` at `ref` (`<dir>/meta/_journal.json`, {entries:[{idx, tag}]}): duplicate idx
+ * or tag, idx or tag number not strictly increasing, a numbered .sql file without a tag. No journal: [].
+ */
+export async function journalProblems({ git, ref = "HEAD", dirs = [] }) {
+  const g = out(git), problems = [];
+  for (const dir of [...new Set(dirs)]) {
+    const jp = posix.join(dir, "meta", "_journal.json");
+    const text = await g(["show", `${ref}:${jp}`]);
+    if (!text.trim()) continue;
+    let entries;
+    try { entries = JSON.parse(text).entries; } catch { problems.push(`${jp}: not valid JSON`); continue; }
+    if (!Array.isArray(entries)) { problems.push(`${jp}: no entries array`); continue; }
+    const tags = new Set();
+    for (let i = 0; i < entries.length; i++) {
+      const e = entries[i] ?? {}, prev = entries[i - 1];
+      if (tags.has(e.tag)) problems.push(`${jp}: duplicate tag ${e.tag}`);
+      tags.add(e.tag);
+      if (prev && !(Number(e.idx) > Number(prev.idx))) problems.push(`${jp}: idx ${e.idx} (${e.tag}) not after ${prev.idx} (${prev.tag})`);
+      const n = numOf(String(e.tag ?? "")), pn = prev ? numOf(String(prev.tag ?? "")) : null;
+      if (n && pn && !(Number(n) > Number(pn))) problems.push(`${jp}: ${e.tag} not numbered after ${prev.tag}`);
+    }
+    for (const l of lines(await g(["-c", "core.quotepath=off", "ls-tree", "--name-only", ref, ...inDir(dir)]))) {
+      if (dirname(l) === dir && l.endsWith(".sql") && numOf(l) && !tags.has(basename(l, ".sql"))) problems.push(`${jp}: no entry for ${basename(l)}`);
+    }
+  }
+  return problems;
+}
+
+/** Lane-added numbered files with their blobs, recorded before a renumber so verifyRenumber can prove the SQL kept. */
+export const snapshotLane = ({ git, base }) => laneNumbered(git, base).catch(() => []);
+
+/**
+ * After the builder renumbered: no collision remains, every recorded non-JSON file (the SQL; snapshots are regenerated)
+ * still exists byte-identical in its directory, and the journals are strictly ordered. @returns {Promise<{ok, problems}>}
+ */
+export async function verifyRenumber({ git, base, before = [] }) {
+  try {
+    const problems = [];
+    const left = await sequenceCollisions({ git, base });
+    for (const c of left.filter((x) => x.base.length)) problems.push(`${c.lane} still shares number ${c.number} with ${c.base.join(", ")}`);
+    const now = await laneNumbered(git, base);
+    for (const b of before.filter((f) => !f.path.endsWith(".json"))) {
+      if (!now.some((f) => dirname(f.path) === dirname(b.path) && f.blob === b.blob)) problems.push(`${b.path}: content changed or file gone after renumbering (keep it byte-identical, rename only)`);
+    }
+    problems.push(...await journalProblems({ git, dirs: [...new Set([...before, ...now].map((f) => dirname(f.path)))] }));
+    return { ok: !problems.length, problems };
+  } catch (e) { return { ok: false, problems: [String(e?.message ?? e)] }; }
+}