forgedock 1.32.1 → 1.34.0

package/bin/lean/batch.test.mjs

@@ -120,6 +120,16 @@ await orchestrateFromCli(["1", "2", "3", "--parallel", "2", "--repo", "o/r", "--
     assert.equal(starts[1].length, 2);
   });
 
+  it("#4518 cli.host_killed deferrals are requeued and never trip the breaker", async () => {
+    const { now, sleep } = fakeClock();
+    const updates = []; let calls = 0;
+    const rows = await orchestrate({ issues: [1], readBody: async () => "", now, sleep, maxDeferrals: 3, breaker: { errors: 1, window_min: 30 },
+      onUpdate: (r) => updates.push(r),
+      runOne: async () => (++calls <= 2 ? { terminalReason: "deferred", code: "cli.host_killed", retryInSeconds: 60, detail: "builder host kill" } : { terminalReason: "merged", pr: 5 }) });
+    assert.equal(rows[0].outcome, "merged");
+    assert.ok(!updates.some((u) => u.outcome === "breaker" || u.outcome === "engine-error"));
+  });
+
   it("the breaker stops admissions after N engine-errors in M minutes and lets running lanes finish", async () => {
     const started = [];
     let releaseSlow;
@@ -203,9 +213,9 @@ await orchestrateFromCli(["1", "2", "3", "--parallel", "2", "--repo", "o/r", "--
     assert.doesNotThrow(() => assertDiskPath("/home/u/.forge/batches", mounts));
     assert.equal(isLaneAlive(process.pid, "--definitely-not-in-my-cmdline"), false);
     assert.equal(isLaneAlive(2 ** 22 + 12345, null), false);
-    assert.deepEqual(readOrchestration(root), { breaker: { errors: 3, window_min: 30 }, maxDeferrals: 3, globalParallel: null, recovery: { maxAttempts: 1 } });
+    assert.deepEqual(readOrchestration(root), { breaker: { errors: 3, window_min: 30 }, maxDeferrals: 3, globalParallel: null, recovery: { maxAttempts: 1 }, baseSync: { enabled: true, unionGlobs: [] } });
     writeFileSync(join(root, "forge.yaml"), "orchestration:\n  breaker:\n    errors: 5\n    window_min: 10\n  max_deferrals: 2\n  global_parallel: 4\n");
-    assert.deepEqual(readOrchestration(root), { breaker: { errors: 5, window_min: 10 }, maxDeferrals: 2, globalParallel: 4, recovery: { maxAttempts: 1 } });
+    assert.deepEqual(readOrchestration(root), { breaker: { errors: 5, window_min: 10 }, maxDeferrals: 2, globalParallel: 4, recovery: { maxAttempts: 1 }, baseSync: { enabled: true, unionGlobs: [] } });
   });
 
   it("run-issue --result-file writes the lane result, also when the run throws", async () => {
@@ -659,3 +669,94 @@ describe("#4504: a resume settles lanes that ended while no dispatcher ran", ()
     assert.deepEqual(launched.sort(), [3, 4], "the dead lane without a result is requeued and run once");
   });
 });
+
+describe("#4528: a full sandbox pool (sandbox.capacity) is never capped", () => {
+  it("sandbox.capacity is uncapped and honours retryInSeconds; other codes still cap at maxDeferrals", () => {
+    assert.deepEqual(deferPolicy({ code: "sandbox.capacity" }, { now: 0, deferrals: 99 }), { delayMs: 60000, host: true });
+    assert.deepEqual(deferPolicy({ code: "sandbox.capacity", retryInSeconds: 30 }, { now: 0, deferrals: 99, maxDeferrals: 3 }), { delayMs: 30000, host: true });
+    for (const code of ["merge.transient", "gh.unavailable", "cli.idle", "rate_limited", "sandbox.start_failed"]) {
+      assert.deepEqual(deferPolicy({ code, retryInSeconds: 60 }, { now: 0, deferrals: 3, maxDeferrals: 3 }), { capped: true }, code);
+    }
+  });
+
+  it("with K above the pool size every issue eventually runs and none is skipped for capacity", async () => {
+    const c = { t: 1_000_000 };
+    const now = () => c.t;
+    // Fake clock that advances after a macrotask, so running lanes progress while deferred ones wait.
+    const sleep = async (ms) => { await new Promise((r) => setImmediate(r)); c.t += ms; };
+    const POOL = 2;
+    let inUse = 0;
+    const capacityDefers = new Map();
+    const rows = await orchestrate({ issues: [1, 2, 3, 4, 5, 6], parallel: 4, maxDeferrals: 3, now, sleep, readBody: async () => "",
+      runOne: async (n) => {
+        if (inUse >= POOL) {
+          capacityDefers.set(n, (capacityDefers.get(n) ?? 0) + 1);
+          return { terminalReason: "deferred", code: "sandbox.capacity", retryInSeconds: 60, detail: "sandbox launch: HTTP 503" };
+        }
+        inUse++;
+        try { await new Promise((r) => setTimeout(r, 20)); } finally { inUse--; }
+        return { terminalReason: "merged", pr: 100 + n };
+      } });
+    assert.deepEqual(rows.map((r) => [r.issue, r.outcome]).sort((a, b) => a[0] - b[0]), [1, 2, 3, 4, 5, 6].map((n) => [n, "merged"]));
+    assert.ok(!rows.some((r) => /gave up/.test(r.detail ?? "")));
+    assert.ok(Math.max(...capacityDefers.values()) > 3, "an issue was deferred for capacity more than maxDeferrals times");
+  });
+});
+
+describe("#4540: remote admissions are capped to the provider's free capacity", () => {
+  const run = async (capacity) => {
+    const c = { t: 1_000_000 };
+    const sleep = async (ms) => { await new Promise((r) => setImmediate(r)); c.t += ms; };
+    let live = 0, peak = 0, bounced = 0;
+    const POOL = 2;
+    const rows = await orchestrate({ issues: [1, 2, 3, 4, 5], parallel: 5, now: () => c.t, sleep, pollMs: 1, readBody: async () => "",
+      remote: { max: 5, isRemote: () => true, capacity: capacity(() => POOL - live) },
+      runOne: async (n) => {
+        if (live >= POOL) { bounced++; return { terminalReason: "deferred", code: "sandbox.capacity", retryInSeconds: 1, detail: "sandbox launch: HTTP 503" }; }
+        live++; peak = Math.max(peak, live);
+        try { await new Promise((r) => setTimeout(r, 10)); } finally { live--; }
+        return { terminalReason: "merged", pr: 100 + n };
+      } });
+    return { rows, peak, bounced };
+  };
+  it("queues beyond the free slots inside the batch: no 503 bounce, every issue still runs", async () => {
+    let at = 0;
+    const { rows, bounced } = await run((free) => () => ({ free: free(), at: ++at })); // a fresh sample on every read
+    assert.equal(bounced, 0);
+    assert.deepEqual(rows.map((r) => r.outcome), ["merged", "merged", "merged", "merged", "merged"]);
+  });
+  it("unknown capacity (null) admits as before and the 503 deferral path is the backstop", async () => {
+    const { rows, bounced } = await run(() => () => null);
+    assert.ok(bounced > 0);
+    assert.deepEqual(rows.map((r) => r.outcome).filter((o) => o !== "merged"), []);
+  });
+});
+
+describe("#4547: opt-in local fallback in mode remote when no pool agent is online", () => {
+  const run = async (remoteExtra, online) => {
+    const c = { t: 1_000_000 };
+    const sleep = async (ms) => { await new Promise((r) => setImmediate(r)); c.t += ms; };
+    const where = new Map();
+    let remoteTries = 0;
+    const rows = await orchestrate({ issues: [1, 2, 3], parallel: 3, now: () => c.t, sleep, pollMs: 1, readBody: async () => "",
+      remote: { max: 3, isRemote: () => true, capacity: () => (online === null ? null : { free: null, online, at: 1 }), ...remoteExtra },
+      runOne: async (n, _ln, ctx = {}) => {
+        where.set(n, ctx.placement?.where ?? "remote");
+        if (ctx.placement?.where !== "local") { remoteTries++; if (online === 0) return { terminalReason: "deferred", code: "sandbox.capacity", retryInSeconds: 1, detail: "HTTP 503" }; }
+        return { terminalReason: "merged", pr: 100 + n };
+      } });
+    return { rows, where, remoteTries };
+  };
+  it("every agent offline: the batch completes on the host", async () => {
+    const { rows, where, remoteTries } = await run({ offlineFallback: true }, 0);
+    assert.deepEqual(rows.map((r) => r.outcome), ["merged", "merged", "merged"]);
+    assert.ok([...where.values()].every((w) => w === "local"));
+    assert.equal(remoteTries, 0);
+  });
+  it("an agent online, or liveness unknown, stays remote", async () => {
+    for (const online of [1, null]) {
+      const { where } = await run({ offlineFallback: true }, online);
+      assert.ok([...where.values()].every((w) => w === "remote"), String(online));
+    }
+  });
+});

package/bin/lean/budget.mjs

@@ -5,7 +5,8 @@
  * else the frozen contract's distinct deliverable paths (S <= 2, M <= 5, else L). The budget for the size is forge.yaml
  * `engine.turn_budget_s|m|l` (defaults 25/60/100; 0 disables that size). The implement stage (the build call and its
  * context/timeout restarts) spends it; on overrun the builder gets one wrap-up grant to finish or stop with a digest, and
- * an unfinished lane ends needs-human `budget.exceeded` (recoverable once by the batch). Pure module: no I/O.
+ * an unfinished lane ends the non-human `budget-exceeded` reason (code `budget.exceeded`, #4505): the batch recovers it once
+ * from the checkpoint at the next size (`recoverySize`); only a final recovery that overruns too ends needs-human. Pure module: no I/O.
  */
 
 export const TURN_BUDGET_DEFAULTS = Object.freeze({ S: 25, M: 60, L: 100 });
@@ -36,6 +37,11 @@ export function laneSize({ contract = null, labels = [] } = {}) {
   return n <= 2 ? "S" : n <= 5 ? "M" : "L";
 }
 
+/** #4505: the next size up (S -> M -> L; L stays L); anything else unchanged. */
+export const nextSize = (size) => (SIZES.includes(size) ? SIZES[Math.min(SIZES.indexOf(size) + 1, SIZES.length - 1)] : size);
+/** #4505: a `budget.exceeded` recovery lane runs at the next size's budget; any other lane keeps its size. */
+export const recoverySize = (size, recover) => (recover?.code === "budget.exceeded" ? nextSize(size) : size);
+
 /** Turns the wrap-up call may use after the budget is spent. */
 export const wrapupGrant = (budget) => Math.max(5, Math.ceil(budget / 5));
 

package/bin/lean/budget.test.mjs

@@ -5,7 +5,7 @@ import { describe, it } from "node:test";
 import assert from "node:assert/strict";
 import { writeFileSync, chmodSync } from "node:fs";
 import { tmpDir } from "../tests/_tmp.mjs";
-import { TURN_BUDGET_DEFAULTS, parseTurnBudgets, laneSize, wrapupGrant, makeTurnBudget, budgetDigest, formatTurnSummary } from "./budget.mjs";
+import { TURN_BUDGET_DEFAULTS, parseTurnBudgets, laneSize, nextSize, recoverySize, wrapupGrant, makeTurnBudget, budgetDigest, formatTurnSummary } from "./budget.mjs";
 import { orchestrate } from "./orchestrate.mjs";
 import { OUTCOME_CODES, RECOVERABLE_CODES } from "../engine/outcome-codes.mjs";
 import { RECOVER_HOW } from "./lean.mjs";
@@ -34,6 +34,14 @@ describe("#4392 turn budget", () => {
     assert.equal(laneSize({}), null);
   });
 
+  it("#4505 a budget.exceeded recovery runs at the next size (S->M->L, L stays L); other lanes keep their size", () => {
+    assert.deepEqual(["S", "M", "L"].map(nextSize), ["M", "L", "L"]);
+    assert.equal(nextSize(null), null); assert.equal(nextSize("XL"), "XL");
+    assert.equal(recoverySize("S", { code: "budget.exceeded", attempt: 1, final: true }), "M");
+    assert.equal(recoverySize("L", { code: "budget.exceeded" }), "L");
+    assert.equal(recoverySize("S", { code: "ci.red" }), "S"); assert.equal(recoverySize("S", null), "S"); assert.equal(recoverySize(null, { code: "budget.exceeded" }), null);
+  });
+
   it("the tally: build turns spend the budget, other attempts count only in total; off or unsized is unlimited", () => {
     const tb = makeTurnBudget({ M: 60 });
     assert.equal(tb.record(), null); assert.equal(tb.remaining(), Infinity);

package/bin/lean/conformance.mjs

@@ -261,18 +261,51 @@ export function validateContract(c, issueText, { repro = false } = {}) {
   return { ok: problems.length === 0, problems };
 }
 
+/** forge#4519: a shell pattern as in scripts/hotspots.sh `case` (`*` crosses `/`, `?` one character), anchored. */
+const shellGlobRe = (g) => new RegExp(`^${String(g).replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".")}$`);
+
+const AMEND_KINDS = ["add-deliverable", "add-requirement", ...TRACKED];
+const AMEND_SLIPS = { "amend-requirement": "defer", limitation: "defer" };
+
+/** forge#4574: common builder slips: `type` for `kind`, amend-requirement/limitation as defer, an `R#` in `id` as the requirement. */
+function normalizeAmendment(a) {
+  if (!a || typeof a !== "object" || Array.isArray(a)) return a;
+  const n = { ...a };
+  if (blank(n.kind) && !blank(n.type)) n.kind = n.type;
+  if (Object.hasOwn(AMEND_SLIPS, n.kind)) n.kind = AMEND_SLIPS[n.kind];
+  if (TRACKED.includes(n.kind) && blank(n.requirement) && /^R\d+$/.test(String(n.id ?? ""))) n.requirement = n.id;
+  return n;
+}
+
+/** The rejection reason, naming the valid kinds or the missing field so the builder's correction round can fix it. */
+function amendmentProblem(a, tag) {
+  const kind = a?.kind;
+  const need = kind === "add-deliverable" ? "needs a path" : kind === "add-requirement" ? "needs requirement {id, quote}"
+    : TRACKED.includes(kind) ? "needs requirement: \"R#\"" : null;
+  const why = need ? `amendment kind '${kind}' ${need}` : `unknown kind '${kind ?? ""}'; valid kinds are ${AMEND_KINDS.join(", ")}`;
+  return `unknown or incomplete amendment: ${why}: ${tag}`;
+}
+
 /**
  * Apply result.json amendments to the frozen contract. Kinds: add-deliverable {path}, add-requirement {requirement},
  * defer|post-deploy {requirement: id}. Every amendment needs a why. Returns {contract, applied[], toTrack[], problems[]}.
+ * `generatedPaths` (forge#4519, forge.yaml maintenance.generated_paths via scripts/hotspots.sh): an add-deliverable on a
+ * matching path is refused; the repo's generator or bot owns that file.
  */
-export function applyAmendments(frozen, amendments, issueText = null) {
+export function applyAmendments(frozen, amendments, issueText = null, { generatedPaths = [] } = {}) {
   const c = structuredClone(frozen);
   c.deliverables = c.deliverables ?? [];
   const applied = [], toTrack = [], problems = [];
-  for (const a of Array.isArray(amendments) ? amendments : []) {
-    const tag = JSON.stringify(a ?? null).slice(0, 100);
+  const genRes = (Array.isArray(generatedPaths) ? generatedPaths : []).filter((g) => typeof g === "string" && g).map(shellGlobRe);
+  for (const raw of Array.isArray(amendments) ? amendments : []) {
+    const a = normalizeAmendment(raw);
+    const tag = JSON.stringify(raw ?? null).slice(0, 400);
     if (blank(a?.why)) { problems.push(`amendment without why: ${tag}`); continue; }
     if (a.kind === "add-deliverable" && !blank(a.path)) {
+      if (genRes.some((re) => re.test(cleanPath(a.path)))) {
+        problems.push(`add-deliverable ${a.path}: generated path (forge.yaml maintenance.generated_paths) is owned by the repo's generator/bot; revert the file instead of amending it in`);
+        continue;
+      }
       const have = c.deliverables.find((d) => d.path === a.path);
       // An existing deliverable keeps its id and gains the why, so an amendment naming a package on an already-listed
       // package.json declares the edge (forge#4159); before, it was reported as applied but changed nothing.
@@ -291,7 +324,7 @@ export function applyAmendments(frozen, amendments, issueText = null) {
       const r = c.requirements.find((x) => x.id === a.requirement);
       if (!r) { problems.push(`amendment names unknown requirement ${a.requirement}`); continue; }
       if (!TRACKED.includes(r.disposition)) { r.disposition = a.kind; r.why = a.why; r.amended = true; toTrack.push(r); }
-    } else { problems.push(`unknown or incomplete amendment: ${tag}`); continue; }
+    } else { problems.push(amendmentProblem(a, tag)); continue; }
     applied.push({ kind: a.kind, target: a.path ?? a.requirement?.id ?? a.requirement, why: String(a.why).trim() });
   }
   return { contract: c, applied, toTrack, problems };
@@ -343,9 +376,11 @@ export function repoTracks(paths) {
  * `frozen` (#4350): the contract before typed amendments. A failed scope-check validate (isScopeCheck) whose only files
  * outside `frozen` are covered by deliverables an amendment added, with no scope creep left, is superseded by the engine's
  * scope gate: listed in `superseded` [{id, files}], not a failure. Without `frozen`, `superseded` is always [].
+ * `ciCovered` (#4527): the validate ids accepted only because the head is CI-verified (no local result, or a missing host
+ * tool), for the trajectory; always an array.
  */
 export function precheck({ contract, result, changedFiles, ignore = [], ciVerified = false, hotspots = null, repro = null, validateRuns = null, repoHas = () => false, frozen = null }) {
-  const failures = [], superseded = [];
+  const failures = [], superseded = [], ciCovered = [];
   const outside = (k) => changedFiles.filter((f) => !ignore.some((p) => f.startsWith(p)) && !authorizationOf(k, f));
   const creep = outside(contract);
   const amendedOnly = frozenAmendedFiles(frozen, contract, outside, creep);
@@ -355,13 +390,13 @@ export function precheck({ contract, result, changedFiles, ignore = [], ciVerifi
     const r = runs.find((x) => x?.id === v.id);
     // #4167: a head the engine verified on CI covers validate ids the builder left to CI (no result, or ci:true without an exit)
     const leftToCi = !r || (r.ci === true && r.exit === undefined);
-    if (ciVerified && leftToCi) continue;
+    if (ciVerified && leftToCi) { ciCovered.push(v.id); continue; }
     if (leftToCi) failures.push({ code: C.CONFORMANCE_UNIMPLEMENTED, kind: "validate_missing", detail: `validate ${v.id} (\`${v.command}\`) has no recorded result` });
     else if (r.exit !== 0) {
       const rerun = validateRuns?.[v.id];
       if (rerun?.rc === 0) continue;
       const miss = rerun ? missingTool(rerun, { repoHas, command: v.command }) : null;
-      if (miss && ciVerified) continue;
+      if (miss && ciVerified) { ciCovered.push(v.id); continue; }
       if (miss) failures.push({ code: C.ENV_TOOL_MISSING, kind: "validate_env", tool: miss.tool, detail: `validate ${v.id} (\`${v.command}\`) could not run: the host is missing ${KIND_NAME[miss.kind]} '${miss.tool}' (environment, not the change)` });
       else if (amendedOnly && isScopeCheck(v.command)) superseded.push({ id: v.id, files: amendedOnly });
       else failures.push({ code: C.CONFORMANCE_UNIMPLEMENTED, kind: "validate_failed", detail: `validate ${v.id} (\`${v.command}\`) failed (exit ${r.exit})` });
@@ -382,7 +417,7 @@ export function precheck({ contract, result, changedFiles, ignore = [], ciVerifi
   }
   failures.push(...hotspotGrowth({ contract, result, hotspots, changedFiles }));
   if (repro) failures.push(...reproFailures(contract, repro));
-  return { ok: failures.length === 0, failures, superseded };
+  return { ok: failures.length === 0, failures, superseded, ciCovered };
 }
 
 /**
@@ -419,7 +454,8 @@ export function reproFailures(contract, { base, head } = {}) {
 
 const EXEMPT_RE = /hotspot-exempt:\s*\S/i;
 /**
- * forge#4195: a changed file already over `file_lines` (not generated, vendored or glob-excluded) that grew by more
+ * forge#4195: a changed file already over `file_lines` (not generated, vendored or glob-excluded; forge#4519: a
+ * `generated` row, from linguist-generated, maintenance.generated_paths or a generated header, never counts) that grew by more
  * than `growth_lines` since the base, with no `hotspot-exempt: <reason>` in a covering deliverable's change or an
  * add-deliverable amendment's why. `hotspots` is scripts/hotspots.sh --base output; null or disabled checks nothing.
  */
@@ -431,7 +467,7 @@ export function hotspotGrowth({ contract, result, hotspots, changedFiles = [] })
   const exempt = (f) => (contract?.deliverables ?? []).some((d) => pathCovers(d.path, f) && EXEMPT_RE.test(String(d.change ?? "")))
     || amends.some((a) => a?.kind === "add-deliverable" && pathCovers(a.path, f) && EXEMPT_RE.test(String(a.why ?? "")));
   return hotspots.files
-    .filter((h) => h?.over && !h.excluded && Number(h.growth) > limit && changed.has(cleanPath(h.path)) && !exempt(h.path))
+    .filter((h) => h?.over && !h.excluded && !h.generated && Number(h.growth) > limit && changed.has(cleanPath(h.path)) && !exempt(h.path))
     .map((h) => ({ code: C.MAINTENANCE_HOTSPOT_GROWTH, kind: "hotspot_growth",
       detail: `${h.path} grew by ${h.growth} lines (now ${h.lines}, limit +${limit} over ${max}) without hotspot-exempt` }));
 }

package/bin/lean/conformance.test.mjs

@@ -60,9 +60,30 @@ describe("amendments", () => {
   });
 });
 
+describe("#4574 amendment slips", () => {
+  it("type for kind, amend-requirement and limitation apply as defer; an R# id fills requirement", () => {
+    const a = applyAmendments(C, [{ kind: "amend-requirement", id: "R2", why: "stored hashed" }, { id: "A1", requirement: "R1", type: "limitation", why: "plugin limit" }]);
+    assert.deepEqual(a.problems, []);
+    assert.deepEqual(a.toTrack.map((r) => r.id).sort(), ["R1", "R2"]);
+    assert.ok(a.contract.requirements.every((r) => r.disposition === "defer"));
+  });
+  it("an unknown kind names the valid kinds and shows the full amendment", () => {
+    const why = "x".repeat(300);
+    const a = applyAmendments(C, [{ kind: "rewrite", why }]);
+    assert.match(a.problems[0], /unknown kind 'rewrite'; valid kinds are add-deliverable, add-requirement, defer, post-deploy/);
+    assert.ok(a.problems[0].includes(why));
+  });
+  it("an Object.prototype key as kind is an unknown kind, not a slip", () => {
+    assert.match(applyAmendments(C, [{ kind: "constructor", why: "w" }]).problems[0], /unknown kind 'constructor'; valid kinds are add-deliverable/);
+  });
+  it("an incomplete known kind names the missing field", () => {
+    assert.match(applyAmendments(C, [{ kind: "defer", why: "w" }]).problems[0], /'defer' needs requirement/);
+  });
+});
+
 describe("precheck (deterministic, no reviewer)", () => {
   const res = { requirements: [{ id: "R1", status: "implemented" }, { id: "R2", status: "implemented" }], validation: [{ id: "V1", exit: 0 }] };
-  it("passes a complete, validated, in-scope change", () => assert.deepEqual(precheck({ contract: C, result: res, changedFiles: files }), { ok: true, failures: [], superseded: [] }));
+  it("passes a complete, validated, in-scope change", () => assert.deepEqual(precheck({ contract: C, result: res, changedFiles: files }), { ok: true, failures: [], superseded: [], ciCovered: [] }));
   it("unauthorized file -> scope_creep; tests and ignored paths are always allowed", () => {
     assert.deepEqual(codes(precheck({ contract: C, result: res, changedFiles: [...files, "src/billing.js", ".forge-lean/x", "tests/test_x.py", "pkg/foo_test.go"], ignore: [".forge-lean/"] })), ["conformance.scope_creep"]);
     for (const t of ["test/a.js", "src/a.test.ts", "tests/test_a.py", "a_test.go", "spec/a.spec.js"]) assert.ok(isTestPath(t), t);
@@ -85,6 +106,15 @@ describe("precheck (deterministic, no reviewer)", () => {
     const local = precheck({ contract: C, result: failed, changedFiles: files, repoHas: repoTracks(["src/leak/__init__.py"]), validateRuns: { V1: { rc: 1, output: "ModuleNotFoundError: No module named 'leak.core'" } } });
     assert.deepEqual(codes(local), ["conformance.unimplemented"], "a repo module the change did not create is a code defect");
   });
+  it("#4527 a CI-verified head lists the validate ids it accepted without a local result; a ran-and-failed validate still fails", () => {
+    const C2 = { ...C, validate: [{ id: "V1", command: "node --test a.test.mjs" }, { id: "V2", command: "tsc" }, { id: "V3", command: "lint" }] };
+    const pc = precheck({ contract: C2, result: { ...res, validation: [{ id: "V2", ci: true }, { id: "V3", exit: 0 }] }, changedFiles: files, ciVerified: true });
+    assert.deepEqual(codes(pc), []); assert.deepEqual(pc.ciCovered, ["V1", "V2"]);
+    const local = precheck({ contract: C2, result: { ...res, validation: [{ id: "V2", ci: true }, { id: "V3", exit: 0 }] }, changedFiles: files });
+    assert.deepEqual(local.failures.map((f) => f.kind), ["validate_missing", "validate_missing"]); assert.deepEqual(local.ciCovered, []);
+    const failed = precheck({ contract: C2, result: { ...res, validation: [{ id: "V1", exit: 1 }, { id: "V2", ci: true }, { id: "V3", exit: 0 }] }, changedFiles: files, ciVerified: true });
+    assert.deepEqual(failed.failures.map((f) => f.kind), ["validate_failed"]); assert.deepEqual(failed.ciCovered, ["V2"]);
+  });
   it("#4323 missingTool: commands, python and node modules; paths, relative and repo-tracked names are not the environment", () => {
     assert.deepEqual(missingTool({ rc: 127, output: "sh: 1: jq: not found" }), { tool: "jq", kind: "command" });
     assert.deepEqual(missingTool({ rc: 127, output: "" }, { command: "uv run pytest -q" }), { tool: "uv", kind: "command" });
@@ -362,6 +392,24 @@ describe("hotspot growth precheck (forge#4195)", () => {
     assert.match(pc.failures[0].detail, /^src\/scrape\.js grew by 200 lines \(now 2600, limit \+150 over 2000\) without hotspot-exempt$/);
   });
 
+  it("#4519 generated paths: applyAmendments refuses add-deliverable on maintenance.generated_paths, hotspotGrowth skips generated rows", () => {
+    const gen = ["web/public/openapi*.json", "web/public/openapi-versions/*"];
+    const amends = [{ kind: "add-deliverable", path: "web/public/openapi-versions/3.29.2.json", why: "sync the spec archive" },
+      { kind: "add-deliverable", path: "./web/public/openapi.json", why: "sync the spec" }, { kind: "add-deliverable", path: "src/shared.js", why: "helper" }];
+    const a = applyAmendments(C, amends, null, { generatedPaths: gen });
+    assert.deepEqual(a.applied.map((x) => x.target), ["src/shared.js"]);
+    assert.equal(a.problems.length, 2);
+    assert.match(a.problems[0], /^add-deliverable web\/public\/openapi-versions\/3\.29\.2\.json: generated path \(forge\.yaml maintenance\.generated_paths\)/);
+    assert.ok(!a.contract.deliverables.some((d) => d.path.includes("openapi")), "a refused path never becomes a deliverable");
+    assert.equal(applyAmendments(C, amends).applied.length, 3, "without generatedPaths every add-deliverable applies as before");
+    assert.equal(applyAmendments(C, amends, null, { generatedPaths: ["web/public/openapi.json"] }).applied.length, 2, "a literal glob matches only that path");
+    // linguist-generated / generated_paths / header rows (hotspots.sh `generated`) never trigger hotspot_growth
+    const big = H({ path: "src/scrape.js", lines: 83974, growth: 83974 });
+    assert.equal(run(hs([big])).ok, false);
+    assert.equal(run(hs([{ ...big, generated: true }])).ok, true);
+    assert.equal(run(hs([{ ...big, generated: true, excluded: true }])).ok, true);
+  });
+
   it("within the limit, under the threshold, excluded, unchanged, disabled or no facts: no failure", () => {
     for (const h of [hs([H({ growth: 150 })]), hs([H({ over: false, lines: 1900 })]), hs([H({ excluded: true })]), hs([H({ path: "src/other.js" })]),
       hs([H()], { enabled: false }), hs([H({ growth: null })]), null, undefined, {}]) assert.equal(run(h).ok, true, JSON.stringify(h));
@@ -398,7 +446,7 @@ describe("repro_test on bug lanes (#4271)", () => {
   });
   it("red on base and green on head passes; anything else fails the precheck as conformance.unimplemented", () => {
     const k = { ...C, repro_test: RT };
-    assert.deepEqual(precheck({ contract: k, result: res, changedFiles: files, repro: { base: "FAILED", head: "RAN" } }), { ok: true, failures: [], superseded: [] });
+    assert.deepEqual(precheck({ contract: k, result: res, changedFiles: files, repro: { base: "FAILED", head: "RAN" } }), { ok: true, failures: [], superseded: [], ciCovered: [] });
     for (const repro of [{ base: "RAN", head: "RAN" }, { base: "SKIPPED", head: "FAILED" }, { base: "UNRUNNABLE", head: "RAN" }, { base: "SKIPPED", head: "RAN_NOTHING" }]) {
       const pc = precheck({ contract: k, result: res, changedFiles: files, repro });
       assert.deepEqual(pc.failures.map((f) => [f.code, f.kind]), [["conformance.unimplemented", "repro_red_green"]], JSON.stringify(repro));

package/bin/lean/depcache.mjs

@@ -11,7 +11,9 @@
  *   <repoKey>/<eco>-<key>.tmp-<pid>   snapshot being published (atomic rename into place)
  *   stores/{pnpm-store,npm-cache,uv-cache,poetry-cache,pip-cache}   the package managers' own shared stores
  * The repo's own package manager does every install (frozen lockfile). Every effect is injected; nothing here throws:
- * a missing tool, another filesystem (EXDEV) or a failed install leaves the lane as it was before.
+ * a missing tool, another filesystem (EXDEV) or a failed install leaves the lane as it was before. A snapshot that fails
+ * to link or repair falls back to a fresh install in the lane (status materialize-fallback, #4522); failed only when that
+ * install fails too.
  */
 import * as nodeFs from "node:fs";
 import { createHash } from "node:crypto";
@@ -28,7 +30,17 @@ const TMP_RE = /^(pnpm|npm|yarn|uv|poetry)-[0-9a-f]{16}\.tmp-(\d+)$/;
 const LOCK_RE = /^(pnpm|npm|yarn|uv|poetry)-[0-9a-f]{16}\.lock$/;
 
 const pidAlive = (pid) => { try { process.kill(pid, 0); return true; } catch (e) { return e?.code === "EPERM"; } };
-const msg = (e) => String(e?.stderr || e?.message || e).trim().split("\n").slice(-3).join(" ").slice(0, 300);
+/**
+ * Error text for a reason: pnpm prints its ERR_PNPM_* errors on stdout (#4522), so the first such line from either
+ * stream leads, then the last lines of stderr and stdout; the message only when both streams are empty.
+ */
+const msg = (e) => {
+  const lines = (s) => String(s ?? "").split("\n").map((l) => l.trim()).filter(Boolean);
+  const err = lines(e?.stderr), out = lines(e?.stdout);
+  const pm = [...err, ...out].find((l) => /\bERR_PNPM_[A-Z0-9_]+/.test(l));
+  const parts = [pm, ...err.slice(-3), ...out.slice(-3)].filter((l, i, a) => l && a.indexOf(l) === i);
+  return (parts.length ? parts.join(" ") : String(e?.message || e).trim().split("\n").slice(-3).join(" ")).slice(0, 500);
+};
 
 /** Lockfile -> ecosystem, in priority order: at most one JS and one Python ecosystem per repo. */
 function detect(fs, wt) {
@@ -215,7 +227,8 @@ export function makeDepCache({ root = depcacheRoot(), exec, fs = nodeFs, now = (
         return done("hit", { key });
       } catch (e) {
         rmTrees(wt, created);
-        return done(e?.code === "EXDEV" ? "skipped" : "failed", { key, reason: e?.code === "EXDEV" ? "cache is on another filesystem than the worktree (EXDEV)" : `materialize failed: ${msg(e)}` });
+        if (e?.code === "EXDEV") return done("skipped", { key, reason: "cache is on another filesystem than the worktree (EXDEV)" });
+        return { fallback: `materialize failed: ${msg(e)}` }; // installed by the caller, outside the key lock
       }
     };
     const install = async () => {
@@ -235,6 +248,13 @@ export function makeDepCache({ root = depcacheRoot(), exec, fs = nodeFs, now = (
       const why = publish(wt, d, key, ins.trees, snap);
       return done("miss-installed", { key, published: !why, reason: why });
     });
+    if (r.locked && r.value?.fallback) {
+      // #4522: a broken snapshot must not leave the lane without dependencies. Install outside the lock so other lanes on
+      // this key are not serialised behind it; never publish (the snapshot exists and is never replaced).
+      const ins = await install();
+      if (ins.failed) return { ...ins.failed, reason: `${r.value.fallback}; ${ins.failed.reason}` };
+      return done("materialize-fallback", { key, published: false, reason: r.value.fallback });
+    }
     if (r.locked) return r.value ?? done("failed", { key, reason: "snapshot manifest unreadable" });
     // The lock stayed busy past the wait: install in this lane without touching the cache.
     const ins = await install();

package/bin/lean/depcache.test.mjs

@@ -38,7 +38,7 @@ const FIXTURES = {
 const LOCKFILE = { pnpm: "pnpm-lock.yaml", npm: "package-lock.json", uv: "uv.lock", poetry: "poetry.lock" };
 
 /** Fake package managers: write what a real install leaves behind, including the absolute paths that need rewriting. */
-function fakeExec({ failInstall = false, installDelayMs = 0 } = {}) {
+function fakeExec({ failInstall = false, failRepair = false, installDelayMs = 0 } = {}) {
   const calls = [];
   const venv = (cwd, lock) => {
     const v = join(cwd, ".venv");
@@ -65,6 +65,10 @@ function fakeExec({ failInstall = false, installDelayMs = 0 } = {}) {
     if (full && installDelayMs) await new Promise((r) => setTimeout(r, installDelayMs));
     if (full && failInstall) { put(join(cwd, cmd === "uv" || cmd === "poetry" ? ".venv/partial" : "node_modules/partial"), "x"); throw Object.assign(new Error("install failed"), { stderr: "ERR_PNPM_FROZEN_LOCKFILE" }); }
     if (cmd === "pnpm") {
+      if (args.includes("--offline") && failRepair) {
+        // pnpm prints its errors on stdout; execFile's error then carries an empty stderr (#4522).
+        throw Object.assign(new Error("Command failed: pnpm install --frozen-lockfile --offline"), { stderr: "", stdout: "Lockfile is up to date\n ERR_PNPM_TEST_REPAIR  some cause\n" });
+      }
       if (args.includes("--offline")) return ""; // warm repair pass: nothing to do
       const lock = readFileSync(join(cwd, "pnpm-lock.yaml"), "utf8");
       put(join(cwd, "node_modules/.pnpm/lodash@4/node_modules/lodash/index.js"), `module.exports = ${JSON.stringify(lock)};\n`);
@@ -222,6 +226,43 @@ describe("depcache (#4220): shared dependency installs keyed by lock hash", () =
     assert.equal(r.published, true, "the stale lock was replaced");
   });
 
+  it("#4522 a snapshot whose repair fails falls back to a fresh install outside the lock, keeps the snapshot and records pnpm's stdout error", async () => {
+    const s = setup("pnpm");
+    const [ra] = await s.cache.prepare({ wt: s.lane("a"), repoKey: "r" });
+    assert.equal(ra.status, "miss-installed", ra.reason);
+    const [snap] = snapDirs(s.root, "pnpm");
+    const before = snapshotOf(snap);
+    const broken = makeDepCache({ root: s.root, exec: fakeExec({ failRepair: true }).exec, env: {}, pollMs: 5 });
+    const wt = s.lane("b");
+    const [rb] = await broken.prepare({ wt, repoKey: "r" });
+    assert.equal(rb.status, "materialize-fallback", rb.reason);
+    assert.equal(rb.published, false);
+    assert.match(rb.reason, /^materialize failed: ERR_PNPM_TEST_REPAIR {2}some cause/);
+    assert.ok(existsSync(join(wt, "node_modules/lodash/index.js")), "the lane has node_modules");
+    const after = snapshotOf(snap);
+    delete before["last-used"]; delete after["last-used"];
+    assert.deepEqual(after, before, "the snapshot is untouched");
+    assert.deepEqual(readdirSync(join(s.root, "r")).filter((n) => !/^pnpm-[0-9a-f]{16}$/.test(n)), [], "no lock or tmp left");
+
+    const both = makeDepCache({ root: s.root, exec: fakeExec({ failRepair: true, failInstall: true }).exec, env: {}, pollMs: 5 });
+    const wt2 = s.lane("c");
+    const [rc] = await both.prepare({ wt: wt2, repoKey: "r" });
+    assert.equal(rc.status, "failed");
+    assert.match(rc.reason, /^materialize failed: ERR_PNPM_TEST_REPAIR.*; install failed: ERR_PNPM_FROZEN_LOCKFILE/);
+    assert.ok(!existsSync(join(wt2, "node_modules")), "no partial tree");
+  });
+
+  it("#4522 a failed pnpm install records the ERR_PNPM line it printed on stdout", async () => {
+    const s = setup("pnpm");
+    const exec = async (cmd, args, o) => {
+      if (cmd === "pnpm" && !args.includes("--offline")) throw Object.assign(new Error("Command failed: pnpm install --frozen-lockfile"), { stderr: "", stdout: "Progress: resolved 1\n ERR_PNPM_OUTDATED_LOCKFILE  Cannot install with frozen-lockfile\n\nNote that in CI environments this setting is true by default.\nA\nB\nC\n" });
+      return s.f.exec(cmd, args, o);
+    };
+    const [r] = await makeDepCache({ root: s.root, exec, env: {} }).prepare({ wt: s.lane("x"), repoKey: "r" });
+    assert.equal(r.status, "failed");
+    assert.match(r.reason, /^install failed: ERR_PNPM_OUTDATED_LOCKFILE {2}Cannot install with frozen-lockfile A B C$/);
+  });
+
   it("a failed install, a missing tool or another filesystem leaves no snapshot and no partial tree", async () => {
     const s = setup("pnpm", { failInstall: true });
     const wt = s.lane("x");

package/bin/lean/effects.mjs

@@ -370,7 +370,7 @@ export function makeLeanIo({ repo }) {
     sequenceSnapshot: ({ cwd, base }) => snapshotLane({ git: (a) => run("git")(a, { cwd }), base }),
     sequenceVerify: ({ cwd, base, before, moves }) => verifyRenumber({ git: (a) => run("git")(a, { cwd }), base, before, moves }),
     /** #4386: the deterministic renumber, sequence-only base-sync resolution, and the host sequence lease at the merge gate. */
-    sequenceRenumber: ({ cwd, base, laneRef, regenerate }) => renumberLane({ git: (a) => run("git")(a, { cwd }), base, cwd, laneRef, regenerate }),
+    sequenceRenumber: ({ cwd, base, laneRef, regenerate, reserve }) => renumberLane({ git: (a) => run("git")(a, { cwd }), base, cwd, laneRef, regenerate, reserve }),
     sequenceResolve: ({ cwd, files }) => resolveSequenceConflicts({ git: (a) => run("git")(a, { cwd }), files }),
     sequenceLease: makeSequenceLeaseIo(),
     /** #4406: the lane's scoped checks after a base merge (bin/lean/checks.mjs; [] on failure), and same-app duplicate exports (bin/lean/postmerge.mjs). */