forgedock 1.32.0 → 1.33.0

package/bin/lean/basesync.test.mjs

@@ -0,0 +1,151 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// #4525: parked-PR base sync. Unit tests for the union resolver, and a real temp git repo (bare origin + clone).
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { tmpDir } from "../tests/_tmp.mjs";
+import { readOrchestration } from "../engine-cli.mjs";
+import { orchestrate } from "./orchestrate.mjs";
+import { unionResolve, matchesGlobs, isSequenceFile, syncParked } from "./basesync.mjs";
+
+const sh = (cwd, ...a) => execFileSync("git", a, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t" } });
+const put = (root, files) => { for (const [p, t] of Object.entries(files)) { mkdirSync(dirname(join(root, p)), { recursive: true }); writeFileSync(join(root, p), t); } };
+const commit = (d, m) => { sh(d, "add", "-A"); sh(d, "commit", "-qm", m); };
+
+const hunk = (ours, base, theirs) => `top\n<<<<<<< HEAD\n${ours}||||||| base\n${base}=======\n${theirs}>>>>>>> origin/main\nbottom\n`;
+
+describe("unionResolve (#4525)", () => {
+  it("keeps both sides when both only added lines", () => {
+    assert.equal(unionResolve(hunk("a\n", "", "b\n")), "top\na\nb\nbottom\n");
+  });
+  it("refuses a hunk that modified or deleted base lines", () => {
+    assert.equal(unionResolve(hunk("a\n", "x\n", "b\n")), null);
+    assert.equal(unionResolve(hunk("", "x\n", "b\n")), null);
+  });
+  it("refuses text with no base section or no conflict", () => {
+    assert.equal(unionResolve("top\n<<<<<<< HEAD\na\n=======\nb\n>>>>>>> x\n"), null);
+    assert.equal(unionResolve("plain\n"), null);
+  });
+  it("refuses a file when any one hunk is not a pure addition", () => {
+    assert.equal(unionResolve(`${hunk("a\n", "", "b\n")}${hunk("c\n", "y\n", "d\n")}`), null);
+  });
+  it("matches globs and excludes sequence files", () => {
+    assert.ok(matchesGlobs("src/registry/index.ts", ["src/registry/*.ts"]));
+    assert.ok(matchesGlobs("a/b/CHANGELOG.md", ["CHANGELOG.md"]));
+    assert.ok(matchesGlobs("a/b/c.txt", ["a/**"]));
+    assert.ok(!matchesGlobs("a/b/c.txt", ["a/*.txt"]));
+    assert.ok(!matchesGlobs("x", []));
+    assert.ok(isSequenceFile("db/drizzle/meta/_journal.json"));
+    assert.ok(isSequenceFile("db/drizzle/0007_usage.sql"));
+    assert.ok(!isSequenceFile("src/registry.ts"));
+  });
+});
+
+/** origin (bare) + clone `d` on main with registry.txt, other.txt; branch `lane` from main. Returns helpers. */
+function fixture() {
+  const root = tmpDir("basesync-");
+  sh(root, "init", "-q", "--bare", "-b", "main", "origin.git");
+  const d = join(root, "work");
+  sh(root, "clone", "-q", "origin.git", "work");
+  sh(d, "checkout", "-qb", "main");
+  put(d, { "reg/registry.txt": "a\nb\n", "other.txt": "one\ntwo\n", "db/_journal.json": "{}\n" });
+  commit(d, "init"); sh(d, "push", "-q", "origin", "main");
+  sh(d, "checkout", "-qb", "lane");
+  const onLane = (files) => { sh(d, "checkout", "-q", "lane"); put(d, files); commit(d, "lane"); sh(d, "push", "-q", "origin", "lane"); };
+  const onMain = (files) => { sh(d, "checkout", "-q", "main"); put(d, files); commit(d, "base"); sh(d, "push", "-q", "origin", "main"); };
+  const git = async (a, o = {}) => sh(o.cwd ?? d, ...a);
+  const tip = (b) => sh(root, "--git-dir", "origin.git", "rev-parse", b).trim();
+  const show = (b, f) => sh(root, "--git-dir", "origin.git", "show", `${b}:${f}`);
+  return { d, onLane, onMain, git, tip, show, tmp: root };
+}
+
+describe("syncParked (#4525)", () => {
+  it("pushes a clean merge of the base into the branch", async () => {
+    const f = fixture();
+    f.onLane({ "lane.txt": "x\n" });
+    f.onMain({ "other.txt": "one\ntwo\nthree\n" });
+    const r = await syncParked({ git: f.git, base: "main", branch: "lane", tmp: f.tmp });
+    assert.equal(r.status, "merged");
+    assert.equal(f.show("lane", "other.txt"), "one\ntwo\nthree\n");
+    assert.equal(f.show("lane", "lane.txt"), "x\n");
+  });
+  it("is up-to-date when the base is already merged, and unchanged for a handled base sha", async () => {
+    const f = fixture();
+    f.onLane({ "lane.txt": "x\n" });
+    const r = await syncParked({ git: f.git, base: "main", branch: "lane", tmp: f.tmp });
+    assert.equal(r.status, "up-to-date");
+    f.onMain({ "m.txt": "m\n" });
+    const r2 = await syncParked({ git: f.git, base: "main", branch: "lane", tmp: f.tmp, last: f.tip("main") });
+    assert.equal(r2.status, "unchanged");
+  });
+  it("union-resolves a two-sided list addition in a matching file and pushes", async () => {
+    const f = fixture();
+    f.onLane({ "reg/registry.txt": "a\nb\nlane\n" });
+    f.onMain({ "reg/registry.txt": "a\nb\nbase\n" });
+    const r = await syncParked({ git: f.git, base: "main", branch: "lane", unionGlobs: ["reg/*.txt"], tmp: f.tmp });
+    assert.equal(r.status, "merged");
+    assert.equal(f.show("lane", "reg/registry.txt"), "a\nb\nlane\nbase\n");
+  });
+  it("reports other conflicts and leaves the branch alone", async () => {
+    const f = fixture();
+    f.onLane({ "other.txt": "one\nLANE\n" });
+    f.onMain({ "other.txt": "one\nBASE\n" });
+    const before = f.tip("lane");
+    const r = await syncParked({ git: f.git, base: "main", branch: "lane", unionGlobs: ["reg/*.txt"], tmp: f.tmp });
+    assert.equal(r.status, "conflict");
+    assert.deepEqual(r.paths, ["other.txt"]);
+    assert.equal(f.tip("lane"), before);
+  });
+  it("does not union-resolve a file outside the globs, or a _journal.json even when matched", async () => {
+    const f = fixture();
+    f.onLane({ "reg/registry.txt": "a\nb\nlane\n", "db/_journal.json": "{\"l\":1}\n" });
+    f.onMain({ "reg/registry.txt": "a\nb\nbase\n", "db/_journal.json": "{\"b\":1}\n" });
+    const r = await syncParked({ git: f.git, base: "main", branch: "lane", unionGlobs: ["**/*.json"], tmp: f.tmp });
+    assert.equal(r.status, "conflict");
+    assert.deepEqual(r.paths.sort(), ["db/_journal.json", "reg/registry.txt"]);
+    assert.equal(readFileSync(join(f.d, "other.txt"), "utf8").length > 0, true);
+  });
+  it("returns an error (never throws) when git fails", async () => {
+    const r = await syncParked({ git: async () => { throw new Error("boom\nmore"); }, base: "main", branch: "lane" });
+    assert.deepEqual(r, { status: "error", detail: "boom" });
+  });
+});
+
+describe("orchestrate parked-PR sync seam (#4525)", () => {
+  it("syncs a parked needs-human PR while the batch runs; comments conflicting paths once per base sha", async () => {
+    const calls = [];
+    let release; const gate = new Promise((r) => { release = r; });
+    const decisions = [];
+    const seam = async (a) => { calls.push(a); if (calls.length >= 3) release(); return a.last ? { status: "unchanged", base: a.last } : { status: "conflict", base: "b".repeat(40), paths: ["x.ts", "y.ts"] }; };
+    const rows = await orchestrate({ issues: [1, 2], parallel: 2, readBody: async () => "", pollMs: 5, syncEveryMs: 10, syncParked: seam,
+      runOne: async (n) => { if (n === 1) return { terminalReason: "needs-human", pr: 101, detail: "stuck" }; await gate; return { terminalReason: "merged", pr: 102 }; },
+      decide: (d) => { decisions.push(d); } });
+    assert.equal(rows.find((r) => r.issue === 1).outcome, "needs-human");
+    assert.ok(calls.length >= 3 && calls.every((c) => c.issue === 1 && c.pr === 101));
+    assert.equal(calls[0].last, null);
+    assert.equal(calls[1].last, "b".repeat(40));
+    const mine = decisions.filter((d) => d.issue === 1 && /conflicts with its base/.test(d.what));
+    assert.equal(mine.length, 1);
+    assert.match(mine[0].what, /x\.ts, y\.ts/);
+  });
+  it("without the seam nothing is synced and rows are unchanged", async () => {
+    const decisions = [];
+    const rows = await orchestrate({ issues: [1, 2], parallel: 2, readBody: async () => "", syncEveryMs: 1, decide: (d) => { decisions.push(d); },
+      runOne: async (n) => (n === 1 ? { terminalReason: "needs-human", pr: 101, detail: "stuck" } : { terminalReason: "merged", pr: 102 }) });
+    assert.deepEqual(rows.map((r) => r.outcome), ["needs-human", "merged"]);
+    assert.equal(decisions.length, 0);
+  });
+});
+
+describe("readOrchestration base_sync (#4525)", () => {
+  const read = (yaml) => { const d = tmpDir("basesync-cfg-"); writeFileSync(join(d, "forge.yaml"), yaml); return readOrchestration(d).baseSync; };
+  it("defaults on with no globs; parses enabled and union_globs; false opts out", () => {
+    assert.deepEqual(read("project:\n  name: x\n"), { enabled: true, unionGlobs: [] });
+    assert.deepEqual(read("orchestration:\n  base_sync:\n    enabled: true\n    union_globs: \"reg/*.ts, CHANGELOG.md\"\n"), { enabled: true, unionGlobs: ["reg/*.ts", "CHANGELOG.md"] });
+    assert.equal(read("orchestration:\n  base_sync:\n    enabled: false\n").enabled, false);
+    assert.equal(read("orchestration:\n  base_sync_enabled: false\n").enabled, false);
+  });
+});

package/bin/lean/batch.mjs

@@ -181,7 +181,10 @@ export async function waitLane({ batch, issue, pid, alive = isLaneAlive, sleep =
   }
 }
 
-/** Lane seams for `orchestrate()`: `runOne` launches a detached lane, `adopt` re-attaches to a live one (else null). */
+/**
+ * Lane seams for `orchestrate()`: `runOne` launches a detached lane, `adopt` re-attaches to a live one (else null). A lane
+ * that already ended resolves at once with its result file; a live one calls `ctx.onLive()` before it is awaited (#4504).
+ */
 export function detachedLanes({ batch, laneArgs, script, cwd, env, bin, alive = isLaneAlive, pollMs, spawnFn }) {
   return {
     runOne: async (n, lane, ctx = {}) => {
@@ -189,9 +192,12 @@ export function detachedLanes({ batch, laneArgs, script, cwd, env, bin, alive =
       ctx.onStart?.({ pid });
       return waitLane({ batch, issue: n, pid, alive, pollMs });
     },
-    adopt: async (n, rec) => {
+    adopt: async (n, rec, ctx = {}) => {
       const rf = batch.resultFile(n);
-      if (!existsSync(rf) && !alive(rec.pid, rf)) return null;
+      if (!existsSync(rf)) {
+        if (!alive(rec.pid, rf)) return null;
+        ctx.onLive?.();
+      }
       return waitLane({ batch, issue: n, pid: rec.pid, alive, pollMs });
     },
   };

package/bin/lean/batch.test.mjs

@@ -7,7 +7,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
 import { orchestrate, deferPolicy, DEFER_BACKOFF_S } from "./orchestrate.mjs";
-import { openBatch, fsTypeOf, assertDiskPath, isLaneAlive, writeAtomic, launchLane } from "./batch.mjs";
+import { openBatch, fsTypeOf, assertDiskPath, isLaneAlive, writeAtomic, launchLane, detachedLanes } from "./batch.mjs";
 import { orchestrateFromCli, runFromCli, readOrchestration, makeRelabelNeedsHuman, makeDispatchCheck } from "../engine-cli.mjs";
 
 // orchestrateFromCli opens the host lane registry (#4191): keep it in a temp dir, also for spawned dispatchers.
@@ -120,6 +120,16 @@ await orchestrateFromCli(["1", "2", "3", "--parallel", "2", "--repo", "o/r", "--
     assert.equal(starts[1].length, 2);
   });
 
+  it("#4518 cli.host_killed deferrals are requeued and never trip the breaker", async () => {
+    const { now, sleep } = fakeClock();
+    const updates = []; let calls = 0;
+    const rows = await orchestrate({ issues: [1], readBody: async () => "", now, sleep, maxDeferrals: 3, breaker: { errors: 1, window_min: 30 },
+      onUpdate: (r) => updates.push(r),
+      runOne: async () => (++calls <= 2 ? { terminalReason: "deferred", code: "cli.host_killed", retryInSeconds: 60, detail: "builder host kill" } : { terminalReason: "merged", pr: 5 }) });
+    assert.equal(rows[0].outcome, "merged");
+    assert.ok(!updates.some((u) => u.outcome === "breaker" || u.outcome === "engine-error"));
+  });
+
   it("the breaker stops admissions after N engine-errors in M minutes and lets running lanes finish", async () => {
     const started = [];
     let releaseSlow;
@@ -203,9 +213,9 @@ await orchestrateFromCli(["1", "2", "3", "--parallel", "2", "--repo", "o/r", "--
     assert.doesNotThrow(() => assertDiskPath("/home/u/.forge/batches", mounts));
     assert.equal(isLaneAlive(process.pid, "--definitely-not-in-my-cmdline"), false);
     assert.equal(isLaneAlive(2 ** 22 + 12345, null), false);
-    assert.deepEqual(readOrchestration(root), { breaker: { errors: 3, window_min: 30 }, maxDeferrals: 3, globalParallel: null, recovery: { maxAttempts: 1 } });
+    assert.deepEqual(readOrchestration(root), { breaker: { errors: 3, window_min: 30 }, maxDeferrals: 3, globalParallel: null, recovery: { maxAttempts: 1 }, baseSync: { enabled: true, unionGlobs: [] } });
     writeFileSync(join(root, "forge.yaml"), "orchestration:\n  breaker:\n    errors: 5\n    window_min: 10\n  max_deferrals: 2\n  global_parallel: 4\n");
-    assert.deepEqual(readOrchestration(root), { breaker: { errors: 5, window_min: 10 }, maxDeferrals: 2, globalParallel: 4, recovery: { maxAttempts: 1 } });
+    assert.deepEqual(readOrchestration(root), { breaker: { errors: 5, window_min: 10 }, maxDeferrals: 2, globalParallel: 4, recovery: { maxAttempts: 1 }, baseSync: { enabled: true, unionGlobs: [] } });
   });
 
   it("run-issue --result-file writes the lane result, also when the run throws", async () => {
@@ -629,3 +639,124 @@ describe("#4481: a closed issue or a merged lane PR is skipped at dispatch", ()
     assert.equal(await mk(new Error("x"), new Error("y"))(9), null);
   });
 });
+
+describe("#4504: a resume settles lanes that ended while no dispatcher ran", () => {
+  it("#4504: a resume settles a dead lane's result and requeues a dead lane without one before scheduling, while an earlier live lane still runs", async () => {
+    const dir = tmpDir("batch-4504-");
+    const batch = { resultFile: (n) => join(dir, `${n}.json`), logFile: (n) => join(dir, `${n}.log`) };
+    writeFileSync(batch.resultFile(2), JSON.stringify({ terminalReason: "merged", pr: 102 }));
+    const { adopt } = detachedLanes({ batch, laneArgs: () => [], script: "x", alive: (pid) => pid === 11, pollMs: 5 });
+    let saved = { v: 1, order: [1, 2, 3, 4], deps: { 1: [], 2: [], 3: [], 4: [] }, lanes: {}, crossSeen: [], results: {},
+      running: { 1: { startedAt: 1, pid: 11 }, 2: { startedAt: 1, pid: 22 }, 3: { startedAt: 1, pid: 33 } }, pending: [4],
+      notBefore: {}, requeues: {}, deferrals: {}, engineErrors: {}, prefilter: {}, errorTimes: [], breaker: null };
+    const store = { load: () => saved, save: (s) => { saved = s; }, log: () => {} };
+    const launched = [];
+    const done = orchestrate({ issues: [], parallel: 3, pollMs: 5, readBody: async () => "", store, adopt,
+      runOne: async (n) => { launched.push(n); return { terminalReason: "merged", pr: 100 + n }; } });
+    let rows;
+    try {
+      await until(() => launched.includes(3) && launched.includes(4) && saved.results?.["2"], 3000);
+      assert.equal(saved.results["2"].outcome, "merged", "the dead lane's result is settled");
+      assert.equal(saved.results["2"].pr, 102);
+      assert.ok(saved.running["1"], "the live lane is still running");
+      assert.equal(saved.running["2"], undefined, "the settled lane left running");
+      assert.ok(!launched.includes(1) && !launched.includes(2), "neither adopted lane is relaunched");
+    } finally {
+      writeFileSync(batch.resultFile(1), JSON.stringify({ terminalReason: "merged", pr: 101 })); // the live lane ends
+      rows = await done;
+    }
+    assert.deepEqual(rows.map((r) => [r.issue, r.outcome, r.pr]), [[1, "merged", 101], [2, "merged", 102], [3, "merged", 103], [4, "merged", 104]]);
+    assert.deepEqual(launched.sort(), [3, 4], "the dead lane without a result is requeued and run once");
+  });
+});
+
+describe("#4528: a full sandbox pool (sandbox.capacity) is never capped", () => {
+  it("sandbox.capacity is uncapped and honours retryInSeconds; other codes still cap at maxDeferrals", () => {
+    assert.deepEqual(deferPolicy({ code: "sandbox.capacity" }, { now: 0, deferrals: 99 }), { delayMs: 60000, host: true });
+    assert.deepEqual(deferPolicy({ code: "sandbox.capacity", retryInSeconds: 30 }, { now: 0, deferrals: 99, maxDeferrals: 3 }), { delayMs: 30000, host: true });
+    for (const code of ["merge.transient", "gh.unavailable", "cli.idle", "rate_limited", "sandbox.start_failed"]) {
+      assert.deepEqual(deferPolicy({ code, retryInSeconds: 60 }, { now: 0, deferrals: 3, maxDeferrals: 3 }), { capped: true }, code);
+    }
+  });
+
+  it("with K above the pool size every issue eventually runs and none is skipped for capacity", async () => {
+    const c = { t: 1_000_000 };
+    const now = () => c.t;
+    // Fake clock that advances after a macrotask, so running lanes progress while deferred ones wait.
+    const sleep = async (ms) => { await new Promise((r) => setImmediate(r)); c.t += ms; };
+    const POOL = 2;
+    let inUse = 0;
+    const capacityDefers = new Map();
+    const rows = await orchestrate({ issues: [1, 2, 3, 4, 5, 6], parallel: 4, maxDeferrals: 3, now, sleep, readBody: async () => "",
+      runOne: async (n) => {
+        if (inUse >= POOL) {
+          capacityDefers.set(n, (capacityDefers.get(n) ?? 0) + 1);
+          return { terminalReason: "deferred", code: "sandbox.capacity", retryInSeconds: 60, detail: "sandbox launch: HTTP 503" };
+        }
+        inUse++;
+        try { await new Promise((r) => setTimeout(r, 20)); } finally { inUse--; }
+        return { terminalReason: "merged", pr: 100 + n };
+      } });
+    assert.deepEqual(rows.map((r) => [r.issue, r.outcome]).sort((a, b) => a[0] - b[0]), [1, 2, 3, 4, 5, 6].map((n) => [n, "merged"]));
+    assert.ok(!rows.some((r) => /gave up/.test(r.detail ?? "")));
+    assert.ok(Math.max(...capacityDefers.values()) > 3, "an issue was deferred for capacity more than maxDeferrals times");
+  });
+});
+
+describe("#4540: remote admissions are capped to the provider's free capacity", () => {
+  const run = async (capacity) => {
+    const c = { t: 1_000_000 };
+    const sleep = async (ms) => { await new Promise((r) => setImmediate(r)); c.t += ms; };
+    let live = 0, peak = 0, bounced = 0;
+    const POOL = 2;
+    const rows = await orchestrate({ issues: [1, 2, 3, 4, 5], parallel: 5, now: () => c.t, sleep, pollMs: 1, readBody: async () => "",
+      remote: { max: 5, isRemote: () => true, capacity: capacity(() => POOL - live) },
+      runOne: async (n) => {
+        if (live >= POOL) { bounced++; return { terminalReason: "deferred", code: "sandbox.capacity", retryInSeconds: 1, detail: "sandbox launch: HTTP 503" }; }
+        live++; peak = Math.max(peak, live);
+        try { await new Promise((r) => setTimeout(r, 10)); } finally { live--; }
+        return { terminalReason: "merged", pr: 100 + n };
+      } });
+    return { rows, peak, bounced };
+  };
+  it("queues beyond the free slots inside the batch: no 503 bounce, every issue still runs", async () => {
+    let at = 0;
+    const { rows, bounced } = await run((free) => () => ({ free: free(), at: ++at })); // a fresh sample on every read
+    assert.equal(bounced, 0);
+    assert.deepEqual(rows.map((r) => r.outcome), ["merged", "merged", "merged", "merged", "merged"]);
+  });
+  it("unknown capacity (null) admits as before and the 503 deferral path is the backstop", async () => {
+    const { rows, bounced } = await run(() => () => null);
+    assert.ok(bounced > 0);
+    assert.deepEqual(rows.map((r) => r.outcome).filter((o) => o !== "merged"), []);
+  });
+});
+
+describe("#4547: opt-in local fallback in mode remote when no pool agent is online", () => {
+  const run = async (remoteExtra, online) => {
+    const c = { t: 1_000_000 };
+    const sleep = async (ms) => { await new Promise((r) => setImmediate(r)); c.t += ms; };
+    const where = new Map();
+    let remoteTries = 0;
+    const rows = await orchestrate({ issues: [1, 2, 3], parallel: 3, now: () => c.t, sleep, pollMs: 1, readBody: async () => "",
+      remote: { max: 3, isRemote: () => true, capacity: () => (online === null ? null : { free: null, online, at: 1 }), ...remoteExtra },
+      runOne: async (n, _ln, ctx = {}) => {
+        where.set(n, ctx.placement?.where ?? "remote");
+        if (ctx.placement?.where !== "local") { remoteTries++; if (online === 0) return { terminalReason: "deferred", code: "sandbox.capacity", retryInSeconds: 1, detail: "HTTP 503" }; }
+        return { terminalReason: "merged", pr: 100 + n };
+      } });
+    return { rows, where, remoteTries };
+  };
+  it("every agent offline: the batch completes on the host", async () => {
+    const { rows, where, remoteTries } = await run({ offlineFallback: true }, 0);
+    assert.deepEqual(rows.map((r) => r.outcome), ["merged", "merged", "merged"]);
+    assert.ok([...where.values()].every((w) => w === "local"));
+    assert.equal(remoteTries, 0);
+  });
+  it("an agent online, or liveness unknown, stays remote", async () => {
+    for (const online of [1, null]) {
+      const { where } = await run({ offlineFallback: true }, online);
+      assert.ok([...where.values()].every((w) => w === "remote"), String(online));
+    }
+  });
+});

package/bin/lean/budget.mjs

@@ -5,7 +5,8 @@
  * else the frozen contract's distinct deliverable paths (S <= 2, M <= 5, else L). The budget for the size is forge.yaml
  * `engine.turn_budget_s|m|l` (defaults 25/60/100; 0 disables that size). The implement stage (the build call and its
  * context/timeout restarts) spends it; on overrun the builder gets one wrap-up grant to finish or stop with a digest, and
- * an unfinished lane ends needs-human `budget.exceeded` (recoverable once by the batch). Pure module: no I/O.
+ * an unfinished lane ends the non-human `budget-exceeded` reason (code `budget.exceeded`, #4505): the batch recovers it once
+ * from the checkpoint at the next size (`recoverySize`); only a final recovery that overruns too ends needs-human. Pure module: no I/O.
  */
 
 export const TURN_BUDGET_DEFAULTS = Object.freeze({ S: 25, M: 60, L: 100 });
@@ -36,6 +37,11 @@ export function laneSize({ contract = null, labels = [] } = {}) {
   return n <= 2 ? "S" : n <= 5 ? "M" : "L";
 }
 
+/** #4505: the next size up (S -> M -> L; L stays L); anything else unchanged. */
+export const nextSize = (size) => (SIZES.includes(size) ? SIZES[Math.min(SIZES.indexOf(size) + 1, SIZES.length - 1)] : size);
+/** #4505: a `budget.exceeded` recovery lane runs at the next size's budget; any other lane keeps its size. */
+export const recoverySize = (size, recover) => (recover?.code === "budget.exceeded" ? nextSize(size) : size);
+
 /** Turns the wrap-up call may use after the budget is spent. */
 export const wrapupGrant = (budget) => Math.max(5, Math.ceil(budget / 5));
 

package/bin/lean/budget.test.mjs

@@ -5,7 +5,7 @@ import { describe, it } from "node:test";
 import assert from "node:assert/strict";
 import { writeFileSync, chmodSync } from "node:fs";
 import { tmpDir } from "../tests/_tmp.mjs";
-import { TURN_BUDGET_DEFAULTS, parseTurnBudgets, laneSize, wrapupGrant, makeTurnBudget, budgetDigest, formatTurnSummary } from "./budget.mjs";
+import { TURN_BUDGET_DEFAULTS, parseTurnBudgets, laneSize, nextSize, recoverySize, wrapupGrant, makeTurnBudget, budgetDigest, formatTurnSummary } from "./budget.mjs";
 import { orchestrate } from "./orchestrate.mjs";
 import { OUTCOME_CODES, RECOVERABLE_CODES } from "../engine/outcome-codes.mjs";
 import { RECOVER_HOW } from "./lean.mjs";
@@ -34,6 +34,14 @@ describe("#4392 turn budget", () => {
     assert.equal(laneSize({}), null);
   });
 
+  it("#4505 a budget.exceeded recovery runs at the next size (S->M->L, L stays L); other lanes keep their size", () => {
+    assert.deepEqual(["S", "M", "L"].map(nextSize), ["M", "L", "L"]);
+    assert.equal(nextSize(null), null); assert.equal(nextSize("XL"), "XL");
+    assert.equal(recoverySize("S", { code: "budget.exceeded", attempt: 1, final: true }), "M");
+    assert.equal(recoverySize("L", { code: "budget.exceeded" }), "L");
+    assert.equal(recoverySize("S", { code: "ci.red" }), "S"); assert.equal(recoverySize("S", null), "S"); assert.equal(recoverySize(null, { code: "budget.exceeded" }), null);
+  });
+
   it("the tally: build turns spend the budget, other attempts count only in total; off or unsized is unlimited", () => {
     const tb = makeTurnBudget({ M: 60 });
     assert.equal(tb.record(), null); assert.equal(tb.remaining(), Infinity);

package/bin/lean/conformance.mjs

@@ -261,18 +261,28 @@ export function validateContract(c, issueText, { repro = false } = {}) {
   return { ok: problems.length === 0, problems };
 }
 
+/** forge#4519: a shell pattern as in scripts/hotspots.sh `case` (`*` crosses `/`, `?` one character), anchored. */
+const shellGlobRe = (g) => new RegExp(`^${String(g).replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".")}$`);
+
 /**
  * Apply result.json amendments to the frozen contract. Kinds: add-deliverable {path}, add-requirement {requirement},
  * defer|post-deploy {requirement: id}. Every amendment needs a why. Returns {contract, applied[], toTrack[], problems[]}.
+ * `generatedPaths` (forge#4519, forge.yaml maintenance.generated_paths via scripts/hotspots.sh): an add-deliverable on a
+ * matching path is refused; the repo's generator or bot owns that file.
  */
-export function applyAmendments(frozen, amendments, issueText = null) {
+export function applyAmendments(frozen, amendments, issueText = null, { generatedPaths = [] } = {}) {
   const c = structuredClone(frozen);
   c.deliverables = c.deliverables ?? [];
   const applied = [], toTrack = [], problems = [];
+  const genRes = (Array.isArray(generatedPaths) ? generatedPaths : []).filter((g) => typeof g === "string" && g).map(shellGlobRe);
   for (const a of Array.isArray(amendments) ? amendments : []) {
     const tag = JSON.stringify(a ?? null).slice(0, 100);
     if (blank(a?.why)) { problems.push(`amendment without why: ${tag}`); continue; }
     if (a.kind === "add-deliverable" && !blank(a.path)) {
+      if (genRes.some((re) => re.test(cleanPath(a.path)))) {
+        problems.push(`add-deliverable ${a.path}: generated path (forge.yaml maintenance.generated_paths) is owned by the repo's generator/bot; revert the file instead of amending it in`);
+        continue;
+      }
       const have = c.deliverables.find((d) => d.path === a.path);
       // An existing deliverable keeps its id and gains the why, so an amendment naming a package on an already-listed
       // package.json declares the edge (forge#4159); before, it was reported as applied but changed nothing.
@@ -343,9 +353,11 @@ export function repoTracks(paths) {
  * `frozen` (#4350): the contract before typed amendments. A failed scope-check validate (isScopeCheck) whose only files
  * outside `frozen` are covered by deliverables an amendment added, with no scope creep left, is superseded by the engine's
  * scope gate: listed in `superseded` [{id, files}], not a failure. Without `frozen`, `superseded` is always [].
+ * `ciCovered` (#4527): the validate ids accepted only because the head is CI-verified (no local result, or a missing host
+ * tool), for the trajectory; always an array.
  */
 export function precheck({ contract, result, changedFiles, ignore = [], ciVerified = false, hotspots = null, repro = null, validateRuns = null, repoHas = () => false, frozen = null }) {
-  const failures = [], superseded = [];
+  const failures = [], superseded = [], ciCovered = [];
   const outside = (k) => changedFiles.filter((f) => !ignore.some((p) => f.startsWith(p)) && !authorizationOf(k, f));
   const creep = outside(contract);
   const amendedOnly = frozenAmendedFiles(frozen, contract, outside, creep);
@@ -355,13 +367,13 @@ export function precheck({ contract, result, changedFiles, ignore = [], ciVerifi
     const r = runs.find((x) => x?.id === v.id);
     // #4167: a head the engine verified on CI covers validate ids the builder left to CI (no result, or ci:true without an exit)
     const leftToCi = !r || (r.ci === true && r.exit === undefined);
-    if (ciVerified && leftToCi) continue;
+    if (ciVerified && leftToCi) { ciCovered.push(v.id); continue; }
     if (leftToCi) failures.push({ code: C.CONFORMANCE_UNIMPLEMENTED, kind: "validate_missing", detail: `validate ${v.id} (\`${v.command}\`) has no recorded result` });
     else if (r.exit !== 0) {
       const rerun = validateRuns?.[v.id];
       if (rerun?.rc === 0) continue;
       const miss = rerun ? missingTool(rerun, { repoHas, command: v.command }) : null;
-      if (miss && ciVerified) continue;
+      if (miss && ciVerified) { ciCovered.push(v.id); continue; }
       if (miss) failures.push({ code: C.ENV_TOOL_MISSING, kind: "validate_env", tool: miss.tool, detail: `validate ${v.id} (\`${v.command}\`) could not run: the host is missing ${KIND_NAME[miss.kind]} '${miss.tool}' (environment, not the change)` });
       else if (amendedOnly && isScopeCheck(v.command)) superseded.push({ id: v.id, files: amendedOnly });
       else failures.push({ code: C.CONFORMANCE_UNIMPLEMENTED, kind: "validate_failed", detail: `validate ${v.id} (\`${v.command}\`) failed (exit ${r.exit})` });
@@ -382,7 +394,7 @@ export function precheck({ contract, result, changedFiles, ignore = [], ciVerifi
   }
   failures.push(...hotspotGrowth({ contract, result, hotspots, changedFiles }));
   if (repro) failures.push(...reproFailures(contract, repro));
-  return { ok: failures.length === 0, failures, superseded };
+  return { ok: failures.length === 0, failures, superseded, ciCovered };
 }
 
 /**
@@ -419,7 +431,8 @@ export function reproFailures(contract, { base, head } = {}) {
 
 const EXEMPT_RE = /hotspot-exempt:\s*\S/i;
 /**
- * forge#4195: a changed file already over `file_lines` (not generated, vendored or glob-excluded) that grew by more
+ * forge#4195: a changed file already over `file_lines` (not generated, vendored or glob-excluded; forge#4519: a
+ * `generated` row, from linguist-generated, maintenance.generated_paths or a generated header, never counts) that grew by more
  * than `growth_lines` since the base, with no `hotspot-exempt: <reason>` in a covering deliverable's change or an
  * add-deliverable amendment's why. `hotspots` is scripts/hotspots.sh --base output; null or disabled checks nothing.
  */
@@ -431,7 +444,7 @@ export function hotspotGrowth({ contract, result, hotspots, changedFiles = [] })
   const exempt = (f) => (contract?.deliverables ?? []).some((d) => pathCovers(d.path, f) && EXEMPT_RE.test(String(d.change ?? "")))
     || amends.some((a) => a?.kind === "add-deliverable" && pathCovers(a.path, f) && EXEMPT_RE.test(String(a.why ?? "")));
   return hotspots.files
-    .filter((h) => h?.over && !h.excluded && Number(h.growth) > limit && changed.has(cleanPath(h.path)) && !exempt(h.path))
+    .filter((h) => h?.over && !h.excluded && !h.generated && Number(h.growth) > limit && changed.has(cleanPath(h.path)) && !exempt(h.path))
     .map((h) => ({ code: C.MAINTENANCE_HOTSPOT_GROWTH, kind: "hotspot_growth",
       detail: `${h.path} grew by ${h.growth} lines (now ${h.lines}, limit +${limit} over ${max}) without hotspot-exempt` }));
 }

package/bin/lean/conformance.test.mjs

@@ -62,7 +62,7 @@ describe("amendments", () => {
 
 describe("precheck (deterministic, no reviewer)", () => {
   const res = { requirements: [{ id: "R1", status: "implemented" }, { id: "R2", status: "implemented" }], validation: [{ id: "V1", exit: 0 }] };
-  it("passes a complete, validated, in-scope change", () => assert.deepEqual(precheck({ contract: C, result: res, changedFiles: files }), { ok: true, failures: [], superseded: [] }));
+  it("passes a complete, validated, in-scope change", () => assert.deepEqual(precheck({ contract: C, result: res, changedFiles: files }), { ok: true, failures: [], superseded: [], ciCovered: [] }));
   it("unauthorized file -> scope_creep; tests and ignored paths are always allowed", () => {
     assert.deepEqual(codes(precheck({ contract: C, result: res, changedFiles: [...files, "src/billing.js", ".forge-lean/x", "tests/test_x.py", "pkg/foo_test.go"], ignore: [".forge-lean/"] })), ["conformance.scope_creep"]);
     for (const t of ["test/a.js", "src/a.test.ts", "tests/test_a.py", "a_test.go", "spec/a.spec.js"]) assert.ok(isTestPath(t), t);
@@ -85,6 +85,15 @@ describe("precheck (deterministic, no reviewer)", () => {
     const local = precheck({ contract: C, result: failed, changedFiles: files, repoHas: repoTracks(["src/leak/__init__.py"]), validateRuns: { V1: { rc: 1, output: "ModuleNotFoundError: No module named 'leak.core'" } } });
     assert.deepEqual(codes(local), ["conformance.unimplemented"], "a repo module the change did not create is a code defect");
   });
+  it("#4527 a CI-verified head lists the validate ids it accepted without a local result; a ran-and-failed validate still fails", () => {
+    const C2 = { ...C, validate: [{ id: "V1", command: "node --test a.test.mjs" }, { id: "V2", command: "tsc" }, { id: "V3", command: "lint" }] };
+    const pc = precheck({ contract: C2, result: { ...res, validation: [{ id: "V2", ci: true }, { id: "V3", exit: 0 }] }, changedFiles: files, ciVerified: true });
+    assert.deepEqual(codes(pc), []); assert.deepEqual(pc.ciCovered, ["V1", "V2"]);
+    const local = precheck({ contract: C2, result: { ...res, validation: [{ id: "V2", ci: true }, { id: "V3", exit: 0 }] }, changedFiles: files });
+    assert.deepEqual(local.failures.map((f) => f.kind), ["validate_missing", "validate_missing"]); assert.deepEqual(local.ciCovered, []);
+    const failed = precheck({ contract: C2, result: { ...res, validation: [{ id: "V1", exit: 1 }, { id: "V2", ci: true }, { id: "V3", exit: 0 }] }, changedFiles: files, ciVerified: true });
+    assert.deepEqual(failed.failures.map((f) => f.kind), ["validate_failed"]); assert.deepEqual(failed.ciCovered, ["V2"]);
+  });
   it("#4323 missingTool: commands, python and node modules; paths, relative and repo-tracked names are not the environment", () => {
     assert.deepEqual(missingTool({ rc: 127, output: "sh: 1: jq: not found" }), { tool: "jq", kind: "command" });
     assert.deepEqual(missingTool({ rc: 127, output: "" }, { command: "uv run pytest -q" }), { tool: "uv", kind: "command" });
@@ -362,6 +371,24 @@ describe("hotspot growth precheck (forge#4195)", () => {
     assert.match(pc.failures[0].detail, /^src\/scrape\.js grew by 200 lines \(now 2600, limit \+150 over 2000\) without hotspot-exempt$/);
   });
 
+  it("#4519 generated paths: applyAmendments refuses add-deliverable on maintenance.generated_paths, hotspotGrowth skips generated rows", () => {
+    const gen = ["web/public/openapi*.json", "web/public/openapi-versions/*"];
+    const amends = [{ kind: "add-deliverable", path: "web/public/openapi-versions/3.29.2.json", why: "sync the spec archive" },
+      { kind: "add-deliverable", path: "./web/public/openapi.json", why: "sync the spec" }, { kind: "add-deliverable", path: "src/shared.js", why: "helper" }];
+    const a = applyAmendments(C, amends, null, { generatedPaths: gen });
+    assert.deepEqual(a.applied.map((x) => x.target), ["src/shared.js"]);
+    assert.equal(a.problems.length, 2);
+    assert.match(a.problems[0], /^add-deliverable web\/public\/openapi-versions\/3\.29\.2\.json: generated path \(forge\.yaml maintenance\.generated_paths\)/);
+    assert.ok(!a.contract.deliverables.some((d) => d.path.includes("openapi")), "a refused path never becomes a deliverable");
+    assert.equal(applyAmendments(C, amends).applied.length, 3, "without generatedPaths every add-deliverable applies as before");
+    assert.equal(applyAmendments(C, amends, null, { generatedPaths: ["web/public/openapi.json"] }).applied.length, 2, "a literal glob matches only that path");
+    // linguist-generated / generated_paths / header rows (hotspots.sh `generated`) never trigger hotspot_growth
+    const big = H({ path: "src/scrape.js", lines: 83974, growth: 83974 });
+    assert.equal(run(hs([big])).ok, false);
+    assert.equal(run(hs([{ ...big, generated: true }])).ok, true);
+    assert.equal(run(hs([{ ...big, generated: true, excluded: true }])).ok, true);
+  });
+
   it("within the limit, under the threshold, excluded, unchanged, disabled or no facts: no failure", () => {
     for (const h of [hs([H({ growth: 150 })]), hs([H({ over: false, lines: 1900 })]), hs([H({ excluded: true })]), hs([H({ path: "src/other.js" })]),
       hs([H()], { enabled: false }), hs([H({ growth: null })]), null, undefined, {}]) assert.equal(run(h).ok, true, JSON.stringify(h));
@@ -398,7 +425,7 @@ describe("repro_test on bug lanes (#4271)", () => {
   });
   it("red on base and green on head passes; anything else fails the precheck as conformance.unimplemented", () => {
     const k = { ...C, repro_test: RT };
-    assert.deepEqual(precheck({ contract: k, result: res, changedFiles: files, repro: { base: "FAILED", head: "RAN" } }), { ok: true, failures: [], superseded: [] });
+    assert.deepEqual(precheck({ contract: k, result: res, changedFiles: files, repro: { base: "FAILED", head: "RAN" } }), { ok: true, failures: [], superseded: [], ciCovered: [] });
     for (const repro of [{ base: "RAN", head: "RAN" }, { base: "SKIPPED", head: "FAILED" }, { base: "UNRUNNABLE", head: "RAN" }, { base: "SKIPPED", head: "RAN_NOTHING" }]) {
       const pc = precheck({ contract: k, result: res, changedFiles: files, repro });
       assert.deepEqual(pc.failures.map((f) => [f.code, f.kind]), [["conformance.unimplemented", "repro_red_green"]], JSON.stringify(repro));

package/bin/lean/depcache.mjs

@@ -11,7 +11,9 @@
  *   <repoKey>/<eco>-<key>.tmp-<pid>   snapshot being published (atomic rename into place)
  *   stores/{pnpm-store,npm-cache,uv-cache,poetry-cache,pip-cache}   the package managers' own shared stores
  * The repo's own package manager does every install (frozen lockfile). Every effect is injected; nothing here throws:
- * a missing tool, another filesystem (EXDEV) or a failed install leaves the lane as it was before.
+ * a missing tool, another filesystem (EXDEV) or a failed install leaves the lane as it was before. A snapshot that fails
+ * to link or repair falls back to a fresh install in the lane (status materialize-fallback, #4522); failed only when that
+ * install fails too.
  */
 import * as nodeFs from "node:fs";
 import { createHash } from "node:crypto";
@@ -28,7 +30,17 @@ const TMP_RE = /^(pnpm|npm|yarn|uv|poetry)-[0-9a-f]{16}\.tmp-(\d+)$/;
 const LOCK_RE = /^(pnpm|npm|yarn|uv|poetry)-[0-9a-f]{16}\.lock$/;
 
 const pidAlive = (pid) => { try { process.kill(pid, 0); return true; } catch (e) { return e?.code === "EPERM"; } };
-const msg = (e) => String(e?.stderr || e?.message || e).trim().split("\n").slice(-3).join(" ").slice(0, 300);
+/**
+ * Error text for a reason: pnpm prints its ERR_PNPM_* errors on stdout (#4522), so the first such line from either
+ * stream leads, then the last lines of stderr and stdout; the message only when both streams are empty.
+ */
+const msg = (e) => {
+  const lines = (s) => String(s ?? "").split("\n").map((l) => l.trim()).filter(Boolean);
+  const err = lines(e?.stderr), out = lines(e?.stdout);
+  const pm = [...err, ...out].find((l) => /\bERR_PNPM_[A-Z0-9_]+/.test(l));
+  const parts = [pm, ...err.slice(-3), ...out.slice(-3)].filter((l, i, a) => l && a.indexOf(l) === i);
+  return (parts.length ? parts.join(" ") : String(e?.message || e).trim().split("\n").slice(-3).join(" ")).slice(0, 500);
+};
 
 /** Lockfile -> ecosystem, in priority order: at most one JS and one Python ecosystem per repo. */
 function detect(fs, wt) {
@@ -215,7 +227,8 @@ export function makeDepCache({ root = depcacheRoot(), exec, fs = nodeFs, now = (
         return done("hit", { key });
       } catch (e) {
         rmTrees(wt, created);
-        return done(e?.code === "EXDEV" ? "skipped" : "failed", { key, reason: e?.code === "EXDEV" ? "cache is on another filesystem than the worktree (EXDEV)" : `materialize failed: ${msg(e)}` });
+        if (e?.code === "EXDEV") return done("skipped", { key, reason: "cache is on another filesystem than the worktree (EXDEV)" });
+        return { fallback: `materialize failed: ${msg(e)}` }; // installed by the caller, outside the key lock
       }
     };
     const install = async () => {
@@ -235,6 +248,13 @@ export function makeDepCache({ root = depcacheRoot(), exec, fs = nodeFs, now = (
       const why = publish(wt, d, key, ins.trees, snap);
       return done("miss-installed", { key, published: !why, reason: why });
     });
+    if (r.locked && r.value?.fallback) {
+      // #4522: a broken snapshot must not leave the lane without dependencies. Install outside the lock so other lanes on
+      // this key are not serialised behind it; never publish (the snapshot exists and is never replaced).
+      const ins = await install();
+      if (ins.failed) return { ...ins.failed, reason: `${r.value.fallback}; ${ins.failed.reason}` };
+      return done("materialize-fallback", { key, published: false, reason: r.value.fallback });
+    }
     if (r.locked) return r.value ?? done("failed", { key, reason: "snapshot manifest unreadable" });
     // The lock stayed busy past the wait: install in this lane without touching the cache.
     const ins = await install();