forgedock 1.32.0 → 1.33.0

package/bin/lean/schedd.mjs

@@ -24,6 +24,9 @@
  * Shutdown (#4454): `close()` aborts every hosted batch's `signal` (its loop admits nothing more and leaves, lanes keep
  * running, no report is written), waits for them up to `stopMs`, and only then releases `schedd.lock` and the socket, so a
  * daemon started next never resumes a batch this one still dispatches. `exitOnSignals` exits 128+signo after that.
+ * Exit reasons (#4503): every exit logs a line: `idle, exiting`, `exiting (<signal>|uncaught: …)`, `lost lock|socket …,
+ * exiting` (checked every `lockCheckMs`: the lock no longer holds this pid, or the socket is gone; nothing it does not own
+ * is removed), then `exited (code N)`. Taking over a dead owner's lock is logged, so a SIGKILLed daemon leaves a trace.
  * `runBatch({argv, cwd, env, sink, keepBreaker, signal})` is injected (engine-cli `runBatch`); `sink` gets `batch(b)`, `row(r)`, `line(text)`.
  */
 import { closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, writeSync } from "node:fs";
@@ -35,6 +38,7 @@ import { batchRoot, listBatches, openBatch } from "./batch.mjs";
 
 export const IDLE_MS = 10 * 60 * 1000;
 export const STOP_MS = 10 * 1000;
+export const LOCK_CHECK_MS = 5 * 1000;
 
 export function scheddPaths(dir = schedDir()) {
   return { dir, sock: join(dir, "schedd.sock"), lock: join(dir, "schedd.lock"), log: join(dir, "schedd.log") };
@@ -54,17 +58,19 @@ export function mergeEnv(base, kept) {
   return { ...out, ...kept };
 }
 
-/** Take the daemon lock; null when another live daemon holds it. */
+/** Take the daemon lock: `{took}` (the dead owner's pid it replaced, or null); null when another live daemon holds it. */
 function takeLock(lock, pid, alive) {
+  let took = null;
   for (let i = 0; i < 5; i++) {
     try {
       const fd = openSync(lock, "wx");
       try { writeSync(fd, String(pid)); } finally { closeSync(fd); }
-      return true;
+      return { took };
     } catch (e) { if (e?.code !== "EEXIST") throw e; }
     let owner = NaN;
     try { owner = parseInt(readFileSync(lock, "utf8"), 10); } catch { continue; }
     if (Number.isInteger(owner) && owner !== pid && alive(owner)) return null;
+    if (Number.isInteger(owner) && owner !== pid) took = owner;
     rmSync(lock, { force: true });
   }
   return null;
@@ -74,10 +80,13 @@ function takeLock(lock, pid, alive) {
  * Start the daemon. @returns {Promise<{running:true, pid, close, closed:Promise} | {running:false, owner:number}>}
  */
 export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(process.env.FORGEDOCK_SCHEDD_IDLE_MS) || IDLE_MS,
-  root = batchRoot(), log = (m) => console.log(m), pid = process.pid, alive = pidAlive, resumeOnStart = true, stopMs = STOP_MS } = {}) {
+  root = batchRoot(), log = (m) => console.log(m), pid = process.pid, alive = pidAlive, resumeOnStart = true, stopMs = STOP_MS,
+  lockCheckMs = LOCK_CHECK_MS } = {}) {
   const p = scheddPaths(dir);
   mkdirSync(dir, { recursive: true });
-  if (!takeLock(p.lock, pid, alive)) {
+  const lock = takeLock(p.lock, pid, alive);
+  if (lock?.took) log(`schedd: took over the lock of dead pid ${lock.took} (it exited without a clean shutdown)`);
+  if (!lock) {
     let owner = NaN;
     try { owner = parseInt(readFileSync(p.lock, "utf8"), 10); } catch { /* gone */ }
     log(`schedd: already running (pid ${owner})`);
@@ -88,7 +97,8 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
   const hosted = new Map(); // batch id -> {id, dir, subs:Set<fn>}
   const conns = new Set();
   const runs = new Map(); // every runBatch in flight (hosted or not yet started) -> its AbortController (#4454)
-  let closing = null, resolveClosed;
+  const starting = new Map(); // batch id -> host() promise of a start-up resume not yet hosted (#4503: an attach waits for it)
+  let closing = null, resolveClosed, keepSock = false;
   let lastActive = Date.now();
   const touch = () => { lastActive = Date.now(); };
 
@@ -143,6 +153,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       return send({ type: "batch", id: ent.id, dir: ent.dir });
     }
     if (req.op === "attach") {
+      if (!hosted.has(req.id) && starting.has(req.id)) await starting.get(req.id).catch(() => {});
       const live = hosted.get(req.id);
       if (live) {
         // Synchronous read + subscribe: no row is lost or doubled between the replay and the live stream.
@@ -191,6 +202,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
   const close = () => {
     if (closing) return closing;
     clearInterval(idleTimer);
+    clearInterval(lockTimer);
     for (const c of conns) c.destroy();
     const stopping = [...runs.keys()];
     for (const ac of runs.values()) ac.abort();
@@ -198,9 +210,11 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     // #4454: the lock goes only once every hosted batch loop has stopped (or `stopMs` passed: the caller exits regardless)
     const stopped = stopping.length ? Promise.race([Promise.allSettled(stopping).then(() => true), new Promise((r) => { timer = setTimeout(() => r(false), stopMs); })])
       .then((ok) => { clearTimeout(timer); if (!ok) log(`schedd: ${runs.size} batch(es) did not stop within ${stopMs} ms`); }) : Promise.resolve();
-    closing = Promise.all([new Promise((r) => server.close(() => r())), stopped]).then(() => {
+    // A lost lock: the socket path may be the successor's, and closing the server unlinks it (libuv), so leave it unref'd.
+    const serverClosed = keepSock ? Promise.resolve(server.unref()) : new Promise((r) => server.close(() => r()));
+    closing = Promise.all([serverClosed, stopped]).then(() => {
       try { if (parseInt(readFileSync(p.lock, "utf8"), 10) === pid) rmSync(p.lock, { force: true }); } catch { /* gone */ }
-      rmSync(p.sock, { force: true });
+      if (!keepSock) rmSync(p.sock, { force: true });
       resolveClosed();
     });
     return closing;
@@ -209,6 +223,15 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     if (!hosted.size && !conns.size && Date.now() - lastActive >= idleMs) { log("schedd: idle, exiting"); close(); }
   }, Math.min(idleMs, 30_000));
   idleTimer.unref?.();
+  // #4503: a daemon whose lock or socket was taken stops (its batches stay resumable) instead of running unreachable.
+  const lockTimer = setInterval(() => {
+    if (closing) return;
+    let owner = NaN;
+    try { owner = parseInt(readFileSync(p.lock, "utf8"), 10); } catch { /* gone */ }
+    if (owner !== pid) { keepSock = true; log(`schedd: lost lock (owner ${Number.isInteger(owner) ? `pid ${owner}` : "none"}), exiting`); close(); }
+    else if (!existsSync(p.sock)) { log(`schedd: lost socket ${p.sock}, exiting`); close(); }
+  }, lockCheckMs);
+  lockTimer.unref?.();
 
   // Daemon restart: resume every daemon-hosted batch that never wrote its report; live lanes are re-adopted.
   if (resumeOnStart) {
@@ -216,22 +239,35 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       const b = openBatch(root, id);
       if (!b.readPlan()?.daemon || b.readReport()) continue;
       log(`schedd: resuming batch ${id}`);
-      host({ argv: ["--resume", id], cwd: process.cwd(), env: {}, keepBreaker: true }).catch((e) => log(`schedd: resume ${id} failed: ${e?.message ?? e}`));
+      const started = host({ argv: ["--resume", id], cwd: process.cwd(), env: {}, keepBreaker: true });
+      starting.set(id, started);
+      started.catch((e) => log(`schedd: resume ${id} failed: ${e?.message ?? e}`)).finally(() => starting.delete(id));
     }
   }
   return { running: true, pid, sock: p.sock, close, closed };
 }
 
-/** #4454: on SIGTERM/SIGINT stop the daemon (`close()`: batch loops first, then lock and socket), then exit 128+signo, once. */
-export function exitOnSignals(d, { on = (s, f) => process.once(s, f), exit = (c) => process.exit(c), signals = ["SIGTERM", "SIGINT"] } = {}) {
-  const SIGNO = { SIGINT: 2, SIGTERM: 15 };
+/**
+ * #4454: on SIGTERM/SIGINT/SIGHUP stop the daemon (`close()`: batch loops first, then lock and socket), then exit 128+signo,
+ * once; an uncaught exception or unhandled rejection does the same and exits 1. #4503: each logs its reason first, and the
+ * process `exit` logs `schedd: exited (code N)`.
+ */
+export function exitOnSignals(d, { on = (s, f) => (s.startsWith("SIG") || s === "exit" ? process.once(s, f) : process.on(s, f)),
+  exit = (c) => process.exit(c), log = (m) => console.log(m), signals = ["SIGTERM", "SIGINT", "SIGHUP"] } = {}) {
+  const SIGNO = { SIGHUP: 1, SIGINT: 2, SIGTERM: 15 };
   let fired = false;
-  for (const sig of signals) on(sig, async () => {
+  const stop = async (reason, code, detail = null) => {
     if (fired) return;
     fired = true;
+    log(`schedd: exiting (${reason})`);
+    if (detail) log(detail);
     try { await d.close(); } catch { /* exit anyway */ }
-    exit(128 + (SIGNO[sig] ?? 0));
-  });
+    exit(code);
+  };
+  for (const sig of signals) on(sig, () => stop(sig, 128 + (SIGNO[sig] ?? 0)));
+  on("uncaughtException", (e) => stop(`uncaught: ${String(e?.message ?? e).split("\n")[0]}`, 1, e?.stack ?? null));
+  on("unhandledRejection", (e) => stop(`unhandled rejection: ${String(e?.message ?? e).split("\n")[0]}`, 1, e?.stack ?? null));
+  on("exit", (code) => log(`schedd: exited (code ${code})`));
 }
 
 /** Connect to the daemon, spawning it (`spawnDaemon()`) when nobody listens. @returns {Promise<import("node:net").Socket>} */

package/bin/lean/schedd.test.mjs

@@ -3,14 +3,23 @@
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
 import { spawn } from "node:child_process";
-import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { closeSync, existsSync, openSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
 import { openBatch, isLaneAlive, listBatches } from "./batch.mjs";
-import { startDaemon, scheddPaths, connectDaemon, request, mergeEnv, batchEnv } from "./schedd.mjs";
+import { startDaemon, scheddPaths, connectDaemon, request, mergeEnv, batchEnv, exitOnSignals } from "./schedd.mjs";
+import { schedDir } from "./sched.mjs";
 import { formatTable, orchestrate } from "./orchestrate.mjs";
 import { orchestrateFromCli } from "../engine-cli.mjs";
 
+// #4503: this process, and the env its clients submit for their lanes, never resolve the host's `~/.forge/sched`.
+process.env.FORGEDOCK_SCHED_DIR = tmpDir("schedd-home-");
+// Stub lanes run detached: a host whose test never reached its `finally` still has them reaped when the file ends.
+const hosts = [];
+process.on("exit", () => { for (const h of hosts) h.reap(); });
+const LANE_MAX_MS = 120_000;
+
 const until = async (cond, ms = 15000) => {
   const end = Date.now() + ms;
   while (!cond()) { if (Date.now() > end) throw new Error("timed out waiting"); await new Promise((r) => setTimeout(r, 25)); }
@@ -24,13 +33,20 @@ const capture = async (fn) => {
 /** A host: sched + batch dirs, a stub lane that waits for `release`, and a daemon script with injected lanes. */
 function host() {
   const work = tmpDir("schedd-");
-  const h = { work, sched: join(work, "s"), root: join(work, "batches"), launches: join(work, "launches"), release: join(work, "release"), procs: [] };
+  const h = { work, sched: join(work, "s"), root: join(work, "batches"), launches: join(work, "launches"), release: join(work, "release"), procs: [],
+    lanePidFile: join(work, "lane-pids"), log: join(work, "schedd.log") };
   writeFileSync(h.launches, "");
+  writeFileSync(h.lanePidFile, "");
+  hosts.push(h);
   const stub = join(work, "stub-lane.mjs");
   writeFileSync(stub, `import { appendFileSync, existsSync, writeFileSync, renameSync } from "node:fs";
 const a = process.argv.slice(2); const n = Number(a[1]); const rf = a[a.indexOf("--result-file") + 1];
 appendFileSync(${JSON.stringify(h.launches)}, n + "\\n");
-const t = setInterval(() => { if (!existsSync(${JSON.stringify(h.release)})) return; clearInterval(t);
+appendFileSync(${JSON.stringify(h.lanePidFile)}, process.pid + "\\n");
+// #4503: a stub whose test run is gone (work dir removed) or that outlived its bound exits instead of polling forever.
+const born = Date.now();
+const t = setInterval(() => { if (!existsSync(${JSON.stringify(work)}) || Date.now() - born > ${LANE_MAX_MS}) process.exit(0);
+  if (!existsSync(${JSON.stringify(h.release)})) return; clearInterval(t);
   writeFileSync(rf + ".tmp", JSON.stringify({ terminalReason: "merged", pr: 100 + n, cost_usd: 0 })); renameSync(rf + ".tmp", rf); }, 20);
 // Like run-issue's #4179 teardown: SIGTERM reports deferred run.interrupted.
 process.once("SIGTERM", () => { writeFileSync(rf + ".tmp", JSON.stringify({ terminalReason: "deferred", code: "run.interrupted", detail: "interrupted by SIGTERM" })); renameSync(rf + ".tmp", rf); process.exit(143); });
@@ -49,7 +65,9 @@ await d.closed;
   h.env = { ...process.env, FORGEDOCK_SCHED_DIR: h.sched, FORGEDOCK_BATCH_DIR: h.root, FORGEDOCK_LIMITS_FILE: join(work, "limits.json") };
   h.daemon = async () => {
     const before = existsSync(scheddPaths(h.sched).lock) ? readFileSync(scheddPaths(h.sched).lock, "utf8") : null;
-    const child = spawn(process.execPath, [h.daemonScript], { env: h.env, stdio: "ignore" });
+    const out = openSync(h.log, "a");
+    let child;
+    try { child = spawn(process.execPath, [h.daemonScript], { env: h.env, stdio: ["ignore", out, out] }); } finally { closeSync(out); }
     h.procs.push(child);
     await until(() => existsSync(scheddPaths(h.sched).sock) && existsSync(scheddPaths(h.sched).lock) && readFileSync(scheddPaths(h.sched).lock, "utf8") === String(child.pid) && readFileSync(scheddPaths(h.sched).lock, "utf8") !== before);
     // Wait for the socket to accept.
@@ -59,9 +77,15 @@ await d.closed;
   h.client = { scheddDir: h.sched, spawnDaemon: null };
   h.launched = () => readFileSync(h.launches, "utf8").trim().split("\n").filter(Boolean).map(Number).sort();
   h.lanePids = () => listBatches(h.root).flatMap((id) => Object.values(openBatch(h.root, id).store.load()?.running ?? {}).map((r) => r.pid).filter(Boolean));
-  h.cleanup = () => {
-    writeFileSync(h.release, "");
+  h.stubPids = () => (existsSync(h.lanePidFile) ? readFileSync(h.lanePidFile, "utf8").trim().split("\n").filter(Boolean).map(Number) : []);
+  h.reap = () => {
     for (const p of h.procs) { try { p.kill("SIGKILL"); } catch { /* gone */ } }
+    for (const pid of h.stubPids()) { try { process.kill(pid, "SIGKILL"); } catch { /* gone */ } }
+  };
+  h.cleanup = async () => {
+    if (existsSync(h.work)) writeFileSync(h.release, "");
+    h.reap();
+    await until(() => h.stubPids().every((pid) => !isLaneAlive(pid, null)), 5000).catch(() => {});
   };
   return h;
 }
@@ -98,7 +122,7 @@ describe("schedd host scheduler (#4192)", () => {
       const id = done[0].batch;
       await withRoot(h, () => capture(() => orchestrateFromCli(["--set-parallel", id, "5"], h.client)));
       assert.equal(openBatch(h.root, id).readParallel(), 5);
-    } finally { h.cleanup(); }
+    } finally { await h.cleanup(); }
   });
 
   it("a client that disconnects does not stop the batch; --resume reattaches, and replays the report once done", async () => {
@@ -127,7 +151,7 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
       assert.deepEqual(again.res.rows, res.rows);
       assert.ok(again.lines.join("\n").includes(formatTable(res.rows)));
       assert.deepEqual(h.launched(), [1, 2]);
-    } finally { h.cleanup(); }
+    } finally { await h.cleanup(); }
   });
 
   it("a daemon restart resumes the batch from state and re-adopts its live lanes without duplicate launches", async () => {
@@ -150,7 +174,7 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
       assert.deepEqual(res.rows.map((r) => [r.issue, r.outcome, r.pr]), [[1, "merged", 101], [2, "merged", 102], [3, "merged", 103]]);
       assert.deepEqual(h.launched(), [1, 2, 3], "adopted, not relaunched");
       assert.deepEqual(openBatch(h.root, id).store.load().running, {});
-    } finally { h.cleanup(); }
+    } finally { await h.cleanup(); }
   });
 
   it("#4408: --add and --resume <id> N add issues to the live batch (no second batch); an unhosted finished batch runs its adds on --resume", async () => {
@@ -181,7 +205,7 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
       }));
       assert.equal(res.rows.length, 3);
       assert.deepEqual(h.launched(), [1, 2, 4, 6], "each issue launched once");
-    } finally { h.cleanup(); }
+    } finally { await h.cleanup(); }
   });
 
   it("#4376: --cancel stops one batch (its lane torn down, the rest held, report written); the other batch and the daemon run on", async () => {
@@ -211,7 +235,7 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
         assert.equal(again.cancelled, false, "a finished batch is left alone");
         await assert.rejects(orchestrateFromCli(["--cancel", "nope"], h.client), /no batch nope/);
       }));
-    } finally { h.cleanup(); }
+    } finally { await h.cleanup(); }
   });
 
   it("#4481: a batch cancelled through the daemon is not resumed by a schedd restart", async () => {
@@ -232,7 +256,7 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
         assert.deepEqual(await statusIds(), [], "the restarted daemon hosts no cancelled batch");
         assert.deepEqual(h.launched(), [1]);
       }));
-    } finally { h.cleanup(); }
+    } finally { await h.cleanup(); }
   });
 
   it("single instance: a live daemon keeps its lock; a dead owner's lock and socket are taken over", async () => {
@@ -252,7 +276,7 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
         assert.equal(st.pid, process.pid);
       } finally { await third.close(); }
       assert.equal(existsSync(scheddPaths(h.sched).lock), false, "lock released on close");
-    } finally { h.cleanup(); }
+    } finally { await h.cleanup(); }
   });
 
   it("#4454: close() stops every hosted batch loop before releasing the lock; no batch admits a lane after close()", async () => {
@@ -309,11 +333,12 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
     const h = host();
     try {
       const d1 = await h.daemon();
-      await withRoot(h, () => capture(async () => {
-        const a = orchestrateFromCli(["1", "2", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client).then(() => null, (e) => e);
+      const { lines } = await withRoot(h, () => capture(async () => {
+        // #4503: a gives up at once (reconnectMs 0) with the resume hint; b re-attaches to the next daemon on its own.
+        const a = orchestrateFromCli(["1", "2", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], { ...h.client, reconnectMs: 0 }).then(() => null, (e) => e);
         await until(() => h.launched().length === 1 && h.lanePids().length === 1);
         const [idA] = listBatches(h.root);
-        const b = orchestrateFromCli(["5", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client).then(() => null, (e) => e);
+        const b = orchestrateFromCli(["5", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client);
         await until(() => h.launched().includes(5) && h.lanePids().length === 2);
         const idB = listBatches(h.root).find((x) => x !== idA);
         const lanes = h.lanePids();
@@ -327,15 +352,102 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
         assert.equal(openBatch(h.root, idA).readReport(), null, "a stopped batch writes no report");
         assert.equal(openBatch(h.root, idB).readReport(), null);
         assert.match(String((await a)?.message), /--resume/);
-        await b;
         await h.daemon(); // resumes both batches
-        const ra = orchestrateFromCli(["--resume", idA], h.client), rb = orchestrateFromCli(["--resume", idB], h.client);
+        const ra = orchestrateFromCli(["--resume", idA], h.client);
         setTimeout(() => writeFileSync(h.release, ""), 200);
         assert.deepEqual((await ra).rows.map((r) => [r.issue, r.outcome]), [[1, "merged"], [2, "merged"]]);
-        assert.deepEqual((await rb).rows.map((r) => [r.issue, r.outcome]), [[5, "merged"]]);
+        assert.deepEqual((await b).rows.map((r) => [r.issue, r.outcome]), [[5, "merged"]], "b re-attached and finished");
         assert.deepEqual(h.launched(), [1, 2, 5], "adopted, not relaunched");
       }));
-    } finally { h.cleanup(); }
+      assert.ok(lines.some((l) => l.startsWith("schedd connection lost; reconnecting to batch ")), "the client said it reconnects");
+      const log = readFileSync(h.log, "utf8");
+      assert.match(log, /^schedd: exiting \(SIGTERM\)$/m);
+      assert.match(log, /^schedd: exited \(code 143\)$/m);
+    } finally { await h.cleanup(); }
+  });
+
+  it("#4503: every schedd exit writes a reason line", async () => {
+    for (const [ev, arg, code, re] of [["SIGTERM", undefined, 143, /^schedd: exiting \(SIGTERM\)$/], ["SIGINT", undefined, 130, /^schedd: exiting \(SIGINT\)$/],
+      ["SIGHUP", undefined, 129, /^schedd: exiting \(SIGHUP\)$/], ["uncaughtException", new Error("boom\nmore"), 1, /^schedd: exiting \(uncaught: boom\)$/],
+      ["unhandledRejection", new Error("nope"), 1, /^schedd: exiting \(unhandled rejection: nope\)$/]]) {
+      const handlers = {}, lines = [], exits = [];
+      exitOnSignals({ close: async () => { lines.push("closed"); } }, { on: (s, f) => { handlers[s] = f; }, exit: (c) => exits.push(c), log: (m) => lines.push(m) });
+      await handlers[ev](arg);
+      assert.match(lines[0], re, `${ev}: the reason is logged before close()`);
+      assert.equal(lines.at(-1), "closed");
+      assert.deepEqual(exits, [code]);
+      handlers.exit(code);
+      assert.equal(lines.at(-1), `schedd: exited (code ${code})`);
+    }
+    const dir = tmpDir("schedd-idle-"), lines = [];
+    const d = await startDaemon({ dir, root: join(dir, "batches"), runBatch: async () => ({}), log: (m) => lines.push(m), resumeOnStart: false, idleMs: 50 });
+    await d.closed;
+    assert.ok(lines.includes("schedd: idle, exiting"));
+  });
+
+  it("#4503: a daemon that lost its lock or socket logs it and stops its batches resumable, never removing the successor's lock or socket; a dead owner's takeover is logged", async () => {
+    const dir = tmpDir("schedd-lost-"), p = scheddPaths(dir), root = join(dir, "batches"), lines = [];
+    let stopped = false;
+    const d = await startDaemon({ dir, root, log: (m) => lines.push(m), resumeOnStart: false, lockCheckMs: 30,
+      runBatch: ({ sink, signal }) => { sink.batch({ id: "x", dir }); return new Promise((_, rej) => signal.addEventListener("abort", () => { stopped = true; rej(Object.assign(new Error("stopped"), { code: "batch.stopped" })); })); } });
+    request(await connectDaemon({ dir, spawnDaemon: null }), { op: "submit", argv: ["x"], cwd: dir, env: {} }).catch(() => {});
+    await until(() => lines.some((l) => l.includes("listening")));
+    await new Promise((r) => setTimeout(r, 50));
+    writeFileSync(p.lock, "999999999"); // a successor (now dead) took the lock
+    await d.closed;
+    assert.ok(lines.includes("schedd: lost lock (owner pid 999999999), exiting"), lines.join("\n"));
+    assert.equal(stopped, true, "the hosted batch loop was stopped (no report: the next daemon resumes it)");
+    assert.equal(readFileSync(p.lock, "utf8"), "999999999", "the successor's lock is left alone");
+    assert.equal(existsSync(p.sock), true, "the successor's socket is left alone");
+
+    const lines2 = [];
+    const d2 = await startDaemon({ dir, root, runBatch: async () => ({}), log: (m) => lines2.push(m), resumeOnStart: false, lockCheckMs: 30 });
+    assert.ok(lines2.includes("schedd: took over the lock of dead pid 999999999 (it exited without a clean shutdown)"), lines2.join("\n"));
+    rmSync(p.sock);
+    await d2.closed;
+    assert.ok(lines2.includes(`schedd: lost socket ${p.sock}, exiting`), lines2.join("\n"));
+    assert.equal(existsSync(p.lock), false, "its own lock is released");
+  });
+
+  it("#4503: a client whose daemon is SIGKILLed re-attaches to the next daemon and finishes, without relaunching a lane", async () => {
+    const h = host();
+    try {
+      const d1 = await h.daemon();
+      await withRoot(h, () => capture(async () => {
+        const a = orchestrateFromCli(["1", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client);
+        await until(() => h.launched().length === 1 && h.lanePids().length === 1);
+        const gone = new Promise((r) => d1.on("exit", r));
+        d1.kill("SIGKILL");
+        await gone;
+        await new Promise((r) => setTimeout(r, 300)); // the client is retrying against the dead socket
+        await h.daemon();
+        setTimeout(() => writeFileSync(h.release, ""), 200);
+        assert.deepEqual((await a).rows.map((r) => [r.issue, r.outcome]), [[1, "merged"]]);
+        assert.deepEqual(h.launched(), [1], "adopted, not relaunched");
+      }));
+      assert.match(readFileSync(h.log, "utf8"), /^schedd: took over the lock of dead pid \d+ \(it exited without a clean shutdown\)$/m);
+    } finally { await h.cleanup(); }
+  });
+
+  it("#4503: test runs never resolve ~/.forge/sched, and teardown leaves no live stub lane", async () => {
+    assert.notEqual(schedDir(), join(homedir(), ".forge", "sched"));
+    assert.ok(schedDir().startsWith(tmpdir()), schedDir());
+    const h = host();
+    let pids = [], a;
+    try {
+      await h.daemon();
+      await withRoot(h, async () => {
+        a = capture(() => orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-prefilter"], { ...h.client, reconnectMs: 0 })).catch((e) => e);
+        await until(() => h.stubPids().length === 2 && h.lanePids().length === 2);
+      });
+      pids = h.stubPids();
+      for (const pid of pids) assert.equal(isLaneAlive(pid, null), true);
+      const [id] = listBatches(h.root);
+      assert.equal(openBatch(h.root, id).readPlan().env.FORGEDOCK_SCHED_DIR, schedDir(), "the lanes' env pins the test sched dir");
+    } finally { await h.cleanup(); }
+    await a;
+    assert.equal(pids.length, 2);
+    for (const pid of pids) assert.equal(isLaneAlive(pid, null), false, `stub lane ${pid} reaped`);
   });
 
   it("lane env: allowlisted keys come from the client only, the rest from the daemon", () => {

package/bin/lean/seqlease.mjs

@@ -6,23 +6,70 @@
  * Each key is a one-slot `heavy.mjs` semaphore in its own registry dir (`schedDir()/seq-lease/<key>/heavy.json`):
  * file-locked, FIFO, dead holders dropped by pid. The heavy-step registry and its env caps are never touched.
  * Fail-open: a wait over `maxWaitS` (or an unreadable registry) proceeds without the lease; CI and base-sync stay the gate.
+ * #4526: number reservations. `reserve` picks the numbers of a renumber under a short host lock per (repo, base, dir) in
+ * its own registry (`schedDir()/seq-pick/<key>/`, never the lease above: its holder renumbers while it holds it) and
+ * records them in `reserved.json` until `unreserve` (run end), the holder's death or 24h, so concurrent lanes renumbering
+ * against the same base get distinct numbers without holding a lock across regenerate and push. Fail-open.
  */
 import { createHash } from "node:crypto";
+import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { acquireHeavy, openHeavy } from "./heavy.mjs";
-import { schedDir } from "./sched.mjs";
+import { pidAlive, schedDir } from "./sched.mjs";
 
 /** The lease key of one sequence dir of one lane branch of one repo. */
 export const sequenceLeaseKey = (repo, lane, dir) => createHash("sha1").update(`${repo}\0${lane}\0${dir}`).digest("hex").slice(0, 16);
 
 const NO_MEM = Object.freeze({ cores: () => 2, memAvailableMb: () => null });
+const RESERVE_TTL_MS = 24 * 3600 * 1000;
 
 /**
  * `acquire({repo, lane, dirs, issue, maxWaitS})` takes every dir's lease in sorted order (no lock-order deadlock).
  * @returns {{acquire: (e:object) => Promise<{waitMs:number, timedOut:boolean, release:Function}>}}
  */
-export function makeSequenceLeaseIo({ dir = join(schedDir(), "seq-lease"), open = openHeavy, acquire = acquireHeavy, pollMs } = {}) {
+export function makeSequenceLeaseIo({ dir = join(schedDir(), "seq-lease"), pickDir = join(schedDir(), "seq-pick"), open = openHeavy, acquire = acquireHeavy, pollMs, alive = pidAlive, now = () => Date.now() } = {}) {
+  const reservedDirs = new Map(); // `${repo}\0${lane}\0${issue}` -> dirs reserved in, for unreserve
+  /** Run `fn(entries) => entries'` on one key's reservations under its pick lock; returns what `fn` set as `out`. */
+  const locked = async ({ repo, lane, dir: d }, fn) => {
+    const kd = join(pickDir, sequenceLeaseKey(repo, lane, d)), file = join(kd, "reserved.json");
+    const h = open(kd, { cfg: { max: 1, max_wait_s: 60 }, env: {}, host: NO_MEM });
+    const g = await acquire(h, { phase: "sequence-pick" }, ...(pollMs ? [{ pollMs }] : []));
+    try {
+      let entries = [];
+      try { const o = JSON.parse(readFileSync(file, "utf8")); if (Array.isArray(o?.entries)) entries = o.entries; } catch { /* none yet */ }
+      const t = now();
+      entries = entries.filter((e) => Array.isArray(e?.numbers) && alive(e.pid) && t - Number(e.at ?? 0) < RESERVE_TTL_MS);
+      const r = fn(entries);
+      mkdirSync(kd, { recursive: true });
+      writeFileSync(`${file}.tmp`, `${JSON.stringify({ entries: r.entries }, null, 2)}\n`);
+      renameSync(`${file}.tmp`, file);
+      return r.out;
+    } finally { try { g.release(); } catch { /* fail-open */ } }
+  };
   return {
+    /**
+     * #4526: `count` consecutive numbers above max(floor, every other issue's live reservation) in (repo, lane, dir),
+     * recorded for `issue` (its earlier entries there replaced). @returns {Promise<{numbers:number[]}|null>} null on error.
+     */
+    async reserve({ repo, lane, dir: d, issue = null, floor = 0, count = 1, pid = process.pid } = {}) {
+      try {
+        const k = `${repo}\0${lane}\0${issue}`;
+        reservedDirs.set(k, new Set([...(reservedDirs.get(k) ?? []), d]));
+        return await locked({ repo, lane, dir: d }, (entries) => {
+          const others = entries.filter((e) => String(e.issue) !== String(issue));
+          let n = Math.max(Number(floor) || 0, ...others.flatMap((e) => e.numbers.map(Number)).filter(Number.isFinite));
+          const numbers = Array.from({ length: Math.max(0, count) }, () => ++n);
+          return { entries: [...others, { issue, pid, numbers, at: now() }], out: { numbers } };
+        });
+      } catch { return null; }
+    },
+    /** #4526: drop `issue`'s reservations in every dir of (repo, lane) it reserved in this process. Never throws. */
+    async unreserve({ repo, lane, issue = null } = {}) {
+      for (const d of [...(reservedDirs.get(`${repo}\0${lane}\0${issue}`) ?? [])]) {
+        try { await locked({ repo, lane, dir: d }, (entries) => ({ entries: entries.filter((e) => String(e.issue) !== String(issue)) })); } catch { /* fail-open */ }
+      }
+      reservedDirs.delete(`${repo}\0${lane}\0${issue}`);
+    },
     async acquire({ repo, lane, dirs = [], issue = null, maxWaitS = 1800 } = {}) {
       const held = [];
       let waitMs = 0, timedOut = false;

package/bin/lean/seqlease.test.mjs

@@ -39,6 +39,20 @@ describe("sequence lease (#4386)", () => {
     assert.equal(b.timedOut, true);
     b.release(); a.release();
   });
+
+  it("#4526: reserve gives each issue distinct numbers above the floor and other issues' live reservations; dead or unreserved entries free them", async () => {
+    const dead = new Set();
+    const io = makeSequenceLeaseIo({ dir: tmpDir("seqlease-"), pickDir: tmpDir("seqpick-"), pollMs: 20, alive: (pid) => !dead.has(pid) });
+    const k = { repo: "o/r", lane: "staging", dir: "db" };
+    const [a, b] = await Promise.all([io.reserve({ ...k, issue: 1, floor: 51, count: 1, pid: 101 }), io.reserve({ ...k, issue: 2, floor: 51, count: 2, pid: 102 })]);
+    assert.deepEqual([...a.numbers, ...b.numbers].sort((x, y) => x - y), [52, 53, 54]);
+    assert.deepEqual((await io.reserve({ ...k, issue: 1, floor: 51, count: 1, pid: 101 })).numbers, [Math.max(...b.numbers) + 1], "own entry replaced, others kept");
+    assert.deepEqual((await io.reserve({ ...k, dir: "docs/adr", issue: 3, floor: 51, pid: 103 })).numbers, [52], "other dir: own registry");
+    assert.deepEqual((await io.reserve({ ...k, lane: "main", issue: 3, floor: 51, pid: 103 })).numbers, [52], "other base: own registry");
+    dead.add(102);
+    await io.unreserve({ repo: "o/r", lane: "staging", issue: 1 });
+    assert.deepEqual((await io.reserve({ ...k, issue: 4, floor: 51, pid: 104 })).numbers, [52], "dead holder and unreserved issue dropped");
+  });
 });
 
 const env = { ...process.env, GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t" };

package/bin/lean/sequence.mjs

@@ -11,6 +11,9 @@
  * #4343: a leading calendar date (YYYY-MM-DD, YYYYMMDD) is not a number, and a directory only counts when it holds a
  * series on the base: a migration journal (`<dir>/meta/_journal.json`, `<dir>/_journal.json`), or >=2 distinct numbers
  * of the lane file's width that fill at least half of their min..max range (sparse issue-numbered docs are no series).
+ * #4526: renumberLane takes its numbers from a host reservation (`reserve`, seqlease.mjs) so concurrent lanes get distinct
+ * ones; verifyRenumber accepts a first-line header that only swaps the file's own tag (`headerOnlyChange`); a hand-written
+ * migration the generator writes nothing for gets the generator's `--custom` snapshot and keeps the lane's SQL.
  */
 import { execFile } from "node:child_process";
 import { randomUUID } from "node:crypto";
@@ -123,6 +126,21 @@ export function numberOnlyChange(was, now, from, to) {
   return new RegExp(`^${parts.map((p, i) => (i % 2 ? alt(p) : esc(p))).join("")}$`).test(String(now));
 }
 
+/**
+ * #4526: everything after the first line is byte-identical, and the first line of `now` is that of `was` with the old
+ * stem (`fromPath` without extension) mapped to the new one, then the old number (padded or bare, not next to another
+ * digit) to the new number: a migration's `-- Migration 0067_x` or an ADR's `# 0029. Title` header following a renumber.
+ */
+export function headerOnlyChange(was, now, fromPath, toPath) {
+  const from = numOf(fromPath), to = numOf(toPath);
+  if (!from || !to) return false;
+  const split = (t) => { const s = String(t), i = s.indexOf("\n"); return i < 0 ? [s, ""] : [s.slice(0, i), s.slice(i)]; };
+  const [h1, r1] = split(was), [h2, r2] = split(now);
+  if (r1 !== r2) return false;
+  const mapped = h1.replaceAll(stemOf(fromPath), stemOf(toPath)).replace(new RegExp(`(?<!\\d)0*${Number(from)}(?!\\d)`, "g"), (tok) => (tok.length >= to.length ? to : String(Number(to))));
+  return mapped === h2;
+}
+
 /** Lane-added numbered files with their blobs, recorded before a renumber so verifyRenumber can prove the SQL kept. */
 export const snapshotLane = ({ git, base }) => laneNumbered(git, base).catch(() => []);
 
@@ -173,7 +191,7 @@ export async function verifyRenumber({ git, base, before = [], moves = null }) {
       for (const f of peers) {
         if (kept || f.path === b.path) continue;
         const text = await g(["cat-file", "-p", f.blob]);
-        kept = numberOnlyChange(was, text, numOf(b.path), numOf(f.path)) || (rest(f.path) === rest(b.path) && text === mapped);
+        kept = numberOnlyChange(was, text, numOf(b.path), numOf(f.path)) || (rest(f.path) === rest(b.path) && text === mapped) || headerOnlyChange(was, text, b.path, f.path);
       }
       if (!kept) problems.push(`${b.path}: content changed or file gone after renumbering (keep it byte-identical, rename only)`);
     }
@@ -269,7 +287,8 @@ const writeJson = (p, o) => writeFileSync(p, `${JSON.stringify(o, null, 2)}\n`);
  * where it started. Never throws.
  * @returns {Promise<{ok:true, moves:Array<{from,to}>, commit:string|null}|{unsupported:string}>}
  */
-export async function renumberLane({ git, base, cwd, laneRef = "HEAD", regenerate = null, run = null, uuid = randomUUID }) {
+export async function renumberLane({ git, base, cwd, laneRef = "HEAD", regenerate = null, regenerateCustom = null, run = null, uuid = randomUUID, reserve = null }) {
+  const custom = regenerateCustom ?? (regenerate ? `${regenerate} --custom` : null);
   const g = out(git);
   const start = (await g(["rev-parse", "HEAD"])).trim();
   const written = [];
@@ -286,6 +305,16 @@ export async function renumberLane({ git, base, cwd, laneRef = "HEAD", regenerat
     if (!collisions.length) return { ok: true, moves: [], commit: null };
     const odd = collisions.find((c) => !SUPPORTED_RE.test(c.lane));
     if (odd) return { unsupported: `${odd.lane}: no deterministic renumber for .${basename(odd.lane).split(".").pop()} migrations` };
+    // #4526: numbers above every other live lane's host reservation, not only above the base (fail-open: base-derived).
+    if (reserve) {
+      for (const dir of [...new Set(collisions.map((c) => c.dir))].sort()) {
+        const cs = collisions.filter((c) => c.dir === dir), width = cs[0].next.length;
+        let got = null;
+        try { got = await reserve({ dir, floor: Number(cs[0].next) - 1, count: cs.length }); } catch { /* fail-open */ }
+        const nums = Array.isArray(got?.numbers) ? got.numbers.map(Number) : [];
+        if (nums.length === cs.length && nums.every(Number.isInteger)) cs.forEach((c, i) => { c.next = String(nums[i]).padStart(width, "0"); });
+      }
+    }
     const moves = collisions.map((c) => ({ from: c.lane, to: posix.join(c.dir, c.next + basename(c.lane).slice(c.number.length)), dir: c.dir, oldNum: c.number, newNum: c.next }));
     // Plan the journals and snapshots before touching anything, so an unsupported migration leaves the branch as it was.
     const baseList = lines(await g(["-c", "core.quotepath=off", "ls-tree", "-r", "--name-only", base]));
@@ -296,12 +325,14 @@ export async function renumberLane({ git, base, cwd, laneRef = "HEAD", regenerat
       if (!jp) continue;
       const meta = dirname(jp), mine = moves.filter((m) => m.dir === dir && /\.sql$/i.test(m.from));
       const snaps = [];
+      let chained = false; // #4526: after a schema-changing move, a data-only one's predecessor is not the base tip
       for (const m of mine) {
         const laneSnap = await json(g, `${lane}:${posix.join(meta, `${m.oldNum}_snapshot.json`)}`);
         if (!laneSnap) continue;
         const laneMeta = lines(await g(["-c", "core.quotepath=off", "ls-tree", "--name-only", lane, `${meta}/`])).filter((p) => SNAPSHOT_RE.test(p) && Number(numOf(p)) < Number(m.oldNum));
         const pred = laneMeta.sort((a, b) => Number(numOf(a)) - Number(numOf(b))).at(-1);
-        const dataOnly = Boolean(pred) && strip(await json(g, `${lane}:${pred}`)) === strip(laneSnap);
+        const dataOnly = !chained && Boolean(pred) && strip(await json(g, `${lane}:${pred}`)) === strip(laneSnap);
+        if (!dataOnly) chained = true;
         if (!dataOnly && !regenerate) return { unsupported: `schema-changing migration ${m.from} needs lean.sequence.regenerate (its snapshot cannot be cloned)` };
         if (!dataOnly) needRegen = true;
         snaps.push({ m, dataOnly });
@@ -339,7 +370,8 @@ export async function renumberLane({ git, base, cwd, laneRef = "HEAD", regenerat
       const tags = new Set(baseEntries.map((e) => e?.tag));
       let idx = Math.max(-1, ...baseEntries.map((e) => Number(e?.idx)).filter(Number.isFinite));
       const add = (Array.isArray(lj.entries) ? lj.entries : []).filter((e) => !tags.has(e?.tag)).map((e) => ({ ...e, idx: ++idx, tag: stem.get(e.tag) ?? e.tag }));
-      writeJson(join(cwd, p.jp), { ...bj, entries: [...baseEntries, ...add] });
+      p.journal = { ...bj, entries: [...baseEntries, ...add] };
+      writeJson(join(cwd, p.jp), p.journal);
       touched.add(p.jp);
       const baseSnaps = baseList.filter((x) => dirname(x) === p.meta && SNAPSHOT_RE.test(x)).sort((a, b) => Number(numOf(a)) - Number(numOf(b)));
       let tip = baseSnaps.length ? await json(g, `${base}:${baseSnaps.at(-1)}`) : null;
@@ -360,11 +392,27 @@ export async function renumberLane({ git, base, cwd, laneRef = "HEAD", regenerat
     }
     if (needRegen) {
       try { await (run ?? defaultRun)(regenerate, { cwd }); } catch (e) { return await fail(`lean.sequence.regenerate failed: ${String(e?.stderr || e?.message || e).trim().split("\n")[0]}`); }
+      const exists = (p) => { try { readFileSync(join(cwd, p)); return true; } catch { return false; } };
       for (const p of plans) {
         touched.add(p.meta);
-        for (const { m, dataOnly } of p.snaps) {
+        // #4526: a hand-written migration (no schema diff) gets nothing from the regular run: the generator's --custom
+        // writes an empty migration and its snapshot at that number (its SQL and later ones withheld), then the lane's
+        // SQL and the planned journal come back.
+        for (const { m, dataOnly } of [...p.snaps].sort((a, b) => Number(a.m.newNum) - Number(b.m.newNum))) {
           const want = posix.join(p.meta, `${m.newNum}_snapshot.json`);
-          if (!dataOnly) { try { readFileSync(join(cwd, want)); } catch { return await fail(`lean.sequence.regenerate did not write ${want}`); } }
+          if (dataOnly || exists(want)) continue;
+          if (!custom) return await fail(`lean.sequence.regenerate did not write ${want}`);
+          const held = moves.filter((w) => w.dir === p.dir && /\.sql$/i.test(w.to) && Number(w.newNum) >= Number(m.newNum));
+          const heldTags = new Set(held.map((w) => stemOf(w.to)));
+          for (const w of held) rmSync(join(cwd, w.to), { force: true });
+          writeJson(join(cwd, p.jp), { ...p.journal, entries: p.journal.entries.filter((e) => !heldTags.has(e?.tag)) });
+          try { await (run ?? defaultRun)(custom, { cwd }); } catch (e) { return await fail(`lean.sequence.regenerate --custom failed: ${String(e?.stderr || e?.message || e).trim().split("\n")[0]}`); }
+          for (const f of lines(await g(["-c", "core.quotepath=off", "ls-files", "--others", "--exclude-standard", "--", `${p.dir}/`]))) {
+            if (dirname(f) === p.dir && /\.sql$/i.test(f) && numOf(f) === m.newNum && f !== m.to) rmSync(join(cwd, f), { force: true });
+          }
+          for (const w of held) writeFileSync(join(cwd, w.to), await g(["cat-file", "blob", `${lane}:${w.from}`]));
+          writeJson(join(cwd, p.jp), p.journal);
+          if (!exists(want)) return await fail(`lean.sequence.regenerate did not write ${want}`);
         }
       }
     }