forgedock 1.31.0 → 1.32.1

package/bin/lean/schedd.mjs

@@ -24,6 +24,9 @@
  * Shutdown (#4454): `close()` aborts every hosted batch's `signal` (its loop admits nothing more and leaves, lanes keep
  * running, no report is written), waits for them up to `stopMs`, and only then releases `schedd.lock` and the socket, so a
  * daemon started next never resumes a batch this one still dispatches. `exitOnSignals` exits 128+signo after that.
+ * Exit reasons (#4503): every exit logs a line: `idle, exiting`, `exiting (<signal>|uncaught: …)`, `lost lock|socket …,
+ * exiting` (checked every `lockCheckMs`: the lock no longer holds this pid, or the socket is gone; nothing it does not own
+ * is removed), then `exited (code N)`. Taking over a dead owner's lock is logged, so a SIGKILLed daemon leaves a trace.
  * `runBatch({argv, cwd, env, sink, keepBreaker, signal})` is injected (engine-cli `runBatch`); `sink` gets `batch(b)`, `row(r)`, `line(text)`.
  */
 import { closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, writeSync } from "node:fs";
@@ -35,6 +38,7 @@ import { batchRoot, listBatches, openBatch } from "./batch.mjs";
 
 export const IDLE_MS = 10 * 60 * 1000;
 export const STOP_MS = 10 * 1000;
+export const LOCK_CHECK_MS = 5 * 1000;
 
 export function scheddPaths(dir = schedDir()) {
   return { dir, sock: join(dir, "schedd.sock"), lock: join(dir, "schedd.lock"), log: join(dir, "schedd.log") };
@@ -54,17 +58,19 @@ export function mergeEnv(base, kept) {
   return { ...out, ...kept };
 }
 
-/** Take the daemon lock; null when another live daemon holds it. */
+/** Take the daemon lock: `{took}` (the dead owner's pid it replaced, or null); null when another live daemon holds it. */
 function takeLock(lock, pid, alive) {
+  let took = null;
   for (let i = 0; i < 5; i++) {
     try {
       const fd = openSync(lock, "wx");
       try { writeSync(fd, String(pid)); } finally { closeSync(fd); }
-      return true;
+      return { took };
     } catch (e) { if (e?.code !== "EEXIST") throw e; }
     let owner = NaN;
     try { owner = parseInt(readFileSync(lock, "utf8"), 10); } catch { continue; }
     if (Number.isInteger(owner) && owner !== pid && alive(owner)) return null;
+    if (Number.isInteger(owner) && owner !== pid) took = owner;
     rmSync(lock, { force: true });
   }
   return null;
@@ -74,10 +80,13 @@ function takeLock(lock, pid, alive) {
  * Start the daemon. @returns {Promise<{running:true, pid, close, closed:Promise} | {running:false, owner:number}>}
  */
 export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(process.env.FORGEDOCK_SCHEDD_IDLE_MS) || IDLE_MS,
-  root = batchRoot(), log = (m) => console.log(m), pid = process.pid, alive = pidAlive, resumeOnStart = true, stopMs = STOP_MS } = {}) {
+  root = batchRoot(), log = (m) => console.log(m), pid = process.pid, alive = pidAlive, resumeOnStart = true, stopMs = STOP_MS,
+  lockCheckMs = LOCK_CHECK_MS } = {}) {
   const p = scheddPaths(dir);
   mkdirSync(dir, { recursive: true });
-  if (!takeLock(p.lock, pid, alive)) {
+  const lock = takeLock(p.lock, pid, alive);
+  if (lock?.took) log(`schedd: took over the lock of dead pid ${lock.took} (it exited without a clean shutdown)`);
+  if (!lock) {
     let owner = NaN;
     try { owner = parseInt(readFileSync(p.lock, "utf8"), 10); } catch { /* gone */ }
     log(`schedd: already running (pid ${owner})`);
@@ -88,7 +97,8 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
   const hosted = new Map(); // batch id -> {id, dir, subs:Set<fn>}
   const conns = new Set();
   const runs = new Map(); // every runBatch in flight (hosted or not yet started) -> its AbortController (#4454)
-  let closing = null, resolveClosed;
+  const starting = new Map(); // batch id -> host() promise of a start-up resume not yet hosted (#4503: an attach waits for it)
+  let closing = null, resolveClosed, keepSock = false;
   let lastActive = Date.now();
   const touch = () => { lastActive = Date.now(); };
 
@@ -143,6 +153,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       return send({ type: "batch", id: ent.id, dir: ent.dir });
     }
     if (req.op === "attach") {
+      if (!hosted.has(req.id) && starting.has(req.id)) await starting.get(req.id).catch(() => {});
       const live = hosted.get(req.id);
       if (live) {
         // Synchronous read + subscribe: no row is lost or doubled between the replay and the live stream.
@@ -191,6 +202,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
   const close = () => {
     if (closing) return closing;
     clearInterval(idleTimer);
+    clearInterval(lockTimer);
     for (const c of conns) c.destroy();
     const stopping = [...runs.keys()];
     for (const ac of runs.values()) ac.abort();
@@ -198,9 +210,11 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     // #4454: the lock goes only once every hosted batch loop has stopped (or `stopMs` passed: the caller exits regardless)
     const stopped = stopping.length ? Promise.race([Promise.allSettled(stopping).then(() => true), new Promise((r) => { timer = setTimeout(() => r(false), stopMs); })])
       .then((ok) => { clearTimeout(timer); if (!ok) log(`schedd: ${runs.size} batch(es) did not stop within ${stopMs} ms`); }) : Promise.resolve();
-    closing = Promise.all([new Promise((r) => server.close(() => r())), stopped]).then(() => {
+    // A lost lock: the socket path may be the successor's, and closing the server unlinks it (libuv), so leave it unref'd.
+    const serverClosed = keepSock ? Promise.resolve(server.unref()) : new Promise((r) => server.close(() => r()));
+    closing = Promise.all([serverClosed, stopped]).then(() => {
       try { if (parseInt(readFileSync(p.lock, "utf8"), 10) === pid) rmSync(p.lock, { force: true }); } catch { /* gone */ }
-      rmSync(p.sock, { force: true });
+      if (!keepSock) rmSync(p.sock, { force: true });
       resolveClosed();
     });
     return closing;
@@ -209,6 +223,15 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     if (!hosted.size && !conns.size && Date.now() - lastActive >= idleMs) { log("schedd: idle, exiting"); close(); }
   }, Math.min(idleMs, 30_000));
   idleTimer.unref?.();
+  // #4503: a daemon whose lock or socket was taken stops (its batches stay resumable) instead of running unreachable.
+  const lockTimer = setInterval(() => {
+    if (closing) return;
+    let owner = NaN;
+    try { owner = parseInt(readFileSync(p.lock, "utf8"), 10); } catch { /* gone */ }
+    if (owner !== pid) { keepSock = true; log(`schedd: lost lock (owner ${Number.isInteger(owner) ? `pid ${owner}` : "none"}), exiting`); close(); }
+    else if (!existsSync(p.sock)) { log(`schedd: lost socket ${p.sock}, exiting`); close(); }
+  }, lockCheckMs);
+  lockTimer.unref?.();
 
   // Daemon restart: resume every daemon-hosted batch that never wrote its report; live lanes are re-adopted.
   if (resumeOnStart) {
@@ -216,22 +239,35 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       const b = openBatch(root, id);
       if (!b.readPlan()?.daemon || b.readReport()) continue;
       log(`schedd: resuming batch ${id}`);
-      host({ argv: ["--resume", id], cwd: process.cwd(), env: {}, keepBreaker: true }).catch((e) => log(`schedd: resume ${id} failed: ${e?.message ?? e}`));
+      const started = host({ argv: ["--resume", id], cwd: process.cwd(), env: {}, keepBreaker: true });
+      starting.set(id, started);
+      started.catch((e) => log(`schedd: resume ${id} failed: ${e?.message ?? e}`)).finally(() => starting.delete(id));
     }
   }
   return { running: true, pid, sock: p.sock, close, closed };
 }
 
-/** #4454: on SIGTERM/SIGINT stop the daemon (`close()`: batch loops first, then lock and socket), then exit 128+signo, once. */
-export function exitOnSignals(d, { on = (s, f) => process.once(s, f), exit = (c) => process.exit(c), signals = ["SIGTERM", "SIGINT"] } = {}) {
-  const SIGNO = { SIGINT: 2, SIGTERM: 15 };
+/**
+ * #4454: on SIGTERM/SIGINT/SIGHUP stop the daemon (`close()`: batch loops first, then lock and socket), then exit 128+signo,
+ * once; an uncaught exception or unhandled rejection does the same and exits 1. #4503: each logs its reason first, and the
+ * process `exit` logs `schedd: exited (code N)`.
+ */
+export function exitOnSignals(d, { on = (s, f) => (s.startsWith("SIG") || s === "exit" ? process.once(s, f) : process.on(s, f)),
+  exit = (c) => process.exit(c), log = (m) => console.log(m), signals = ["SIGTERM", "SIGINT", "SIGHUP"] } = {}) {
+  const SIGNO = { SIGHUP: 1, SIGINT: 2, SIGTERM: 15 };
   let fired = false;
-  for (const sig of signals) on(sig, async () => {
+  const stop = async (reason, code, detail = null) => {
     if (fired) return;
     fired = true;
+    log(`schedd: exiting (${reason})`);
+    if (detail) log(detail);
     try { await d.close(); } catch { /* exit anyway */ }
-    exit(128 + (SIGNO[sig] ?? 0));
-  });
+    exit(code);
+  };
+  for (const sig of signals) on(sig, () => stop(sig, 128 + (SIGNO[sig] ?? 0)));
+  on("uncaughtException", (e) => stop(`uncaught: ${String(e?.message ?? e).split("\n")[0]}`, 1, e?.stack ?? null));
+  on("unhandledRejection", (e) => stop(`unhandled rejection: ${String(e?.message ?? e).split("\n")[0]}`, 1, e?.stack ?? null));
+  on("exit", (code) => log(`schedd: exited (code ${code})`));
 }
 
 /** Connect to the daemon, spawning it (`spawnDaemon()`) when nobody listens. @returns {Promise<import("node:net").Socket>} */

package/bin/lean/schedd.test.mjs

@@ -3,14 +3,23 @@
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
 import { spawn } from "node:child_process";
-import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { closeSync, existsSync, openSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
 import { openBatch, isLaneAlive, listBatches } from "./batch.mjs";
-import { startDaemon, scheddPaths, connectDaemon, request, mergeEnv, batchEnv } from "./schedd.mjs";
+import { startDaemon, scheddPaths, connectDaemon, request, mergeEnv, batchEnv, exitOnSignals } from "./schedd.mjs";
+import { schedDir } from "./sched.mjs";
 import { formatTable, orchestrate } from "./orchestrate.mjs";
 import { orchestrateFromCli } from "../engine-cli.mjs";
 
+// #4503: this process, and the env its clients submit for their lanes, never resolve the host's `~/.forge/sched`.
+process.env.FORGEDOCK_SCHED_DIR = tmpDir("schedd-home-");
+// Stub lanes run detached: a host whose test never reached its `finally` still has them reaped when the file ends.
+const hosts = [];
+process.on("exit", () => { for (const h of hosts) h.reap(); });
+const LANE_MAX_MS = 120_000;
+
 const until = async (cond, ms = 15000) => {
   const end = Date.now() + ms;
   while (!cond()) { if (Date.now() > end) throw new Error("timed out waiting"); await new Promise((r) => setTimeout(r, 25)); }
@@ -24,13 +33,20 @@ const capture = async (fn) => {
 /** A host: sched + batch dirs, a stub lane that waits for `release`, and a daemon script with injected lanes. */
 function host() {
   const work = tmpDir("schedd-");
-  const h = { work, sched: join(work, "s"), root: join(work, "batches"), launches: join(work, "launches"), release: join(work, "release"), procs: [] };
+  const h = { work, sched: join(work, "s"), root: join(work, "batches"), launches: join(work, "launches"), release: join(work, "release"), procs: [],
+    lanePidFile: join(work, "lane-pids"), log: join(work, "schedd.log") };
   writeFileSync(h.launches, "");
+  writeFileSync(h.lanePidFile, "");
+  hosts.push(h);
   const stub = join(work, "stub-lane.mjs");
   writeFileSync(stub, `import { appendFileSync, existsSync, writeFileSync, renameSync } from "node:fs";
 const a = process.argv.slice(2); const n = Number(a[1]); const rf = a[a.indexOf("--result-file") + 1];
 appendFileSync(${JSON.stringify(h.launches)}, n + "\\n");
-const t = setInterval(() => { if (!existsSync(${JSON.stringify(h.release)})) return; clearInterval(t);
+appendFileSync(${JSON.stringify(h.lanePidFile)}, process.pid + "\\n");
+// #4503: a stub whose test run is gone (work dir removed) or that outlived its bound exits instead of polling forever.
+const born = Date.now();
+const t = setInterval(() => { if (!existsSync(${JSON.stringify(work)}) || Date.now() - born > ${LANE_MAX_MS}) process.exit(0);
+  if (!existsSync(${JSON.stringify(h.release)})) return; clearInterval(t);
   writeFileSync(rf + ".tmp", JSON.stringify({ terminalReason: "merged", pr: 100 + n, cost_usd: 0 })); renameSync(rf + ".tmp", rf); }, 20);
 // Like run-issue's #4179 teardown: SIGTERM reports deferred run.interrupted.
 process.once("SIGTERM", () => { writeFileSync(rf + ".tmp", JSON.stringify({ terminalReason: "deferred", code: "run.interrupted", detail: "interrupted by SIGTERM" })); renameSync(rf + ".tmp", rf); process.exit(143); });
@@ -49,7 +65,9 @@ await d.closed;
   h.env = { ...process.env, FORGEDOCK_SCHED_DIR: h.sched, FORGEDOCK_BATCH_DIR: h.root, FORGEDOCK_LIMITS_FILE: join(work, "limits.json") };
   h.daemon = async () => {
     const before = existsSync(scheddPaths(h.sched).lock) ? readFileSync(scheddPaths(h.sched).lock, "utf8") : null;
-    const child = spawn(process.execPath, [h.daemonScript], { env: h.env, stdio: "ignore" });
+    const out = openSync(h.log, "a");
+    let child;
+    try { child = spawn(process.execPath, [h.daemonScript], { env: h.env, stdio: ["ignore", out, out] }); } finally { closeSync(out); }
     h.procs.push(child);
     await until(() => existsSync(scheddPaths(h.sched).sock) && existsSync(scheddPaths(h.sched).lock) && readFileSync(scheddPaths(h.sched).lock, "utf8") === String(child.pid) && readFileSync(scheddPaths(h.sched).lock, "utf8") !== before);
     // Wait for the socket to accept.
@@ -59,9 +77,15 @@ await d.closed;
   h.client = { scheddDir: h.sched, spawnDaemon: null };
   h.launched = () => readFileSync(h.launches, "utf8").trim().split("\n").filter(Boolean).map(Number).sort();
   h.lanePids = () => listBatches(h.root).flatMap((id) => Object.values(openBatch(h.root, id).store.load()?.running ?? {}).map((r) => r.pid).filter(Boolean));
-  h.cleanup = () => {
-    writeFileSync(h.release, "");
+  h.stubPids = () => (existsSync(h.lanePidFile) ? readFileSync(h.lanePidFile, "utf8").trim().split("\n").filter(Boolean).map(Number) : []);
+  h.reap = () => {
     for (const p of h.procs) { try { p.kill("SIGKILL"); } catch { /* gone */ } }
+    for (const pid of h.stubPids()) { try { process.kill(pid, "SIGKILL"); } catch { /* gone */ } }
+  };
+  h.cleanup = async () => {
+    if (existsSync(h.work)) writeFileSync(h.release, "");
+    h.reap();
+    await until(() => h.stubPids().every((pid) => !isLaneAlive(pid, null)), 5000).catch(() => {});
   };
   return h;
 }
@@ -98,7 +122,7 @@ describe("schedd host scheduler (#4192)", () => {
       const id = done[0].batch;
       await withRoot(h, () => capture(() => orchestrateFromCli(["--set-parallel", id, "5"], h.client)));
       assert.equal(openBatch(h.root, id).readParallel(), 5);
-    } finally { h.cleanup(); }
+    } finally { await h.cleanup(); }
   });
 
   it("a client that disconnects does not stop the batch; --resume reattaches, and replays the report once done", async () => {
@@ -127,7 +151,7 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
       assert.deepEqual(again.res.rows, res.rows);
       assert.ok(again.lines.join("\n").includes(formatTable(res.rows)));
       assert.deepEqual(h.launched(), [1, 2]);
-    } finally { h.cleanup(); }
+    } finally { await h.cleanup(); }
   });
 
   it("a daemon restart resumes the batch from state and re-adopts its live lanes without duplicate launches", async () => {
@@ -150,7 +174,7 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
       assert.deepEqual(res.rows.map((r) => [r.issue, r.outcome, r.pr]), [[1, "merged", 101], [2, "merged", 102], [3, "merged", 103]]);
       assert.deepEqual(h.launched(), [1, 2, 3], "adopted, not relaunched");
       assert.deepEqual(openBatch(h.root, id).store.load().running, {});
-    } finally { h.cleanup(); }
+    } finally { await h.cleanup(); }
   });
 
   it("#4408: --add and --resume <id> N add issues to the live batch (no second batch); an unhosted finished batch runs its adds on --resume", async () => {
@@ -181,7 +205,7 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
       }));
       assert.equal(res.rows.length, 3);
       assert.deepEqual(h.launched(), [1, 2, 4, 6], "each issue launched once");
-    } finally { h.cleanup(); }
+    } finally { await h.cleanup(); }
   });
 
   it("#4376: --cancel stops one batch (its lane torn down, the rest held, report written); the other batch and the daemon run on", async () => {
@@ -211,7 +235,28 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
         assert.equal(again.cancelled, false, "a finished batch is left alone");
         await assert.rejects(orchestrateFromCli(["--cancel", "nope"], h.client), /no batch nope/);
       }));
-    } finally { h.cleanup(); }
+    } finally { await h.cleanup(); }
+  });
+
+  it("#4481: a batch cancelled through the daemon is not resumed by a schedd restart", async () => {
+    const h = host();
+    try {
+      const statusIds = async () => (await request(await connectDaemon({ dir: h.sched, spawnDaemon: null }), { op: "status" })).batches.map((x) => x.id);
+      const kill = async (d) => { const gone = new Promise((r) => d.on("exit", r)); d.kill("SIGKILL"); await gone; };
+      let d = await h.daemon();
+      await withRoot(h, () => capture(async () => {
+        const a = orchestrateFromCli(["1", "2", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client);
+        await until(() => h.launched().length === 1 && h.lanePids().length === 1);
+        const [idA] = listBatches(h.root);
+        assert.equal((await orchestrateFromCli(["--cancel", idA], h.client)).cancelled, true);
+        await a;
+        await kill(d);
+        d = await h.daemon();
+        await new Promise((r) => setTimeout(r, 300));
+        assert.deepEqual(await statusIds(), [], "the restarted daemon hosts no cancelled batch");
+        assert.deepEqual(h.launched(), [1]);
+      }));
+    } finally { await h.cleanup(); }
   });
 
   it("single instance: a live daemon keeps its lock; a dead owner's lock and socket are taken over", async () => {
@@ -231,7 +276,7 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
         assert.equal(st.pid, process.pid);
       } finally { await third.close(); }
       assert.equal(existsSync(scheddPaths(h.sched).lock), false, "lock released on close");
-    } finally { h.cleanup(); }
+    } finally { await h.cleanup(); }
   });
 
   it("#4454: close() stops every hosted batch loop before releasing the lock; no batch admits a lane after close()", async () => {
@@ -288,11 +333,12 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
     const h = host();
     try {
       const d1 = await h.daemon();
-      await withRoot(h, () => capture(async () => {
-        const a = orchestrateFromCli(["1", "2", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client).then(() => null, (e) => e);
+      const { lines } = await withRoot(h, () => capture(async () => {
+        // #4503: a gives up at once (reconnectMs 0) with the resume hint; b re-attaches to the next daemon on its own.
+        const a = orchestrateFromCli(["1", "2", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], { ...h.client, reconnectMs: 0 }).then(() => null, (e) => e);
         await until(() => h.launched().length === 1 && h.lanePids().length === 1);
         const [idA] = listBatches(h.root);
-        const b = orchestrateFromCli(["5", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client).then(() => null, (e) => e);
+        const b = orchestrateFromCli(["5", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client);
         await until(() => h.launched().includes(5) && h.lanePids().length === 2);
         const idB = listBatches(h.root).find((x) => x !== idA);
         const lanes = h.lanePids();
@@ -306,15 +352,102 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
         assert.equal(openBatch(h.root, idA).readReport(), null, "a stopped batch writes no report");
         assert.equal(openBatch(h.root, idB).readReport(), null);
         assert.match(String((await a)?.message), /--resume/);
-        await b;
         await h.daemon(); // resumes both batches
-        const ra = orchestrateFromCli(["--resume", idA], h.client), rb = orchestrateFromCli(["--resume", idB], h.client);
+        const ra = orchestrateFromCli(["--resume", idA], h.client);
         setTimeout(() => writeFileSync(h.release, ""), 200);
         assert.deepEqual((await ra).rows.map((r) => [r.issue, r.outcome]), [[1, "merged"], [2, "merged"]]);
-        assert.deepEqual((await rb).rows.map((r) => [r.issue, r.outcome]), [[5, "merged"]]);
+        assert.deepEqual((await b).rows.map((r) => [r.issue, r.outcome]), [[5, "merged"]], "b re-attached and finished");
         assert.deepEqual(h.launched(), [1, 2, 5], "adopted, not relaunched");
       }));
-    } finally { h.cleanup(); }
+      assert.ok(lines.some((l) => l.startsWith("schedd connection lost; reconnecting to batch ")), "the client said it reconnects");
+      const log = readFileSync(h.log, "utf8");
+      assert.match(log, /^schedd: exiting \(SIGTERM\)$/m);
+      assert.match(log, /^schedd: exited \(code 143\)$/m);
+    } finally { await h.cleanup(); }
+  });
+
+  it("#4503: every schedd exit writes a reason line", async () => {
+    for (const [ev, arg, code, re] of [["SIGTERM", undefined, 143, /^schedd: exiting \(SIGTERM\)$/], ["SIGINT", undefined, 130, /^schedd: exiting \(SIGINT\)$/],
+      ["SIGHUP", undefined, 129, /^schedd: exiting \(SIGHUP\)$/], ["uncaughtException", new Error("boom\nmore"), 1, /^schedd: exiting \(uncaught: boom\)$/],
+      ["unhandledRejection", new Error("nope"), 1, /^schedd: exiting \(unhandled rejection: nope\)$/]]) {
+      const handlers = {}, lines = [], exits = [];
+      exitOnSignals({ close: async () => { lines.push("closed"); } }, { on: (s, f) => { handlers[s] = f; }, exit: (c) => exits.push(c), log: (m) => lines.push(m) });
+      await handlers[ev](arg);
+      assert.match(lines[0], re, `${ev}: the reason is logged before close()`);
+      assert.equal(lines.at(-1), "closed");
+      assert.deepEqual(exits, [code]);
+      handlers.exit(code);
+      assert.equal(lines.at(-1), `schedd: exited (code ${code})`);
+    }
+    const dir = tmpDir("schedd-idle-"), lines = [];
+    const d = await startDaemon({ dir, root: join(dir, "batches"), runBatch: async () => ({}), log: (m) => lines.push(m), resumeOnStart: false, idleMs: 50 });
+    await d.closed;
+    assert.ok(lines.includes("schedd: idle, exiting"));
+  });
+
+  it("#4503: a daemon that lost its lock or socket logs it and stops its batches resumable, never removing the successor's lock or socket; a dead owner's takeover is logged", async () => {
+    const dir = tmpDir("schedd-lost-"), p = scheddPaths(dir), root = join(dir, "batches"), lines = [];
+    let stopped = false;
+    const d = await startDaemon({ dir, root, log: (m) => lines.push(m), resumeOnStart: false, lockCheckMs: 30,
+      runBatch: ({ sink, signal }) => { sink.batch({ id: "x", dir }); return new Promise((_, rej) => signal.addEventListener("abort", () => { stopped = true; rej(Object.assign(new Error("stopped"), { code: "batch.stopped" })); })); } });
+    request(await connectDaemon({ dir, spawnDaemon: null }), { op: "submit", argv: ["x"], cwd: dir, env: {} }).catch(() => {});
+    await until(() => lines.some((l) => l.includes("listening")));
+    await new Promise((r) => setTimeout(r, 50));
+    writeFileSync(p.lock, "999999999"); // a successor (now dead) took the lock
+    await d.closed;
+    assert.ok(lines.includes("schedd: lost lock (owner pid 999999999), exiting"), lines.join("\n"));
+    assert.equal(stopped, true, "the hosted batch loop was stopped (no report: the next daemon resumes it)");
+    assert.equal(readFileSync(p.lock, "utf8"), "999999999", "the successor's lock is left alone");
+    assert.equal(existsSync(p.sock), true, "the successor's socket is left alone");
+
+    const lines2 = [];
+    const d2 = await startDaemon({ dir, root, runBatch: async () => ({}), log: (m) => lines2.push(m), resumeOnStart: false, lockCheckMs: 30 });
+    assert.ok(lines2.includes("schedd: took over the lock of dead pid 999999999 (it exited without a clean shutdown)"), lines2.join("\n"));
+    rmSync(p.sock);
+    await d2.closed;
+    assert.ok(lines2.includes(`schedd: lost socket ${p.sock}, exiting`), lines2.join("\n"));
+    assert.equal(existsSync(p.lock), false, "its own lock is released");
+  });
+
+  it("#4503: a client whose daemon is SIGKILLed re-attaches to the next daemon and finishes, without relaunching a lane", async () => {
+    const h = host();
+    try {
+      const d1 = await h.daemon();
+      await withRoot(h, () => capture(async () => {
+        const a = orchestrateFromCli(["1", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client);
+        await until(() => h.launched().length === 1 && h.lanePids().length === 1);
+        const gone = new Promise((r) => d1.on("exit", r));
+        d1.kill("SIGKILL");
+        await gone;
+        await new Promise((r) => setTimeout(r, 300)); // the client is retrying against the dead socket
+        await h.daemon();
+        setTimeout(() => writeFileSync(h.release, ""), 200);
+        assert.deepEqual((await a).rows.map((r) => [r.issue, r.outcome]), [[1, "merged"]]);
+        assert.deepEqual(h.launched(), [1], "adopted, not relaunched");
+      }));
+      assert.match(readFileSync(h.log, "utf8"), /^schedd: took over the lock of dead pid \d+ \(it exited without a clean shutdown\)$/m);
+    } finally { await h.cleanup(); }
+  });
+
+  it("#4503: test runs never resolve ~/.forge/sched, and teardown leaves no live stub lane", async () => {
+    assert.notEqual(schedDir(), join(homedir(), ".forge", "sched"));
+    assert.ok(schedDir().startsWith(tmpdir()), schedDir());
+    const h = host();
+    let pids = [], a;
+    try {
+      await h.daemon();
+      await withRoot(h, async () => {
+        a = capture(() => orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-prefilter"], { ...h.client, reconnectMs: 0 })).catch((e) => e);
+        await until(() => h.stubPids().length === 2 && h.lanePids().length === 2);
+      });
+      pids = h.stubPids();
+      for (const pid of pids) assert.equal(isLaneAlive(pid, null), true);
+      const [id] = listBatches(h.root);
+      assert.equal(openBatch(h.root, id).readPlan().env.FORGEDOCK_SCHED_DIR, schedDir(), "the lanes' env pins the test sched dir");
+    } finally { await h.cleanup(); }
+    await a;
+    assert.equal(pids.length, 2);
+    for (const pid of pids) assert.equal(isLaneAlive(pid, null), false, `stub lane ${pid} reaped`);
   });
 
   it("lane env: allowlisted keys come from the client only, the rest from the daemon", () => {

package/bin/stats/efficiency.mjs

@@ -59,14 +59,18 @@ function parseLog(text) {
   return out;
 }
 
-/** Every RUN_START..RUN_TERMINAL slice of one run log (all of them, not only the last). */
+export const LANE_SHAPES = Object.freeze(["light", "upgraded", "full"]);
+
+/** Every RUN_START..RUN_TERMINAL slice of one run log (all of them, not only the last). `shape` (#4452): light (LIGHT_DONE), upgraded (LIGHT_UPGRADE) or full. */
 export function issueSlices(events) {
   const out = [];
   let cur = null;
   for (const e of Array.isArray(events) ? events : []) {
     if (!e || typeof e !== "object") continue;
-    if (e.event === "RUN_START") { cur = { startTs: e.ts ?? null, repo: e.repo ?? null, terminal: null, attempts: [] }; out.push(cur); continue; }
+    if (e.event === "RUN_START") { cur = { startTs: e.ts ?? null, repo: e.repo ?? null, terminal: null, shape: "full", attempts: [] }; out.push(cur); continue; }
     if (!cur) continue;
+    if (e.event === "LIGHT_DONE") cur.shape = "light";
+    else if (e.event === "LIGHT_UPGRADE") cur.shape = "upgraded";
     if (e.event === "PHASE_ATTEMPT_USAGE") {
       cur.attempts.push({ phase: e.phase ?? "other", trigger: e.trigger ?? null, restart_of: e.restart_of ?? null, model: e.model ?? null,
         turns: num(e.turns), cacheRead: num(e.usage?.cache_read_input_tokens), cost: costOf(e),
@@ -145,9 +149,18 @@ const group = (xs) => {
  */
 export function efficiencyMetrics({ entries = [], logs = new Map(), issues = null, repo = null, now = 0, notes = [] } = {}) {
   const merged = [];
+  const inScope = (row, settleAt) => issueSlices(logs.get(row.issue) ?? []).filter((s) => (!repo || !s.repo || s.repo === repo) && (!settleAt || !s.startTs || s.startTs <= settleAt));
+  // #4452: light/full split and upgrade rate over every lane of every row (all outcomes); cost is the #4407 delta sum
+  const shapes = Object.fromEntries(LANE_SHAPES.map((k) => [k, { n: 0, cost_usd: 0 }]));
+  for (const { row, settleAt } of entries) {
+    if (!Number.isSafeInteger(row?.issue)) continue;
+    for (const s of inScope(row, settleAt)) { const g = shapes[s.shape] ?? shapes.full; g.n++; g.cost_usd += sum(s.attempts.map((a) => a.cost)); }
+  }
+  for (const g of Object.values(shapes)) g.cost_usd = r(g.cost_usd, 1e6);
+  const lanes = { ...shapes, upgrade_rate: r(per(shapes.upgraded.n, shapes.light.n + shapes.upgraded.n)) };
   for (const { row, settleAt } of entries) {
     if (row?.outcome !== "merged") continue;
-    const slices = issueSlices(logs.get(row.issue) ?? []).filter((s) => (!repo || !s.repo || s.repo === repo) && (!settleAt || !s.startTs || s.startTs <= settleAt));
+    const slices = inScope(row, settleAt);
     const atts = slices.flatMap((s) => s.attempts), impl = atts.filter((a) => a.phase === "implement");
     const retries = Object.fromEntries(RETRY_CAUSES.map((c) => [c, 0]));
     for (const s of slices) for (const [c, k] of Object.entries(sliceRetries(s))) retries[c] += k;
@@ -156,7 +169,7 @@ export function efficiencyMetrics({ entries = [], logs = new Map(), issues = nul
     const term = [...slices].reverse().find((s) => s.terminal?.reason === "merged")?.terminal;
     merged.push({ issue: row.issue, slices: slices.length, cost: sum(atts.map((a) => a.cost)), implCost: sum(impl.map((a) => a.cost)),
       reviewCost: sum(atts.filter((a) => a.phase === "review").map((a) => a.cost)), implTurns: sum(impl.map((a) => a.turns)), implCache: sum(impl.map((a) => a.cacheRead)),
-      size: term?.size ?? laneSize({ labels: info?.labels ?? [] }) ?? "unknown", impl, retries, info, tools: laneTools(impl) });
+      size: term?.size ?? laneSize({ labels: info?.labels ?? [] }) ?? "unknown", impl, retries, info, tools: laneTools(impl), shape: slices.at(-1)?.shape ?? null });
   }
   const all = group(merged);
   const bySize = {};
@@ -202,10 +215,10 @@ export function efficiencyMetrics({ entries = [], logs = new Map(), issues = nul
     retries: { total, per_merge: r(per(total, merged.length), 100), by_cause: retries, lane_reruns: sum(merged.map((m) => Math.max(0, m.slices - 1))) },
     no_action: { lanes: sum(Object.values(noAction).map((g) => g.n)), cost_usd: r(sum(Object.values(noAction).map((g) => g.cost_usd)), 1e6),
       by_kind: Object.fromEntries(Object.entries(noAction).map(([k, g]) => [k, { ...g, cost_usd: r(g.cost_usd, 1e6) }])) },
-    quality, acceptance,
+    quality, acceptance, lanes,
     tool_calls_per_turn_p50: median(withTools.map((m) => m.tools.per_turn)), rereads_per_merge: withTools.length ? r(per(sum(withTools.map((m) => m.tools.rereads)), withTools.length), 100) : null,
     per_issue: merged.map((m) => ({ issue: m.issue, slices: m.slices, cost_usd: r(m.cost, 1e6), implement_usd: r(m.implCost, 1e6), implement_turns: m.implTurns,
-      implement_cache_read: m.implCache, size: m.size, retries: m.retries, ...(m.info ? { quality: issueQuality(m.info.body).tag } : {}),
+      implement_cache_read: m.implCache, size: m.size, retries: m.retries, ...(m.shape ? { shape: m.shape } : {}), ...(m.info ? { quality: issueQuality(m.info.body).tag } : {}),
       ...(m.tools ? { implement_tool_calls_per_turn: m.tools.per_turn, implement_rereads: m.tools.rereads } : {}) })),
     notes,
   };
@@ -315,5 +328,8 @@ export function formatEfficiencyLines(m, label = "") {
   lines.push(a ? `acceptance (7d after promotion): pass ${a.pass}/${a.pass + a.fail}${a.pass_rate == null ? "" : ` (${Math.round(a.pass_rate * 100)}%)`}, pending ${a.pending}, unprobed ${a.unprobed}`
     : "acceptance (7d after promotion): n/a");
   lines.push(`tools: calls/turn p50 ${f(m.tool_calls_per_turn_p50)}, re-reads (>=3x same path)/merge ${f(m.rereads_per_merge)}`); // #4410
+  const ln = m.lanes; // #4452
+  lines.push(ln ? `lanes: ${LANE_SHAPES.map((k) => `${k} ${ln[k]?.n ?? 0} (${usd(ln[k]?.cost_usd ?? 0)})`).join(", ")}; upgrade rate ${ln.upgrade_rate == null ? "n/a"
+    : `${Math.round(ln.upgrade_rate * 100)}% (${ln.upgraded?.n ?? 0}/${(ln.light?.n ?? 0) + (ln.upgraded?.n ?? 0)})`}` : "lanes: n/a");
   return lines;
 }

package/bin/stats/efficiency.test.mjs

@@ -115,11 +115,44 @@ describe("#4402 efficiencyMetrics", () => {
   });
   it("renders n/a, never NaN, for an empty cohort", () => {
     const lines = formatEfficiencyLines(efficiencyMetrics({}));
-    assert.equal(lines.length, 8);
+    assert.equal(lines.length, 9);
     assert.ok(!lines.some((l) => /NaN|undefined/.test(l)), lines.join("\n"));
     assert.match(lines[0], /^efficiency: merged 0, n\/a\/merge/);
     assert.equal(lines[6], "acceptance (7d after promotion): n/a");
     assert.equal(lines[7], "tools: calls/turn p50 n/a, re-reads (>=3x same path)/merge n/a");
+    assert.equal(lines[8], "lanes: light 0 ($0.00), upgraded 0 ($0.00), full 0 ($0.00); upgrade rate n/a");
+  });
+});
+
+describe("#4452 light/full split and upgrade rate", () => {
+  // issue 1: light, merged; 2: light then upgraded in the same run, merged; 3: full, merged; 4: light, failed; 5: upgraded, failed
+  const logs = new Map([
+    [1, [start("2026-10-22T10:00:00Z"), { event: "LIGHT_ELIGIBILITY", light: true }, att("implement", { trigger: "build", cost_usd: 0.9, cost_usd_delta: 0.4 }), { event: "LIGHT_DONE" }, att("review", { cost_usd: 0.05 }), term("merged", "merged")]],
+    [2, [start("2026-10-22T10:00:00Z"), att("implement", { trigger: "build", cost_usd_delta: 0.3 }), { event: "LIGHT_UPGRADE", reasons: [{ code: "diff_lines" }] }, att("implement", { trigger: "plan", cost_usd_delta: 0.5 }), term("merged", "merged")]],
+    [3, [start("2026-10-22T10:00:00Z"), att("implement", { trigger: "plan", cost_usd_delta: 0.6 }), att("implement", { trigger: "build", cost_usd: 1.2, cost_usd_delta: 0.4 }), term("merged", "merged")]],
+    [4, [start("2026-10-22T10:00:00Z"), att("implement", { trigger: "build", cost_usd_delta: 0.2 }), { event: "LIGHT_DONE" }, term("failed", "ci.red")]],
+    [5, [start("2026-10-22T10:00:00Z"), att("implement", { trigger: "build", cost_usd_delta: 0.1 }), { event: "LIGHT_UPGRADE" }, term("failed", "conformance.unimplemented")]],
+  ]);
+  const entries = [1, 2, 3, 4, 5].map((issue) => ({ row: { issue, outcome: issue <= 3 ? "merged" : "failed" }, settleAt: null }));
+  const m = efficiencyMetrics({ entries, logs, repo: "o/r" });
+
+  it("tags each lane slice by shape from LIGHT_DONE / LIGHT_UPGRADE", () => {
+    assert.deepEqual(issueSlices(logs.get(1)).map((s) => s.shape), ["light"]);
+    assert.deepEqual(issueSlices(logs.get(2)).map((s) => s.shape), ["upgraded"]);
+    assert.deepEqual(issueSlices(logs.get(3)).map((s) => s.shape), ["full"]);
+  });
+  it("splits every lane (all outcomes) by shape with true #4407 delta costs and computes the upgrade rate", () => {
+    assert.deepEqual(m.lanes, {
+      light: { n: 2, cost_usd: 0.65 }, // 0.4 delta (not the 0.9 running total) + 0.05 review + 0.2
+      upgraded: { n: 2, cost_usd: 0.9 },
+      full: { n: 1, cost_usd: 1 }, // 0.6 + 0.4 delta, not 1.2
+      upgrade_rate: 0.5,
+    });
+    assert.deepEqual(m.per_issue.map((x) => [x.issue, x.shape]), [[1, "light"], [2, "upgraded"], [3, "full"]]);
+  });
+  it("the batch report prints the split and the upgrade rate", () => {
+    const lines = formatEfficiencyLines(m);
+    assert.equal(lines.at(-1), "lanes: light 2 ($0.65), upgraded 2 ($0.90), full 1 ($1.00); upgrade rate 50% (2/4)");
   });
 });
 
@@ -142,7 +175,7 @@ describe("#4410 tool calls per turn and re-reads", () => {
     const p = Object.fromEntries(m.per_issue.map((x) => [x.issue, x]));
     assert.equal(p[1].implement_tool_calls_per_turn, 1.5); assert.equal(p[1].implement_rereads, 2);
     assert.equal("implement_tool_calls_per_turn" in p[4], false);
-    assert.equal(formatEfficiencyLines(m).at(-1), "tools: calls/turn p50 1.50, re-reads (>=3x same path)/merge 1.00");
+    assert.equal(formatEfficiencyLines(m).find((l) => l.startsWith("tools:")), "tools: calls/turn p50 1.50, re-reads (>=3x same path)/merge 1.00");
   });
   it("null without tool data (older logs)", () => {
     const e = efficiencyMetrics({ entries: [entries[3]], logs, repo: "o/r", now: NOW });