forgedock 1.42.0 → 1.43.1

package/bin/lean/schedd.mjs

@@ -20,7 +20,7 @@
  *                                   {type:"ok", cancelled:false, finished:true}, nothing written
  *   {op:"hold"|"release", id, issues} → {type:"ok", changed, hosted} (#4592): merges into / removes from the batch's `hold` file; a hosted
  *                                   runner admits no held issue from its next pass (running lanes are not stopped)
- *   {op:"status"}                 → {type:"status", pid, lastTick, batches:[{id, running}]} (#4780: `lastTick` = epoch ms of the last heartbeat; see schedd-health.mjs)
+ *   {op:"status"}                 → {type:"status", pid, lastTick, batches:[{id, running, lastProgressAt, lastProgressAgeSec, pendingAdds}]} (#4775: also an unhosted batch with pending adds)
  * Failures answer {type:"error", message}. A client closing its connection never cancels a batch (only `cancel` stops one), and stopping the daemon
  * never stops a lane (lanes are detached sessions). On start the daemon resumes every daemon-hosted batch without a
  * report, so live lanes are re-adopted through the batch runner's `adopt`. It exits after `idleMs` with no batch and no client.
@@ -34,8 +34,8 @@
  */
 import { closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, writeSync } from "node:fs";
 import { createConnection, createServer } from "node:net";
-import { spawn } from "node:child_process";
-import { tmpdir } from "node:os";
+import { execFileSync, spawn } from "node:child_process";
+import { homedir, tmpdir } from "node:os";
 import { createHash } from "node:crypto";
 import { join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -46,6 +46,10 @@ import { batchRoot, listBatches, openBatch, SECRET_ENV } from "./batch.mjs";
 export const IDLE_MS = 10 * 60 * 1000;
 export const STOP_MS = 10 * 1000;
 export const LOCK_CHECK_MS = 5 * 1000;
+export const ADMIT_MS = 120 * 1000; // #4775: how long one add/requeue/attach/resume waits for its batch to start before answering an error
+export const LAG_MS = 5 * 1000; // #4775: an event-loop stall longer than this is logged
+export const REQUEST_MS = 15 * 1000; // #4775: default bound of a control/status request
+export const UNIT = "forgedock-schedd";
 export const DUE_MS = 15 * 60 * 1000; // #4627: how often the due callback (windowed post-deploy monitoring) runs while the daemon is up
 
 /** #4762: win32 `net` servers only accept named pipes, so the socket is a per-dir pipe there (it vanishes with its server). */
@@ -130,7 +134,7 @@ function takeLock(lock, pid, alive) {
  */
 export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(process.env.FORGEDOCK_SCHEDD_IDLE_MS) || IDLE_MS,
   root = batchRoot(), log = (m) => console.log(m), pid = process.pid, alive = pidAlive, resumeOnStart = true, stopMs = STOP_MS,
-  lockCheckMs = LOCK_CHECK_MS, due = null, dueMs = Number(process.env.FORGEDOCK_SCHEDD_DUE_MS) || DUE_MS, preflight = null } = {}) {
+  lockCheckMs = LOCK_CHECK_MS, admitMs = Number(process.env.FORGEDOCK_SCHEDD_ADMIT_MS) || ADMIT_MS, lagMs = LAG_MS, due = null, dueMs = Number(process.env.FORGEDOCK_SCHEDD_DUE_MS) || DUE_MS, preflight = null } = {}) {
   const p = scheddPaths(dir);
   mkdirSync(dir, { recursive: true });
   const lock = takeLock(p.lock, pid, alive);
@@ -168,6 +172,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     const run = Promise.resolve().then(() => runBatch({ argv, cwd, env, sink, keepBreaker, signal: ac.signal })).then((result) => {
       const report = ent.dir ? openBatch(root, ent.id).readReport()?.report ?? null : null;
       emit({ type: "done", result, report });
+      if (ent.id) rehostLater(ent.id); // #4917
     }, (e) => {
       const message = e?.message ?? String(e);
       if (e?.code === "batch.stopped") { log(`schedd: batch ${ent.id ?? argv.join(" ")} stopped (lanes keep running; resumed by the next daemon)`); if (!started) reject(new Error(message)); return; }
@@ -176,8 +181,61 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     runs.set(run, ac);
   });
 
+  // #4917: a batch that ended with a deferred issue still waiting (no report) is re-hosted once its retry-at is near, without a restart
+  const rehostTimers = new Set();
+  const rehostLater = (id) => {
+    if (closing) return;
+    let b, st;
+    try { b = openBatch(root, id); if (b.readReport() || b.readCancel() || !b.pendingIssues().length) return; st = b.store.load(); } catch { return; }
+    if (st?.breaker && !st.breaker.transport) return; // a tripped breaker holds the batch for an operator --resume (a re-host would stop again at once)
+    const at = Math.min(...b.pendingIssues().map((n) => Number(st?.notBefore?.[n]) || Infinity));
+    if (!Number.isFinite(at)) return; // nothing waits on the clock: a deliberate hold stays for --resume
+    if (at <= Date.now()) return; // the run that just ended already saw this retry-at pass and did not admit: re-hosting would loop every second
+    const t = setTimeout(() => {
+      rehostTimers.delete(t);
+      if (closing || hosted.has(id) || starting.has(id)) return;
+      log(`schedd: re-hosting batch ${id} (pending issues waiting on a retry)`);
+      const started = host({ argv: ["--resume", id], cwd: process.cwd(), env: {}, keepBreaker: true });
+      starting.set(id, started);
+      started.catch((e) => log(`schedd: re-host ${id} failed: ${e?.message ?? e}`)).finally(() => starting.delete(id));
+    }, Math.min(60_000, Math.max(1000, at - Date.now())));
+    t.unref?.();
+    rehostTimers.add(t);
+  };
+
+  /** #4775: wait for a batch start at most `admitMs`; the start itself goes on (a second host() would duplicate the batch). */
+  const bound = (p, what) => {
+    let t;
+    return Promise.race([p, new Promise((_, rej) => { t = setTimeout(() => rej(new Error(`${what} did not start within ${admitMs} ms (schedd is busy; retry)`)), admitMs); t.unref?.(); })]).finally(() => clearTimeout(t));
+  };
+  const hostBounded = (args, first = null, what = "batch") => bound(host(args, first), what);
+
+  /** #4775: per batch id, last progress and pending adds: every hosted batch, plus any daemon batch holding adds it never took. */
+  const statusBatches = () => {
+    const now = Date.now(), out = [];
+    const ids = new Set(hosted.keys());
+    for (const id of starting.keys()) ids.add(id);
+    try { for (const id of listBatches(root)) if (!ids.has(id) && openBatch(root, id).readPlan()?.daemon) ids.add(id); } catch { /* unreadable root */ }
+    for (const id of [...ids].sort()) {
+      const isHosted = hosted.has(id) || starting.has(id);
+      let last = null, pending = [];
+      try { const b = openBatch(root, id); last = b.lastProgressMs(); pending = b.unrunAdds(); } catch { /* a half-written batch */ }
+      if (!isHosted && !pending.length) continue;
+      out.push({ id, running: isHosted, lastProgressAt: last === null ? null : new Date(last).toISOString(), lastProgressAgeSec: last === null ? null : Math.max(0, Math.round((now - last) / 1000)), pendingAdds: pending });
+    }
+    return out;
+  };
+
   const handle = async (req, send) => {
-    if (req.op === "status") return send({ type: "status", pid, lastTick, batches: [...hosted.keys()].map((id) => ({ id, running: true })) }, true);
+    if (req.op === "status") {
+      // #4917: a batch dir that still holds pending issues but is not hosted (finalised early, or never resumed)
+      const unhosted = [];
+      for (const id of listBatches(root)) {
+        if (hosted.has(id) || starting.has(id)) continue;
+        try { const pending = openBatch(root, id).pendingIssues(); if (pending.length) unhosted.push({ id, pending }); } catch { /* advisory */ }
+      }
+      return send({ type: "status", pid, lastTick, batches: statusBatches(), unhosted }, true);
+    }
     if (req.op === "set-parallel") {
       const b = openBatch(root, req.id);
       if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
@@ -188,9 +246,9 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       const b = openBatch(root, req.id);
       if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
       const { added, skipped, requeued = [] } = b.addIssues(req.issues ?? []);
-      // #4687: a re-queued skipped issue reopens a finished, unhosted batch like --requeue does
-      const reopen = requeued.length > 0 && !hosted.has(req.id) && !starting.has(req.id);
-      if (reopen) await host({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} });
+      // #4687/#5000/#4917: a re-queued skipped issue or a newly added one reopens a finished, unhosted batch like --requeue does
+      const reopen = (added.length > 0 || requeued.length > 0 || b.unrunAdds().length > 0 || b.unrunRequeues().length > 0) && !hosted.has(req.id) && !starting.has(req.id);
+      if (reopen) await hostBounded({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} }, null, `batch ${req.id}`);
       return send({ type: "ok", added, skipped, requeued, hosted: hosted.has(req.id), reopened: reopen }, true);
     }
     if (req.op === "requeue") {
@@ -198,8 +256,9 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       const b = openBatch(root, req.id);
       if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
       const { requeued, refused } = b.requeue(req.issues ?? [], { reason: req.reason, by: req.by });
-      const reopen = requeued.length > 0 && !hosted.has(req.id) && !starting.has(req.id);
-      if (reopen) await host({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} });
+      // #4917: a refused repeat requeue still reopens when an earlier entry was never applied
+      const reopen = (requeued.length > 0 || b.unrunRequeues().length > 0) && !hosted.has(req.id) && !starting.has(req.id);
+      if (reopen) await hostBounded({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} }, null, `batch ${req.id}`);
       return send({ type: "ok", requeued, refused, hosted: hosted.has(req.id), reopened: reopen }, true);
     }
     if (req.op === "hold" || req.op === "release") {
@@ -222,7 +281,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     }
     if (req.op === "attach") {
       if (req.resume) { const rb = openBatch(root, req.id); if (rb.exists()) rb.clearCancel(); } // #4592: the operator's resume continues a cancelled batch the daemon still hosts
-      if (!hosted.has(req.id) && starting.has(req.id)) await starting.get(req.id).catch(() => {});
+      if (!hosted.has(req.id) && starting.has(req.id)) await bound(starting.get(req.id), `batch ${req.id}`).catch((e) => { if (/did not start within/.test(e?.message)) throw e; });
       const live = hosted.get(req.id);
       if (live) {
         // Synchronous read + subscribe: no row is lost or doubled between the replay and the live stream.
@@ -236,7 +295,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       if (rep && !(rep.rows ?? []).some((r) => r.outcome === "held") && !b.unrunAdds().length && !b.unrunRequeues().length) {
         return send({ type: "done", replay: true, result: { rows: rep.rows ?? [], crossLane: rep.crossLane ?? [], batch: b.id, ok: !!rep.ok }, report: rep.report ?? null }, true);
       }
-      const ent = await host({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} }, sub);
+      const ent = await hostBounded({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} }, sub, `batch ${req.id}`);
       return send({ type: "batch", id: ent.id, dir: ent.dir });
     }
     throw new Error(`unknown op ${JSON.stringify(req.op)}`);
@@ -277,7 +336,9 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     if (closing) return closing;
     clearInterval(idleTimer);
     clearInterval(lockTimer);
+    clearInterval(lagTimer);
     clearInterval(beatTimer);
+    for (const t of rehostTimers) clearTimeout(t);
     if (dueTimer) clearInterval(dueTimer);
     for (const c of conns) c.destroy();
     const stopping = [...runs.keys()];
@@ -309,6 +370,14 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     else if (process.platform !== "win32" && !existsSync(p.sock)) { log(`schedd: lost socket ${p.sock}, exiting`); close(); }
   }, lockCheckMs);
   lockTimer.unref?.();
+  // #4775: an event loop starved by host load leaves sockets unanswered and children unreaped; leave a trace of it.
+  let lagAt = Date.now(), lagLogged = 0;
+  const lagTimer = setInterval(() => {
+    const now = Date.now(), lag = now - lagAt - 1000;
+    lagAt = now;
+    if (lag >= lagMs && now - lagLogged >= 60_000) { lagLogged = now; log(`schedd: event loop stalled ${Math.round(lag / 1000)}s (host overloaded?)`); }
+  }, 1000);
+  lagTimer.unref?.();
   // #4627: windowed post-deploy checks. One run at a time; a failure is logged and never stops the daemon.
   let dueRunning = false;
   const dueTimer = due ? setInterval(async () => {
@@ -333,6 +402,9 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       log(`schedd: resuming batch ${id}`);
       const started = host({ argv: ["--resume", id], cwd: process.cwd(), env: {}, keepBreaker: true });
       starting.set(id, started);
+      const stall = setTimeout(() => log(`schedd: batch ${id} has not started after ${admitMs} ms (still resuming; other batches are served)`), admitMs);
+      stall.unref?.();
+      started.finally(() => clearTimeout(stall)).catch(() => {});
       started.catch((e) => log(`schedd: resume ${id} failed: ${e?.message ?? e}`)).finally(() => starting.delete(id));
     }
   }
@@ -362,8 +434,29 @@ export function exitOnSignals(d, { on = (s, f) => (s.startsWith("SIG") || s ===
   on("exit", (code) => log(`schedd: exited (code ${code})`));
 }
 
-/** Connect to the daemon, spawning it (`spawnDaemon()`) when nobody listens. @returns {Promise<import("node:net").Socket>} */
-export async function connectDaemon({ dir = schedDir(), spawnDaemon = defaultSpawn, waitMs = 10_000, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) } = {}) {
+/**
+ * #4775: state of the systemd-user unit (`active`, `activating`, `inactive`, ...), or null when no unit is installed or
+ * systemd is not usable here (then a client may spawn the daemon itself).
+ */
+export function unitState({ unitFile = join(homedir(), ".config", "systemd", "user", `${UNIT}.service`), exists = existsSync, run = execFileSync } = {}) {
+  if (!exists(unitFile)) return null;
+  try { return String(run("systemctl", ["--user", "is-active", UNIT], { encoding: "utf8", timeout: 3000, stdio: ["ignore", "pipe", "ignore"] })).trim() || null; }
+  catch (e) { return e?.code === "ENOENT" || e?.killed ? null : String(e?.stdout ?? "").trim() || null; } // is-active exits 3 and prints the state when not active
+}
+
+/** Start the installed unit (best effort: the connect retry reports a failure). */
+export function startUnit({ run = execFileSync } = {}) {
+  try { run("systemctl", ["--user", "start", "--no-block", UNIT], { timeout: 5000, stdio: "ignore" }); } catch { /* reported by the connect retry */ }
+}
+
+/**
+ * Connect to the daemon. When nobody listens it is started: a systemd-managed instance (`unitState()` active, activating
+ * or reloading: restarting) is waited for and never raced by a second daemon, an installed but stopped unit is started
+ * (`startUnit()`), and only without a unit does `spawnDaemon()` run (#4775). `spawnDaemon: null` never starts anything.
+ * @returns {Promise<import("node:net").Socket>}
+ */
+export async function connectDaemon({ dir = schedDir(), spawnDaemon = defaultSpawn, waitMs = 10_000, sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
+  unit = unitState, start = startUnit } = {}) {
   const { sock } = scheddPaths(dir);
   const tryConnect = () => new Promise((res, rej) => {
     const c = createConnection(sock);
@@ -371,7 +464,9 @@ export async function connectDaemon({ dir = schedDir(), spawnDaemon = defaultSpa
     c.once("error", rej);
   });
   try { return await tryConnect(); } catch (e) { if (!["ENOENT", "ECONNREFUSED"].includes(e?.code) || !spawnDaemon) throw e; }
-  spawnDaemon({ dir });
+  const state = unit();
+  if (state === null) spawnDaemon({ dir });
+  else if (!["active", "activating", "reloading"].includes(state)) start();
   const end = Date.now() + waitMs;
   for (;;) {
     await sleep(100);
@@ -386,7 +481,7 @@ export function defaultSpawn({ dir = schedDir(), script = SCHEDD_SCRIPT, env = p
   mkdirSync(dir, { recursive: true });
   const out = openSync(scheddPaths(dir).log, "a");
   try {
-    const child = spawn(process.execPath, [script, "schedd"], { env, detached: true, stdio: ["ignore", out, out] });
+    const child = spawn(process.execPath, [script, "schedd"], { env, detached: true, windowsHide: true, stdio: ["ignore", out, out] });
     child.on?.("error", () => { /* the connect retry reports it */ });
     child.unref?.();
   } finally { closeSync(out); }
@@ -396,10 +491,12 @@ export function defaultSpawn({ dir = schedDir(), script = SCHEDD_SCRIPT, env = p
  * Send one request and feed every reply to `onMsg`. Resolves with the last message (`done`, `ok`, `status`), rejects on
  * `error` or when the daemon closes the stream first.
  */
-export function request(sock, msg, onMsg = () => {}) {
+export function request(sock, msg, onMsg = () => {}, { timeoutMs = 0 } = {}) {
   return new Promise((resolve, reject) => {
-    let buf = "", last = null, settled = false;
-    const finish = (fn, v) => { if (!settled) { settled = true; fn(v); } };
+    let buf = "", last = null, settled = false, timer = null;
+    const finish = (fn, v) => { if (!settled) { settled = true; clearTimeout(timer); fn(v); } };
+    // #4775: a bound for one-shot requests; a stalled daemon answers an error instead of hanging the caller (streams pass none)
+    if (timeoutMs > 0) timer = setTimeout(() => { sock.destroy(); finish(reject, Object.assign(new Error(`schedd did not answer within ${Math.round(timeoutMs / 1000)}s (not responding)`), { code: "schedd.timeout" })); }, timeoutMs);
     sock.on("data", (d) => {
       buf += d;
       let i;

package/bin/lean/schedd.test.mjs

@@ -8,11 +8,11 @@ import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
 import { openBatch, isLaneAlive, listBatches, SECRET_ENV } from "./batch.mjs";
-import { startDaemon, scheddPaths, connectDaemon, request, SCHEDD_SCRIPT, mergeEnv, batchEnv, droppedEnv, sanitizePath, redactEnv, exitOnSignals } from "./schedd.mjs";
+import { startDaemon, scheddPaths, connectDaemon, request, unitState, SCHEDD_SCRIPT, mergeEnv, batchEnv, droppedEnv, sanitizePath, redactEnv, exitOnSignals } from "./schedd.mjs";
 import { schedDir } from "./sched.mjs";
 import { readHeartbeat } from "./schedd-health.mjs";
 import { formatTable, orchestrate } from "./orchestrate.mjs";
-import { orchestrateFromCli } from "../engine-cli.mjs";
+import { orchestrateFromCli, scheddFromCli } from "../engine-cli.mjs";
 
 // #4503: this process, and the env its clients submit for their lanes, never resolve the host's `~/.forge/sched`.
 process.env.FORGEDOCK_SCHED_DIR = tmpDir("schedd-home-");
@@ -249,6 +249,55 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
     } finally { await h.cleanup(); }
   });
 
+  it("#5000: add to a finished unhosted batch reopens it through the daemon and runs the new issue", async () => {
+    const h = host();
+    try {
+      await h.daemon();
+      const client = { ...h.client, io: { gh: async (a) => (a[0] === "issue" ? JSON.stringify({ state: "OPEN", labels: [] }) : "o/r\n") } };
+      await withRoot(h, () => capture(async () => {
+        writeFileSync(h.release, "");
+        await orchestrateFromCli(["1", "--parallel", "2", "--repo", "o/r", "--no-prefilter"], h.client);
+        const [id] = listBatches(h.root);
+        const b = openBatch(h.root, id);
+        const res = await orchestrateFromCli(["--add", id, "6"], client);
+        assert.deepEqual(res.added, [6]);
+        assert.equal(res.reopened, true, "the finished batch is hosted again");
+        await until(() => h.launched().includes(6));
+        await until(() => b.store.load()?.results?.[6]?.outcome === "merged");
+        assert.deepEqual(listBatches(h.root), [id], "no second batch");
+      }));
+    } finally { await h.cleanup(); }
+  });
+
+  it("#4917: a held (pending) issue is flagged unhosted by status once its report is gone; --requeue re-hosts it and says so", async () => {
+    const h = host();
+    try {
+      await h.daemon();
+      const client = { ...h.client, io: { gh: async (a) => (a[0] === "issue" ? JSON.stringify({ state: "OPEN", labels: [] }) : "o/r\n") } };
+      await withRoot(h, () => capture(async () => {
+        const run = orchestrateFromCli(["1", "2", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client);
+        await until(() => h.launched().length === 1 && listBatches(h.root).length === 1);
+        const [id] = listBatches(h.root);
+        const b = openBatch(h.root, id);
+        b.holdIssues([2]);
+        writeFileSync(h.release, "");
+        const res = await run;
+        assert.equal(res.rows.find((r) => r.issue === 2).outcome, "held");
+        assert.deepEqual(b.pendingIssues(), [2]);
+        b.clearReport(); // a batch finalised before the fix, or one never resumed
+        const st = await request(await connectDaemon({ dir: h.sched, spawnDaemon: null }), { op: "status" });
+        assert.deepEqual(st.unhosted, [{ id, pending: [2] }]);
+        // finalised by an older version: a report plus unapplied requeue entries still reopens, and the CLI says so
+        b.releaseIssues([2]);
+        b.store.save({ ...b.store.load(), results: { ...b.store.load().results, 1: { issue: 1, outcome: "needs-human", pr: null, cost_usd: 0, minutes: 0 } } });
+        b.writeReport({ rows: [{ issue: 1, outcome: "needs-human" }] });
+        const rq = await capture(() => orchestrateFromCli(["--requeue", id, "1"], client));
+        assert.deepEqual(rq.res.requeued, [1]);
+        assert.ok(rq.lines.some((l) => l.includes("reopened and hosted by the schedd")), rq.lines.join("\n"));
+      }));
+    } finally { await h.cleanup(); }
+  });
+
   it("#4376: --cancel stops one batch (its lane torn down, the rest held, report written); the other batch and the daemon run on", async () => {
     const h = host();
     try {
@@ -350,6 +399,66 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
     } finally { await h.cleanup(); }
   });
 
+  it("#4917: a batch that ends unreported with a deferred issue waiting is re-hosted by the running daemon once its retry-at is near", async () => {
+    const dir = tmpDir("schedd-rehost-"), root = join(dir, "batches"), id = "b20261031-000000-4917";
+    const calls = [];
+    const runBatch = async ({ argv, sink }) => {
+      calls.push(argv);
+      const b = openBatch(root, id);
+      if (argv[0] !== "--resume") {
+        b.writePlan({ daemon: true, repo: "o/r" });
+        b.store.save({ v: 1, order: [1], results: {}, running: {}, pending: [1], notBefore: { 1: Date.now() + 1200 } });
+      } else b.store.save({ ...b.store.load(), pending: [], results: { 1: { issue: 1, outcome: "merged" } } });
+      sink.batch({ id, dir: b.dir });
+      return {};
+    };
+    const d = await startDaemon({ dir, root, runBatch, log: () => {}, resumeOnStart: false });
+    try {
+      await request(await connectDaemon({ dir, spawnDaemon: null }), { op: "submit", argv: ["1"], cwd: dir, env: {} });
+      await until(() => calls.length === 2, 8000);
+      assert.deepEqual(calls[1], ["--resume", id]);
+    } finally { await d.close(); }
+  });
+
+  it("#4917: a resume that leaves the waiting issue unadmitted (breaker tripped, retry-at passed) is not re-hosted again", async () => {
+    const dir = tmpDir("schedd-rehost-loop-"), root = join(dir, "batches"), id = "b20261031-000000-4918";
+    const calls = [];
+    const runBatch = async ({ argv, sink }) => {
+      calls.push(argv);
+      const b = openBatch(root, id);
+      // first run: deferred issue 1 waits 300 ms; every resume ends at once (a tripped breaker), leaving it pending with a past retry-at
+      if (argv[0] !== "--resume") { b.writePlan({ daemon: true, repo: "o/r" }); b.store.save({ v: 1, order: [1], results: {}, running: {}, pending: [1], notBefore: { 1: Date.now() + 300 } }); }
+      sink.batch({ id, dir: b.dir });
+      return {};
+    };
+    const d = await startDaemon({ dir, root, runBatch, log: () => {}, resumeOnStart: false });
+    try {
+      await request(await connectDaemon({ dir, spawnDaemon: null }), { op: "submit", argv: ["1"], cwd: dir, env: {} });
+      await until(() => calls.length >= 2, 8000);
+      await new Promise((r) => setTimeout(r, 3500));
+      assert.equal(calls.length, 2, `re-hosted ${calls.length - 1} times; a past retry-at the resume did not act on must not loop`);
+    } finally { await d.close(); }
+  });
+
+  it("#4917: a batch whose breaker tripped (not transport) is not re-hosted by the timer", async () => {
+    const dir = tmpDir("schedd-rehost-brk-"), root = join(dir, "batches"), id = "b20261031-000000-4919";
+    const calls = [];
+    const runBatch = async ({ argv, sink }) => {
+      calls.push(argv);
+      const b = openBatch(root, id);
+      b.writePlan({ daemon: true, repo: "o/r" });
+      b.store.save({ v: 1, order: [1], results: {}, running: {}, pending: [1], notBefore: { 1: Date.now() + 300 }, breaker: { at: Date.now(), errors: 3 } });
+      sink.batch({ id, dir: b.dir });
+      return {};
+    };
+    const d = await startDaemon({ dir, root, runBatch, log: () => {}, resumeOnStart: false });
+    try {
+      await request(await connectDaemon({ dir, spawnDaemon: null }), { op: "submit", argv: ["1"], cwd: dir, env: {} });
+      await new Promise((r) => setTimeout(r, 2000));
+      assert.equal(calls.length, 1, "a tripped breaker holds the batch for an operator --resume");
+    } finally { await d.close(); }
+  });
+
   it("#4454: close() stops every hosted batch loop before releasing the lock; no batch admits a lane after close()", async () => {
     const dir = tmpDir("schedd-stop-"), root = join(dir, "batches"), lock = scheddPaths(dir).lock;
     const started = { a: [], b: [] }, stoppedWithLock = {};
@@ -637,6 +746,96 @@ describe("schedd due timer (#4627)", () => {
   });
 });
 
+// #4775: a client never races a systemd-managed daemon; a stalled daemon is visible and bounded.
+describe("schedd under load and restart (#4775)", () => {
+  const noSleep = () => Promise.resolve();
+  it("connectDaemon never spawns while the unit is active or activating, and starts an installed stopped unit instead of spawning", async () => {
+    for (const state of ["active", "activating", "reloading"]) {
+      let spawned = 0;
+      await assert.rejects(connectDaemon({ dir: tmpDir("schedd-unit-"), waitMs: 0, sleep: noSleep, spawnDaemon: () => { spawned++; }, unit: () => state, start: () => { spawned++; } }), /did not start/);
+      assert.equal(spawned, 0, `${state}: nothing is started`);
+    }
+    const calls = [];
+    await assert.rejects(connectDaemon({ dir: tmpDir("schedd-unit-"), waitMs: 0, sleep: noSleep, spawnDaemon: () => calls.push("spawn"), unit: () => "inactive", start: () => calls.push("unit") }), /did not start/);
+    assert.deepEqual(calls, ["unit"]);
+  });
+
+  it("connectDaemon spawns the daemon itself only when no unit is installed; spawnDaemon:null starts nothing", async () => {
+    const calls = [];
+    await assert.rejects(connectDaemon({ dir: tmpDir("schedd-unit-"), waitMs: 0, sleep: noSleep, spawnDaemon: () => calls.push("spawn"), unit: () => null, start: () => calls.push("unit") }), /did not start/);
+    assert.deepEqual(calls, ["spawn"]);
+    await assert.rejects(connectDaemon({ dir: tmpDir("schedd-unit-"), spawnDaemon: null, unit: () => { throw new Error("not consulted"); } }), { code: "ENOENT" });
+  });
+
+  it("unitState reads the systemctl state (also from a non-zero exit) and is null without a unit file", () => {
+    assert.equal(unitState({ exists: () => false }), null);
+    assert.equal(unitState({ exists: () => true, run: () => "activating\n" }), "activating");
+    assert.equal(unitState({ exists: () => true, run: () => { throw Object.assign(new Error("exit 3"), { stdout: "inactive\n" }); } }), "inactive");
+    assert.equal(unitState({ exists: () => true, run: () => { throw Object.assign(new Error("none"), { code: "ENOENT" }); } }), null);
+  });
+
+  it("request(timeoutMs) rejects a daemon that never answers instead of hanging", async () => {
+    const dir = tmpDir("schedd-mute-"), sock = scheddPaths(dir).sock;
+    const server = (await import("node:net")).createServer(() => { /* accepts, never answers */ });
+    await new Promise((r) => server.listen(sock, r));
+    try {
+      const c = await connectDaemon({ dir, spawnDaemon: null });
+      await assert.rejects(request(c, { op: "status" }, undefined, { timeoutMs: 80 }), { code: "schedd.timeout" });
+    } finally { server.close(); }
+  });
+
+  it("status reports per-batch last-progress age and pending adds, including an unhosted batch with un-admitted adds", async () => {
+    const root = tmpDir("schedd-st-b-"), dir = tmpDir("schedd-st-");
+    const mk = (id, files) => { mkdirSync(join(root, id), { recursive: true }); writeFileSync(join(root, id, "plan.json"), JSON.stringify({ daemon: true })); for (const [n, t] of Object.entries(files)) writeFileSync(join(root, id, n), t); };
+    mk("b-stuck", { "state.json": JSON.stringify({ order: [1] }), add: "[7,8]" });
+    mk("b-quiet", { "state.json": JSON.stringify({ order: [1] }) });
+    const d = await startDaemon({ dir, root, runBatch: async () => ({}), resumeOnStart: false, log: () => {} });
+    try {
+      const st = await request(await connectDaemon({ dir, spawnDaemon: null }), { op: "status" });
+      assert.equal(st.batches.length, 1, "a quiet unhosted batch is not listed");
+      const [b] = st.batches;
+      assert.equal(b.id, "b-stuck");
+      assert.equal(b.running, false);
+      assert.deepEqual(b.pendingAdds, [7, 8]);
+      assert.ok(b.lastProgressAgeSec >= 0 && b.lastProgressAgeSec < 60);
+    } finally { await d.close(); }
+  });
+
+  it("a batch whose start stalls does not block another batch's request, and the stalled request answers an error", async () => {
+    const root = tmpDir("schedd-stall-b-"), dir = tmpDir("schedd-stall-");
+    for (const id of ["b-hung", "b-ok"]) { mkdirSync(join(root, id), { recursive: true }); writeFileSync(join(root, id, "plan.json"), JSON.stringify({ daemon: true })); writeFileSync(join(root, id, "state.json"), JSON.stringify({ order: [1] })); }
+    const d = await startDaemon({ dir, root, resumeOnStart: false, admitMs: 150, log: () => {},
+      runBatch: ({ argv, sink }) => (argv[1] === "b-hung" ? new Promise(() => {}) : Promise.resolve().then(() => { sink.batch({ id: argv[1], dir: join(root, argv[1]) }); return {}; })) });
+    try {
+      const ask = async (id) => request(await connectDaemon({ dir, spawnDaemon: null }), { op: "attach", id }).catch((e) => e);
+      const hung = ask("b-hung");
+      const ok = await ask("b-ok");
+      assert.notEqual(ok instanceof Error && /did not start/.test(ok.message), true, "the healthy batch is served");
+      const e = await hung;
+      assert.match(e.message, /did not start within 150 ms/);
+    } finally { await d.close(); }
+  });
+
+  it("the unit stands by while another daemon holds the lock instead of exiting 0 (no Restart=always loop)", async () => {
+    const dir = tmpDir("schedd-sb-"), root = tmpDir("schedd-sb-b-");
+    const owner = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60000)"], { stdio: "ignore" });
+    const gone = new Promise((r) => owner.on("exit", r));
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(scheddPaths(dir).lock, String(owner.pid)); // a live daemon of another process
+    const slept = [];
+    process.env.FORGEDOCK_SCHEDD_IDLE_MS = "100"; // the successor daemon exits when idle, so the call returns
+    process.env.FORGEDOCK_BATCH_DIR = root;
+    try {
+      const r = await Promise.race([
+        scheddFromCli([], { scheddDir: dir, standby: true, standbyMs: 1, sleep: async () => { slept.push(1); if (slept.length === 3) { owner.kill("SIGKILL"); await gone; } }, processOn: () => {}, exit: () => {}, batchDeps: {}, due: null }),
+        new Promise((res) => setTimeout(() => res("timeout"), 5000)),
+      ]);
+      assert.ok(slept.length >= 3, "waited for the owner before retrying");
+      assert.notEqual(r, "timeout");
+    } finally { delete process.env.FORGEDOCK_SCHEDD_IDLE_MS; delete process.env.FORGEDOCK_BATCH_DIR; owner.kill("SIGKILL"); }
+  });
+});
+
 describe("#4762 win32 socket path", () => {
   it("win32 derives a stable per-dir named pipe; other platforms keep the unix socket file", () => {
     const a = scheddPaths("/tmp/a", "win32"), b = scheddPaths("/tmp/b", "win32");

package/bin/lean/selftest-live.mjs

@@ -158,7 +158,7 @@ const CLI = join(dirname(fileURLToPath(import.meta.url)), "..", "forgedock.mjs")
 export function realIo(env = process.env, { pool = false } = {}) {
   let self;
   const root = batchRoot(env);
-  const detached = (args) => { const p = spawn(process.execPath, [CLI, ...args], { detached: true, stdio: "ignore", env }); p.unref(); return { kill: async () => { try { process.kill(-p.pid, "SIGTERM"); } catch { /* gone */ } } }; };
+  const detached = (args) => { const p = spawn(process.execPath, [CLI, ...args], { detached: true, windowsHide: true, stdio: "ignore", env }); p.unref(); return { kill: async () => { try { process.kill(-p.pid, "SIGTERM"); } catch { /* gone */ } } }; };
   const waitBatch = async (before) => { for (let i = 0; i < 60; i++) { const id = listBatches(root).find((b) => !before.has(b)); if (id) return id; await new Promise((r) => setTimeout(r, 1000)); } throw new Error("orchestrate did not create a batch within 60s"); };
   return (self = {
     now: () => Date.now(), sleep: (ms) => new Promise((r) => setTimeout(r, ms)),

package/bin/lean/selftest-nightly.mjs

@@ -37,7 +37,7 @@ function writeLast(dir, repo, ts) {
  */
 export function launchNightly({ cli, repo, cwd = process.cwd(), spawnFn = spawn }) {
   try {
-    const c = spawnFn(process.execPath, [cli, "selftest", "--live", "--nightly-report", repo], { cwd, detached: true, stdio: "ignore" });
+    const c = spawnFn(process.execPath, [cli, "selftest", "--live", "--nightly-report", repo], { cwd, detached: true, windowsHide: true, stdio: "ignore" });
     c.on("error", () => {});
     c.unref();
     return c;

package/bin/lean/signoff.mjs

@@ -8,6 +8,7 @@
  * never in the range, so they are never rewritten. Deterministic, 0 LLM calls.
  */
 const SEP = "\x1f", END = "\x1e";
+const STASH_REF = "refs/forgedock/dco-stash";
 const PLACEHOLDER_RE = /@localhost$/i;
 
 /** True when the commit must be rewritten: placeholder author or committer, or no sign-off naming its author. */
@@ -59,8 +60,17 @@ export async function ensureSignedOff({ git, lane, branch, env = process.env })
     throw Object.assign(new Error(`${bad.length} unpushed commit(s) fail DCO (placeholder author or no sign-off: ${bad.slice(0, 5).map((c) => c.sha.slice(0, 8)).join(", ")}) and the worktree has no usable git identity to rewrite them`), { noIdentity: true });
   }
   const fenv = { FD_BAD: bad.map((c) => c.sha).join(" "), FD_NAME: name, FD_EMAIL: email, FILTER_BRANCH_SQUELCH_WARNING: "1" };
+  // filter-branch refuses a dirty tree (lockfile churn, build output): park tracked changes on a private ref, never the shared stash stack
+  const saved = await ok(["status", "--porcelain", "--untracked-files=no"]) ? await ok(["stash", "create"]) : "";
+  if (saved) { await git(["update-ref", STASH_REF, saved]); await git(["reset", "-q", "--hard", "HEAD"]); }
+  let failure = null;
   try { await git(["filter-branch", "-f", "--env-filter", ENV_FILTER, "--msg-filter", MSG_FILTER, "--", "HEAD", ...exclude], { env: fenv }); }
-  catch (e) { throw Object.assign(new Error(`could not rewrite ${bad.length} unpushed commit(s) for DCO: ${String(e?.stderr ?? e?.message ?? e).slice(0, 300)}`), { noIdentity: false }); }
+  catch (e) { failure = `could not rewrite ${bad.length} unpushed commit(s) for DCO: ${String(e?.stderr ?? e?.message ?? e).slice(0, 300)}`; }
+  if (saved) {
+    try { await git(["stash", "apply", saved]); await ok(["update-ref", "-d", STASH_REF]); }
+    catch { failure = `${failure ? `${failure}; ` : ""}could not restore the lane's uncommitted changes after the DCO rewrite; they are kept as ${saved} (${STASH_REF})`; }
+  }
+  if (failure) throw Object.assign(new Error(failure), { noIdentity: false, engineCode: "engine.dco_rewrite" });
   await ok(["for-each-ref", "--format=delete %(refname)", "refs/original/"]).then(async (l) => { for (const r of l.split("\n").map((x) => x.replace(/^delete /, "")).filter(Boolean)) await ok(["update-ref", "-d", r]); });
   return { rewritten: bad.length };
 }

package/bin/lean/signoff.test.mjs

@@ -4,7 +4,7 @@
 import { describe, it } from "node:test";
 import assert from "node:assert/strict";
 import { execFileSync } from "node:child_process";
-import { writeFileSync } from "node:fs";
+import { readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
 import { ensureSignedOff, needsSignoff, offendersOf } from "./signoff.mjs";
@@ -99,4 +99,27 @@ describe("ensureSignedOff", () => {
     await assert.rejects(ensureSignedOff({ git: f.git, lane: "staging", branch: "feat/x", env: {} }), (e) => e.noIdentity === true);
     assert.equal(f.sh("rev-parse", "HEAD"), bad);
   });
+
+  it("rewrites on a dirty tree (#4997): unstaged and staged tracked changes survive, no stash entry is left", async () => {
+    const f = fixture();
+    f.commit("a.txt", "unpushed placeholder", "placeholder");
+    writeFileSync(join(f.wt, "base.txt"), "churn\n");
+    writeFileSync(join(f.wt, "a.txt"), "staged\n"); f.sh("add", "a.txt");
+    const r = await ensureSignedOff({ git: f.git, lane: "staging", branch: "feat/x", env: {} });
+    assert.equal(r.rewritten, 1);
+    assert.equal(readFileSync(join(f.wt, "base.txt"), "utf8"), "churn\n");
+    assert.equal(readFileSync(join(f.wt, "a.txt"), "utf8"), "staged\n");
+    assert.equal(f.sh("stash", "list"), "");
+    assert.equal(f.sh("for-each-ref", "refs/forgedock/"), "");
+    assert.match(f.sh("log", "-1", "--format=%B"), /Signed-off-by: Lane Bot/);
+  });
+
+  it("a failed rewrite carries engineCode engine.dco_rewrite, restores the tree, and is not noIdentity (#4997)", async () => {
+    const f = fixture();
+    f.commit("a.txt", "unpushed placeholder", "placeholder");
+    writeFileSync(join(f.wt, "base.txt"), "churn\n");
+    const git = async (a, o) => { if (a[0] === "filter-branch") throw new Error("boom"); return f.git(a, o); };
+    await assert.rejects(ensureSignedOff({ git, lane: "staging", branch: "feat/x", env: {} }), (e) => e.engineCode === "engine.dco_rewrite" && e.noIdentity === false && /boom/.test(e.message));
+    assert.equal(readFileSync(join(f.wt, "base.txt"), "utf8"), "churn\n");
+  });
 });

package/bin/lean/toolstate.mjs

@@ -51,14 +51,30 @@ export async function unstageToolState(git) {
   return stray.length ? (await git(["diff", "--cached", "--name-only"])).trim() !== "" : staged.length > 0;
 }
 
+const REQUIREMENTS_EXPORT = /^requirements([-_.][\w.-]+)?\.txt$/;
+
+/**
+ * #4959: `requirements*.txt` files the branch added to a Poetry/uv project (pyproject [tool.poetry], or poetry.lock / uv.lock at
+ * the base) that the base does not track and the contract does not list (`keep`): an export generated to run local tests.
+ */
+export async function strayGenerated(git, ref, added, keep = []) {
+  const cands = added.filter((p) => REQUIREMENTS_EXPORT.test(base(p)) && !keep.includes(p));
+  if (!cands.length) return [];
+  const tracked = zlist(await git(["ls-tree", "-r", "--name-only", "-z", ref]));
+  const managed = tracked.some((f) => /(^|\/)(poetry|uv)\.lock$/.test(f))
+    || (await Promise.all(tracked.filter((f) => /(^|\/)pyproject\.toml$/.test(f)).map((f) => git(["show", `${ref}:${f}`]).catch(() => "")))).some((t) => /^\[tool\.poetry[\].]/m.test(String(t)));
+  return managed ? cands : [];
+}
+
 /**
  * Remove from the index (and commit) the deny-listed files this branch added against `base`. Returns the removed paths ([]
- * when the branch is clean). Never throws: a failure leaves the branch as it was and the push proceeds.
+ * when the branch is clean). `keep`: contract deliverable paths, never removed as generated files. Never throws: a failure
+ * leaves the branch as it was and the push proceeds.
  */
-export async function untrackToolState({ git, base: ref, ev = () => {} }) {
+export async function untrackToolState({ git, base: ref, ev = () => {}, keep = [] }) {
   try {
     const added = zlist(await git(["diff", "--diff-filter=A", "--name-only", "-z", `${ref}...HEAD`]));
-    const stray = await strayPaths(git, added);
+    const stray = [...await strayPaths(git, added), ...await strayGenerated(git, ref, added, keep)];
     if (!stray.length) return [];
     await git(["rm", "--cached", "-r", "-q", "--ignore-unmatch", "--", ...stray]);
     await git(["commit", "--no-verify", "-m", "chore: untrack lane tool state (#4590)", "-m", stray.slice(0, 20).join("\n")]);