forgedock 1.29.1 → 1.31.0

package/bin/lean/routing.mjs

@@ -0,0 +1,74 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Builder model routing (#4394). The build runs on a mid-tier model when the frozen contract is small or touches only
+ * docs, config and tests, and on the top model (the configured builder model, else the CLI default) when it is large or
+ * hits a risk path (the built-in risk words or forge.yaml review.risk_paths). The route is computed once from the
+ * contract's deliverable paths; a failed validate round on the mid tier escalates the lane to the top model.
+ * Pure: no I/O, no model call. Repo-agnostic: paths are classified by extension and directory names only.
+ */
+import { riskClassOf, riskPathHits } from "./review-trigger.mjs";
+
+export const ROUTING_MODES = Object.freeze(["on", "off"]);
+export const DEFAULT_MID_MODEL = "claude-sonnet-5-5";
+/** Deliverables of at most this many non-test files make a small change. */
+export const SMALL_MAX_FILES = 2;
+/** Fix-round triggers that are validate rounds: a failure on the mid tier escalates the lane to the top model. */
+export const ESCALATE_TRIGGERS = Object.freeze(["precheck", "ci_red", "merge_ci"]);
+
+const MODEL_RE = /^[A-Za-z0-9._:[\]-]+$/;
+
+/** {mode, midModel}: an unknown mode is on, an invalid model id falls back to the default mid model. */
+export function parseBuilderRouting({ mode, midModel } = {}) {
+  const m = String(mode ?? "").trim().toLowerCase();
+  const id = String(midModel ?? "").trim();
+  return { mode: ROUTING_MODES.includes(m) ? m : "on", midModel: MODEL_RE.test(id) ? id : DEFAULT_MID_MODEL };
+}
+
+const LOCKFILE_RE = /(^|\/)(package-lock\.json|npm-shrinkwrap\.json|yarn\.lock|pnpm-lock\.yaml|bun\.lockb?|Cargo\.lock|poetry\.lock|uv\.lock|Pipfile\.lock|Gemfile\.lock|composer\.lock|go\.sum|[^/]*\.lock)$/i;
+const TEST_FILE_RE = /(\.(test|spec)\.[A-Za-z0-9]+$)|(_test\.(go|py|rb|exs?)$)|((^|\/)test_[^/]+\.py$)|(_spec\.rb$)|(Tests?\.(java|kt|cs|swift)$)/;
+const TEST_DIR_RE = /(^|\/)(tests?|__tests__|spec|specs|fixtures|__snapshots__)\//i;
+const DOCS_EXT_RE = /\.(md|mdx|markdown|rst|txt|adoc)$/i;
+const DOCS_DIR_RE = /(^|\/)(docs?|devdocs)\//i;
+const CONFIG_EXT_RE = /\.(json|jsonc|ya?ml|toml|ini|cfg|conf|properties|editorconfig)$/i;
+const DOTFILE_RE = /(^|\/)\.[^/]+$/;
+const ENV_EXAMPLE_RE = /(^|\/)\.env\.(example|sample|template)$/i;
+
+/** 'test' | 'docs' | 'config' | 'code' for one repo-relative path. Lockfiles are code (they change what runs). */
+export function classifyPath(p) {
+  const s = String(p ?? "").trim().replace(/^\.\//, "");
+  if (LOCKFILE_RE.test(s)) return "code";
+  if (TEST_FILE_RE.test(s) || TEST_DIR_RE.test(s)) return "test";
+  if (DOCS_EXT_RE.test(s) || DOCS_DIR_RE.test(s)) return "docs";
+  if (ENV_EXAMPLE_RE.test(s) || CONFIG_EXT_RE.test(s) || DOTFILE_RE.test(s)) return "config";
+  return "code";
+}
+
+/** Deliverable paths of a frozen contract (trimmed, unique, in order). */
+export function deliverablePaths(contract) {
+  const ds = Array.isArray(contract?.deliverables) ? contract.deliverables : [];
+  return [...new Set(ds.map((d) => String(d?.path ?? "").trim()).filter(Boolean))];
+}
+
+/**
+ * The route of one lane: {class, tier, model, risk}. class: risk | docs-config-test | small | large; tier: mid | top.
+ * `topModel` null means the CLI default. Mode off keeps every lane on the top model but still reports its class.
+ */
+export function routeBuilder({ contract = null, riskPaths = [], cfg = {}, topModel = null } = {}) {
+  const c = parseBuilderRouting(cfg);
+  const paths = deliverablePaths(contract);
+  const declared = new Set(riskPathHits(paths, riskPaths));
+  const risk = paths.filter((p) => riskClassOf(p) || declared.has(p));
+  const kinds = paths.map(classifyPath);
+  const cls = !paths.length ? "large"
+    : risk.length ? "risk"
+      : kinds.every((k) => k !== "code") ? "docs-config-test"
+        : kinds.filter((k) => k !== "test").length <= SMALL_MAX_FILES ? "small" : "large";
+  const tier = c.mode === "on" && (cls === "docs-config-test" || cls === "small") ? "mid" : "top";
+  return { mode: c.mode, class: cls, tier, model: tier === "mid" ? c.midModel : topModel, risk };
+}
+
+/** True when a failed `trigger` round on the mid tier escalates the lane (once). */
+export function shouldEscalate(route, escalated, trigger) {
+  return route?.tier === "mid" && !escalated && ESCALATE_TRIGGERS.includes(trigger);
+}

package/bin/lean/routing.test.mjs

@@ -0,0 +1,67 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { parseBuilderRouting, classifyPath, routeBuilder, shouldEscalate, ESCALATE_TRIGGERS, DEFAULT_MID_MODEL } from "./routing.mjs";
+
+const contract = (...paths) => ({ deliverables: paths.map((path, i) => ({ id: `D${i + 1}`, path, change: "x", requirements: ["R1"] })) });
+
+describe("parseBuilderRouting (#4394)", () => {
+  it("defaults to on with the default mid model; unknown modes are on; invalid ids fall back", () => {
+    assert.deepEqual(parseBuilderRouting(), { mode: "on", midModel: DEFAULT_MID_MODEL });
+    assert.equal(DEFAULT_MID_MODEL, "claude-sonnet-5-5");
+    assert.equal(parseBuilderRouting({ mode: "OFF" }).mode, "off");
+    assert.equal(parseBuilderRouting({ mode: "shadow" }).mode, "on");
+    assert.equal(parseBuilderRouting({ midModel: "claude-haiku-5-5" }).midModel, "claude-haiku-5-5");
+    assert.equal(parseBuilderRouting({ midModel: "openai/gpt[1m]" }).midModel, DEFAULT_MID_MODEL);
+    assert.equal(parseBuilderRouting({ midModel: "sonnet; rm -rf /" }).midModel, DEFAULT_MID_MODEL);
+    assert.equal(parseBuilderRouting({ midModel: "opus[1m]" }).midModel, "opus[1m]");
+  });
+});
+
+describe("classifyPath (#4394)", () => {
+  it("classifies tests across ecosystems", () => {
+    for (const p of ["src/a.test.ts", "web/b.spec.tsx", "pkg/c_test.go", "tests/test_d.py", "app/test_e.py", "crates/x/tests/it.rs", "src/__tests__/f.js", "spec/models/user_spec.rb", "src/test/java/FooTest.java"]) {
+      assert.equal(classifyPath(p), "test", p);
+    }
+  });
+  it("classifies docs, config and code; lockfiles are code", () => {
+    for (const p of ["README.md", "docs/guide.mdx", "devdocs/modules/lean.md", "CHANGES.rst", "notes.txt", "docs/img/arch.svg"]) assert.equal(classifyPath(p), "docs", p);
+    for (const p of ["forge.yaml", ".github/workflows/ci.yml", "pyproject.toml", "setup.cfg", "tsconfig.json", ".env.example", ".eslintrc", "Cargo.toml"]) assert.equal(classifyPath(p), "config", p);
+    for (const p of ["package-lock.json", "pnpm-lock.yaml", "yarn.lock", "Cargo.lock", "poetry.lock", "go.sum", "src/main.rs", "lib/x.py", "cmd/main.go", "src/foo.ts", "Makefile"]) assert.equal(classifyPath(p), "code", p);
+  });
+});
+
+describe("routeBuilder (#4394)", () => {
+  it("risk: a built-in risk word or a declared review.risk_paths glob routes to the top model", () => {
+    const r = routeBuilder({ contract: contract("src/auth/session.ts", "README.md"), topModel: "opus" });
+    assert.deepEqual(r, { mode: "on", class: "risk", tier: "top", model: "opus", risk: ["src/auth/session.ts"] });
+    const g = routeBuilder({ contract: contract("detect/rules.py"), riskPaths: ["detect/**"] });
+    assert.deepEqual([g.class, g.tier, g.model, g.risk], ["risk", "top", null, ["detect/rules.py"]]);
+    assert.equal(routeBuilder({ contract: contract("detect/rules.py") }).class, "small", "detection is not a built-in risk word");
+  });
+  it("docs-config-test, small and large classes", () => {
+    assert.deepEqual(routeBuilder({ contract: contract("docs/a.md", "forge.yaml", "test/a.test.js"), topModel: "opus" }),
+      { mode: "on", class: "docs-config-test", tier: "mid", model: DEFAULT_MID_MODEL, risk: [] });
+    const s = routeBuilder({ contract: contract("src/a.js", "src/b.js", "test/a.test.js", "test/b.test.js"), cfg: { midModel: "haiku" } });
+    assert.deepEqual([s.class, s.tier, s.model], ["small", "mid", "haiku"]);
+    const l = routeBuilder({ contract: contract("src/a.js", "src/b.js", "src/c.js"), topModel: "opus" });
+    assert.deepEqual([l.class, l.tier, l.model], ["large", "top", "opus"]);
+    const lock = routeBuilder({ contract: contract("package.json", "package-lock.json") });
+    assert.equal(lock.class, "small", "a lockfile change is code, not config");
+  });
+  it("no deliverables is large; mode off keeps the top tier but still reports the class", () => {
+    assert.deepEqual([routeBuilder({ contract: { deliverables: [] } }).class, routeBuilder({}).tier], ["large", "top"]);
+    const off = routeBuilder({ contract: contract("docs/a.md"), cfg: { mode: "off" }, topModel: "opus" });
+    assert.deepEqual([off.mode, off.class, off.tier, off.model], ["off", "docs-config-test", "top", "opus"]);
+  });
+  it("escalates once, on validate rounds only, from the mid tier", () => {
+    assert.deepEqual(ESCALATE_TRIGGERS, ["precheck", "ci_red", "merge_ci"]);
+    const mid = { tier: "mid" };
+    for (const t of ESCALATE_TRIGGERS) assert.equal(shouldEscalate(mid, false, t), true, t);
+    for (const t of ["conformance", "review", "build", "recovery"]) assert.equal(shouldEscalate(mid, false, t), false, t);
+    assert.equal(shouldEscalate(mid, true, "precheck"), false);
+    assert.equal(shouldEscalate({ tier: "top" }, false, "precheck"), false);
+    assert.equal(shouldEscalate(null, false, "precheck"), false);
+  });
+});

package/bin/lean/sched.test.mjs

@@ -6,7 +6,7 @@ import { writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
 import { grantOrder, openSched, priorityFromLabels } from "./sched.mjs";
-import { orchestrate } from "./orchestrate.mjs";
+import { orchestrate, estimateLaneMinutes, laneHistory, formatEstimateSummary, EST_DEFAULT_MIN, POST_DEPLOY_EST_MIN } from "./orchestrate.mjs";
 import { schedFromCli } from "../engine-cli.mjs";
 
 const quiet = async (fn) => { const l = console.log; console.log = () => {}; try { return await fn(); } finally { console.log = l; } };
@@ -308,3 +308,129 @@ describe("cross-repo fairness weights (#4193)", () => {
     assert.throws(() => a.setWeight("", 2), /non-empty/);
   });
 });
+
+// #4409: a virtual clock. Lane durations and sleeps are timers in virtual ms; the clock jumps to the next timer once the
+// event loop has drained every microtask, so the orchestrator's async chains settle before time moves.
+const vclock = () => {
+  let t = 0, done = false;
+  const timers = [];
+  const at = (ms, signal) => new Promise((res) => {
+    const x = { when: t + ms, res };
+    timers.push(x);
+    signal?.addEventListener("abort", () => { const i = timers.indexOf(x); if (i >= 0) timers.splice(i, 1); res(); }, { once: true });
+  });
+  const tick = () => setImmediate(() => {
+    if (done) return;
+    if (timers.length) { timers.sort((a, b) => a.when - b.when); const x = timers.shift(); t = x.when; x.res(); }
+    tick();
+  });
+  tick();
+  return { now: () => t, at, sleep: (ms, signal) => at(ms, signal), stop: () => { done = true; } };
+};
+const MIN = 60000;
+
+describe("longest-first admission (#4409)", () => {
+  it("estimateLaneMinutes: size label, unsized = M, post-deploy floor, history only with enough merged samples of the repo", () => {
+    assert.equal(estimateLaneMinutes({ labels: ["size:S"] }), EST_DEFAULT_MIN.S);
+    assert.equal(estimateLaneMinutes({ labels: [{ name: "size:L" }] }), EST_DEFAULT_MIN.L);
+    assert.equal(estimateLaneMinutes({}), EST_DEFAULT_MIN.M);
+    assert.equal(estimateLaneMinutes({ labels: ["size:L"], body: "- [ ] `R1: smoke` — post-deploy: needs prod" }), POST_DEPLOY_EST_MIN);
+    const run = (repo, size, min, reason = "merged") => ({ repo, size, reason, wallMs: min * MIN });
+    const h = laneHistory([run("o/r", "S", 4), run("o/r", "S", 6), run("o/r", "S", 8), run("o/r", "S", 100, "needs-human"), run("x/y", "S", 1),
+      run("o/r", "L", 90), run("o/r", "L", 70)], "o/r");
+    assert.deepEqual(h, { S: 6 }, "L has only 2 samples; other repos and unmerged runs never count");
+    assert.equal(estimateLaneMinutes({ labels: ["size:S"], history: h }), 6);
+    assert.equal(estimateLaneMinutes({ labels: ["size:L"], history: h }), EST_DEFAULT_MIN.L);
+  });
+
+  it("admits by estimate, descending, within a priority level; priority across levels is unchanged", async () => {
+    const est = { 1: 5, 2: 50, 3: 20 };
+    const order = [];
+    const rows = await orchestrate({ issues: [1, 2, 3], parallel: 1, readBody: async () => "", estimate: async (n) => est[n],
+      runOne: async (n) => { order.push(n); return { terminalReason: "merged" }; } });
+    assert.deepEqual(order, [2, 3, 1]);
+    assert.deepEqual(rows.map((r) => r.est_minutes), [5, 50, 20]);
+    const { handle } = world();
+    const order2 = [];
+    await orchestrate({ issues: [1, 2, 3], parallel: 1, readBody: async () => "", estimate: async (n) => est[n], pollMs: 2, batchId: "lpt",
+      sched: { registry: handle(7101), batch: "lpt", repo: "o/r", cap: 4 }, priorityOf: async (n) => (n === 1 ? 1 : 2),
+      runOne: async (n) => { order2.push(n); return { terminalReason: "merged" }; } });
+    assert.deepEqual(order2, [1, 2, 3], "the short P1 still precedes the long P2s");
+  });
+
+  it("decomposed sub-issues are admitted before same-priority FIFO items, with or without an estimator", async () => {
+    for (const estimate of [null, async () => 10]) {
+      const order = [];
+      let store = null;
+      await orchestrate({ issues: [1, 2, 3], parallel: 1, readBody: async () => "", estimate, store: { load: () => null, save: (x) => { store = x; } },
+        runOne: async (n) => { order.push(n); return n === 1 ? { terminalReason: "decomposed", subissues: [8, 9] } : { terminalReason: "merged" }; } });
+      assert.deepEqual(order, [1, 8, 9, 2, 3]);
+      assert.equal("estimates" in store, !!estimate, "no estimates key without an estimator");
+      if (estimate) assert.equal(store.estimates[8], EST_DEFAULT_MIN.M, "parent estimate ÷ N, floored at M");
+    }
+  });
+
+  it("starvation guard: an issue queued past starveMs is bumped ahead of a longer estimate", async () => {
+    // #2 runs 0-10 min; #3 (longer) is added at 1 min; at 10 min #1 has waited 10 min.
+    const run = async (starveMs) => {
+      const c = vclock();
+      const order = [];
+      try {
+        await orchestrate({ issues: [1, 2], parallel: 1, readBody: async () => "", now: c.now, sleep: c.sleep, pollMs: MIN, starveMs,
+          getAdded: () => (c.now() >= MIN ? [3] : []), estimate: async (n) => ({ 1: 5, 2: 30, 3: 40 })[n],
+          runOne: async (n) => { order.push(n); await c.at(10 * MIN); return { terminalReason: "merged" }; } });
+      } finally { c.stop(); }
+      return order;
+    };
+    assert.deepEqual(await run(8 * MIN), [2, 1, 3], "#1 waited past the limit");
+    assert.deepEqual(await run(30 * MIN), [2, 3, 1], "under the limit: longest first");
+  });
+
+  it("critical path: an issue with an in-batch dependent outranks an equal estimate without one", async () => {
+    const order = [];
+    await orchestrate({ issues: [1, 2, 3], parallel: 1, readBody: async (n) => (n === 3 ? "Depends on #2" : ""), estimate: async () => 10,
+      runOne: async (n) => { order.push(n); return { terminalReason: "merged" }; } });
+    assert.deepEqual(order, [2, 1, 3]);
+  });
+
+  it("replay of the 2026-10-22 batch: makespan is about the longest chain, not the FIFO tail", async () => {
+    // K=6; 41 short lanes; #41 (size:L, 40 min) is listed last; #3 decomposes 4 min in; its sub-issue #104 runs 48 min.
+    const c = vclock();
+    const issues = Array.from({ length: 41 }, (_, i) => i + 1);
+    const dur = (n) => (n === 41 ? 40 : n === 3 ? 4 : n === 104 ? 48 : 2) * MIN;
+    const start = new Map();
+    let rows;
+    try {
+      rows = await orchestrate({ issues, parallel: 6, readBody: async () => "", now: c.now, sleep: c.sleep,
+        estimate: async (n) => estimateLaneMinutes({ labels: [n === 41 ? "size:L" : "size:S"] }),
+        runOne: async (n) => {
+          start.set(n, c.now());
+          await c.at(dur(n));
+          return n === 3 ? { terminalReason: "decomposed", subissues: [101, 102, 103, 104] } : { terminalReason: "merged" };
+        } });
+    } finally { c.stop(); }
+    const makespan = c.now() / MIN;
+    assert.equal(rows.filter((r) => r.outcome === "merged").length, 44);
+    assert.equal(start.get(41), 0, "the long lane is admitted first");
+    assert.ok(start.get(104) <= 5 * MIN, `sub-issue #104 started at ${start.get(104) / MIN} min`);
+    assert.ok(makespan <= 56, `makespan ${makespan} min (longest chain 4 + 48)`);
+  });
+
+  it("formatEstimateSummary: estimated vs. actual over lanes that ran", () => {
+    assert.equal(formatEstimateSummary([{ issue: 1, outcome: "merged", minutes: 3 }]), null);
+    assert.equal(formatEstimateSummary([{ issue: 1, outcome: "merged", minutes: 12, est_minutes: 10 }, { issue: 2, outcome: "merged", minutes: 40, est_minutes: 25 },
+      { issue: 3, outcome: "held", minutes: 0, est_minutes: 10 }]),
+    "estimates: 2 lanes, est 35.0 min vs actual 52.0 min; median |error| 8.5 min; worst #2 est 25 actual 40");
+  });
+
+  it("a resume keeps estimates, queue times and the head band from the store", async () => {
+    const saved = { v: 1, order: [1, 2, 3], deps: { 1: [], 2: [], 3: [] }, lanes: {}, crossSeen: [], results: {}, running: {}, pending: [2, 3, 1],
+      estimates: { 1: 50, 2: 5, 3: 20 }, enqueued: { 1: 0, 2: 0, 3: 0 }, heads: [3] };
+    const order = [];
+    let estCalls = 0;
+    await orchestrate({ issues: [], parallel: 1, readBody: async () => "", estimate: async () => { estCalls++; return 1; }, now: () => 0,
+      store: { load: () => saved, save: () => {} }, runOne: async (n) => { order.push(n); return { terminalReason: "merged" }; } });
+    assert.deepEqual(order, [3, 1, 2]);
+    assert.equal(estCalls, 0);
+  });
+});

package/bin/lean/schedd.mjs

@@ -9,13 +9,22 @@
  * Protocol: newline-delimited JSON, one request per connection.
  *   {op:"submit", argv, cwd, env} → {type:"batch", id, dir}, {type:"row", row}…, {type:"line", text}…, {type:"done", result, report}
  *   {op:"attach", id}             → log.jsonl rows replayed, then live rows/lines and done; a finished batch replays report.json
- *                                   ({type:"done", replay:true}); a batch nobody hosts is resumed here
+ *                                   ({type:"done", replay:true}) unless it has unrun added issues; a batch nobody hosts is resumed here
  *   {op:"set-parallel", id, k}    → {type:"ok", parallel}
+ *   {op:"add", id, issues}        → {type:"ok", added, skipped, hosted} (#4408): merged into the batch's `add` file, which a
+ *                                   hosted runner reads on its next pass and an unhosted batch on its next resume
+ *   {op:"cancel", id}             → {type:"ok", cancelled:true, hosted} (#4376): writes the batch's `cancel` file; that batch
+ *                                   admits nothing more, its running local lanes get SIGTERM (their teardown saves the work)
+ *                                   and the rest end held; no other batch and not the daemon. A finished, unhosted batch:
+ *                                   {type:"ok", cancelled:false, finished:true}, nothing written
  *   {op:"status"}                 → {type:"status", pid, batches:[{id, running}]}
- * Failures answer {type:"error", message}. A client closing its connection never cancels a batch, and stopping the daemon
+ * Failures answer {type:"error", message}. A client closing its connection never cancels a batch (only `cancel` stops one), and stopping the daemon
  * never stops a lane (lanes are detached sessions). On start the daemon resumes every daemon-hosted batch without a
  * report, so live lanes are re-adopted through the batch runner's `adopt`. It exits after `idleMs` with no batch and no client.
- * `runBatch({argv, cwd, env, sink, keepBreaker})` is injected (engine-cli `runBatch`); `sink` gets `batch(b)`, `row(r)`, `line(text)`.
+ * Shutdown (#4454): `close()` aborts every hosted batch's `signal` (its loop admits nothing more and leaves, lanes keep
+ * running, no report is written), waits for them up to `stopMs`, and only then releases `schedd.lock` and the socket, so a
+ * daemon started next never resumes a batch this one still dispatches. `exitOnSignals` exits 128+signo after that.
+ * `runBatch({argv, cwd, env, sink, keepBreaker, signal})` is injected (engine-cli `runBatch`); `sink` gets `batch(b)`, `row(r)`, `line(text)`.
  */
 import { closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, writeSync } from "node:fs";
 import { createConnection, createServer } from "node:net";
@@ -25,6 +34,7 @@ import { schedDir, pidAlive } from "./sched.mjs";
 import { batchRoot, listBatches, openBatch } from "./batch.mjs";
 
 export const IDLE_MS = 10 * 60 * 1000;
+export const STOP_MS = 10 * 1000;
 
 export function scheddPaths(dir = schedDir()) {
   return { dir, sock: join(dir, "schedd.sock"), lock: join(dir, "schedd.lock"), log: join(dir, "schedd.log") };
@@ -64,7 +74,7 @@ function takeLock(lock, pid, alive) {
  * Start the daemon. @returns {Promise<{running:true, pid, close, closed:Promise} | {running:false, owner:number}>}
  */
 export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(process.env.FORGEDOCK_SCHEDD_IDLE_MS) || IDLE_MS,
-  root = batchRoot(), log = (m) => console.log(m), pid = process.pid, alive = pidAlive, resumeOnStart = true } = {}) {
+  root = batchRoot(), log = (m) => console.log(m), pid = process.pid, alive = pidAlive, resumeOnStart = true, stopMs = STOP_MS } = {}) {
   const p = scheddPaths(dir);
   mkdirSync(dir, { recursive: true });
   if (!takeLock(p.lock, pid, alive)) {
@@ -77,11 +87,15 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
 
   const hosted = new Map(); // batch id -> {id, dir, subs:Set<fn>}
   const conns = new Set();
+  const runs = new Map(); // every runBatch in flight (hosted or not yet started) -> its AbortController (#4454)
+  let closing = null, resolveClosed;
   let lastActive = Date.now();
   const touch = () => { lastActive = Date.now(); };
 
   /** Run one batch here. Resolves with {id, dir} once the batch exists, rejects when it fails before that. */
   const host = ({ argv, cwd, env, keepBreaker = false }, first = null) => new Promise((resolve, reject) => {
+    if (closing) return reject(new Error("schedd is shutting down"));
+    const ac = new AbortController();
     const ent = { id: null, dir: null, subs: new Set(first ? [first] : []) };
     const emit = (m) => { for (const s of ent.subs) s(m); };
     let started = false;
@@ -91,13 +105,15 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       line: (text) => emit({ type: "line", text }),
     };
     touch();
-    Promise.resolve().then(() => runBatch({ argv, cwd, env, sink, keepBreaker })).then((result) => {
+    const run = Promise.resolve().then(() => runBatch({ argv, cwd, env, sink, keepBreaker, signal: ac.signal })).then((result) => {
       const report = ent.dir ? openBatch(root, ent.id).readReport()?.report ?? null : null;
       emit({ type: "done", result, report });
     }, (e) => {
       const message = e?.message ?? String(e);
+      if (e?.code === "batch.stopped") { log(`schedd: batch ${ent.id ?? argv.join(" ")} stopped (lanes keep running; resumed by the next daemon)`); if (!started) reject(new Error(message)); return; }
       if (!started) reject(new Error(message)); else { log(`schedd: batch ${ent.id} failed: ${message}`); emit({ type: "error", message }); }
-    }).finally(() => { if (ent.id) hosted.delete(ent.id); ent.subs.clear(); touch(); });
+    }).finally(() => { if (ent.id) hosted.delete(ent.id); ent.subs.clear(); runs.delete(run); touch(); });
+    runs.set(run, ac);
   });
 
   const handle = async (req, send) => {
@@ -108,6 +124,19 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       b.setParallel(Number(req.k));
       return send({ type: "ok", parallel: Number(req.k) }, true);
     }
+    if (req.op === "add") {
+      const b = openBatch(root, req.id);
+      if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
+      const { added, skipped } = b.addIssues(req.issues ?? []);
+      return send({ type: "ok", added, skipped, hosted: hosted.has(req.id) }, true);
+    }
+    if (req.op === "cancel") {
+      const b = openBatch(root, req.id);
+      if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
+      if (!hosted.has(req.id) && b.readReport()) return send({ type: "ok", cancelled: false, finished: true }, true);
+      b.requestCancel({ reason: "operator" });
+      return send({ type: "ok", cancelled: true, hosted: hosted.has(req.id) }, true);
+    }
     const sub = (m) => send(m, m.type === "done" || m.type === "error");
     if (req.op === "submit") {
       const ent = await host({ argv: req.argv ?? [], cwd: req.cwd ?? process.cwd(), env: req.env ?? {} }, sub);
@@ -124,7 +153,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       const b = openBatch(root, req.id);
       if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
       const rep = b.readReport();
-      if (rep && !(rep.rows ?? []).some((r) => r.outcome === "held")) {
+      if (rep && !(rep.rows ?? []).some((r) => r.outcome === "held") && !b.unrunAdds().length) {
         return send({ type: "done", replay: true, result: { rows: rep.rows ?? [], crossLane: rep.crossLane ?? [], batch: b.id, ok: !!rep.ok }, report: rep.report ?? null }, true);
       }
       const ent = await host({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} }, sub);
@@ -158,13 +187,18 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
   await new Promise((res, rej) => { server.once("error", rej); server.listen(p.sock, () => { server.off("error", rej); res(); }); });
   log(`schedd: pid ${pid} listening on ${p.sock}`);
 
-  let closing = null, resolveClosed;
   const closed = new Promise((r) => { resolveClosed = r; });
   const close = () => {
     if (closing) return closing;
     clearInterval(idleTimer);
     for (const c of conns) c.destroy();
-    closing = new Promise((r) => server.close(() => r())).then(() => {
+    const stopping = [...runs.keys()];
+    for (const ac of runs.values()) ac.abort();
+    let timer;
+    // #4454: the lock goes only once every hosted batch loop has stopped (or `stopMs` passed: the caller exits regardless)
+    const stopped = stopping.length ? Promise.race([Promise.allSettled(stopping).then(() => true), new Promise((r) => { timer = setTimeout(() => r(false), stopMs); })])
+      .then((ok) => { clearTimeout(timer); if (!ok) log(`schedd: ${runs.size} batch(es) did not stop within ${stopMs} ms`); }) : Promise.resolve();
+    closing = Promise.all([new Promise((r) => server.close(() => r())), stopped]).then(() => {
       try { if (parseInt(readFileSync(p.lock, "utf8"), 10) === pid) rmSync(p.lock, { force: true }); } catch { /* gone */ }
       rmSync(p.sock, { force: true });
       resolveClosed();
@@ -188,6 +222,18 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
   return { running: true, pid, sock: p.sock, close, closed };
 }
 
+/** #4454: on SIGTERM/SIGINT stop the daemon (`close()`: batch loops first, then lock and socket), then exit 128+signo, once. */
+export function exitOnSignals(d, { on = (s, f) => process.once(s, f), exit = (c) => process.exit(c), signals = ["SIGTERM", "SIGINT"] } = {}) {
+  const SIGNO = { SIGINT: 2, SIGTERM: 15 };
+  let fired = false;
+  for (const sig of signals) on(sig, async () => {
+    if (fired) return;
+    fired = true;
+    try { await d.close(); } catch { /* exit anyway */ }
+    exit(128 + (SIGNO[sig] ?? 0));
+  });
+}
+
 /** Connect to the daemon, spawning it (`spawnDaemon()`) when nobody listens. @returns {Promise<import("node:net").Socket>} */
 export async function connectDaemon({ dir = schedDir(), spawnDaemon = defaultSpawn, waitMs = 10_000, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) } = {}) {
   const { sock } = scheddPaths(dir);

package/bin/lean/schedd.test.mjs

@@ -8,7 +8,7 @@ import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
 import { openBatch, isLaneAlive, listBatches } from "./batch.mjs";
 import { startDaemon, scheddPaths, connectDaemon, request, mergeEnv, batchEnv } from "./schedd.mjs";
-import { formatTable } from "./orchestrate.mjs";
+import { formatTable, orchestrate } from "./orchestrate.mjs";
 import { orchestrateFromCli } from "../engine-cli.mjs";
 
 const until = async (cond, ms = 15000) => {
@@ -32,16 +32,18 @@ const a = process.argv.slice(2); const n = Number(a[1]); const rf = a[a.indexOf(
 appendFileSync(${JSON.stringify(h.launches)}, n + "\\n");
 const t = setInterval(() => { if (!existsSync(${JSON.stringify(h.release)})) return; clearInterval(t);
   writeFileSync(rf + ".tmp", JSON.stringify({ terminalReason: "merged", pr: 100 + n, cost_usd: 0 })); renameSync(rf + ".tmp", rf); }, 20);
+// Like run-issue's #4179 teardown: SIGTERM reports deferred run.interrupted.
+process.once("SIGTERM", () => { writeFileSync(rf + ".tmp", JSON.stringify({ terminalReason: "deferred", code: "run.interrupted", detail: "interrupted by SIGTERM" })); renameSync(rf + ".tmp", rf); process.exit(143); });
 `);
   h.daemonScript = join(work, "daemon.mjs");
-  writeFileSync(h.daemonScript, `import { startDaemon } from ${JSON.stringify(new URL("./schedd.mjs", import.meta.url).href)};
+  writeFileSync(h.daemonScript, `import { startDaemon, exitOnSignals } from ${JSON.stringify(new URL("./schedd.mjs", import.meta.url).href)};
 import { runBatch } from ${JSON.stringify(new URL("../engine-cli.mjs", import.meta.url).href)};
 const deps = { allowTmpfs: true, laneScript: ${JSON.stringify(stub)}, pollMs: 30, orchestration: { breaker: null, maxDeferrals: 3 }, io: { gh: async () => "o/r\\n" },
   leanIo: { gh: async (a) => a.includes("milestone") ? JSON.stringify({ milestone: null }) : a[0] === "repo" ? "main\\n" : JSON.stringify({ body: a[2] === "3" ? "Depends on #1" : "", labels: [] }),
     git: async (a) => (a[0] === "ls-remote" ? "abc\\trefs/heads/staging\\n" : "") } };
 const d = await startDaemon({ runBatch: (a) => runBatch({ ...a, daemon: true, deps }), idleMs: 600000 });
 if (!d.running) process.exit(3);
-process.once("SIGTERM", () => d.close());
+exitOnSignals(d); // the forgedock schedd handler (#4454)
 await d.closed;
 `);
   h.env = { ...process.env, FORGEDOCK_SCHED_DIR: h.sched, FORGEDOCK_BATCH_DIR: h.root, FORGEDOCK_LIMITS_FILE: join(work, "limits.json") };
@@ -151,6 +153,67 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
     } finally { h.cleanup(); }
   });
 
+  it("#4408: --add and --resume <id> N add issues to the live batch (no second batch); an unhosted finished batch runs its adds on --resume", async () => {
+    const h = host();
+    try {
+      await h.daemon();
+      const client = { ...h.client, io: { gh: async (a) => (a[0] === "issue" ? JSON.stringify({ state: a[2] === "9" ? "CLOSED" : "OPEN" }) : "o/r\n") } };
+      const { res } = await withRoot(h, () => capture(async () => {
+        const first = orchestrateFromCli(["1", "--parallel", "3", "--repo", "o/r", "--no-prefilter"], h.client);
+        await until(() => h.launched().length === 1 && h.lanePids().length === 1);
+        const [id] = listBatches(h.root);
+        const add = await orchestrateFromCli(["--add", id, "2", "1", "x", "9"], client);
+        assert.deepEqual(add.added, [2]);
+        assert.deepEqual(add.skipped.map((x) => [x.issue, x.reason]).sort(), [[1, "already in batch"], [9, "closed"], ["x", "not an integer"]]);
+        await until(() => h.launched().includes(2));
+        const resumed = orchestrateFromCli(["--resume", id, "4"], client);
+        await until(() => h.launched().includes(4));
+        assert.deepEqual(listBatches(h.root), [id], "no second batch");
+        writeFileSync(h.release, "");
+        const [done] = await Promise.all([first, resumed]);
+        assert.deepEqual(done.rows.map((r) => [r.issue, r.outcome]), [[1, "merged"], [2, "merged"], [4, "merged"]]);
+        // Finished and no longer hosted: the add is persisted and the next --resume runs it instead of replaying the report.
+        const later = await orchestrateFromCli(["--add", id, "6"], client);
+        assert.deepEqual(later.added, [6]);
+        const again = await orchestrateFromCli(["--resume", id], client);
+        assert.deepEqual(again.rows.map((r) => [r.issue, r.outcome]), [[1, "merged"], [2, "merged"], [4, "merged"], [6, "merged"]]);
+        return done;
+      }));
+      assert.equal(res.rows.length, 3);
+      assert.deepEqual(h.launched(), [1, 2, 4, 6], "each issue launched once");
+    } finally { h.cleanup(); }
+  });
+
+  it("#4376: --cancel stops one batch (its lane torn down, the rest held, report written); the other batch and the daemon run on", async () => {
+    const h = host();
+    try {
+      const d = await h.daemon();
+      await withRoot(h, () => capture(async () => {
+        const a = orchestrateFromCli(["1", "2", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client);
+        await until(() => h.launched().length === 1 && h.lanePids().length === 1);
+        const [idA] = listBatches(h.root);
+        const b = orchestrateFromCli(["5", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client);
+        await until(() => h.launched().includes(5) && h.lanePids().length === 2);
+        const idB = listBatches(h.root).find((x) => x !== idA);
+        const res = await orchestrateFromCli(["--cancel", idA], h.client);
+        assert.deepEqual(res, { ok: true, batch: idA, cancelled: true });
+        const doneA = await a;
+        assert.deepEqual(doneA.rows.map((r) => [r.issue, r.outcome, r.detail]), [[1, "held", "cancelled by operator"], [2, "held", "cancelled by operator"]]);
+        assert.ok(openBatch(h.root, idA).readReport(), "the cancelled batch wrote its report");
+        assert.deepEqual(h.launched(), [1, 5], "#2 never launched");
+        const st = await request(await connectDaemon({ dir: h.sched, spawnDaemon: null }), { op: "status" });
+        assert.equal(st.pid, d.pid, "the daemon kept running");
+        assert.deepEqual(st.batches.map((x) => x.id), [idB]);
+        writeFileSync(h.release, "");
+        const doneB = await b;
+        assert.deepEqual(doneB.rows.map((r) => [r.issue, r.outcome]), [[5, "merged"]]);
+        const again = await orchestrateFromCli(["--cancel", idA], h.client);
+        assert.equal(again.cancelled, false, "a finished batch is left alone");
+        await assert.rejects(orchestrateFromCli(["--cancel", "nope"], h.client), /no batch nope/);
+      }));
+    } finally { h.cleanup(); }
+  });
+
   it("single instance: a live daemon keeps its lock; a dead owner's lock and socket are taken over", async () => {
     const h = host();
     try {
@@ -171,6 +234,89 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
     } finally { h.cleanup(); }
   });
 
+  it("#4454: close() stops every hosted batch loop before releasing the lock; no batch admits a lane after close()", async () => {
+    const dir = tmpDir("schedd-stop-"), root = join(dir, "batches"), lock = scheddPaths(dir).lock;
+    const started = { a: [], b: [] }, stoppedWithLock = {};
+    let releaseLane;
+    const lane = new Promise((r) => { releaseLane = r; });
+    // An unref'd cancellable sleep: a base without the stop cannot hang the test process on the 30-min requeue delay.
+    const sleep = (ms, sig) => new Promise((r) => { const t = setTimeout(r, ms); t.unref(); sig?.addEventListener("abort", () => { clearTimeout(t); r(); }, { once: true }); });
+    const runBatch = async ({ argv, sink, signal }) => {
+      const id = argv[0];
+      sink.batch({ id, dir: join(root, id) });
+      try {
+        return await orchestrate({ issues: id === "a" ? [1, 2] : [9], parallel: 1, readBody: async () => "", signal, sleep,
+          runOne: async (n) => {
+            started[id].push(n);
+            if (n === 1) { await lane; return { terminalReason: "merged" }; } // a detached lane, still running at shutdown
+            if (n === 9) return { terminalReason: "deferred", detail: `leased by x until ${new Date(Date.now() + 3600e3).toISOString()}` };
+            return { terminalReason: "merged" };
+          } });
+      } finally { stoppedWithLock[id] = existsSync(lock); }
+    };
+    const d = await startDaemon({ dir, root, runBatch, log: () => {}, resumeOnStart: false });
+    const submit = async (id) => request(await connectDaemon({ dir, spawnDaemon: null }), { op: "submit", argv: [id], cwd: dir, env: {} }).catch((e) => e);
+    const clients = [submit("a"), submit("b")]; // each resolves (or rejects) once its connection ends
+    await until(() => started.a.length === 1 && started.b.length === 1);
+    await new Promise((r) => setTimeout(r, 100)); // b is asleep on its requeue delay
+    const t0 = Date.now();
+    await d.close();
+    assert.ok(Date.now() - t0 < 3000, `close() took ${Date.now() - t0} ms`);
+    assert.deepEqual(stoppedWithLock, { a: true, b: true }, "both batch loops stopped while the lock was still held");
+    assert.equal(existsSync(lock), false, "lock released after the loops stopped");
+    assert.equal(existsSync(scheddPaths(dir).sock), false);
+    releaseLane();
+    await new Promise((r) => setTimeout(r, 200));
+    assert.deepEqual(started, { a: [1], b: [9] }, "no hosted batch admits a lane after close()");
+    await Promise.all(clients);
+  });
+
+  it("#4454: close() waits at most stopMs for a batch that ignores the stop, then releases the lock", async () => {
+    const dir = tmpDir("schedd-bound-");
+    const d = await startDaemon({ dir, root: join(dir, "batches"), log: () => {}, resumeOnStart: false, stopMs: 200,
+      runBatch: ({ sink }) => { sink.batch({ id: "x", dir }); return new Promise(() => {}); } });
+    request(await connectDaemon({ dir, spawnDaemon: null }), { op: "submit", argv: ["x"], cwd: dir, env: {} }).catch(() => {});
+    await new Promise((r) => setTimeout(r, 50));
+    const t0 = Date.now();
+    await d.close();
+    const took = Date.now() - t0;
+    assert.ok(took >= 150 && took < 2000, `close() took ${took} ms`);
+    assert.equal(existsSync(scheddPaths(dir).lock), false);
+  });
+
+  it("#4454: kill -TERM stops both batches and exits 143 within the bound; lanes keep running; the next daemon resumes and adopts them", async () => {
+    const h = host();
+    try {
+      const d1 = await h.daemon();
+      await withRoot(h, () => capture(async () => {
+        const a = orchestrateFromCli(["1", "2", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client).then(() => null, (e) => e);
+        await until(() => h.launched().length === 1 && h.lanePids().length === 1);
+        const [idA] = listBatches(h.root);
+        const b = orchestrateFromCli(["5", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client).then(() => null, (e) => e);
+        await until(() => h.launched().includes(5) && h.lanePids().length === 2);
+        const idB = listBatches(h.root).find((x) => x !== idA);
+        const lanes = h.lanePids();
+        const exit = new Promise((r) => d1.on("exit", (code) => r(code)));
+        const t0 = Date.now();
+        d1.kill("SIGTERM");
+        assert.equal(await exit, 143);
+        assert.ok(Date.now() - t0 < 12000, "exits within the bound");
+        for (const pid of lanes) assert.equal(isLaneAlive(pid, null), true, "lanes keep running");
+        assert.equal(existsSync(scheddPaths(h.sched).lock), false);
+        assert.equal(openBatch(h.root, idA).readReport(), null, "a stopped batch writes no report");
+        assert.equal(openBatch(h.root, idB).readReport(), null);
+        assert.match(String((await a)?.message), /--resume/);
+        await b;
+        await h.daemon(); // resumes both batches
+        const ra = orchestrateFromCli(["--resume", idA], h.client), rb = orchestrateFromCli(["--resume", idB], h.client);
+        setTimeout(() => writeFileSync(h.release, ""), 200);
+        assert.deepEqual((await ra).rows.map((r) => [r.issue, r.outcome]), [[1, "merged"], [2, "merged"]]);
+        assert.deepEqual((await rb).rows.map((r) => [r.issue, r.outcome]), [[5, "merged"]]);
+        assert.deepEqual(h.launched(), [1, 2, 5], "adopted, not relaunched");
+      }));
+    } finally { h.cleanup(); }
+  });
+
   it("lane env: allowlisted keys come from the client only, the rest from the daemon", () => {
     const kept = batchEnv({ PATH: "/c", CLAUDE_CONFIG_DIR: "/acct-b", SECRET_X: "no", FORGEDOCK_SCHED_DIR: "/s" });
     assert.deepEqual(kept, { PATH: "/c", CLAUDE_CONFIG_DIR: "/acct-b", FORGEDOCK_SCHED_DIR: "/s" });