forgedock 1.30.0 → 1.32.0

package/bin/lean/routing.test.mjs

@@ -0,0 +1,67 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { parseBuilderRouting, classifyPath, routeBuilder, shouldEscalate, ESCALATE_TRIGGERS, DEFAULT_MID_MODEL } from "./routing.mjs";
+
+const contract = (...paths) => ({ deliverables: paths.map((path, i) => ({ id: `D${i + 1}`, path, change: "x", requirements: ["R1"] })) });
+
+describe("parseBuilderRouting (#4394)", () => {
+  it("defaults to on with the default mid model; unknown modes are on; invalid ids fall back", () => {
+    assert.deepEqual(parseBuilderRouting(), { mode: "on", midModel: DEFAULT_MID_MODEL });
+    assert.equal(DEFAULT_MID_MODEL, "claude-sonnet-5-5");
+    assert.equal(parseBuilderRouting({ mode: "OFF" }).mode, "off");
+    assert.equal(parseBuilderRouting({ mode: "shadow" }).mode, "on");
+    assert.equal(parseBuilderRouting({ midModel: "claude-haiku-5-5" }).midModel, "claude-haiku-5-5");
+    assert.equal(parseBuilderRouting({ midModel: "openai/gpt[1m]" }).midModel, DEFAULT_MID_MODEL);
+    assert.equal(parseBuilderRouting({ midModel: "sonnet; rm -rf /" }).midModel, DEFAULT_MID_MODEL);
+    assert.equal(parseBuilderRouting({ midModel: "opus[1m]" }).midModel, "opus[1m]");
+  });
+});
+
+describe("classifyPath (#4394)", () => {
+  it("classifies tests across ecosystems", () => {
+    for (const p of ["src/a.test.ts", "web/b.spec.tsx", "pkg/c_test.go", "tests/test_d.py", "app/test_e.py", "crates/x/tests/it.rs", "src/__tests__/f.js", "spec/models/user_spec.rb", "src/test/java/FooTest.java"]) {
+      assert.equal(classifyPath(p), "test", p);
+    }
+  });
+  it("classifies docs, config and code; lockfiles are code", () => {
+    for (const p of ["README.md", "docs/guide.mdx", "devdocs/modules/lean.md", "CHANGES.rst", "notes.txt", "docs/img/arch.svg"]) assert.equal(classifyPath(p), "docs", p);
+    for (const p of ["forge.yaml", ".github/workflows/ci.yml", "pyproject.toml", "setup.cfg", "tsconfig.json", ".env.example", ".eslintrc", "Cargo.toml"]) assert.equal(classifyPath(p), "config", p);
+    for (const p of ["package-lock.json", "pnpm-lock.yaml", "yarn.lock", "Cargo.lock", "poetry.lock", "go.sum", "src/main.rs", "lib/x.py", "cmd/main.go", "src/foo.ts", "Makefile"]) assert.equal(classifyPath(p), "code", p);
+  });
+});
+
+describe("routeBuilder (#4394)", () => {
+  it("risk: a built-in risk word or a declared review.risk_paths glob routes to the top model", () => {
+    const r = routeBuilder({ contract: contract("src/auth/session.ts", "README.md"), topModel: "opus" });
+    assert.deepEqual(r, { mode: "on", class: "risk", tier: "top", model: "opus", risk: ["src/auth/session.ts"] });
+    const g = routeBuilder({ contract: contract("detect/rules.py"), riskPaths: ["detect/**"] });
+    assert.deepEqual([g.class, g.tier, g.model, g.risk], ["risk", "top", null, ["detect/rules.py"]]);
+    assert.equal(routeBuilder({ contract: contract("detect/rules.py") }).class, "small", "detection is not a built-in risk word");
+  });
+  it("docs-config-test, small and large classes", () => {
+    assert.deepEqual(routeBuilder({ contract: contract("docs/a.md", "forge.yaml", "test/a.test.js"), topModel: "opus" }),
+      { mode: "on", class: "docs-config-test", tier: "mid", model: DEFAULT_MID_MODEL, risk: [] });
+    const s = routeBuilder({ contract: contract("src/a.js", "src/b.js", "test/a.test.js", "test/b.test.js"), cfg: { midModel: "haiku" } });
+    assert.deepEqual([s.class, s.tier, s.model], ["small", "mid", "haiku"]);
+    const l = routeBuilder({ contract: contract("src/a.js", "src/b.js", "src/c.js"), topModel: "opus" });
+    assert.deepEqual([l.class, l.tier, l.model], ["large", "top", "opus"]);
+    const lock = routeBuilder({ contract: contract("package.json", "package-lock.json") });
+    assert.equal(lock.class, "small", "a lockfile change is code, not config");
+  });
+  it("no deliverables is large; mode off keeps the top tier but still reports the class", () => {
+    assert.deepEqual([routeBuilder({ contract: { deliverables: [] } }).class, routeBuilder({}).tier], ["large", "top"]);
+    const off = routeBuilder({ contract: contract("docs/a.md"), cfg: { mode: "off" }, topModel: "opus" });
+    assert.deepEqual([off.mode, off.class, off.tier, off.model], ["off", "docs-config-test", "top", "opus"]);
+  });
+  it("escalates once, on validate rounds only, from the mid tier", () => {
+    assert.deepEqual(ESCALATE_TRIGGERS, ["precheck", "ci_red", "merge_ci"]);
+    const mid = { tier: "mid" };
+    for (const t of ESCALATE_TRIGGERS) assert.equal(shouldEscalate(mid, false, t), true, t);
+    for (const t of ["conformance", "review", "build", "recovery"]) assert.equal(shouldEscalate(mid, false, t), false, t);
+    assert.equal(shouldEscalate(mid, true, "precheck"), false);
+    assert.equal(shouldEscalate({ tier: "top" }, false, "precheck"), false);
+    assert.equal(shouldEscalate(null, false, "precheck"), false);
+  });
+});

package/bin/lean/sched.mjs

@@ -17,6 +17,9 @@
  * (`forgedock sched cap N`), else the caller's forge.yaml `orchestration.global_parallel`, else none. Fail-open: an
  * unreadable registry or a lock not taken within `lockWaitMs` grants (`fallback: true`), since a stuck registry would
  * stall every batch on the host.
+ * Per-issue lock (#4481), asked for per request with `exclusive: true` in the entry (not stored; the batch runner sets it):
+ * such a key is never granted while a live slot of another batch holds the same (non-null) repo and issue, so two batches that both include issue N (a resumed batch and a new one) never run two lanes for N at once;
+ * `tryAcquire` answers `{ok:false, conflict:<batch>}` and drops the key from the queue (it would hold a grant rank).
  */
 import { closeSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeSync } from "node:fs";
 import { homedir } from "node:os";
@@ -50,6 +53,14 @@ const posInt = (k) => (Number.isInteger(k) && k >= 1 ? k : null);
 const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 const byRank = ([ka, a], [kb, b]) => (a.priority - b.priority) || (a.seq - b.seq) || (ka < kb ? -1 : ka > kb ? 1 : 0);
 const groupOf = (k, e) => e?.repo ?? e?.batch ?? k;
+/** #4481: the batch of another live slot running the same repo and issue as an `exclusive` `entry`, or null (no repo: never). */
+const holderOf = (s, key, entry) => {
+  if (entry?.exclusive !== true || !entry.repo || entry.issue == null) return null;
+  for (const [k, e] of Object.entries(s.slots)) {
+    if (k !== key && e?.repo === entry.repo && e?.issue === entry.issue && (e.batch ?? null) !== (entry.batch ?? null)) return e.batch ?? k;
+  }
+  return null;
+};
 
 /**
  * Queue entries `[key, entry]` in grant order: priority ascending, then a weighted fair share across repos within each
@@ -88,7 +99,7 @@ export function grantOrder(s) {
 
 /**
  * Handle on the registry. `pid` is this dispatcher; `alive(pid)` is the liveness probe (injected in tests).
- * @returns {{dir, tryAcquire, offer, withdraw, release, setLanePid, adoptSlot, cap, setCap, weights, setWeight, list}}
+ * @returns {{dir, tryAcquire, holder, offer, withdraw, release, setLanePid, adoptSlot, cap, setCap, weights, setWeight, list}}
  */
 export function openSched(dir = schedDir(), { now = () => Date.now(), pid = process.pid, alive = pidAlive, lockWaitMs = LOCK_WAIT_MS } = {}) {
   const file = join(dir, "registry.json"), lockFile = join(dir, "registry.lock");
@@ -167,6 +178,8 @@ export function openSched(dir = schedDir(), { now = () => Date.now(), pid = proc
       try {
         return mutate((s) => {
           if (s.slots[key]) return { ok: true };
+          const holder = holderOf(s, key, entry);
+          if (holder !== null) { delete s.queue[key]; return { ok: false, conflict: holder, reason: `#${entry.issue} has a live lane in batch ${holder}` }; }
           enqueue(s, key, entry);
           const cap = effCap(s, callerCap);
           if (cap !== null) {
@@ -182,6 +195,8 @@ export function openSched(dir = schedDir(), { now = () => Date.now(), pid = proc
         });
       } catch (e) { return { ok: true, fallback: true, reason: e?.message ?? String(e) }; }
     },
+    /** #4481: the other batch holding a live slot for `{repo, issue}` (not `batch`), or null; null on any error. */
+    holder: (entry = {}) => { try { return mutate((s) => holderOf(s, null, entry)); } catch { return null; } },
     /** Queue entries `[{key, batch, repo, issue, priority}]` (their FIFO position is kept on a re-offer). */
     offer: (entries) => { try { mutate((s) => { for (const e of entries) enqueue(s, e.key, e); }); } catch { /* fail-open */ } },
     withdraw: (keys) => { if (!keys.length) return; try { mutate((s) => { for (const k of keys) delete s.queue[k]; }); } catch { /* fail-open */ } },

package/bin/lean/sched.test.mjs

@@ -6,7 +6,7 @@ import { writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
 import { grantOrder, openSched, priorityFromLabels } from "./sched.mjs";
-import { orchestrate } from "./orchestrate.mjs";
+import { orchestrate, estimateLaneMinutes, laneHistory, formatEstimateSummary, EST_DEFAULT_MIN, POST_DEPLOY_EST_MIN } from "./orchestrate.mjs";
 import { schedFromCli } from "../engine-cli.mjs";
 
 const quiet = async (fn) => { const l = console.log; console.log = () => {}; try { return await fn(); } finally { console.log = l; } };
@@ -308,3 +308,177 @@ describe("cross-repo fairness weights (#4193)", () => {
     assert.throws(() => a.setWeight("", 2), /non-empty/);
   });
 });
+
+// #4409: a virtual clock. Lane durations and sleeps are timers in virtual ms; the clock jumps to the next timer once the
+// event loop has drained every microtask, so the orchestrator's async chains settle before time moves.
+const vclock = () => {
+  let t = 0, done = false;
+  const timers = [];
+  const at = (ms, signal) => new Promise((res) => {
+    const x = { when: t + ms, res };
+    timers.push(x);
+    signal?.addEventListener("abort", () => { const i = timers.indexOf(x); if (i >= 0) timers.splice(i, 1); res(); }, { once: true });
+  });
+  const tick = () => setImmediate(() => {
+    if (done) return;
+    if (timers.length) { timers.sort((a, b) => a.when - b.when); const x = timers.shift(); t = x.when; x.res(); }
+    tick();
+  });
+  tick();
+  return { now: () => t, at, sleep: (ms, signal) => at(ms, signal), stop: () => { done = true; } };
+};
+const MIN = 60000;
+
+describe("longest-first admission (#4409)", () => {
+  it("estimateLaneMinutes: size label, unsized = M, post-deploy floor, history only with enough merged samples of the repo", () => {
+    assert.equal(estimateLaneMinutes({ labels: ["size:S"] }), EST_DEFAULT_MIN.S);
+    assert.equal(estimateLaneMinutes({ labels: [{ name: "size:L" }] }), EST_DEFAULT_MIN.L);
+    assert.equal(estimateLaneMinutes({}), EST_DEFAULT_MIN.M);
+    assert.equal(estimateLaneMinutes({ labels: ["size:L"], body: "- [ ] `R1: smoke` — post-deploy: needs prod" }), POST_DEPLOY_EST_MIN);
+    const run = (repo, size, min, reason = "merged") => ({ repo, size, reason, wallMs: min * MIN });
+    const h = laneHistory([run("o/r", "S", 4), run("o/r", "S", 6), run("o/r", "S", 8), run("o/r", "S", 100, "needs-human"), run("x/y", "S", 1),
+      run("o/r", "L", 90), run("o/r", "L", 70)], "o/r");
+    assert.deepEqual(h, { S: 6 }, "L has only 2 samples; other repos and unmerged runs never count");
+    assert.equal(estimateLaneMinutes({ labels: ["size:S"], history: h }), 6);
+    assert.equal(estimateLaneMinutes({ labels: ["size:L"], history: h }), EST_DEFAULT_MIN.L);
+  });
+
+  it("admits by estimate, descending, within a priority level; priority across levels is unchanged", async () => {
+    const est = { 1: 5, 2: 50, 3: 20 };
+    const order = [];
+    const rows = await orchestrate({ issues: [1, 2, 3], parallel: 1, readBody: async () => "", estimate: async (n) => est[n],
+      runOne: async (n) => { order.push(n); return { terminalReason: "merged" }; } });
+    assert.deepEqual(order, [2, 3, 1]);
+    assert.deepEqual(rows.map((r) => r.est_minutes), [5, 50, 20]);
+    const { handle } = world();
+    const order2 = [];
+    await orchestrate({ issues: [1, 2, 3], parallel: 1, readBody: async () => "", estimate: async (n) => est[n], pollMs: 2, batchId: "lpt",
+      sched: { registry: handle(7101), batch: "lpt", repo: "o/r", cap: 4 }, priorityOf: async (n) => (n === 1 ? 1 : 2),
+      runOne: async (n) => { order2.push(n); return { terminalReason: "merged" }; } });
+    assert.deepEqual(order2, [1, 2, 3], "the short P1 still precedes the long P2s");
+  });
+
+  it("decomposed sub-issues are admitted before same-priority FIFO items, with or without an estimator", async () => {
+    for (const estimate of [null, async () => 10]) {
+      const order = [];
+      let store = null;
+      await orchestrate({ issues: [1, 2, 3], parallel: 1, readBody: async () => "", estimate, store: { load: () => null, save: (x) => { store = x; } },
+        runOne: async (n) => { order.push(n); return n === 1 ? { terminalReason: "decomposed", subissues: [8, 9] } : { terminalReason: "merged" }; } });
+      assert.deepEqual(order, [1, 8, 9, 2, 3]);
+      assert.equal("estimates" in store, !!estimate, "no estimates key without an estimator");
+      if (estimate) assert.equal(store.estimates[8], EST_DEFAULT_MIN.M, "parent estimate ÷ N, floored at M");
+    }
+  });
+
+  it("starvation guard: an issue queued past starveMs is bumped ahead of a longer estimate", async () => {
+    // #2 runs 0-10 min; #3 (longer) is added at 1 min; at 10 min #1 has waited 10 min.
+    const run = async (starveMs) => {
+      const c = vclock();
+      const order = [];
+      try {
+        await orchestrate({ issues: [1, 2], parallel: 1, readBody: async () => "", now: c.now, sleep: c.sleep, pollMs: MIN, starveMs,
+          getAdded: () => (c.now() >= MIN ? [3] : []), estimate: async (n) => ({ 1: 5, 2: 30, 3: 40 })[n],
+          runOne: async (n) => { order.push(n); await c.at(10 * MIN); return { terminalReason: "merged" }; } });
+      } finally { c.stop(); }
+      return order;
+    };
+    assert.deepEqual(await run(8 * MIN), [2, 1, 3], "#1 waited past the limit");
+    assert.deepEqual(await run(30 * MIN), [2, 3, 1], "under the limit: longest first");
+  });
+
+  it("critical path: an issue with an in-batch dependent outranks an equal estimate without one", async () => {
+    const order = [];
+    await orchestrate({ issues: [1, 2, 3], parallel: 1, readBody: async (n) => (n === 3 ? "Depends on #2" : ""), estimate: async () => 10,
+      runOne: async (n) => { order.push(n); return { terminalReason: "merged" }; } });
+    assert.deepEqual(order, [2, 1, 3]);
+  });
+
+  it("replay of the 2026-10-22 batch: makespan is about the longest chain, not the FIFO tail", async () => {
+    // K=6; 41 short lanes; #41 (size:L, 40 min) is listed last; #3 decomposes 4 min in; its sub-issue #104 runs 48 min.
+    const c = vclock();
+    const issues = Array.from({ length: 41 }, (_, i) => i + 1);
+    const dur = (n) => (n === 41 ? 40 : n === 3 ? 4 : n === 104 ? 48 : 2) * MIN;
+    const start = new Map();
+    let rows;
+    try {
+      rows = await orchestrate({ issues, parallel: 6, readBody: async () => "", now: c.now, sleep: c.sleep,
+        estimate: async (n) => estimateLaneMinutes({ labels: [n === 41 ? "size:L" : "size:S"] }),
+        runOne: async (n) => {
+          start.set(n, c.now());
+          await c.at(dur(n));
+          return n === 3 ? { terminalReason: "decomposed", subissues: [101, 102, 103, 104] } : { terminalReason: "merged" };
+        } });
+    } finally { c.stop(); }
+    const makespan = c.now() / MIN;
+    assert.equal(rows.filter((r) => r.outcome === "merged").length, 44);
+    assert.equal(start.get(41), 0, "the long lane is admitted first");
+    assert.ok(start.get(104) <= 5 * MIN, `sub-issue #104 started at ${start.get(104) / MIN} min`);
+    assert.ok(makespan <= 56, `makespan ${makespan} min (longest chain 4 + 48)`);
+  });
+
+  it("formatEstimateSummary: estimated vs. actual over lanes that ran", () => {
+    assert.equal(formatEstimateSummary([{ issue: 1, outcome: "merged", minutes: 3 }]), null);
+    assert.equal(formatEstimateSummary([{ issue: 1, outcome: "merged", minutes: 12, est_minutes: 10 }, { issue: 2, outcome: "merged", minutes: 40, est_minutes: 25 },
+      { issue: 3, outcome: "held", minutes: 0, est_minutes: 10 }]),
+    "estimates: 2 lanes, est 35.0 min vs actual 52.0 min; median |error| 8.5 min; worst #2 est 25 actual 40");
+  });
+
+  it("a resume keeps estimates, queue times and the head band from the store", async () => {
+    const saved = { v: 1, order: [1, 2, 3], deps: { 1: [], 2: [], 3: [] }, lanes: {}, crossSeen: [], results: {}, running: {}, pending: [2, 3, 1],
+      estimates: { 1: 50, 2: 5, 3: 20 }, enqueued: { 1: 0, 2: 0, 3: 0 }, heads: [3] };
+    const order = [];
+    let estCalls = 0;
+    await orchestrate({ issues: [], parallel: 1, readBody: async () => "", estimate: async () => { estCalls++; return 1; }, now: () => 0,
+      store: { load: () => saved, save: () => {} }, runOne: async (n) => { order.push(n); return { terminalReason: "merged" }; } });
+    assert.deepEqual(order, [3, 1, 2]);
+    assert.equal(estCalls, 0);
+  });
+});
+
+describe("#4481: one lane per repo and issue across batches", () => {
+  it("the registry refuses a second batch's key for a live repo+issue slot, names the batch and drops it from the queue", () => {
+    const { handle, live } = world();
+    const a = handle(201), b = handle(202);
+    assert.equal(a.tryAcquire("A#5", { batch: "A", repo: "o/r", issue: 5, exclusive: true }, 4).ok, true);
+    const r = b.tryAcquire("B#5", { batch: "B", repo: "o/r", issue: 5, exclusive: true }, 4);
+    assert.deepEqual([r.ok, r.conflict], [false, "A"]);
+    assert.match(r.reason, /#5 has a live lane in batch A/);
+    assert.deepEqual(b.list().queue, [], "a refused key holds no grant rank");
+    assert.equal(b.holder({ batch: "B", repo: "o/r", issue: 5, exclusive: true }), "A");
+    assert.equal(a.holder({ batch: "A", repo: "o/r", issue: 5, exclusive: true }), null, "its own batch is no conflict");
+    assert.equal(b.tryAcquire("B#6", { batch: "B", repo: "o/r", issue: 6, exclusive: true }, 4).ok, true, "another issue");
+    assert.equal(b.tryAcquire("C#5", { batch: "C", repo: "o/other", issue: 5, exclusive: true }, 4).ok, true, "another repo");
+    assert.equal(b.tryAcquire("D#5", { batch: "D", repo: null, issue: 5, exclusive: true }, 4).ok, true, "no repo: unchanged");
+    assert.equal(a.tryAcquire("A#5", { batch: "A", repo: "o/r", issue: 5, exclusive: true }, 4).ok, true, "a granted key stays granted");
+    assert.equal(b.tryAcquire("E#5", { batch: "E", repo: "o/r", issue: 5 }, null).ok, true, "not exclusive: unchanged");
+    b.release("E#5");
+    live.delete(201); // A's dispatcher dies with no lane: its slot is reclaimed
+    assert.equal(b.tryAcquire("B#5", { batch: "B", repo: "o/r", issue: 5, exclusive: true }, 4).ok, true);
+  });
+
+  it("two batches that both include issue N never run two lanes for N; the second defers naming the first and runs its other issues", async () => {
+    const { handle } = world();
+    const regA = handle(301), regB = handle(302);
+    const active = new Map(), maxActive = new Map(), ranB = [];
+    let releaseA;
+    const gateA = new Promise((r) => { releaseA = r; });
+    const lane = (n) => { active.set(n, (active.get(n) ?? 0) + 1); maxActive.set(n, Math.max(maxActive.get(n) ?? 0, active.get(n))); };
+    const pA = orchestrate({ issues: [5], readBody: async () => "", pollMs: 2, batchId: "bA", sched: { registry: regA, batch: "bA", repo: "o/r", cap: null, exclusive: true },
+      runOne: async (n) => { lane(n); await gateA; active.set(n, active.get(n) - 1); return { terminalReason: "merged", cost_usd: 0 }; } });
+    await until(() => active.get(5) === 1, "batch A's lane");
+    const rowsB = [];
+    const pB = orchestrate({ issues: [5, 6], readBody: async () => "", pollMs: 2, batchId: "bB", sched: { registry: regB, batch: "bB", repo: "o/r", cap: null, exclusive: true },
+      onUpdate: (r) => rowsB.push(r),
+      runOne: async (n) => { lane(n); ranB.push(n); active.set(n, active.get(n) - 1); return { terminalReason: "merged", cost_usd: 0 }; } });
+    await until(() => ranB.includes(6), "batch B's other issue");
+    const deferred = rowsB.filter((r) => r.issue === 5 && r.outcome === "deferred");
+    assert.equal(deferred.length, 1, "one row per other batch");
+    assert.match(deferred[0].detail, /live lane in batch bA/);
+    assert.ok(!ranB.includes(5), "no second lane while batch A runs #5");
+    releaseA();
+    await pA;
+    const done = await pB;
+    assert.deepEqual(done.map((r) => [r.issue, r.outcome]).sort(), [[5, "merged"], [6, "merged"]]);
+    assert.equal(maxActive.get(5), 1);
+  });
+});

package/bin/lean/schedd.mjs

@@ -9,13 +9,22 @@
  * Protocol: newline-delimited JSON, one request per connection.
  *   {op:"submit", argv, cwd, env} → {type:"batch", id, dir}, {type:"row", row}…, {type:"line", text}…, {type:"done", result, report}
  *   {op:"attach", id}             → log.jsonl rows replayed, then live rows/lines and done; a finished batch replays report.json
- *                                   ({type:"done", replay:true}); a batch nobody hosts is resumed here
+ *                                   ({type:"done", replay:true}) unless it has unrun added issues; a batch nobody hosts is resumed here
  *   {op:"set-parallel", id, k}    → {type:"ok", parallel}
+ *   {op:"add", id, issues}        → {type:"ok", added, skipped, hosted} (#4408): merged into the batch's `add` file, which a
+ *                                   hosted runner reads on its next pass and an unhosted batch on its next resume
+ *   {op:"cancel", id}             → {type:"ok", cancelled:true, hosted} (#4376): writes the batch's `cancel` file; that batch
+ *                                   admits nothing more, its running local lanes get SIGTERM (their teardown saves the work)
+ *                                   and the rest end held; no other batch and not the daemon. A finished, unhosted batch:
+ *                                   {type:"ok", cancelled:false, finished:true}, nothing written
  *   {op:"status"}                 → {type:"status", pid, batches:[{id, running}]}
- * Failures answer {type:"error", message}. A client closing its connection never cancels a batch, and stopping the daemon
+ * Failures answer {type:"error", message}. A client closing its connection never cancels a batch (only `cancel` stops one), and stopping the daemon
  * never stops a lane (lanes are detached sessions). On start the daemon resumes every daemon-hosted batch without a
  * report, so live lanes are re-adopted through the batch runner's `adopt`. It exits after `idleMs` with no batch and no client.
- * `runBatch({argv, cwd, env, sink, keepBreaker})` is injected (engine-cli `runBatch`); `sink` gets `batch(b)`, `row(r)`, `line(text)`.
+ * Shutdown (#4454): `close()` aborts every hosted batch's `signal` (its loop admits nothing more and leaves, lanes keep
+ * running, no report is written), waits for them up to `stopMs`, and only then releases `schedd.lock` and the socket, so a
+ * daemon started next never resumes a batch this one still dispatches. `exitOnSignals` exits 128+signo after that.
+ * `runBatch({argv, cwd, env, sink, keepBreaker, signal})` is injected (engine-cli `runBatch`); `sink` gets `batch(b)`, `row(r)`, `line(text)`.
  */
 import { closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, writeSync } from "node:fs";
 import { createConnection, createServer } from "node:net";
@@ -25,6 +34,7 @@ import { schedDir, pidAlive } from "./sched.mjs";
 import { batchRoot, listBatches, openBatch } from "./batch.mjs";
 
 export const IDLE_MS = 10 * 60 * 1000;
+export const STOP_MS = 10 * 1000;
 
 export function scheddPaths(dir = schedDir()) {
   return { dir, sock: join(dir, "schedd.sock"), lock: join(dir, "schedd.lock"), log: join(dir, "schedd.log") };
@@ -64,7 +74,7 @@ function takeLock(lock, pid, alive) {
  * Start the daemon. @returns {Promise<{running:true, pid, close, closed:Promise} | {running:false, owner:number}>}
  */
 export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(process.env.FORGEDOCK_SCHEDD_IDLE_MS) || IDLE_MS,
-  root = batchRoot(), log = (m) => console.log(m), pid = process.pid, alive = pidAlive, resumeOnStart = true } = {}) {
+  root = batchRoot(), log = (m) => console.log(m), pid = process.pid, alive = pidAlive, resumeOnStart = true, stopMs = STOP_MS } = {}) {
   const p = scheddPaths(dir);
   mkdirSync(dir, { recursive: true });
   if (!takeLock(p.lock, pid, alive)) {
@@ -77,11 +87,15 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
 
   const hosted = new Map(); // batch id -> {id, dir, subs:Set<fn>}
   const conns = new Set();
+  const runs = new Map(); // every runBatch in flight (hosted or not yet started) -> its AbortController (#4454)
+  let closing = null, resolveClosed;
   let lastActive = Date.now();
   const touch = () => { lastActive = Date.now(); };
 
   /** Run one batch here. Resolves with {id, dir} once the batch exists, rejects when it fails before that. */
   const host = ({ argv, cwd, env, keepBreaker = false }, first = null) => new Promise((resolve, reject) => {
+    if (closing) return reject(new Error("schedd is shutting down"));
+    const ac = new AbortController();
     const ent = { id: null, dir: null, subs: new Set(first ? [first] : []) };
     const emit = (m) => { for (const s of ent.subs) s(m); };
     let started = false;
@@ -91,13 +105,15 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       line: (text) => emit({ type: "line", text }),
     };
     touch();
-    Promise.resolve().then(() => runBatch({ argv, cwd, env, sink, keepBreaker })).then((result) => {
+    const run = Promise.resolve().then(() => runBatch({ argv, cwd, env, sink, keepBreaker, signal: ac.signal })).then((result) => {
       const report = ent.dir ? openBatch(root, ent.id).readReport()?.report ?? null : null;
       emit({ type: "done", result, report });
     }, (e) => {
       const message = e?.message ?? String(e);
+      if (e?.code === "batch.stopped") { log(`schedd: batch ${ent.id ?? argv.join(" ")} stopped (lanes keep running; resumed by the next daemon)`); if (!started) reject(new Error(message)); return; }
       if (!started) reject(new Error(message)); else { log(`schedd: batch ${ent.id} failed: ${message}`); emit({ type: "error", message }); }
-    }).finally(() => { if (ent.id) hosted.delete(ent.id); ent.subs.clear(); touch(); });
+    }).finally(() => { if (ent.id) hosted.delete(ent.id); ent.subs.clear(); runs.delete(run); touch(); });
+    runs.set(run, ac);
   });
 
   const handle = async (req, send) => {
@@ -108,6 +124,19 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       b.setParallel(Number(req.k));
       return send({ type: "ok", parallel: Number(req.k) }, true);
     }
+    if (req.op === "add") {
+      const b = openBatch(root, req.id);
+      if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
+      const { added, skipped } = b.addIssues(req.issues ?? []);
+      return send({ type: "ok", added, skipped, hosted: hosted.has(req.id) }, true);
+    }
+    if (req.op === "cancel") {
+      const b = openBatch(root, req.id);
+      if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
+      if (!hosted.has(req.id) && b.readReport()) return send({ type: "ok", cancelled: false, finished: true }, true);
+      b.requestCancel({ reason: "operator" });
+      return send({ type: "ok", cancelled: true, hosted: hosted.has(req.id) }, true);
+    }
     const sub = (m) => send(m, m.type === "done" || m.type === "error");
     if (req.op === "submit") {
       const ent = await host({ argv: req.argv ?? [], cwd: req.cwd ?? process.cwd(), env: req.env ?? {} }, sub);
@@ -124,7 +153,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       const b = openBatch(root, req.id);
       if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
       const rep = b.readReport();
-      if (rep && !(rep.rows ?? []).some((r) => r.outcome === "held")) {
+      if (rep && !(rep.rows ?? []).some((r) => r.outcome === "held") && !b.unrunAdds().length) {
         return send({ type: "done", replay: true, result: { rows: rep.rows ?? [], crossLane: rep.crossLane ?? [], batch: b.id, ok: !!rep.ok }, report: rep.report ?? null }, true);
       }
       const ent = await host({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} }, sub);
@@ -158,13 +187,18 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
   await new Promise((res, rej) => { server.once("error", rej); server.listen(p.sock, () => { server.off("error", rej); res(); }); });
   log(`schedd: pid ${pid} listening on ${p.sock}`);
 
-  let closing = null, resolveClosed;
   const closed = new Promise((r) => { resolveClosed = r; });
   const close = () => {
     if (closing) return closing;
     clearInterval(idleTimer);
     for (const c of conns) c.destroy();
-    closing = new Promise((r) => server.close(() => r())).then(() => {
+    const stopping = [...runs.keys()];
+    for (const ac of runs.values()) ac.abort();
+    let timer;
+    // #4454: the lock goes only once every hosted batch loop has stopped (or `stopMs` passed: the caller exits regardless)
+    const stopped = stopping.length ? Promise.race([Promise.allSettled(stopping).then(() => true), new Promise((r) => { timer = setTimeout(() => r(false), stopMs); })])
+      .then((ok) => { clearTimeout(timer); if (!ok) log(`schedd: ${runs.size} batch(es) did not stop within ${stopMs} ms`); }) : Promise.resolve();
+    closing = Promise.all([new Promise((r) => server.close(() => r())), stopped]).then(() => {
       try { if (parseInt(readFileSync(p.lock, "utf8"), 10) === pid) rmSync(p.lock, { force: true }); } catch { /* gone */ }
       rmSync(p.sock, { force: true });
       resolveClosed();
@@ -188,6 +222,18 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
   return { running: true, pid, sock: p.sock, close, closed };
 }
 
+/** #4454: on SIGTERM/SIGINT stop the daemon (`close()`: batch loops first, then lock and socket), then exit 128+signo, once. */
+export function exitOnSignals(d, { on = (s, f) => process.once(s, f), exit = (c) => process.exit(c), signals = ["SIGTERM", "SIGINT"] } = {}) {
+  const SIGNO = { SIGINT: 2, SIGTERM: 15 };
+  let fired = false;
+  for (const sig of signals) on(sig, async () => {
+    if (fired) return;
+    fired = true;
+    try { await d.close(); } catch { /* exit anyway */ }
+    exit(128 + (SIGNO[sig] ?? 0));
+  });
+}
+
 /** Connect to the daemon, spawning it (`spawnDaemon()`) when nobody listens. @returns {Promise<import("node:net").Socket>} */
 export async function connectDaemon({ dir = schedDir(), spawnDaemon = defaultSpawn, waitMs = 10_000, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) } = {}) {
   const { sock } = scheddPaths(dir);