forgedock 1.27.0 → 1.28.0

package/bin/lean/schedd.mjs

@@ -0,0 +1,247 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Host scheduler daemon (#4192): one process hosts the `orchestrate()` loop of every batch on the host, instead of one
+ * dispatcher per `forgedock orchestrate`. Single instance: `schedd.lock` (the daemon pid, created with O_EXCL; a dead
+ * owner's lock is taken over) plus the unix socket `schedd.sock`, both in `schedDir()` (`~/.forge/sched/`, or
+ * `FORGEDOCK_SCHED_DIR`). Auto-spawned by the first `orchestrate` client, or run in the foreground by a systemd-user unit.
+ *
+ * Protocol: newline-delimited JSON, one request per connection.
+ *   {op:"submit", argv, cwd, env} → {type:"batch", id, dir}, {type:"row", row}…, {type:"line", text}…, {type:"done", result, report}
+ *   {op:"attach", id}             → log.jsonl rows replayed, then live rows/lines and done; a finished batch replays report.json
+ *                                   ({type:"done", replay:true}); a batch nobody hosts is resumed here
+ *   {op:"set-parallel", id, k}    → {type:"ok", parallel}
+ *   {op:"status"}                 → {type:"status", pid, batches:[{id, running}]}
+ * Failures answer {type:"error", message}. A client closing its connection never cancels a batch, and stopping the daemon
+ * never stops a lane (lanes are detached sessions). On start the daemon resumes every daemon-hosted batch without a
+ * report, so live lanes are re-adopted through the batch runner's `adopt`. It exits after `idleMs` with no batch and no client.
+ * `runBatch({argv, cwd, env, sink, keepBreaker})` is injected (engine-cli `runBatch`); `sink` gets `batch(b)`, `row(r)`, `line(text)`.
+ */
+import { closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, writeSync } from "node:fs";
+import { createConnection, createServer } from "node:net";
+import { spawn } from "node:child_process";
+import { join } from "node:path";
+import { schedDir, pidAlive } from "./sched.mjs";
+import { batchRoot, listBatches, openBatch } from "./batch.mjs";
+
+export const IDLE_MS = 10 * 60 * 1000;
+
+export function scheddPaths(dir = schedDir()) {
+  return { dir, sock: join(dir, "schedd.sock"), lock: join(dir, "schedd.lock"), log: join(dir, "schedd.log") };
+}
+
+// Env keys a batch keeps for its lanes (account and backend selectors, tool paths); persisted in plan.json.
+const ENV_KEEP = /^(PATH|HOME|USER|SHELL|LANG|LC_ALL|TMPDIR|FORGEDOCK_.*|CLAUDE_.*|ANTHROPIC_.*|GH_.*|GITHUB_.*)$/;
+
+/** The allowlisted part of `env`. */
+export function batchEnv(env = process.env) {
+  return Object.fromEntries(Object.entries(env).filter(([k, v]) => ENV_KEEP.test(k) && typeof v === "string"));
+}
+
+/** `base` without its allowlisted keys, plus `kept`: an account selector the client did not set never leaks in from the daemon. */
+export function mergeEnv(base, kept) {
+  const out = Object.fromEntries(Object.entries(base).filter(([k]) => !ENV_KEEP.test(k)));
+  return { ...out, ...kept };
+}
+
+/** Take the daemon lock; null when another live daemon holds it. */
+function takeLock(lock, pid, alive) {
+  for (let i = 0; i < 5; i++) {
+    try {
+      const fd = openSync(lock, "wx");
+      try { writeSync(fd, String(pid)); } finally { closeSync(fd); }
+      return true;
+    } catch (e) { if (e?.code !== "EEXIST") throw e; }
+    let owner = NaN;
+    try { owner = parseInt(readFileSync(lock, "utf8"), 10); } catch { continue; }
+    if (Number.isInteger(owner) && owner !== pid && alive(owner)) return null;
+    rmSync(lock, { force: true });
+  }
+  return null;
+}
+
+/**
+ * Start the daemon. @returns {Promise<{running:true, pid, close, closed:Promise} | {running:false, owner:number}>}
+ */
+export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(process.env.FORGEDOCK_SCHEDD_IDLE_MS) || IDLE_MS,
+  root = batchRoot(), log = (m) => console.log(m), pid = process.pid, alive = pidAlive, resumeOnStart = true } = {}) {
+  const p = scheddPaths(dir);
+  mkdirSync(dir, { recursive: true });
+  if (!takeLock(p.lock, pid, alive)) {
+    let owner = NaN;
+    try { owner = parseInt(readFileSync(p.lock, "utf8"), 10); } catch { /* gone */ }
+    log(`schedd: already running (pid ${owner})`);
+    return { running: false, owner };
+  }
+  rmSync(p.sock, { force: true });
+
+  const hosted = new Map(); // batch id -> {id, dir, subs:Set<fn>}
+  const conns = new Set();
+  let lastActive = Date.now();
+  const touch = () => { lastActive = Date.now(); };
+
+  /** Run one batch here. Resolves with {id, dir} once the batch exists, rejects when it fails before that. */
+  const host = ({ argv, cwd, env, keepBreaker = false }, first = null) => new Promise((resolve, reject) => {
+    const ent = { id: null, dir: null, subs: new Set(first ? [first] : []) };
+    const emit = (m) => { for (const s of ent.subs) s(m); };
+    let started = false;
+    const sink = {
+      batch: (b) => { ent.id = b.id; ent.dir = b.dir; hosted.set(b.id, ent); started = true; resolve(ent); },
+      row: (row) => emit({ type: "row", row }),
+      line: (text) => emit({ type: "line", text }),
+    };
+    touch();
+    Promise.resolve().then(() => runBatch({ argv, cwd, env, sink, keepBreaker })).then((result) => {
+      const report = ent.dir ? openBatch(root, ent.id).readReport()?.report ?? null : null;
+      emit({ type: "done", result, report });
+    }, (e) => {
+      const message = e?.message ?? String(e);
+      if (!started) reject(new Error(message)); else { log(`schedd: batch ${ent.id} failed: ${message}`); emit({ type: "error", message }); }
+    }).finally(() => { if (ent.id) hosted.delete(ent.id); ent.subs.clear(); touch(); });
+  });
+
+  const handle = async (req, send) => {
+    if (req.op === "status") return send({ type: "status", pid, batches: [...hosted.keys()].map((id) => ({ id, running: true })) }, true);
+    if (req.op === "set-parallel") {
+      const b = openBatch(root, req.id);
+      if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
+      b.setParallel(Number(req.k));
+      return send({ type: "ok", parallel: Number(req.k) }, true);
+    }
+    const sub = (m) => send(m, m.type === "done" || m.type === "error");
+    if (req.op === "submit") {
+      const ent = await host({ argv: req.argv ?? [], cwd: req.cwd ?? process.cwd(), env: req.env ?? {} }, sub);
+      return send({ type: "batch", id: ent.id, dir: ent.dir });
+    }
+    if (req.op === "attach") {
+      const live = hosted.get(req.id);
+      if (live) {
+        // Synchronous read + subscribe: no row is lost or doubled between the replay and the live stream.
+        for (const row of openBatch(root, req.id).readLog()) send({ type: "row", row, replay: true });
+        live.subs.add(sub);
+        return send({ type: "batch", id: live.id, dir: live.dir });
+      }
+      const b = openBatch(root, req.id);
+      if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
+      const rep = b.readReport();
+      if (rep && !(rep.rows ?? []).some((r) => r.outcome === "held")) {
+        return send({ type: "done", replay: true, result: { rows: rep.rows ?? [], crossLane: rep.crossLane ?? [], batch: b.id, ok: !!rep.ok }, report: rep.report ?? null }, true);
+      }
+      const ent = await host({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} }, sub);
+      return send({ type: "batch", id: ent.id, dir: ent.dir });
+    }
+    throw new Error(`unknown op ${JSON.stringify(req.op)}`);
+  };
+
+  const server = createServer((conn) => {
+    conns.add(conn);
+    touch();
+    let buf = "", seen = false, ended = false;
+    const send = (m, last = false) => {
+      if (ended || conn.destroyed) return;
+      conn.write(JSON.stringify(m) + "\n");
+      if (last) { ended = true; conn.end(); }
+    };
+    conn.on("error", () => { /* a client went away */ });
+    conn.on("close", () => { conns.delete(conn); ended = true; touch(); });
+    conn.on("data", (d) => {
+      if (seen) return;
+      buf += d;
+      const i = buf.indexOf("\n");
+      if (i < 0) return;
+      seen = true;
+      let req;
+      try { req = JSON.parse(buf.slice(0, i)); } catch { return send({ type: "error", message: "bad request" }, true); }
+      handle(req, send).catch((e) => send({ type: "error", message: e?.message ?? String(e) }, true));
+    });
+  });
+  await new Promise((res, rej) => { server.once("error", rej); server.listen(p.sock, () => { server.off("error", rej); res(); }); });
+  log(`schedd: pid ${pid} listening on ${p.sock}`);
+
+  let closing = null, resolveClosed;
+  const closed = new Promise((r) => { resolveClosed = r; });
+  const close = () => {
+    if (closing) return closing;
+    clearInterval(idleTimer);
+    for (const c of conns) c.destroy();
+    closing = new Promise((r) => server.close(() => r())).then(() => {
+      try { if (parseInt(readFileSync(p.lock, "utf8"), 10) === pid) rmSync(p.lock, { force: true }); } catch { /* gone */ }
+      rmSync(p.sock, { force: true });
+      resolveClosed();
+    });
+    return closing;
+  };
+  const idleTimer = setInterval(() => {
+    if (!hosted.size && !conns.size && Date.now() - lastActive >= idleMs) { log("schedd: idle, exiting"); close(); }
+  }, Math.min(idleMs, 30_000));
+  idleTimer.unref?.();
+
+  // Daemon restart: resume every daemon-hosted batch that never wrote its report; live lanes are re-adopted.
+  if (resumeOnStart) {
+    for (const id of listBatches(root)) {
+      const b = openBatch(root, id);
+      if (!b.readPlan()?.daemon || b.readReport()) continue;
+      log(`schedd: resuming batch ${id}`);
+      host({ argv: ["--resume", id], cwd: process.cwd(), env: {}, keepBreaker: true }).catch((e) => log(`schedd: resume ${id} failed: ${e?.message ?? e}`));
+    }
+  }
+  return { running: true, pid, sock: p.sock, close, closed };
+}
+
+/** Connect to the daemon, spawning it (`spawnDaemon()`) when nobody listens. @returns {Promise<import("node:net").Socket>} */
+export async function connectDaemon({ dir = schedDir(), spawnDaemon = defaultSpawn, waitMs = 10_000, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) } = {}) {
+  const { sock } = scheddPaths(dir);
+  const tryConnect = () => new Promise((res, rej) => {
+    const c = createConnection(sock);
+    c.once("connect", () => { c.off("error", rej); res(c); });
+    c.once("error", rej);
+  });
+  try { return await tryConnect(); } catch (e) { if (!["ENOENT", "ECONNREFUSED"].includes(e?.code) || !spawnDaemon) throw e; }
+  spawnDaemon({ dir });
+  const end = Date.now() + waitMs;
+  for (;;) {
+    await sleep(100);
+    try { return await tryConnect(); } catch (e) { if (Date.now() > end) throw new Error(`schedd did not start within ${waitMs} ms (see ${scheddPaths(dir).log}): ${e?.message ?? e}`); }
+  }
+}
+
+/** `forgedock schedd` in its own session, output to `schedd.log`. */
+export function defaultSpawn({ dir = schedDir(), script = new URL("../forgedock.mjs", import.meta.url).pathname, env = process.env } = {}) {
+  mkdirSync(dir, { recursive: true });
+  const out = openSync(scheddPaths(dir).log, "a");
+  try {
+    const child = spawn(process.execPath, [script, "schedd"], { env, detached: true, stdio: ["ignore", out, out] });
+    child.on?.("error", () => { /* the connect retry reports it */ });
+    child.unref?.();
+  } finally { closeSync(out); }
+}
+
+/**
+ * Send one request and feed every reply to `onMsg`. Resolves with the last message (`done`, `ok`, `status`), rejects on
+ * `error` or when the daemon closes the stream first.
+ */
+export function request(sock, msg, onMsg = () => {}) {
+  return new Promise((resolve, reject) => {
+    let buf = "", last = null, settled = false;
+    const finish = (fn, v) => { if (!settled) { settled = true; fn(v); } };
+    sock.on("data", (d) => {
+      buf += d;
+      let i;
+      while ((i = buf.indexOf("\n")) >= 0) {
+        const line = buf.slice(0, i); buf = buf.slice(i + 1);
+        if (!line) continue;
+        let m;
+        try { m = JSON.parse(line); } catch { continue; }
+        last = m;
+        if (m.type === "error") return finish(reject, new Error(m.message));
+        onMsg(m);
+        if (["done", "ok", "status"].includes(m.type)) { sock.end(); return finish(resolve, m); }
+      }
+    });
+    sock.on("error", (e) => finish(reject, e));
+    sock.on("close", () => finish(reject, Object.assign(new Error("schedd closed the connection before the batch finished"), { last })));
+    sock.write(JSON.stringify(msg) + "\n");
+  });
+}
+
+export const scheddExists = (dir = schedDir()) => existsSync(scheddPaths(dir).sock);

package/bin/lean/schedd.test.mjs

@@ -0,0 +1,180 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpDir } from "../tests/_tmp.mjs";
+import { openBatch, isLaneAlive, listBatches } from "./batch.mjs";
+import { startDaemon, scheddPaths, connectDaemon, request, mergeEnv, batchEnv } from "./schedd.mjs";
+import { formatTable } from "./orchestrate.mjs";
+import { orchestrateFromCli } from "../engine-cli.mjs";
+
+const until = async (cond, ms = 15000) => {
+  const end = Date.now() + ms;
+  while (!cond()) { if (Date.now() > end) throw new Error("timed out waiting"); await new Promise((r) => setTimeout(r, 25)); }
+};
+const capture = async (fn) => {
+  const lines = [], l = console.log;
+  console.log = (...a) => lines.push(a.join(" "));
+  try { return { res: await fn(), lines }; } finally { console.log = l; }
+};
+
+/** A host: sched + batch dirs, a stub lane that waits for `release`, and a daemon script with injected lanes. */
+function host() {
+  const work = tmpDir("schedd-");
+  const h = { work, sched: join(work, "s"), root: join(work, "batches"), launches: join(work, "launches"), release: join(work, "release"), procs: [] };
+  writeFileSync(h.launches, "");
+  const stub = join(work, "stub-lane.mjs");
+  writeFileSync(stub, `import { appendFileSync, existsSync, writeFileSync, renameSync } from "node:fs";
+const a = process.argv.slice(2); const n = Number(a[1]); const rf = a[a.indexOf("--result-file") + 1];
+appendFileSync(${JSON.stringify(h.launches)}, n + "\\n");
+const t = setInterval(() => { if (!existsSync(${JSON.stringify(h.release)})) return; clearInterval(t);
+  writeFileSync(rf + ".tmp", JSON.stringify({ terminalReason: "merged", pr: 100 + n, cost_usd: 0 })); renameSync(rf + ".tmp", rf); }, 20);
+`);
+  h.daemonScript = join(work, "daemon.mjs");
+  writeFileSync(h.daemonScript, `import { startDaemon } from ${JSON.stringify(new URL("./schedd.mjs", import.meta.url).href)};
+import { runBatch } from ${JSON.stringify(new URL("../engine-cli.mjs", import.meta.url).href)};
+const deps = { allowTmpfs: true, laneScript: ${JSON.stringify(stub)}, pollMs: 30, orchestration: { breaker: null, maxDeferrals: 3 }, io: { gh: async () => "o/r\\n" },
+  leanIo: { gh: async (a) => a.includes("milestone") ? JSON.stringify({ milestone: null }) : a[0] === "repo" ? "main\\n" : JSON.stringify({ body: a[2] === "3" ? "Depends on #1" : "", labels: [] }),
+    git: async (a) => (a[0] === "ls-remote" ? "abc\\trefs/heads/staging\\n" : "") } };
+const d = await startDaemon({ runBatch: (a) => runBatch({ ...a, daemon: true, deps }), idleMs: 600000 });
+if (!d.running) process.exit(3);
+process.once("SIGTERM", () => d.close());
+await d.closed;
+`);
+  h.env = { ...process.env, FORGEDOCK_SCHED_DIR: h.sched, FORGEDOCK_BATCH_DIR: h.root, FORGEDOCK_LIMITS_FILE: join(work, "limits.json") };
+  h.daemon = async () => {
+    const before = existsSync(scheddPaths(h.sched).lock) ? readFileSync(scheddPaths(h.sched).lock, "utf8") : null;
+    const child = spawn(process.execPath, [h.daemonScript], { env: h.env, stdio: "ignore" });
+    h.procs.push(child);
+    await until(() => existsSync(scheddPaths(h.sched).sock) && existsSync(scheddPaths(h.sched).lock) && readFileSync(scheddPaths(h.sched).lock, "utf8") === String(child.pid) && readFileSync(scheddPaths(h.sched).lock, "utf8") !== before);
+    // Wait for the socket to accept.
+    for (let i = 0; i < 100; i++) { try { (await connectDaemon({ dir: h.sched, spawnDaemon: null })).destroy(); break; } catch { await new Promise((r) => setTimeout(r, 30)); } }
+    return child;
+  };
+  h.client = { scheddDir: h.sched, spawnDaemon: null };
+  h.launched = () => readFileSync(h.launches, "utf8").trim().split("\n").filter(Boolean).map(Number).sort();
+  h.lanePids = () => listBatches(h.root).flatMap((id) => Object.values(openBatch(h.root, id).store.load()?.running ?? {}).map((r) => r.pid).filter(Boolean));
+  h.cleanup = () => {
+    writeFileSync(h.release, "");
+    for (const p of h.procs) { try { p.kill("SIGKILL"); } catch { /* gone */ } }
+  };
+  return h;
+}
+
+describe("schedd host scheduler (#4192)", () => {
+  // The client and the daemon both resolve the batch root from FORGEDOCK_BATCH_DIR.
+  const withRoot = async (h, fn) => {
+    const prev = process.env.FORGEDOCK_BATCH_DIR;
+    process.env.FORGEDOCK_BATCH_DIR = h.root;
+    try { return await fn(); } finally { if (prev === undefined) delete process.env.FORGEDOCK_BATCH_DIR; else process.env.FORGEDOCK_BATCH_DIR = prev; }
+  };
+
+  it("one daemon hosts every batch: registry slots are the daemon's, its RSS stays ≤ 150 MB, set-parallel goes through it", async () => {
+    const h = host();
+    try {
+      const d = await h.daemon();
+      const submit = (issues) => orchestrateFromCli([...issues, "--parallel", "2", "--repo", "o/r", "--no-prefilter"], h.client);
+      // One capture for the three concurrent clients (console.log is process-wide).
+      const all = capture(() => Promise.all([submit(["1", "2"]), submit(["4"]), submit(["5"])]));
+      await until(() => h.launched().length === 4 && h.lanePids().length === 4);
+      const reg = JSON.parse(readFileSync(join(h.sched, "registry.json"), "utf8"));
+      assert.equal(Object.keys(reg.slots).length, 4);
+      for (const s of Object.values(reg.slots)) assert.equal(s.pid, d.pid, "every lane slot belongs to the one daemon");
+      const st = await request(await connectDaemon({ dir: h.sched, spawnDaemon: null }), { op: "status" });
+      assert.equal(st.pid, d.pid);
+      assert.equal(st.batches.length, 3);
+      const rssKb = Number(/VmRSS:\s+(\d+)/.exec(readFileSync(`/proc/${d.pid}/status`, "utf8"))[1]);
+      assert.ok(rssKb <= 150 * 1024, `daemon RSS ${rssKb} kB hosting 3 batches`);
+      writeFileSync(h.release, "");
+      const { res: done, lines } = await all;
+      assert.deepEqual(done.map((res) => res.rows.map((r) => [r.issue, r.outcome, r.pr])), [[[1, "merged", 101], [2, "merged", 102]], [[4, "merged", 104]], [[5, "merged", 105]]]);
+      assert.ok(lines.some((l) => /^#1 → merged$/.test(l)), "rows stream to the client");
+      for (const res of done) assert.ok(lines.some((l) => l.includes(formatTable(res.rows))), "each report table is printed");
+      const id = done[0].batch;
+      await withRoot(h, () => capture(() => orchestrateFromCli(["--set-parallel", id, "5"], h.client)));
+      assert.equal(openBatch(h.root, id).readParallel(), 5);
+    } finally { h.cleanup(); }
+  });
+
+  it("a client that disconnects does not stop the batch; --resume reattaches, and replays the report once done", async () => {
+    const h = host();
+    try {
+      await h.daemon();
+      const clientScript = join(h.work, "client.mjs");
+      writeFileSync(clientScript, `import { orchestrateFromCli } from ${JSON.stringify(new URL("../engine-cli.mjs", import.meta.url).href)};
+await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-prefilter"], { scheddDir: ${JSON.stringify(h.sched)}, spawnDaemon: null });
+`);
+      const client = spawn(process.execPath, [clientScript], { env: h.env, stdio: "ignore" });
+      h.procs.push(client);
+      const exited = new Promise((r) => client.on("exit", r));
+      await until(() => h.launched().length === 2 && h.lanePids().length === 2);
+      client.kill("SIGKILL");
+      await exited;
+      const [id] = listBatches(h.root);
+      for (const pid of h.lanePids()) assert.equal(isLaneAlive(pid, null), true, "lanes outlive the client");
+      const attached = capture(() => orchestrateFromCli(["--resume", id], h.client));
+      setTimeout(() => writeFileSync(h.release, ""), 200);
+      const { res } = await attached;
+      assert.deepEqual(res.rows.map((r) => [r.issue, r.outcome, r.pr]), [[1, "merged", 101], [2, "merged", 102]]);
+      assert.deepEqual(h.launched(), [1, 2], "no lane relaunched");
+      // Finished: the report is replayed from report.json, nothing runs again.
+      const again = await capture(() => orchestrateFromCli(["--resume", id], h.client));
+      assert.deepEqual(again.res.rows, res.rows);
+      assert.ok(again.lines.join("\n").includes(formatTable(res.rows)));
+      assert.deepEqual(h.launched(), [1, 2]);
+    } finally { h.cleanup(); }
+  });
+
+  it("a daemon restart resumes the batch from state and re-adopts its live lanes without duplicate launches", async () => {
+    const h = host();
+    try {
+      const d1 = await h.daemon();
+      const first = capture(() => orchestrateFromCli(["1", "2", "3", "--parallel", "2", "--repo", "o/r", "--no-prefilter"], h.client)).then(() => null, (e) => e);
+      await until(() => h.launched().length === 2 && h.lanePids().length === 2);
+      const lanes = h.lanePids();
+      const d1exit = new Promise((r) => d1.on("exit", r));
+      d1.kill("SIGKILL");
+      await d1exit;
+      assert.match(String((await first)?.message), /--resume/, "the client is told how to reattach");
+      for (const pid of lanes) assert.equal(isLaneAlive(pid, null), true, "lanes outlive the daemon");
+      const [id] = listBatches(h.root);
+      await h.daemon(); // takes over the dead daemon's lock and socket, resumes the batch
+      const attached = capture(() => orchestrateFromCli(["--resume", id], h.client));
+      setTimeout(() => writeFileSync(h.release, ""), 200);
+      const { res } = await attached;
+      assert.deepEqual(res.rows.map((r) => [r.issue, r.outcome, r.pr]), [[1, "merged", 101], [2, "merged", 102], [3, "merged", 103]]);
+      assert.deepEqual(h.launched(), [1, 2, 3], "adopted, not relaunched");
+      assert.deepEqual(openBatch(h.root, id).store.load().running, {});
+    } finally { h.cleanup(); }
+  });
+
+  it("single instance: a live daemon keeps its lock; a dead owner's lock and socket are taken over", async () => {
+    const h = host();
+    try {
+      const d = await h.daemon();
+      const second = await startDaemon({ dir: h.sched, root: h.root, runBatch: async () => ({}), log: () => {}, resumeOnStart: false });
+      assert.deepEqual(second, { running: false, owner: d.pid });
+      const dexit = new Promise((r) => d.on("exit", r));
+      d.kill("SIGKILL");
+      await dexit;
+      assert.ok(existsSync(scheddPaths(h.sched).lock) && existsSync(scheddPaths(h.sched).sock), "stale lock and socket left behind");
+      const third = await startDaemon({ dir: h.sched, root: h.root, runBatch: async () => ({}), log: () => {}, resumeOnStart: false });
+      try {
+        assert.equal(third.running, true);
+        const st = await request(await connectDaemon({ dir: h.sched, spawnDaemon: null }), { op: "status" });
+        assert.equal(st.pid, process.pid);
+      } finally { await third.close(); }
+      assert.equal(existsSync(scheddPaths(h.sched).lock), false, "lock released on close");
+    } finally { h.cleanup(); }
+  });
+
+  it("lane env: allowlisted keys come from the client only, the rest from the daemon", () => {
+    const kept = batchEnv({ PATH: "/c", CLAUDE_CONFIG_DIR: "/acct-b", SECRET_X: "no", FORGEDOCK_SCHED_DIR: "/s" });
+    assert.deepEqual(kept, { PATH: "/c", CLAUDE_CONFIG_DIR: "/acct-b", FORGEDOCK_SCHED_DIR: "/s" });
+    const env = mergeEnv({ PATH: "/d", ANTHROPIC_API_KEY: "daemon-key", NODE_OPTIONS: "x" }, kept);
+    assert.deepEqual(env, { NODE_OPTIONS: "x", PATH: "/c", CLAUDE_CONFIG_DIR: "/acct-b", FORGEDOCK_SCHED_DIR: "/s" });
+  });
+});

package/bin/lean/toolchain.mjs

@@ -0,0 +1,228 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Toolchain pins for a lane (#4163). Reads the target repo's own pins (package.json `packageManager` / `engines.node`,
+ * `.nvmrc`, `.node-version`, `.tool-versions`) and writes a per-run shim dir for the pinned package manager, prepended
+ * to the runner's PATH so the builder, its validate commands and git hooks all resolve the pinned version. Zero LLM.
+ * A repo without pins gets nothing: no shim dir, no env, no warning.
+ */
+import { readFile, writeFile, mkdir, chmod } from "node:fs/promises";
+import { accessSync, constants } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { join, delimiter, resolve } from "node:path";
+
+const PMS = ["pnpm", "yarn", "npm"];
+const VERSION_RE = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/;
+
+/** "pnpm@10.4.1+sha512.ab" -> {name, version}; null for anything not a known manager at an exact version. */
+export function parsePackageManager(s) {
+  const m = /^(pnpm|yarn|npm)@([^+\s]+)(?:\+\S*)?$/.exec(String(s ?? "").trim());
+  return m && VERSION_RE.test(m[2]) ? { name: m[1], version: m[2] } : null;
+}
+
+/** `.tool-versions` lines -> {tool: firstVersion}. */
+function parseToolVersions(text) {
+  const out = {};
+  for (const l of String(text ?? "").split("\n")) {
+    const [tool, ver] = l.replace(/#.*/, "").trim().split(/\s+/);
+    if (tool && ver && !(tool in out)) out[tool] = ver;
+  }
+  return out;
+}
+
+/**
+ * The repo's pins. `override` is forge.yaml `env.toolchain`: "<pm>@<version>" replaces the package-manager pin, "off"
+ * disables detection. Returns {pm:{name,version,source}|null, node:{spec,source}|null, notes:[]}.
+ */
+export async function detectPins(dir, { override = null, read = (p) => readFile(p, "utf8") } = {}) {
+  const res = { pm: null, node: null, notes: [] };
+  const ov = typeof override === "string" ? override.trim() : "";
+  if (ov.toLowerCase() === "off") return res;
+  const tryRead = async (f) => { try { return String(await read(join(dir, f))); } catch { return null; } };
+  let pkg = null;
+  const pkgText = await tryRead("package.json");
+  if (pkgText !== null) { try { pkg = JSON.parse(pkgText); } catch { res.notes.push("package.json is not valid JSON; its pins are ignored"); } }
+  const tv = parseToolVersions(await tryRead(".tool-versions"));
+  if (ov) {
+    const pm = parsePackageManager(ov);
+    if (pm) res.pm = { ...pm, source: "forge.yaml env.toolchain" };
+    else res.notes.push(`forge.yaml env.toolchain "${ov.slice(0, 60)}" is not <pnpm|yarn|npm>@<x.y.z> or "off"; ignored`);
+  }
+  if (!res.pm && typeof pkg?.packageManager === "string") {
+    const pm = parsePackageManager(pkg.packageManager);
+    if (pm) res.pm = { ...pm, source: "package.json packageManager" };
+    else res.notes.push(`package.json packageManager "${pkg.packageManager.slice(0, 60)}" is not <pnpm|yarn|npm>@<x.y.z>; not pinned`);
+  }
+  if (!res.pm) {
+    const name = PMS.find((n) => VERSION_RE.test(tv[n] ?? ""));
+    if (name) res.pm = { name, version: tv[name], source: ".tool-versions" };
+  }
+  for (const [f, get] of [[".nvmrc", (t) => t], [".node-version", (t) => t]]) {
+    if (res.node) break;
+    const t = await tryRead(f);
+    const spec = t === null ? "" : get(t).split("\n")[0].trim();
+    if (spec) res.node = { spec, source: f };
+  }
+  if (!res.node && (tv.nodejs || tv.node)) res.node = { spec: tv.nodejs || tv.node, source: ".tool-versions" };
+  if (!res.node && typeof pkg?.engines?.node === "string" && pkg.engines.node.trim()) res.node = { spec: pkg.engines.node.trim(), source: "package.json engines.node" };
+  return res;
+}
+
+// ---- minimal semver satisfaction for node pins ----
+const parseV = (s) => {
+  const m = /^v?(\d+)(?:\.(\d+|x|\*))?(?:\.(\d+|x|\*))?$/i.exec(String(s).trim());
+  if (!m) return null;
+  const n = (x) => (x === undefined || /^[x*]$/i.test(x) ? null : Number(x));
+  return [Number(m[1]), n(m[2]), n(m[3])];
+};
+const cmp = (a, b) => { for (let i = 0; i < 3; i++) { if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1; } return 0; };
+const fill = (p, v = 0) => p.map((x) => (x === null ? v : x));
+/** host [M,m,p] within partial p (20 -> 20.x.x, 20.11 -> 20.11.x)? */
+const within = (h, p) => p.every((x, i) => x === null || x === h[i]);
+const upper = (p) => (p[1] === null ? [p[0] + 1, 0, 0] : p[2] === null ? [p[0], p[1] + 1, 0] : null);
+
+/** true/false, or null when the comparator can't be evaluated. */
+function satisfiesOne(h, c) {
+  const m = /^(\^|~|>=|<=|>|<|=)?\s*(\S+)$/.exec(c.trim());
+  if (!m) return null;
+  const p = parseV(m[2]);
+  if (!p) return null;
+  const lo = fill(p);
+  switch (m[1] ?? "") {
+    case "": case "=": return within(h, p);
+    case ">=": return cmp(h, lo) >= 0;
+    case ">": return p[1] === null || p[2] === null ? cmp(h, upper(p)) >= 0 : cmp(h, lo) > 0;
+    case "<": return cmp(h, lo) < 0;
+    case "<=": return p[1] === null || p[2] === null ? cmp(h, upper(p)) < 0 : cmp(h, lo) <= 0;
+    case "~": return cmp(h, lo) >= 0 && cmp(h, p[1] === null ? [p[0] + 1, 0, 0] : [p[0], p[1] + 1, 0]) < 0;
+    case "^": return cmp(h, lo) >= 0 && cmp(h, lo[0] > 0 ? [lo[0] + 1, 0, 0] : lo[1] > 0 ? [0, lo[1] + 1, 0] : [0, 0, lo[2] + 1]) < 0;
+  }
+  return null;
+}
+
+/** Does host version `v` satisfy node spec? true/false, or null when the spec can't be evaluated (aliases, unknown syntax). */
+export function nodeSatisfies(spec, v) {
+  const h = parseV(v);
+  if (!h || h.includes(null)) return null;
+  const alts = String(spec).split("||").map((s) => s.trim()).filter(Boolean);
+  if (!alts.length) return null;
+  let unknown = false;
+  for (const alt of alts) {
+    const hy = /^(\S+)\s+-\s+(\S+)$/.exec(alt); // hyphen range "18 - 20" = ">=18 <=20"
+    const parts = hy ? [`>=${hy[1]}`, `<=${hy[2]}`] : alt.replace(/(\^|~|>=|<=|>|<|=)\s+/g, "$1").split(/\s+/);
+    let ok = true;
+    for (const c of parts) {
+      const r = satisfiesOne(h, c);
+      if (r === null) { unknown = true; ok = false; break; }
+      if (!r) { ok = false; break; }
+    }
+    if (ok) return true;
+  }
+  return unknown ? null : false;
+}
+
+/** One-line trajectory warning when the host node does not satisfy the pin; null when it does or nothing is pinned. */
+export function nodeWarning(pin, hostVersion = process.version) {
+  if (!pin?.spec) return null;
+  if (/^(lts\/|node$|stable$|latest$|current$|system$)/i.test(pin.spec)) return null;
+  const ok = nodeSatisfies(pin.spec, hostVersion);
+  if (ok === true) return null;
+  if (ok === null) return `Toolchain: cannot evaluate node pin "${pin.spec}" (${pin.source}) against host node ${hostVersion}; check it manually`;
+  return `host node ${hostVersion} does not satisfy the ${pin.source} pin "${pin.spec}"; installs, builds and tests may behave differently from the repo's CI`;
+}
+
+/** First executable `name` on `pathStr`, skipping `exclude` dirs (the shim dir itself, so a shim never calls itself). */
+export function which(name, pathStr = process.env.PATH ?? "", exclude = []) {
+  const skip = new Set(exclude.map((d) => resolve(d)));
+  for (const d of String(pathStr).split(delimiter)) {
+    if (!d || skip.has(resolve(d))) continue;
+    const p = join(d, name);
+    try { accessSync(p, constants.X_OK); return p; } catch { /* next */ }
+  }
+  return null;
+}
+
+const q = (s) => `'${String(s).replace(/'/g, `'\\''`)}'`;
+const HEADER = "#!/bin/sh\n# forgedock lane toolchain shim (#4163): pinned by the target repo; do not edit.\nexport COREPACK_ENABLE_DOWNLOAD_PROMPT=0 COREPACK_ENABLE_STRICT=0\n";
+
+/**
+ * Shim scripts {name: content} for the pinned manager: `corepack <pm>@<ver>` when corepack is on the host PATH, else
+ * `npx -y`. Absolute paths are resolved here, so a shim never re-enters the shim dir. Returns {} when neither exists.
+ */
+export function shimScripts(pm, { corepack = null, npx = null } = {}) {
+  if (!pm || !PMS.includes(pm.name) || !VERSION_RE.test(pm.version)) return {};
+  const at = `${pm.name}@${pm.version}`;
+  const out = {};
+  if (pm.name === "npm") {
+    if (npx) { out.npm = `${HEADER}exec ${q(npx)} -y ${q(at)} "$@"\n`; out.npx = `${HEADER}exec ${q(npx)} -y -p ${q(at)} npx "$@"\n`; }
+    return out;
+  }
+  if (corepack) {
+    out[pm.name] = `${HEADER}exec ${q(corepack)} ${q(at)} "$@"\n`;
+    if (pm.name === "pnpm") out.pnpx = `${HEADER}exec ${q(corepack)} ${q(at)} dlx "$@"\n`;
+    return out;
+  }
+  if (!npx) return out;
+  // yarn >= 2 is published as @yarnpkg/cli-dist; the npm "yarn" package is 1.x only.
+  const pkg = pm.name === "yarn" && Number(pm.version.split(".")[0]) >= 2 ? `@yarnpkg/cli-dist@${pm.version}` : at;
+  out[pm.name] = `${HEADER}exec ${q(npx)} -y -p ${q(pkg)} ${pm.name} "$@"\n`;
+  if (pm.name === "pnpm") out.pnpx = `${HEADER}exec ${q(npx)} -y -p ${q(pkg)} pnpm dlx "$@"\n`;
+  return out;
+}
+
+/** Runner env for a shim dir: PATH with the shim dir first, corepack prompts off. */
+export function toolchainEnv(binDir, basePath = process.env.PATH ?? "") {
+  return { PATH: basePath ? `${binDir}${delimiter}${basePath}` : binDir, COREPACK_ENABLE_DOWNLOAD_PROMPT: "0", COREPACK_ENABLE_STRICT: "0" };
+}
+
+/** Version of the host `node` the builder's shells resolve (PATH), falling back to this process. */
+export function hostNodeVersion(pathStr = process.env.PATH ?? "") {
+  const n = which("node", pathStr);
+  if (n) { try { return String(execFileSync(n, ["--version"], { timeout: 10000 })).trim(); } catch { /* fall through */ } }
+  return process.version;
+}
+
+async function writeShimFile(p, content) { await writeFile(p, content); await chmod(p, 0o755); }
+
+/**
+ * Write executable (0755) shims for the pinned manager into `binDir`; launchers are resolved from `path` with `binDir`
+ * excluded. Returns the shim names written ([] when neither corepack nor npx is on `path`).
+ */
+export async function writeShims(binDir, pm, { path = process.env.PATH ?? "", find = which, writeShim = writeShimFile, mkdirp = (d) => mkdir(d, { recursive: true }) } = {}) {
+  const scripts = shimScripts(pm, { corepack: find("corepack", path, [binDir]), npx: find("npx", path, [binDir]) });
+  if (!Object.keys(scripts).length) return [];
+  await mkdirp(binDir);
+  for (const [name, content] of Object.entries(scripts)) await writeShim(join(binDir, name), content);
+  return Object.keys(scripts);
+}
+
+/** Short pin text for prompts and decisions, e.g. "pnpm@10.4.1 (package.json packageManager), node 20 (.nvmrc)"; null when nothing is pinned. */
+export function summary(pins) {
+  const parts = [...(pins?.pm ? [`${pins.pm.name}@${pins.pm.version} (${pins.pm.source})`] : []), ...(pins?.node ? [`node ${pins.node.spec} (${pins.node.source})`] : [])];
+  return parts.length ? parts.join(", ") : null;
+}
+
+/**
+ * Detect pins in `dir` and, for a pinned package manager, write shims to `binDir`.
+ * @returns {Promise<{pins, env:?object, shims:string[], summary:?string, warning:?string, notes:string[]}>}
+ *   env is null and shims empty without a usable pm pin; summary is null when nothing is pinned: the caller changes nothing.
+ */
+export async function prepareToolchain({ dir, binDir, override = null, read, basePath = process.env.PATH ?? "",
+  writeShim = writeShimFile, mkdirp = (d) => mkdir(d, { recursive: true }), find = which, hostNode = () => hostNodeVersion(basePath) } = {}) {
+  const pins = await detectPins(dir, { override, ...(read ? { read } : {}) });
+  const out = { pins, env: null, shims: [], summary: null, warning: null, notes: [...pins.notes] };
+  if (pins.node) {
+    let v = null;
+    try { v = hostNode(); } catch { v = null; }
+    out.warning = v ? nodeWarning(pins.node, v) : null;
+  }
+  const pm = pins.pm;
+  let effective = pins;
+  if (pm && binDir) {
+    out.shims = await writeShims(binDir, pm, { path: basePath, find, writeShim, mkdirp });
+    if (out.shims.length) out.env = toolchainEnv(binDir, basePath);
+    else { out.notes.push(`Toolchain: ${pm.name}@${pm.version} is pinned but neither corepack nor npx is on PATH; the host ${pm.name} is used`); effective = { ...pins, pm: null }; }
+  }
+  out.summary = summary(effective);
+  return out;
+}