forgedock 1.41.0 → 1.43.0

package/bin/lean/schedd.test.mjs

@@ -8,11 +8,11 @@ import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
 import { openBatch, isLaneAlive, listBatches, SECRET_ENV } from "./batch.mjs";
-import { startDaemon, scheddPaths, connectDaemon, request, SCHEDD_SCRIPT, mergeEnv, batchEnv, droppedEnv, sanitizePath, redactEnv, exitOnSignals } from "./schedd.mjs";
+import { startDaemon, scheddPaths, connectDaemon, request, unitState, SCHEDD_SCRIPT, mergeEnv, batchEnv, droppedEnv, sanitizePath, redactEnv, exitOnSignals } from "./schedd.mjs";
 import { schedDir } from "./sched.mjs";
 import { readHeartbeat } from "./schedd-health.mjs";
 import { formatTable, orchestrate } from "./orchestrate.mjs";
-import { orchestrateFromCli } from "../engine-cli.mjs";
+import { orchestrateFromCli, scheddFromCli } from "../engine-cli.mjs";
 
 // #4503: this process, and the env its clients submit for their lanes, never resolve the host's `~/.forge/sched`.
 process.env.FORGEDOCK_SCHED_DIR = tmpDir("schedd-home-");
@@ -637,6 +637,96 @@ describe("schedd due timer (#4627)", () => {
   });
 });
 
+// #4775: a client never races a systemd-managed daemon; a stalled daemon is visible and bounded.
+describe("schedd under load and restart (#4775)", () => {
+  const noSleep = () => Promise.resolve();
+  it("connectDaemon never spawns while the unit is active or activating, and starts an installed stopped unit instead of spawning", async () => {
+    for (const state of ["active", "activating", "reloading"]) {
+      let spawned = 0;
+      await assert.rejects(connectDaemon({ dir: tmpDir("schedd-unit-"), waitMs: 0, sleep: noSleep, spawnDaemon: () => { spawned++; }, unit: () => state, start: () => { spawned++; } }), /did not start/);
+      assert.equal(spawned, 0, `${state}: nothing is started`);
+    }
+    const calls = [];
+    await assert.rejects(connectDaemon({ dir: tmpDir("schedd-unit-"), waitMs: 0, sleep: noSleep, spawnDaemon: () => calls.push("spawn"), unit: () => "inactive", start: () => calls.push("unit") }), /did not start/);
+    assert.deepEqual(calls, ["unit"]);
+  });
+
+  it("connectDaemon spawns the daemon itself only when no unit is installed; spawnDaemon:null starts nothing", async () => {
+    const calls = [];
+    await assert.rejects(connectDaemon({ dir: tmpDir("schedd-unit-"), waitMs: 0, sleep: noSleep, spawnDaemon: () => calls.push("spawn"), unit: () => null, start: () => calls.push("unit") }), /did not start/);
+    assert.deepEqual(calls, ["spawn"]);
+    await assert.rejects(connectDaemon({ dir: tmpDir("schedd-unit-"), spawnDaemon: null, unit: () => { throw new Error("not consulted"); } }), { code: "ENOENT" });
+  });
+
+  it("unitState reads the systemctl state (also from a non-zero exit) and is null without a unit file", () => {
+    assert.equal(unitState({ exists: () => false }), null);
+    assert.equal(unitState({ exists: () => true, run: () => "activating\n" }), "activating");
+    assert.equal(unitState({ exists: () => true, run: () => { throw Object.assign(new Error("exit 3"), { stdout: "inactive\n" }); } }), "inactive");
+    assert.equal(unitState({ exists: () => true, run: () => { throw Object.assign(new Error("none"), { code: "ENOENT" }); } }), null);
+  });
+
+  it("request(timeoutMs) rejects a daemon that never answers instead of hanging", async () => {
+    const dir = tmpDir("schedd-mute-"), sock = scheddPaths(dir).sock;
+    const server = (await import("node:net")).createServer(() => { /* accepts, never answers */ });
+    await new Promise((r) => server.listen(sock, r));
+    try {
+      const c = await connectDaemon({ dir, spawnDaemon: null });
+      await assert.rejects(request(c, { op: "status" }, undefined, { timeoutMs: 80 }), { code: "schedd.timeout" });
+    } finally { server.close(); }
+  });
+
+  it("status reports per-batch last-progress age and pending adds, including an unhosted batch with un-admitted adds", async () => {
+    const root = tmpDir("schedd-st-b-"), dir = tmpDir("schedd-st-");
+    const mk = (id, files) => { mkdirSync(join(root, id), { recursive: true }); writeFileSync(join(root, id, "plan.json"), JSON.stringify({ daemon: true })); for (const [n, t] of Object.entries(files)) writeFileSync(join(root, id, n), t); };
+    mk("b-stuck", { "state.json": JSON.stringify({ order: [1] }), add: "[7,8]" });
+    mk("b-quiet", { "state.json": JSON.stringify({ order: [1] }) });
+    const d = await startDaemon({ dir, root, runBatch: async () => ({}), resumeOnStart: false, log: () => {} });
+    try {
+      const st = await request(await connectDaemon({ dir, spawnDaemon: null }), { op: "status" });
+      assert.equal(st.batches.length, 1, "a quiet unhosted batch is not listed");
+      const [b] = st.batches;
+      assert.equal(b.id, "b-stuck");
+      assert.equal(b.running, false);
+      assert.deepEqual(b.pendingAdds, [7, 8]);
+      assert.ok(b.lastProgressAgeSec >= 0 && b.lastProgressAgeSec < 60);
+    } finally { await d.close(); }
+  });
+
+  it("a batch whose start stalls does not block another batch's request, and the stalled request answers an error", async () => {
+    const root = tmpDir("schedd-stall-b-"), dir = tmpDir("schedd-stall-");
+    for (const id of ["b-hung", "b-ok"]) { mkdirSync(join(root, id), { recursive: true }); writeFileSync(join(root, id, "plan.json"), JSON.stringify({ daemon: true })); writeFileSync(join(root, id, "state.json"), JSON.stringify({ order: [1] })); }
+    const d = await startDaemon({ dir, root, resumeOnStart: false, admitMs: 150, log: () => {},
+      runBatch: ({ argv, sink }) => (argv[1] === "b-hung" ? new Promise(() => {}) : Promise.resolve().then(() => { sink.batch({ id: argv[1], dir: join(root, argv[1]) }); return {}; })) });
+    try {
+      const ask = async (id) => request(await connectDaemon({ dir, spawnDaemon: null }), { op: "attach", id }).catch((e) => e);
+      const hung = ask("b-hung");
+      const ok = await ask("b-ok");
+      assert.notEqual(ok instanceof Error && /did not start/.test(ok.message), true, "the healthy batch is served");
+      const e = await hung;
+      assert.match(e.message, /did not start within 150 ms/);
+    } finally { await d.close(); }
+  });
+
+  it("the unit stands by while another daemon holds the lock instead of exiting 0 (no Restart=always loop)", async () => {
+    const dir = tmpDir("schedd-sb-"), root = tmpDir("schedd-sb-b-");
+    const owner = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60000)"], { stdio: "ignore" });
+    const gone = new Promise((r) => owner.on("exit", r));
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(scheddPaths(dir).lock, String(owner.pid)); // a live daemon of another process
+    const slept = [];
+    process.env.FORGEDOCK_SCHEDD_IDLE_MS = "100"; // the successor daemon exits when idle, so the call returns
+    process.env.FORGEDOCK_BATCH_DIR = root;
+    try {
+      const r = await Promise.race([
+        scheddFromCli([], { scheddDir: dir, standby: true, standbyMs: 1, sleep: async () => { slept.push(1); if (slept.length === 3) { owner.kill("SIGKILL"); await gone; } }, processOn: () => {}, exit: () => {}, batchDeps: {}, due: null }),
+        new Promise((res) => setTimeout(() => res("timeout"), 5000)),
+      ]);
+      assert.ok(slept.length >= 3, "waited for the owner before retrying");
+      assert.notEqual(r, "timeout");
+    } finally { delete process.env.FORGEDOCK_SCHEDD_IDLE_MS; delete process.env.FORGEDOCK_BATCH_DIR; owner.kill("SIGKILL"); }
+  });
+});
+
 describe("#4762 win32 socket path", () => {
   it("win32 derives a stable per-dir named pipe; other platforms keep the unix socket file", () => {
     const a = scheddPaths("/tmp/a", "win32"), b = scheddPaths("/tmp/b", "win32");

package/bin/lean/selftest-gate.mjs

@@ -0,0 +1,46 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// FORGE:SELFTEST marker (#4865): the live selftest result on the staging->main PR, and the release gate over it.
+import { isTrustedComment } from "../engine/phases.mjs";
+
+// Anchored: the marker leads the comment (postMarker writes it first), so prose that quotes a marker never counts.
+const MARKER_RE = /^\s*<!--\s*FORGE:SELFTEST\s+([^>]*?)\s*-->/;
+
+/** @param {{result:"pass"|"fail", sha:string, step?:string}} m */
+export function buildMarker({ result, sha, step }) {
+  return `<!-- FORGE:SELFTEST result=${result} sha=${sha} step=${step || "none"} -->`;
+}
+
+/** @returns {{result:string, sha:string, step:string}|null} null for a body without a well-formed marker. */
+export function parseMarker(body) {
+  const m = MARKER_RE.exec(String(body ?? ""));
+  if (!m) return null;
+  const kv = Object.fromEntries(m[1].split(/\s+/).map((p) => p.split("=")).filter((p) => p.length === 2));
+  if (!/^(pass|fail)$/.test(kv.result ?? "") || !/^[0-9a-f]{40}$/.test(kv.sha ?? "")) return null;
+  return { result: kv.result, sha: kv.sha, step: kv.step ?? "none" };
+}
+
+/** Fail closed: the newest marker must be a pass for exactly `headSha`. @param {string[]} comments comment bodies, oldest first */
+export function selftestGate(comments, headSha) {
+  const marks = comments.map(parseMarker).filter(Boolean);
+  const last = marks[marks.length - 1];
+  if (!last) return { ok: false, reason: "no FORGE:SELFTEST result on the PR; run `forgedock selftest --live --repo <sandbox> --pr <N>`" };
+  if (last.result !== "pass") return { ok: false, reason: `the latest FORGE:SELFTEST failed at step ${last.step}` };
+  if (last.sha !== headSha) return { ok: false, reason: `the passing FORGE:SELFTEST is for ${last.sha.slice(0, 7)}, not the PR head ${String(headSha).slice(0, 7)}` };
+  return { ok: true };
+}
+
+/**
+ * Bodies of the trusted comments of PR `pr` that carry a marker, oldest first (one JSON object per gh output line).
+ * Only comments `isTrustedComment` accepts count (maintainer association, a Bot, FORGE_TRAIL_TRUSTED_LOGINS): anyone can
+ * comment on a public PR, and a forged pass must not open the release gate. A gh failure throws (the caller fails closed).
+ */
+export async function readMarkerComments(gh, repo, pr) {
+  const out = await gh(["api", "--paginate", `repos/${repo}/issues/${pr}/comments`, "--jq", '.[] | select(.body | test("FORGE:SELFTEST")) | {body, author_association, user: {login: .user.login, type: .user.type}} | @json']);
+  return String(out).split("\n").filter(Boolean).flatMap((l) => { try { return [JSON.parse(l)]; } catch { return []; } })
+    .filter(isTrustedComment).map((c) => String(c.body ?? ""));
+}
+
+export async function postMarker(gh, repo, pr, m) {
+  await gh(["pr", "comment", String(pr), "-R", repo, "--body", `${buildMarker(m)}\nLive selftest: ${m.result}${m.step ? ` (step ${m.step})` : ""}`]);
+}

package/bin/lean/selftest-gate.test.mjs

@@ -0,0 +1,40 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { buildMarker, parseMarker, selftestGate, readMarkerComments, postMarker } from "./selftest-gate.mjs";
+
+const H = "a".repeat(40), O = "b".repeat(40);
+describe("FORGE:SELFTEST gate (#4865)", () => {
+  it("round-trips a marker", () => {
+    assert.deepEqual(parseMarker(buildMarker({ result: "fail", sha: H, step: "add" })), { result: "fail", sha: H, step: "add" });
+  });
+  it("accepts a passing marker for the head", () => assert.equal(selftestGate([buildMarker({ result: "pass", sha: H })], H).ok, true));
+  it("refuses missing, failing, stale and malformed markers", () => {
+    assert.equal(selftestGate([], H).ok, false);
+    assert.equal(selftestGate(["lgtm"], H).ok, false);
+    assert.match(selftestGate([buildMarker({ result: "fail", sha: H, step: "add" })], H).reason, /step add/);
+    assert.match(selftestGate([buildMarker({ result: "pass", sha: O })], H).reason, /not the PR head/);
+    assert.equal(selftestGate(["<!-- FORGE:SELFTEST result=pass sha=abc -->"], H).ok, false);
+  });
+  it("a marker quoted inside prose does not count", () => {
+    assert.equal(parseMarker(`see: ${buildMarker({ result: "pass", sha: H })}`), null);
+    assert.equal(selftestGate([buildMarker({ result: "fail", sha: H, step: "add" }), `quoting ${buildMarker({ result: "pass", sha: H })}`], H).ok, false);
+  });
+  it("a gh failure is not read as no marker", async () => {
+    await assert.rejects(readMarkerComments(async () => { throw new Error("HTTP 502"); }, "o/r", 5), /502/);
+  });
+  it("the newest marker wins", () => {
+    assert.equal(selftestGate([buildMarker({ result: "pass", sha: H }), buildMarker({ result: "fail", sha: H, step: "merge" })], H).ok, false);
+    assert.equal(selftestGate([buildMarker({ result: "fail", sha: H, step: "merge" }), buildMarker({ result: "pass", sha: H })], H).ok, true);
+  });
+  it("reads and posts through gh", async () => {
+    const calls = [];
+    const rows = [{ body: "x", author_association: "OWNER", user: { login: "m", type: "User" } }, { body: "forged", author_association: "NONE", user: { login: "drive-by", type: "User" } }, { body: "bot", author_association: "NONE", user: { login: "app[bot]", type: "Bot" } }];
+    const gh = async (a) => { calls.push(a); return a[0] === "api" ? rows.map((r) => JSON.stringify(r)).join("\n") + "\nnot json\n" : ""; };
+    assert.deepEqual(await readMarkerComments(gh, "o/r", 5), ["x", "bot"], "an untrusted commenter's marker is dropped");
+    await postMarker(gh, "o/r", 5, { result: "pass", sha: H });
+    assert.match(calls[1].at(-1), /FORGE:SELFTEST result=pass/);
+  });
+});
+

package/bin/lean/selftest-live.mjs

@@ -0,0 +1,248 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// `forgedock selftest --live --repo <owner/sandbox> [--lang node|python] [--pr N]` (#4865): drives the real binaries
+// (gh, git, forgedock orchestrate) against a disposable sandbox repo. The mocked unit tests missed #4859 (--add skipped
+// every issue), #4860 (issue body wiped to FORGE:STATE), #4817 and #4818; each step below fails on one of those classes.
+// All process access goes through `io` so the step logic is unit-tested with fakes; `realIo` is the live binding.
+import { readFileSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawn, spawnSync } from "node:child_process";
+import { batchRoot, listBatches, openBatch } from "./batch.mjs";
+import { staticCheck, formatStatic, realGh } from "./selftest-static.mjs";
+import { postMarker } from "./selftest-gate.mjs";
+import { reportNightly } from "./selftest-nightly.mjs";
+import { parseForgeYaml } from "../forge-utils.mjs";
+
+export class StepError extends Error {
+  constructor(step, message) { super(message); this.step = step; }
+}
+const fail = (step, msg) => { throw new StepError(step, msg); };
+
+/** Fixture per toolchain: base files plus the two trivial issues (a test that fails on the old code). */
+export const FIXTURES = {
+  node: {
+    files: { "package.json": '{"name":"selftest-sandbox","private":true,"type":"module","scripts":{"test":"node --test"}}\n', "README.md": "ForgeDock selftest sandbox\n" },
+    a: { title: "selftest A: add a()", body: "Create `src/a.js` exporting `a()` that returns `\"a\"`, and `src/a.test.js` (node:test) asserting it. The test must fail on the current code." },
+    b: { title: "selftest B: add b()", body: "Create `src/b.js` exporting `b()` that returns `\"b\"`, and `src/b.test.js` (node:test) asserting it. The test must fail on the current code." },
+  },
+  python: {
+    files: { "pyproject.toml": '[project]\nname = "selftest-sandbox"\nversion = "0.0.0"\n', "README.md": "ForgeDock selftest sandbox\n" },
+    a: { title: "selftest A: add a()", body: "Create `a.py` with `def a(): return \"a\"` and `test_a.py` (pytest) asserting it. The test must fail on the current code." },
+    b: { title: "selftest B: add b()", body: "Create `b.py` with `def b(): return \"b\"` and `test_b.py` (pytest) asserting it. The test must fail on the current code." },
+  },
+};
+
+/** The issue body with its FORGE:STATE annotation removed. */
+export const stripState = (body) => String(body ?? "").replace(/<!--\s*FORGE:STATE[\s\S]*?-->/g, "").trim();
+
+const num = (out) => { const m = /\/issues\/(\d+)/.exec(String(out)) ?? /^(\d+)$/m.exec(String(out)); if (!m) throw new Error(`no issue number in: ${String(out).slice(0, 120)}`); return Number(m[1]); };
+
+async function waitFor(ctx, step, what, cond, onTick) {
+  const { io, timeoutMs } = ctx, t0 = io.now();
+  for (;;) {
+    if (onTick) await onTick();
+    const v = await cond();
+    if (v) return v;
+    if (io.now() - t0 > timeoutMs) fail(step, `timed out waiting for ${what}`);
+    await io.sleep(ctx.pollMs);
+  }
+}
+
+/** Step "body": every issue body still holds its creation text (#4860). */
+async function checkBodies(ctx, step = "body") {
+  for (const [name, n] of Object.entries(ctx.issues)) {
+    const now = JSON.parse(await ctx.io.gh(["issue", "view", String(n), "-R", ctx.repo, "--json", "body"])).body;
+    if (!stripState(now).includes(ctx.bodies[n])) fail(step, `issue ${name} (#${n}) lost its body text; now: ${JSON.stringify(stripState(now).slice(0, 80))}`);
+  }
+}
+
+export const STEPS = [
+  { name: "create", run: async (ctx) => {
+    const { io, repo, fx } = ctx;
+    for (const [path, content] of Object.entries(fx.files)) {
+      await io.gh(["api", "-X", "PUT", `repos/${repo}/contents/${path}`, "-f", `message=selftest fixture ${path}`, "-f", `content=${Buffer.from(content).toString("base64")}`]).catch(() => { /* already present */ });
+    }
+    const mk = async (t, body) => num(await io.gh(["issue", "create", "-R", repo, "--title", t, "--body", body]));
+    const a = await mk(fx.a.title, fx.a.body);
+    const b = await mk(fx.b.title, `${fx.b.body}\n\n**Depends on**: #${a}`);
+    ctx.issues = { A: a, B: b };
+    ctx.bodies = { [a]: fx.a.body, [b]: fx.b.body };
+    ctx.created.push(a, b);
+  } },
+  { name: "add", run: async (ctx) => {
+    const { io, repo, issues } = ctx;
+    ctx.run = await io.orchestrate.start(repo, [issues.A], 2);
+    const r = await io.cli(["orchestrate", "--add", ctx.run.id, String(issues.B)]);
+    if (r.code !== 0) fail("add", `orchestrate --add exited ${r.code}: ${String(r.stderr || r.stdout).slice(0, 200)}`);
+    if (/unreadable|skipped/i.test(`${r.stdout}\n${r.stderr}`)) fail("add", `--add skipped #${issues.B}: ${String(r.stdout || r.stderr).slice(0, 200)}`);
+    const st = await waitFor(ctx, "add", `#${issues.B} in the batch`, async () => { const s = await io.orchestrate.state(ctx.run.id); return s?.order?.includes(issues.B) ? s : null; });
+    if (!(st.deps?.[issues.B] ?? []).includes(issues.A)) fail("add", `#${issues.B} does not wait on #${issues.A} in the DAG (deps: ${JSON.stringify(st.deps?.[issues.B] ?? [])})`);
+  } },
+  { name: "body", run: (ctx) => checkBodies(ctx) },
+  { name: "resume", run: async (ctx) => {
+    const { io, issues } = ctx;
+    const pid = async () => (await io.orchestrate.state(ctx.run.id))?.running?.[issues.A]?.pid ?? null;
+    const before = await waitFor(ctx, "resume", `a live lane for #${issues.A}`, pid);
+    await ctx.run.kill();
+    ctx.run = await io.orchestrate.resume(ctx.run.id);
+    await io.sleep(ctx.pollMs);
+    const st = await io.orchestrate.state(ctx.run.id), after = st?.running?.[issues.A]?.pid ?? null;
+    const done = st?.results?.[issues.A];
+    if (!done && after !== before) fail("resume", `#${issues.A} was launched again after --resume (pid ${before} -> ${after}), not re-adopted`);
+    await checkBodies(ctx, "body");
+  } },
+  { name: "merge", run: async (ctx) => {
+    const { io, repo, issues } = ctx;
+    const res = await waitFor(ctx, "merge", "both lanes to finish", async () => {
+      const s = await io.orchestrate.state(ctx.run.id);
+      return s?.results?.[issues.A] && s?.results?.[issues.B] ? s.results : null;
+    }, () => checkBodies(ctx, "body"));
+    const merged = {};
+    for (const [k, n] of Object.entries(issues)) {
+      if (res[n].outcome !== "merged" || !res[n].pr) fail("merge", `#${n} ended ${res[n].outcome}, not merged`);
+      const pr = JSON.parse(await io.gh(["pr", "view", String(res[n].pr), "-R", repo, "--json", "mergedAt,state"]));
+      if (pr.state !== "MERGED") fail("merge", `PR #${res[n].pr} for ${k} is ${pr.state}`);
+      const is = JSON.parse(await io.gh(["issue", "view", String(n), "-R", repo, "--json", "state,labels"]));
+      if (is.state !== "CLOSED") fail("merge", `#${n} is not closed`);
+      if (!is.labels.some((l) => l.name === "workflow:merged")) fail("merge", `#${n} lacks the workflow:merged label`);
+      merged[k] = Date.parse(pr.mergedAt);
+    }
+    if (!(merged.A <= merged.B)) fail("merge", `B merged before A (${merged.B} < ${merged.A})`);
+  } },
+  { name: "remote", run: async (ctx) => {
+    const { io } = ctx;
+    if (!io.remote) { ctx.notes.push("remote: skipped (pool not configured)"); return; }
+    const c = num(await io.gh(["issue", "create", "-R", ctx.repo, "--title", `${ctx.fx.a.title.replace("A:", "C:")}`, "--body", ctx.fx.a.body]));
+    ctx.created.push(c);
+    const r = await io.remote.run({ repo: ctx.repo, issue: c, timeoutMs: ctx.timeoutMs, pollMs: ctx.pollMs });
+    if (r.where !== "remote") fail("remote", `#${c} ran ${r.where ?? "unknown"}, not in the pool`);
+    if (!r.authenticated) fail("remote", "the remote lane did not authenticate");
+    const cli = await io.cliVersion();
+    if (r.engineVersion !== cli) fail("remote", `sandbox engine ${r.engineVersion} != CLI ${cli}`);
+  } },
+];
+
+/**
+ * Run the steps in order; the first failure stops them, cleanup always runs.
+ * @returns {{ok:boolean, step:string|null, message:string|null, notes:string[], cleanupErrors:string[]}}
+ */
+export async function runSelftestLive({ repo, lang = "node", io, timeoutMs = 45 * 60 * 1000, pollMs = 5000 }) {
+  const fx = FIXTURES[lang];
+  if (!fx) return { ok: false, step: "setup", message: `unknown fixture language ${lang} (node|python)`, notes: [], cleanupErrors: [] };
+  const ctx = { io, repo, fx, timeoutMs, pollMs, issues: {}, bodies: {}, created: [], notes: [], run: null };
+  let out = { ok: true, step: null, message: null };
+  for (const s of STEPS) {
+    try { await s.run(ctx); } catch (e) { out = { ok: false, step: e.step ?? s.name, message: String(e.message ?? e) }; break; }
+  }
+  const cleanupErrors = [];
+  const tryC = async (what, f) => { try { await f(); } catch (e) { cleanupErrors.push(`${what}: ${String(e.message ?? e).slice(0, 120)}`); } };
+  if (ctx.run) await tryC("stop orchestrate", () => ctx.run.kill());
+  for (const n of ctx.created) {
+    await tryC(`close #${n}`, () => io.gh(["issue", "close", String(n), "-R", repo]));
+    await tryC(`delete branches of #${n}`, async () => {
+      const refs = String(await io.gh(["api", `repos/${repo}/git/matching-refs/heads/forge/issue-${n}`, "--jq", ".[].ref"])).split("\n").filter(Boolean);
+      for (const ref of refs) await io.gh(["api", "-X", "DELETE", `repos/${repo}/git/${ref}`]);
+    });
+    await tryC(`close PRs of #${n}`, async () => {
+      const prs = String(await io.gh(["pr", "list", "-R", repo, "--search", `#${n} in:title,body`, "--state", "open", "--json", "number", "--jq", ".[].number"])).split("\n").filter(Boolean);
+      for (const p of prs) await io.gh(["pr", "close", p, "-R", repo, "--delete-branch"]);
+    });
+  }
+  return { ...out, notes: ctx.notes, cleanupErrors };
+}
+
+// ── real io + CLI ───────────────────────────────────────────────────────────────────────────────────────────
+const CLI = join(dirname(fileURLToPath(import.meta.url)), "..", "forgedock.mjs");
+
+export function realIo(env = process.env, { pool = false } = {}) {
+  let self;
+  const root = batchRoot(env);
+  const detached = (args) => { const p = spawn(process.execPath, [CLI, ...args], { detached: true, stdio: "ignore", env }); p.unref(); return { kill: async () => { try { process.kill(-p.pid, "SIGTERM"); } catch { /* gone */ } } }; };
+  const waitBatch = async (before) => { for (let i = 0; i < 60; i++) { const id = listBatches(root).find((b) => !before.has(b)); if (id) return id; await new Promise((r) => setTimeout(r, 1000)); } throw new Error("orchestrate did not create a batch within 60s"); };
+  return (self = {
+    now: () => Date.now(), sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
+    gh: async (args) => { const r = spawnSync("gh", args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); if (r.status !== 0) throw new Error(r.stderr.trim() || `gh exited ${r.status}`); return r.stdout; },
+    cli: async (args) => { const r = spawnSync(process.execPath, [CLI, ...args], { encoding: "utf8", env }); return { code: r.status ?? 1, stdout: r.stdout, stderr: r.stderr }; },
+    cliVersion: async () => JSON.parse(readFileSync(join(dirname(CLI), "..", "package.json"), "utf8")).version,
+    orchestrate: {
+      start: async (repo, issues, parallel) => {
+        const before = new Set(listBatches(root)), h = detached(["orchestrate", ...issues.map(String), "--parallel", String(parallel), "--repo", repo]);
+        return { id: await waitBatch(before), kill: h.kill };
+      },
+      resume: async (id) => ({ id, ...detached(["orchestrate", "--resume", id]) }),
+      state: async (id) => openBatch(root, id).store.load(),
+    },
+    // Step 6: one lane through the pool. The issue's FORGE:STATE records the placement and the lane's FORGE:TELEMETRY rows
+    // carry the engine's forge_version (the sandbox's, not the CLI's: the #4818 stale-image defect).
+    remote: pool ? { run: async ({ repo, issue, timeoutMs, pollMs }) => {
+      const run = await self.orchestrate.start(repo, [issue], 1), t0 = Date.now();
+      let row = null;
+      try {
+        while (!row && Date.now() - t0 < timeoutMs) { row = (await self.orchestrate.state(run.id))?.results?.[issue] ?? null; if (!row) await self.sleep(pollMs); }
+      } finally { await run.kill(); }
+      if (!row) throw new Error(`#${issue} produced no result within ${Math.round(timeoutMs / 1000)}s`);
+      const v = JSON.parse(await self.gh(["issue", "view", String(issue), "-R", repo, "--json", "body,comments"]));
+      const text = `${v.body}\n${v.comments.map((c) => c.body).join("\n")}`;
+      const versions = [...text.matchAll(/"forge_version":"([^"]+)"/g)].map((m) => m[1]);
+      return { where: /"placement":\{"where":"(\w+)"/.exec(text)?.[1] ?? null, authenticated: versions.length > 0 && !/^env\./.test(String(row.code ?? row.outcome ?? "")), engineVersion: versions.at(-1) ?? null };
+    } } : null,
+  });
+}
+
+/** The value after flag `f`, or null when absent. A flag with no value (end of argv, or another `--flag` next) throws. */
+const flag = (argv, f) => {
+  const i = argv.indexOf(f);
+  if (i < 0) return null;
+  const v = argv[i + 1];
+  if (v == null || String(v).startsWith("--") || !String(v).trim()) throw new Error(`${f} needs a value`);
+  return v;
+};
+const REPO_RE = /^[\w.-]+\/[\w.-]+$/;
+const repoOrNull = (v) => (REPO_RE.test(String(v ?? "").trim()) ? String(v).trim() : null);
+
+/** `forgedock selftest [--static] | --live --repo <o/sandbox> [--lang node|python] [--pr N]`; non-zero exit names the failed step. */
+export async function selftestFromCli(argv, deps = {}) {
+  const cwd = deps.cwd ?? process.cwd(), env = deps.env ?? process.env, print = deps.print ?? ((l) => console.log(l));
+  const usage = "usage: forgedock selftest [--static] | --live [--repo <owner/sandbox>] [--lang node|python] [--pr <staging->main PR>]  (sandbox repo from --repo or forge.yaml selftest.sandbox_repo)";
+  if (!argv.includes("--live") && !argv.includes("--static")) throw new Error(usage);
+  const nightlyRepo = argv.includes("--nightly-report") ? flag(argv, "--nightly-report") : null;
+  if (nightlyRepo) {
+    // #4894: the detached nightly run. Any failure, including one before the harness starts (no sandbox repo), is filed.
+    const rest = argv.filter((a, i) => a !== "--nightly-report" && argv[i - 1] !== "--nightly-report");
+    const lines = [];
+    let r;
+    try { r = await selftestFromCli(rest, { ...deps, print: (l) => { lines.push(l); print(l); } }); } catch (e) { r = { ok: false, step: "config", message: String(e?.message ?? e) }; }
+    if (!r.ok) await reportNightly({ repo: nightlyRepo, io: { gh: async (a) => (deps.io?.gh ?? realGh)(a) }, result: { step: r.step, message: r.message ?? lines.filter((l) => /FAILED|FAIL/.test(l)).join("\n") } });
+    return r;
+  }
+  const gh = deps.io?.gh ?? realGh;
+  const st = staticCheck({ root: deps.binRoot ?? join(dirname(CLI)), io: { gh: (a) => gh(a) } });
+  print(formatStatic(st));
+  let exitCode = st.ok ? 0 : 1, step = st.ok ? null : "static";
+  if (argv.includes("--live")) {
+    let cfg = {}; try { cfg = parseForgeYaml(readFileSync(join(cwd, "forge.yaml"), "utf8")) ?? {}; } catch { /* none */ }
+    const repo = repoOrNull(flag(argv, "--repo") ?? cfg.selftest?.sandbox_repo);
+    if (!repo) throw new Error(`no sandbox repo: pass --repo owner/sandbox or set selftest.sandbox_repo in forge.yaml\n${usage}`);
+    const project = cfg.project?.owner && cfg.project?.repo ? `${cfg.project.owner}/${cfg.project.repo}` : null;
+    if (project && project.toLowerCase() === repo.toLowerCase()) throw new Error(`refusing to run in ${repo}: it is this project's repo (forge.yaml project), and the harness creates, merges, closes and deletes there; use a disposable sandbox repo`);
+    const lang = flag(argv, "--lang") ?? cfg.selftest?.fixture ?? "node";
+    const io = deps.io ?? realIo(env, { pool: ["remote", "hybrid"].includes(String(cfg.execution?.mode ?? "")) });
+    // The marker binds the head that was tested: read it (from the release repo) before the run, never after.
+    const pr = flag(argv, "--pr");
+    let releaseRepo = null, head = null;
+    if (pr) {
+      releaseRepo = flag(argv, "--release-repo") ?? JSON.parse(String(await io.gh(["repo", "view", "--json", "nameWithOwner"]))).nameWithOwner;
+      if (releaseRepo.toLowerCase() === repo.toLowerCase()) throw new Error(`refusing to run in ${repo}: it is the release repo`);
+      head = String(await io.gh(["pr", "view", String(pr), "-R", releaseRepo, "--json", "headRefOid", "--jq", ".headRefOid"])).trim();
+    }
+    const r = await runSelftestLive({ repo, lang, io, ...(deps.timing ?? {}) });
+    for (const n of r.notes) print(n);
+    for (const e of r.cleanupErrors) print(`cleanup: ${e}`);
+    if (r.ok) print(`selftest --live: all steps passed on ${repo}`);
+    else { print(`selftest --live FAILED at step ${r.step}: ${r.message}`); exitCode = 1; step = r.step; }
+    if (pr) await postMarker((a) => io.gh(a), releaseRepo, pr, { result: exitCode === 0 ? "pass" : "fail", sha: head, step });
+    if (!r.ok) return { ok: false, step, message: r.message };
+  }
+  return { ok: exitCode === 0, step };
+}

package/bin/lean/selftest-live.test.mjs

@@ -0,0 +1,154 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpDir } from "../tests/_tmp.mjs";
+import { runSelftestLive, stripState, selftestFromCli, STEPS } from "./selftest-live.mjs";
+
+/** A fake sandbox world. `d` toggles a defect: add4859 (--add skips), wipe4860 (body wiped), double (resume relaunches), orderBA, noRemoteAuth. */
+function world(d = {}) {
+  const w = { issues: {}, closed: [], calls: [], killed: 0, pid: 100, results: {}, order: [], deps: {}, running: {}, clock: 0, pool: !!d.pool };
+  const gh = async (a) => {
+    w.calls.push(a.join(" "));
+    if (a[0] === "api" && a[1] === "-X" && a[2] === "PUT") return "{}";
+    if (a[0] === "issue" && a[1] === "create") {
+      const n = 10 + Object.keys(w.issues).length; w.issues[n] = { body: a[a.indexOf("--body") + 1], state: "OPEN", labels: [] }; return `https://github.com/o/r/issues/${n}\n`;
+    }
+    if (a[0] === "issue" && a[1] === "view") {
+      const n = a[2], i = w.issues[n];
+      if (a.includes("body")) return JSON.stringify({ body: d.wipe4860 && w.wiped ? "<!-- FORGE:STATE {} -->" : `${i.body}\n<!-- FORGE:STATE {"v":1} -->` });
+      return JSON.stringify({ state: i.state, labels: i.labels.map((name) => ({ name })) });
+    }
+    if (a[0] === "issue" && a[1] === "close") { w.closed.push(a[2]); return ""; }
+    if (a[0] === "pr" && a[1] === "view") return JSON.stringify({ state: "MERGED", mergedAt: `2026-01-01T00:0${a[2] === "101" ? (d.orderBA ? 2 : 1) : (d.orderBA ? 1 : 2)}:00Z` });
+    if (a[0] === "pr" && a[1] === "list") return "";
+    if (a[0] === "api") { throw new Error("fail cleanup api"); }
+    throw new Error(`unexpected gh ${a.join(" ")}`);
+  };
+  const io = {
+    now: () => (w.clock += 1000), sleep: async () => { w.tick = (w.tick ?? 0) + 1; if (d.wipe4860 && w.tick >= 1) w.wiped = true; finish(); },
+    gh, cliVersion: async () => "1.39.0",
+    cli: async (a) => {
+      if (a[1] === "--add") { const n = Number(a[3]); if (d.add4859) return { code: 0, stdout: `#${n} skipped: unreadable`, stderr: "" }; w.order.push(n); w.deps[n] = [Number(Object.keys(w.issues)[0])]; return { code: 0, stdout: "added", stderr: "" }; }
+      throw new Error("cli " + a);
+    },
+    orchestrate: {
+      start: async (repo, issues) => { w.order.push(...issues); w.running[issues[0]] = { pid: ++w.pid }; w.a = issues[0]; return { id: "b1", kill: async () => { w.killed++; } }; },
+      resume: async (id) => { w.resumedAt = w.tick ?? 0; if (d.double) w.running[w.a] = { pid: ++w.pid }; return { id, kill: async () => { w.killed++; } }; },
+      state: async () => ({ order: w.order, deps: w.deps, running: w.running, results: w.results }),
+    },
+    remote: w.pool ? { run: async ({ issue }) => { w.remoteIssue = issue; return { where: d.local ? "local" : "remote", authenticated: !d.noRemoteAuth, engineVersion: d.oldEngine ? "1.37.0" : "1.39.0" }; } } : null,
+  };
+  function finish() {
+    if (w.resumedAt !== undefined && !w.finished && (w.tick ?? 0) >= w.resumedAt + 2) {
+      w.finished = true;
+      const [a, b] = Object.keys(w.issues).map(Number);
+      w.results = { [a]: { outcome: "merged", pr: 101 }, [b]: { outcome: "merged", pr: 102 } };
+      for (const n of [a, b]) w.issues[n].labels.push("workflow:merged"), w.issues[n].state = "CLOSED";
+    }
+  }
+  return { w, io };
+}
+const run = (d, extra = {}) => { const { w, io } = world(d); return runSelftestLive({ repo: "o/sandbox", io, pollMs: 1, timeoutMs: 60000, ...extra }).then((r) => ({ w, r })); };
+
+describe("selftest --live steps (#4865)", () => {
+  it("all steps pass and cleanup runs", async () => {
+    const { w, r } = await run({});
+    assert.equal(r.ok, true, JSON.stringify(r));
+    assert.deepEqual(w.closed, ["10", "11"]);
+    assert.match(r.notes.join(), /remote: skipped/);
+  });
+  it("fixtures: B depends on A and the issue bodies are the fixture text", async () => {
+    const { w } = await run({});
+    assert.match(w.issues[11].body, /\*\*Depends on\*\*: #10/);
+  });
+  it("the #4859 defect (--add skips the issue) fails the add step", async () => {
+    const { r, w } = await run({ add4859: true });
+    assert.equal(r.ok, false); assert.equal(r.step, "add"); assert.match(r.message, /skipped #11/);
+    assert.deepEqual(w.closed, ["10", "11"], "cleanup ran after the failure");
+  });
+  it("the #4860 defect (body wiped to FORGE:STATE) fails the body step", async () => {
+    const { r } = await run({ wipe4860: true });
+    assert.equal(r.ok, false); assert.equal(r.step, "body"); assert.match(r.message, /lost its body text/);
+  });
+  it("a lane launched twice after --resume fails the resume step", async () => {
+    const { r } = await run({ double: true });
+    assert.equal(r.step, "resume"); assert.match(r.message, /launched again/);
+  });
+  it("B merging before A fails the merge step", async () => {
+    const { r } = await run({ orderBA: true });
+    assert.equal(r.step, "merge"); assert.match(r.message, /B merged before A/);
+  });
+  it("remote step: stale sandbox engine or no auth fails it; matching passes", async () => {
+    assert.equal((await run({ pool: true })).r.ok, true);
+    assert.match((await run({ pool: true, oldEngine: true })).r.message, /1\.37\.0 != CLI 1\.39\.0/);
+    assert.equal((await run({ pool: true, noRemoteAuth: true })).r.step, "remote");
+    assert.match((await run({ pool: true, local: true })).r.message, /not in the pool/);
+    const ok = await run({ pool: true });
+    assert.equal(ok.w.remoteIssue, 12); assert.deepEqual(ok.w.closed, ["10", "11", "12"]);
+  });
+  it("unknown fixture language fails at setup", async () => {
+    assert.equal((await runSelftestLive({ repo: "o/r", lang: "ruby", io: {} })).step, "setup");
+  });
+  it("stripState drops only the annotation", () => assert.equal(stripState("hi\n<!-- FORGE:STATE\n{}\n-->"), "hi"));
+  it("has the six ordered phases (cleanup runs after them)", () => assert.deepEqual(STEPS.map((s) => s.name), ["create", "add", "body", "resume", "merge", "remote"]));
+});
+
+describe("selftestFromCli", () => {
+  it("requires --live or --static and a sandbox repo", async () => {
+    await assert.rejects(selftestFromCli([]), /usage/);
+    await assert.rejects(selftestFromCli(["--live"], { cwd: "/nonexistent", print: () => {}, io: { gh: async () => "" } }), /no sandbox repo/);
+  });
+  it("a failing step is named in the output, returned not-ok, and the marker is posted as fail", async () => {
+    const { io } = world({ add4859: true }); const out = [], posted = [];
+    const gh = io.gh;
+    io.gh = async (a) => {
+      if (a[0] === "pr" && a[1] === "view" && a.includes("headRefOid")) return "a".repeat(40);
+      if (a[0] === "repo") return JSON.stringify({ nameWithOwner: "o/r" });
+      if (a[0] === "pr" && a[1] === "comment") { posted.push(a.at(-1)); return ""; }
+      if (a[1] === "--help" || a[2] === "--help") return "JSON FIELDS\n  body, number, title\n";
+      return gh(a);
+    };
+    const r = await selftestFromCli(["--live", "--repo", "o/sandbox", "--pr", "9"], { cwd: "/nonexistent", print: (l) => out.push(l), io, timing: { pollMs: 1 } });
+    assert.equal(r.ok, false); assert.equal(r.step, "add");
+    assert.ok(out.some((l) => /FAILED at step add/.test(l)));
+    assert.match(posted[0], /FORGE:SELFTEST result=fail sha=a{40} step=add/);
+  });
+  it("the PR head is read from the release repo before the run", async () => {
+    const { io } = world({}); const order = [];
+    const gh = io.gh;
+    io.gh = async (a) => {
+      if (a[0] === "pr" && a[1] === "view" && a.includes("headRefOid")) { order.push(`head ${a[a.indexOf("-R") + 1]}`); return "b".repeat(40); }
+      if (a[0] === "pr" && a[1] === "comment") { order.push("post"); return ""; }
+      if (a[1] === "--help" || a[2] === "--help") return "JSON FIELDS\n  body, number, title\n";
+      if (a[0] === "issue" && a[1] === "create" && !order.includes("run")) order.push("run");
+      return gh(a);
+    };
+    await selftestFromCli(["--live", "--repo", "o/sandbox", "--pr", "9", "--release-repo", "o/rel"], { cwd: "/nonexistent", print: () => {}, io, timing: { pollMs: 1 } });
+    assert.deepEqual(order.slice(0, 2), ["head o/rel", "run"]);
+  });
+  it("refuses the project repo or the release repo as the sandbox", async () => {
+    const cwd = tmpDir("st-"); writeFileSync(join(cwd, "forge.yaml"), "project:\n  owner: o\n  repo: R\n");
+    const io = { gh: async (a) => (a[1] === "--help" || a[2] === "--help" ? "JSON FIELDS\n  body\n" : "") };
+    await assert.rejects(selftestFromCli(["--live", "--repo", "o/r"], { cwd, print: () => {}, io }), /this project's repo/);
+    await assert.rejects(selftestFromCli(["--live", "--repo", "o/x", "--pr", "3", "--release-repo", "o/x"], { cwd: "/nonexistent", print: () => {}, io }), /release repo/);
+  });
+  it("a flag without a value throws instead of being skipped", async () => {
+    await assert.rejects(selftestFromCli(["--live", "--repo", "o/s", "--pr"], { cwd: "/nonexistent", print: () => {}, io: { gh: async () => "" } }), /--pr needs a value/);
+    await assert.rejects(selftestFromCli(["--live", "--repo", "--pr", "4"], { cwd: "/nonexistent", print: () => {}, io: { gh: async () => "" } }), /--repo needs a value/);
+  });
+  it("forge.yaml is read per section: another section's mode or sandbox_repo is not picked up", async () => {
+    const cwd = tmpDir("st-"); writeFileSync(join(cwd, "forge.yaml"), "selftest:\n  fixture: node\nother:\n  sandbox_repo: x/y\n");
+    await assert.rejects(selftestFromCli(["--live"], { cwd, print: () => {}, io: { gh: async () => "" } }), /no sandbox repo/);
+  });
+  it("--nightly-report files a failure that happens before the harness starts (no sandbox repo)", async () => {
+    const calls = [];
+    const gh = async (a) => { calls.push(a); if (a[1] === "list") return "[]"; if (a[1] === "create") return "https://github.com/o/r/issues/5\n"; return a[1] === "--help" || a[2] === "--help" ? "JSON FIELDS\n  body\n" : ""; };
+    const r = await selftestFromCli(["--live", "--nightly-report", "o/r"], { cwd: "/nonexistent", print: () => {}, io: { gh } });
+    assert.equal(r.ok, false); assert.equal(r.step, "config");
+    const create = calls.find((a) => a[0] === "issue" && a[1] === "create");
+    assert.ok(create && create.includes("o/r")); assert.match(create.join(" "), /step config/);
+  });
+});