forgedock 1.43.1 → 1.45.0

package/bin/lean/sandbox-remote.mjs

@@ -60,8 +60,14 @@ export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fet
     return { status: res.status, ok: res.ok, data, retryAfter: res.headers?.get?.("retry-after") ?? null };
   };
   const fail = (what, r) => {
-    const e = new Error(`sandbox ${what}: HTTP ${r.status}`);
+    // #5053: a 401 names the hop that answered it (router or agent; the pool tags the body), never the token.
+    const hop = r.status === 401 && ["router", "agent"].includes(r.data?.hop) ? r.data.hop : null;
+    const who = hop === "agent" && typeof r.data?.agent === "string" ? `agent ${r.data.agent.slice(0, 64)}` : hop;
+    const reason = hop && typeof r.data?.reason === "string" ? r.data.reason.replace(/^(router|agent) auth: /, "").replace(/[^\x20-\x7e]/g, "").slice(0, 80) : "";
+    const e = new Error(`sandbox ${what}: HTTP ${r.status}${who ? ` (${who} auth${reason ? `: ${reason}` : ""})` : ""}`);
     e.status = r.status;
+    if (hop) e.hop = hop;
+    if (reason) e.reason = reason;
     if ([502, 503, 504].includes(r.status)) e.transient = true; // #4672: a router restart or gateway blip, retried by the lane poll
     if (r.status === 429 || r.status === 503) { e.retryable = true; const n = Number(r.retryAfter); if (Number.isFinite(n) && n > 0) e.retryInSeconds = n; }
     return e;
@@ -74,12 +80,12 @@ export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fet
     async status(id) {
       const r = await call("GET", p(id));
       if (r.status === 404) return "gone";
-      if (!r.ok) throw fail("status", r);
+      if (!r.ok) throw Object.assign(fail("status", r), r.status === 401 ? { transient: true } : {}); // #5053: the sandbox launched, so a 401 is retried like an unreachable pool
       const s = String(r.data?.state ?? "");
       return s === "running" || s === "pending" || s === "starting" ? "running" : s === "gone" ? "gone" : "exited";
     },
     async logs(id, cursor = 0) { const r = await call("GET", `${p(id)}/logs?cursor=${encodeURIComponent(cursor)}`); return r.ok ? { text: String(r.data?.text ?? ""), cursor: r.data?.cursor ?? cursor } : { text: "", cursor }; },
-    async result(id) { const r = await call("GET", `${p(id)}/result`); return r.ok && r.data && typeof r.data === "object" ? r.data : null; },
+    async result(id) { const r = await call("GET", `${p(id)}/result`); if (r.status === 401) throw Object.assign(fail("result", r), { transient: true }); return r.ok && r.data && typeof r.data === "object" ? r.data : null; },
     async usage(id) { const r = await call("GET", `${p(id)}/usage`); return r.ok && Number.isFinite(Number(r.data?.minutes)) ? { minutes: Number(r.data.minutes), ...(typeof r.data.agent === "string" && r.data.agent ? { agent: r.data.agent } : {}) } : null; }, // agent: #4549, optional
     async stop(id) { const r = await call("POST", `${p(id)}/stop`); if (r.ok) return true; if (r.status === 404 || r.status === 405) return false; throw fail("stop", r); },
     async destroy(id) { const r = await call("DELETE", p(id)); if (r.ok || r.status === 404) return true; throw fail("destroy", r); },
@@ -88,19 +94,23 @@ export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fet
     async capacity() {
       // `free` (#4540) and agent liveness `online`/`total` (#4547) are each optional; none valid is unknown.
       const ok = (v) => Number.isInteger(v) && v >= 0;
-      let out = {};
+      let out = {}, reached = false, lastErr = null;
       try {
         const r = await call("GET", "/status");
+        reached = true;
         if (r.ok) { const { free, online, total } = r.data ?? {}; out = { ...(ok(free) ? { free } : {}), ...(ok(online) ? { online } : {}), ...(ok(total) ? { total } : {}) }; }
-      } catch { out = {}; }
+      } catch (e) { out = {}; lastErr = e; }
       if (out.online === undefined) {
         // #4788: a router without /status liveness: online = agents answering and not paused, as bin/pool/router.mjs status() does.
         try {
           const r = await call("GET", "/agents");
+          reached = true;
           const list = r.ok && Array.isArray(r.data?.agents) ? r.data.agents : null;
           if (list) out = { ...out, online: list.filter((a) => a?.ok === true && !a.paused).length, total: list.length };
-        } catch { /* liveness stays unknown */ }
+        } catch (e) { lastErr = e; /* liveness stays unknown */ }
       }
+      // #5048 (release review REG-2): a router that answered neither call is unreachable: throw, so the pool is marked down
+      if (!reached) throw Object.assign(new Error(`pool router unreachable: ${String(lastErr?.message ?? lastErr).slice(0, 160)}`), { unreachable: true });
       return Object.keys(out).length ? out : null;
     },
   };
@@ -218,7 +228,9 @@ export function remoteProvider({ batch, repo, settings = {}, name = "remote", tr
       return r ?? null;
     },
     async capacity() {
-      try { return (await transport.capacity?.()) ?? null; } catch { return null; }
+      // #5048: no capacity() is unknown (null); a transport error propagates, so the caller can report the pool down
+      if (typeof transport.capacity !== "function") return null;
+      return (await transport.capacity()) ?? null;
     },
     /** #4612: a graceful stop at the lane's checkpoint; false when unsupported or failed. Never destroys the sandbox. */
     async stop(h) {

package/bin/lean/sched.mjs

@@ -144,12 +144,14 @@ export function openSched(dir = schedDir(), { now = () => Date.now(), pid = proc
     }
     const obj = (x) => (x && typeof x === "object" && !Array.isArray(x) ? x : {});
     const weights = Object.fromEntries(Object.entries(obj(o?.weights)).filter(([, n]) => posInt(n)));
-    return { v: 1, cap: posInt(o?.cap), weights, seq: Number.isInteger(o?.seq) ? o.seq : 0, slots: obj(o?.slots), queue: obj(o?.queue) };
+    return { v: 1, cap: posInt(o?.cap), weights, seq: Number.isInteger(o?.seq) ? o.seq : 0, slots: obj(o?.slots), queue: obj(o?.queue), remote: obj(o?.remote), pool: obj(o?.pool) };
   };
   // Drop queued entries of dead dispatchers, and slots whose dispatcher and lane are both gone.
   const reclaim = (s) => {
     for (const [k, e] of Object.entries(s.queue)) if (!alive(e.pid)) delete s.queue[k];
     for (const [k, e] of Object.entries(s.slots)) if (!alive(e.pid) && !(e.lanePid && alive(e.lanePid))) delete s.slots[k];
+    for (const [k, e] of Object.entries(s.remote)) if (!alive(e?.pid)) delete s.remote[k]; // #4964
+    for (const [k, e] of Object.entries(s.pool)) if (!alive(e?.pid)) delete s.pool[k];
   };
   const mutate = (fn) => {
     lock();
@@ -200,7 +202,18 @@ export function openSched(dir = schedDir(), { now = () => Date.now(), pid = proc
     /** Queue entries `[{key, batch, repo, issue, priority}]` (their FIFO position is kept on a re-offer). */
     offer: (entries) => { try { mutate((s) => { for (const e of entries) enqueue(s, e.key, e); }); } catch { /* fail-open */ } },
     withdraw: (keys) => { if (!keys.length) return; try { mutate((s) => { for (const k of keys) delete s.queue[k]; }); } catch { /* fail-open */ } },
-    release: (key) => { try { mutate((s) => { delete s.slots[key]; delete s.queue[key]; }); } catch { /* fail-open */ } },
+    release: (key) => { try { mutate((s) => { delete s.slots[key]; delete s.queue[key]; delete s.remote[key]; }); } catch { /* fail-open */ } },
+    /** #4964: publish a running remote (pool) lane for `sched show`; it never takes a host slot. */
+    noteRemote: (key, entry = {}) => {
+      try {
+        mutate((s) => { s.remote[key] = { pid, batch: entry.batch ?? null, repo: entry.repo ?? null, issue: entry.issue ?? null, sandbox: entry.sandbox ?? null, agent: entry.agent ?? null, at: now() }; });
+      } catch { /* fail-open */ }
+    },
+    dropRemote: (key) => { try { mutate((s) => { delete s.remote[key]; }); } catch { /* fail-open */ } },
+    /** #4964: this batch's pool is unreachable (`{reason}`), or null to clear it. */
+    notePool: (batch, state) => {
+      try { mutate((s) => { if (state) s.pool[batch] = { pid, state: "unreachable", reason: state.reason ?? null, at: now() }; else delete s.pool[batch]; }); } catch { /* fail-open */ }
+    },
     setLanePid: (key, lanePid) => { try { mutate((s) => { if (s.slots[key] && posInt(lanePid)) s.slots[key].lanePid = lanePid; }); } catch { /* fail-open */ } },
     /** Record a lane this dispatcher re-attached to: it runs already, so the cap does not gate it. */
     adoptSlot: (key, entry = {}) => {
@@ -231,6 +244,16 @@ export function openSched(dir = schedDir(), { now = () => Date.now(), pid = proc
       weights: { ...s.weights },
       slots: Object.entries(s.slots).map(([key, e]) => ({ key, ...e })),
       queue: grantOrder(s).map(([key, e]) => ({ key, ...e })),
+      remote: Object.entries(s.remote).map(([key, e]) => ({ key, ...e })),
+      pool: Object.entries(s.pool).map(([batch, e]) => ({ batch, ...e })),
     })),
   };
 }
+
+/** #4964: `sched show` lines for remote (pool) lanes and an unreachable pool; empty when there are none. */
+export function remoteLines(st) {
+  const out = (st?.remote ?? []).map((e) => `  remote #${e.issue} ${e.repo ?? "-"} batch ${e.batch ?? "-"} sandbox ${e.sandbox ?? "-"} agent ${e.agent ?? "-"}  dispatcher ${e.pid}`);
+  const down = st?.pool ?? [];
+  if (down.length) out.push(`pool: unreachable (${down.map((e) => `batch ${e.batch}${e.reason ? `: ${e.reason}` : ""}`).join("; ")})`);
+  return out;
+}

package/bin/lean/sched.test.mjs

@@ -5,7 +5,7 @@ import assert from "node:assert/strict";
 import { writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
-import { grantOrder, openSched, priorityFromLabels } from "./sched.mjs";
+import { grantOrder, openSched, priorityFromLabels, remoteLines } from "./sched.mjs";
 import { orchestrate, estimateLaneMinutes, laneHistory, formatEstimateSummary, EST_DEFAULT_MIN, POST_DEPLOY_EST_MIN } from "./orchestrate.mjs";
 import { schedFromCli } from "../engine-cli.mjs";
 
@@ -512,3 +512,60 @@ describe("live local lane cap (#4777)", () => {
     assert.equal(peak, 2);
   });
 });
+
+describe("sched show lists remote (pool) lanes (#4964)", () => {
+  const show = async (sched) => { const lines = []; const l = console.log; console.log = (m) => lines.push(String(m)); try { const st = await schedFromCli(["show"], { sched }); return { st, lines }; } finally { console.log = l; } };
+
+  it("remote lanes get their own rows and a remote N count, and take no host slot or cap", async () => {
+    const { handle } = world();
+    const a = handle(501), b = handle(502);
+    a.setCap(1);
+    a.noteRemote("B#2", { batch: "B", repo: "o/r", issue: 2, sandbox: "fdrsbx-B-2", agent: "legion" });
+    a.noteRemote("B#3", { batch: "B", repo: "o/r", issue: 3, sandbox: "fdrsbx-B-3" });
+    const { st, lines } = await show(b);
+    assert.equal(st.slots.length, 0);
+    assert.equal(st.remote.length, 2);
+    assert.ok(lines.some((m) => /running 0, queued 0, remote 2$/.test(m)), lines.join("\n"));
+    assert.ok(lines.includes("  remote #2 o/r batch B sandbox fdrsbx-B-2 agent legion  dispatcher 501"), lines.join("\n"));
+    assert.equal(b.tryAcquire("C#1", {}, null).ok, true, "cap 1 is still free: remote lanes hold no slot");
+    a.dropRemote("B#2");
+    assert.deepEqual(b.list().remote.map((r) => r.key), ["B#3"]);
+    a.release("B#3");
+    assert.equal(b.list().remote.length, 0);
+  });
+
+  it("a dead dispatcher's remote lane and pool notice are reclaimed", () => {
+    const { handle, live } = world();
+    const a = handle(511), b = handle(512);
+    a.noteRemote("B#2", { batch: "B", issue: 2 });
+    a.notePool("B", { reason: "capacity probe failed" });
+    assert.equal(b.list().remote.length, 1);
+    live.delete(511);
+    assert.deepEqual([b.list().remote.length, b.list().pool.length], [0, 0]);
+  });
+
+  it("an unreachable pool is one line; none once cleared", async () => {
+    const { handle } = world();
+    const a = handle(521);
+    a.notePool("B", { reason: "capacity probe failed" });
+    const down = (await show(a)).lines.filter((m) => m.startsWith("pool:"));
+    assert.deepEqual(down, ["pool: unreachable (batch B: capacity probe failed)"]);
+    a.notePool("B", null);
+    assert.equal((await show(a)).lines.filter((m) => m.startsWith("pool:")).length, 0);
+    assert.deepEqual(remoteLines({}), []);
+  });
+
+  it("orchestrate publishes a remote lane while it runs, drops it after, and never acquires a slot", async () => {
+    const { handle } = world();
+    const reg = handle(531);
+    let seen = null, acquired = 0;
+    const spy = { ...reg, tryAcquire: (...x) => { acquired++; return reg.tryAcquire(...x); } };
+    await orchestrate({ issues: [7], parallel: 1, pollMs: 5, trusted: () => true, readBody: async () => "", sched: { registry: spy, batch: "B", repo: "o/r" },
+      remote: { max: 1, isRemote: () => true, capacity: () => ({ free: 1, online: 1 }) },
+      runOne: async (n, lane, ctx) => { ctx.onStart?.({ handle: { id: "fdrsbx-B-7", agent: "legion" }, remote: true }); seen = reg.list(); return { terminalReason: "merged" }; } });
+    assert.equal(acquired, 0);
+    assert.deepEqual(seen.remote.map((r) => [r.issue, r.sandbox, r.agent]), [[7, "fdrsbx-B-7", "legion"]]);
+    assert.equal(seen.slots.length, 0);
+    assert.equal(reg.list().remote.length, 0);
+  });
+});

package/bin/lean/selftest-live.mjs

@@ -10,10 +10,14 @@ import { fileURLToPath } from "node:url";
 import { spawn, spawnSync } from "node:child_process";
 import { batchRoot, listBatches, openBatch } from "./batch.mjs";
 import { staticCheck, formatStatic, realGh } from "./selftest-static.mjs";
+import { pollUntil } from "./retry.mjs";
 import { postMarker } from "./selftest-gate.mjs";
 import { reportNightly } from "./selftest-nightly.mjs";
 import { parseForgeYaml } from "../forge-utils.mjs";
 
+const BATCH_WAIT_TRIES = 60, BATCH_WAIT_MS = 1000;
+const DEFAULT_TIMEOUT_MS = 45 * 60 * 1000, DEFAULT_POLL_MS = 5000, GH_MAX_BUFFER = 64 * 1024 * 1024;
+
 export class StepError extends Error {
   constructor(step, message) { super(message); this.step = step; }
 }
@@ -33,20 +37,31 @@ export const FIXTURES = {
   },
 };
 
+/**
+ * The fixture with every path and title made unique to this run (#4952): a merged run leaves its files on the sandbox
+ * default branch, and the next run's "the test fails on the current code" premise must still hold.
+ */
+export function uniqueFixture(fx, lang, runId) {
+  const re = lang === "python" ? /`((?:test_)?[ab]\.py)`/g : /`src\/([ab](?:\.test)?\.js)`/g;
+  const dir = (m, f) => (lang === "python" ? `\`${runId}/${f}\`` : `\`src/${runId}/${f}\``);
+  const one = (i) => ({ title: `${i.title} (${runId})`, body: i.body.replace(re, dir) });
+  return { files: fx.files, a: one(fx.a), b: one(fx.b) };
+}
+
+/** Whether a batch plan is the one this harness started: the sandbox repo and every issue it passed (#4952). */
+export const isOwnBatch = (plan, repo, issues) =>
+  String(plan?.repo ?? "").toLowerCase() === repo.toLowerCase() && issues.every((n) => (plan?.argv ?? []).map(String).includes(String(n)));
+
 /** The issue body with its FORGE:STATE annotation removed. */
 export const stripState = (body) => String(body ?? "").replace(/<!--\s*FORGE:STATE[\s\S]*?-->/g, "").trim();
 
 const num = (out) => { const m = /\/issues\/(\d+)/.exec(String(out)) ?? /^(\d+)$/m.exec(String(out)); if (!m) throw new Error(`no issue number in: ${String(out).slice(0, 120)}`); return Number(m[1]); };
 
 async function waitFor(ctx, step, what, cond, onTick) {
-  const { io, timeoutMs } = ctx, t0 = io.now();
-  for (;;) {
-    if (onTick) await onTick();
-    const v = await cond();
-    if (v) return v;
-    if (io.now() - t0 > timeoutMs) fail(step, `timed out waiting for ${what}`);
-    await io.sleep(ctx.pollMs);
-  }
+  const { io, timeoutMs, pollMs } = ctx;
+  const v = await pollUntil(async () => { if (onTick) await onTick(); return cond(); }, { timeoutMs, pollMs, sleep: io.sleep, now: io.now });
+  if (!v) fail(step, `timed out waiting for ${what}`);
+  return v;
 }
 
 /** Step "body": every issue body still holds its creation text (#4860). */
@@ -59,16 +74,16 @@ async function checkBodies(ctx, step = "body") {
 
 export const STEPS = [
   { name: "create", run: async (ctx) => {
-    const { io, repo, fx } = ctx;
+    const { io, repo } = ctx, fx = ctx.fx = uniqueFixture(ctx.fx, ctx.lang, ctx.runId);
     for (const [path, content] of Object.entries(fx.files)) {
       await io.gh(["api", "-X", "PUT", `repos/${repo}/contents/${path}`, "-f", `message=selftest fixture ${path}`, "-f", `content=${Buffer.from(content).toString("base64")}`]).catch(() => { /* already present */ });
     }
-    const mk = async (t, body) => num(await io.gh(["issue", "create", "-R", repo, "--title", t, "--body", body]));
+    // Tracked the moment it exists: if B's create fails, A is still cleaned up.
+    const mk = async (t, body) => { const n = num(await io.gh(["issue", "create", "-R", repo, "--title", t, "--body", body])); ctx.created.push(n); return n; };
     const a = await mk(fx.a.title, fx.a.body);
     const b = await mk(fx.b.title, `${fx.b.body}\n\n**Depends on**: #${a}`);
     ctx.issues = { A: a, B: b };
     ctx.bodies = { [a]: fx.a.body, [b]: fx.b.body };
-    ctx.created.push(a, b);
   } },
   { name: "add", run: async (ctx) => {
     const { io, repo, issues } = ctx;
@@ -84,12 +99,13 @@ export const STEPS = [
     const { io, issues } = ctx;
     const pid = async () => (await io.orchestrate.state(ctx.run.id))?.running?.[issues.A]?.pid ?? null;
     const before = await waitFor(ctx, "resume", `a live lane for #${issues.A}`, pid);
+    // The batch runs inside schedd: killing the client stops nothing, so the batch itself is cancelled (#4952).
     await ctx.run.kill();
+    await io.orchestrate.stop(ctx.run.id);
+    await waitFor(ctx, "resume", `the lane of #${issues.A} (pid ${before}) to stop after the cancel`, async () => ((await pid()) ? null : { stopped: true }));
     ctx.run = await io.orchestrate.resume(ctx.run.id);
-    await io.sleep(ctx.pollMs);
-    const st = await io.orchestrate.state(ctx.run.id), after = st?.running?.[issues.A]?.pid ?? null;
-    const done = st?.results?.[issues.A];
-    if (!done && after !== before) fail("resume", `#${issues.A} was launched again after --resume (pid ${before} -> ${after}), not re-adopted`);
+    const st = await waitFor(ctx, "resume", `the batch to pick #${issues.A} up again after --resume`, async () => { const s = await io.orchestrate.state(ctx.run.id); return s?.results?.[issues.A] || s?.running?.[issues.A]?.pid ? s : null; });
+    if (!st.results?.[issues.A] && st.running?.[issues.A]?.pid === before) fail("resume", `#${issues.A} still has its pre-cancel lane (pid ${before}) after --resume: the batch was never stopped`);
     await checkBodies(ctx, "body");
   } },
   { name: "merge", run: async (ctx) => {
@@ -127,26 +143,31 @@ export const STEPS = [
  * Run the steps in order; the first failure stops them, cleanup always runs.
  * @returns {{ok:boolean, step:string|null, message:string|null, notes:string[], cleanupErrors:string[]}}
  */
-export async function runSelftestLive({ repo, lang = "node", io, timeoutMs = 45 * 60 * 1000, pollMs = 5000 }) {
+export async function runSelftestLive({ repo, lang = "node", io, timeoutMs = DEFAULT_TIMEOUT_MS, pollMs = DEFAULT_POLL_MS }) {
   const fx = FIXTURES[lang];
   if (!fx) return { ok: false, step: "setup", message: `unknown fixture language ${lang} (node|python)`, notes: [], cleanupErrors: [] };
-  const ctx = { io, repo, fx, timeoutMs, pollMs, issues: {}, bodies: {}, created: [], notes: [], run: null };
+  const ctx = { io, repo, fx, lang, runId: `r${io.now?.().toString(36) ?? "0"}`, timeoutMs, pollMs, issues: {}, bodies: {}, created: [], notes: [], run: null };
   let out = { ok: true, step: null, message: null };
   for (const s of STEPS) {
     try { await s.run(ctx); } catch (e) { out = { ok: false, step: e.step ?? s.name, message: String(e.message ?? e) }; break; }
   }
   const cleanupErrors = [];
   const tryC = async (what, f) => { try { await f(); } catch (e) { cleanupErrors.push(`${what}: ${String(e.message ?? e).slice(0, 120)}`); } };
-  if (ctx.run) await tryC("stop orchestrate", () => ctx.run.kill());
+  if (ctx.run) {
+    await tryC("kill orchestrate client", () => ctx.run.kill());
+    await tryC("stop batch", () => io.orchestrate.stop(ctx.run.id));
+  }
   for (const n of ctx.created) {
     await tryC(`close #${n}`, () => io.gh(["issue", "close", String(n), "-R", repo]));
-    await tryC(`delete branches of #${n}`, async () => {
-      const refs = String(await io.gh(["api", `repos/${repo}/git/matching-refs/heads/forge/issue-${n}`, "--jq", ".[].ref"])).split("\n").filter(Boolean);
-      for (const ref of refs) await io.gh(["api", "-X", "DELETE", `repos/${repo}/git/${ref}`]);
-    });
-    await tryC(`close PRs of #${n}`, async () => {
-      const prs = String(await io.gh(["pr", "list", "-R", repo, "--search", `#${n} in:title,body`, "--state", "open", "--json", "number", "--jq", ".[].number"])).split("\n").filter(Boolean);
-      for (const p of prs) await io.gh(["pr", "close", p, "-R", repo, "--delete-branch"]);
+    // The prefix search over-matches (issue-5 lists issue-50): keep only this issue's own branches, then find PRs by head.
+    const own = new RegExp(`^refs/heads/forge/issue-${n}(?![0-9])`);
+    const lines = async (args) => String(await io.gh(args)).split("\n").filter(Boolean);
+    await tryC(`clean up branches of #${n}`, async () => {
+      const refs = (await lines(["api", `repos/${repo}/git/matching-refs/heads/forge/issue-${n}`, "--jq", ".[].ref"])).filter((r) => own.test(r));
+      for (const ref of refs) {
+        for (const p of await lines(["pr", "list", "-R", repo, "--head", ref.replace(/^refs\/heads\//, ""), "--state", "open", "--json", "number", "--jq", ".[].number"])) await io.gh(["pr", "close", p, "-R", repo]);
+        await io.gh(["api", "-X", "DELETE", `repos/${repo}/git/${ref}`]);
+      }
     });
   }
   return { ...out, notes: ctx.notes, cleanupErrors };
@@ -159,28 +180,32 @@ export function realIo(env = process.env, { pool = false } = {}) {
   let self;
   const root = batchRoot(env);
   const detached = (args) => { const p = spawn(process.execPath, [CLI, ...args], { detached: true, windowsHide: true, stdio: "ignore", env }); p.unref(); return { kill: async () => { try { process.kill(-p.pid, "SIGTERM"); } catch { /* gone */ } } }; };
-  const waitBatch = async (before) => { for (let i = 0; i < 60; i++) { const id = listBatches(root).find((b) => !before.has(b)); if (id) return id; await new Promise((r) => setTimeout(r, 1000)); } throw new Error("orchestrate did not create a batch within 60s"); };
+  // The batch of THIS start: new on disk, for the sandbox repo, and naming the issue we passed. A concurrent real batch is never adopted (#4952).
+  const ownBatch = (id, repo, issues) => { try { return isOwnBatch(openBatch(root, id).readPlan(), repo, issues); } catch { return false; } };
+  const waitBatch = async (before, repo, issues) => { const id = await pollUntil(() => listBatches(root).find((b) => !before.has(b) && ownBatch(b, repo, issues)), { timeoutMs: BATCH_WAIT_TRIES * BATCH_WAIT_MS, pollMs: BATCH_WAIT_MS }); if (id) return id; throw new Error(`orchestrate did not create a batch for ${repo} ${issues.join(",")} within ${BATCH_WAIT_TRIES * BATCH_WAIT_MS / 1000}s`); };
   return (self = {
     now: () => Date.now(), sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
-    gh: async (args) => { const r = spawnSync("gh", args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); if (r.status !== 0) throw new Error(r.stderr.trim() || `gh exited ${r.status}`); return r.stdout; },
+    gh: async (args) => { const r = spawnSync("gh", args, { encoding: "utf8", maxBuffer: GH_MAX_BUFFER }); if (r.status !== 0) throw new Error(r.stderr.trim() || `gh exited ${r.status}`); return r.stdout; },
     cli: async (args) => { const r = spawnSync(process.execPath, [CLI, ...args], { encoding: "utf8", env }); return { code: r.status ?? 1, stdout: r.stdout, stderr: r.stderr }; },
     cliVersion: async () => JSON.parse(readFileSync(join(dirname(CLI), "..", "package.json"), "utf8")).version,
     orchestrate: {
       start: async (repo, issues, parallel) => {
         const before = new Set(listBatches(root)), h = detached(["orchestrate", ...issues.map(String), "--parallel", String(parallel), "--repo", repo]);
-        return { id: await waitBatch(before), kill: h.kill };
+        return { id: await waitBatch(before, repo, issues), kill: h.kill };
       },
+      // The batch lives in schedd, not in the client: `--cancel` is what stops it (its lanes tear down, the rest is held for --resume).
+      stop: async (id) => { const r = await self.cli(["orchestrate", "--cancel", id]); if (r.code !== 0) throw new Error(`orchestrate --cancel ${id} exited ${r.code}: ${String(r.stderr || r.stdout).slice(0, 160)}`); },
       resume: async (id) => ({ id, ...detached(["orchestrate", "--resume", id]) }),
       state: async (id) => openBatch(root, id).store.load(),
     },
     // Step 6: one lane through the pool. The issue's FORGE:STATE records the placement and the lane's FORGE:TELEMETRY rows
     // carry the engine's forge_version (the sandbox's, not the CLI's: the #4818 stale-image defect).
     remote: pool ? { run: async ({ repo, issue, timeoutMs, pollMs }) => {
-      const run = await self.orchestrate.start(repo, [issue], 1), t0 = Date.now();
+      const run = await self.orchestrate.start(repo, [issue], 1);
       let row = null;
       try {
-        while (!row && Date.now() - t0 < timeoutMs) { row = (await self.orchestrate.state(run.id))?.results?.[issue] ?? null; if (!row) await self.sleep(pollMs); }
-      } finally { await run.kill(); }
+        row = await pollUntil(async () => (await self.orchestrate.state(run.id))?.results?.[issue] ?? null, { timeoutMs, pollMs, sleep: self.sleep, now: self.now });
+      } finally { await run.kill(); await self.orchestrate.stop(run.id).catch(() => {}); }
       if (!row) throw new Error(`#${issue} produced no result within ${Math.round(timeoutMs / 1000)}s`);
       const v = JSON.parse(await self.gh(["issue", "view", String(issue), "-R", repo, "--json", "body,comments"]));
       const text = `${v.body}\n${v.comments.map((c) => c.body).join("\n")}`;

package/bin/lean/selftest-live.test.mjs

@@ -5,11 +5,11 @@ import assert from "node:assert/strict";
 import { writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
-import { runSelftestLive, stripState, selftestFromCli, STEPS } from "./selftest-live.mjs";
+import { runSelftestLive, stripState, selftestFromCli, STEPS, FIXTURES, uniqueFixture, isOwnBatch } from "./selftest-live.mjs";
 
 /** A fake sandbox world. `d` toggles a defect: add4859 (--add skips), wipe4860 (body wiped), double (resume relaunches), orderBA, noRemoteAuth. */
 function world(d = {}) {
-  const w = { issues: {}, closed: [], calls: [], killed: 0, pid: 100, results: {}, order: [], deps: {}, running: {}, clock: 0, pool: !!d.pool };
+  const w = { issues: {}, closed: [], calls: [], killed: 0, stops: [], pid: 100, results: {}, order: [], deps: {}, running: {}, clock: 0, pool: !!d.pool };
   const gh = async (a) => {
     w.calls.push(a.join(" "));
     if (a[0] === "api" && a[1] === "-X" && a[2] === "PUT") return "{}";
@@ -36,7 +36,9 @@ function world(d = {}) {
     },
     orchestrate: {
       start: async (repo, issues) => { w.order.push(...issues); w.running[issues[0]] = { pid: ++w.pid }; w.a = issues[0]; return { id: "b1", kill: async () => { w.killed++; } }; },
-      resume: async (id) => { w.resumedAt = w.tick ?? 0; if (d.double) w.running[w.a] = { pid: ++w.pid }; return { id, kill: async () => { w.killed++; } }; },
+      // stop = `orchestrate --cancel`: the lane is torn down (d.nostop: the defect where only the client dies and the lane lives on)
+      stop: async (id) => { w.stops.push(id); if (!d.nostop) delete w.running[w.a]; },
+      resume: async (id) => { w.resumedAt = w.tick ?? 0; w.running[w.a] = { pid: d.nostop ? w.running[w.a]?.pid ?? ++w.pid : ++w.pid }; return { id, kill: async () => { w.killed++; } }; },
       state: async () => ({ order: w.order, deps: w.deps, running: w.running, results: w.results }),
     },
     remote: w.pool ? { run: async ({ issue }) => { w.remoteIssue = issue; return { where: d.local ? "local" : "remote", authenticated: !d.noRemoteAuth, engineVersion: d.oldEngine ? "1.37.0" : "1.39.0" }; } } : null,
@@ -73,9 +75,14 @@ describe("selftest --live steps (#4865)", () => {
     const { r } = await run({ wipe4860: true });
     assert.equal(r.ok, false); assert.equal(r.step, "body"); assert.match(r.message, /lost its body text/);
   });
-  it("a lane launched twice after --resume fails the resume step", async () => {
-    const { r } = await run({ double: true });
-    assert.equal(r.step, "resume"); assert.match(r.message, /launched again/);
+  it("resume really stops the batch (not just the client) and cleanup stops it again (#4952)", async () => {
+    const { r, w } = await run({});
+    assert.equal(r.ok, true, JSON.stringify(r));
+    assert.deepEqual(w.stops, ["b1", "b1"], "once in the resume step, once in cleanup");
+  });
+  it("a stop that leaves the lane running fails the resume step instead of passing vacuously (#4952)", async () => {
+    const { r } = await run({ nostop: true }, { timeoutMs: 20000 });
+    assert.equal(r.step, "resume"); assert.match(r.message, /to stop after the cancel/);
   });
   it("B merging before A fails the merge step", async () => {
     const { r } = await run({ orderBA: true });
@@ -89,6 +96,41 @@ describe("selftest --live steps (#4865)", () => {
     const ok = await run({ pool: true });
     assert.equal(ok.w.remoteIssue, 12); assert.deepEqual(ok.w.closed, ["10", "11", "12"]);
   });
+  it("fixtures are run-unique, so a merged run leaves nothing the next run trips on (#4952)", async () => {
+    const x = uniqueFixture(FIXTURES.node, "node", "r1"), y = uniqueFixture(FIXTURES.python, "python", "r1");
+    assert.match(x.a.body, /`src\/r1\/a\.js`.*`src\/r1\/a\.test\.js`/); assert.match(x.a.title, /\(r1\)/);
+    assert.match(y.b.body, /`r1\/b\.py`.*`r1\/test_b\.py`/);
+    const { w } = await run({});
+    assert.match(w.issues[10].body, /src\/r[0-9a-z]+\/a\.js/);
+  });
+  it("the created issue is tracked at once: A is closed when B's create fails (#4952)", async () => {
+    const { w, io } = world({}), gh = io.gh; let creates = 0;
+    io.gh = async (a) => { if (a[0] === "issue" && a[1] === "create" && ++creates === 2) throw new Error("rate limited"); return gh(a); };
+    const r = await runSelftestLive({ repo: "o/sandbox", io, pollMs: 1, timeoutMs: 60000 });
+    assert.equal(r.step, "create"); assert.deepEqual(w.closed, ["10"]);
+  });
+  it("cleanup matches branches exactly and finds PRs by the head branch it created (#4952)", async () => {
+    const { w, io } = world({ add4859: true }), gh = io.gh, log = [];
+    io.gh = async (a) => {
+      if (a[0] === "api" && a[1].includes("matching-refs/heads/forge/issue-10")) return "refs/heads/forge/issue-10\nrefs/heads/forge/issue-100\nrefs/heads/forge/issue-10-x\n";
+      if (a[0] === "api" && a[2] === "DELETE") { log.push(`del ${a[3].split("git/")[1]}`); return ""; }
+      if (a[0] === "api") return "";
+      if (a[0] === "pr" && a[1] === "list") { log.push(`list ${a[a.indexOf("--head") + 1]}`); return a.includes("forge/issue-10") ? "7\n" : ""; }
+      if (a[0] === "pr" && a[1] === "close") { log.push(`close ${a[2]}`); return ""; }
+      return gh(a);
+    };
+    const r = await runSelftestLive({ repo: "o/sandbox", io, pollMs: 1, timeoutMs: 60000 });
+    assert.deepEqual(r.cleanupErrors, []);
+    assert.deepEqual(log, ["list forge/issue-10", "close 7", "del refs/heads/forge/issue-10", "list forge/issue-10-x", "del refs/heads/forge/issue-10-x"]);
+    assert.ok(!w.calls.some((c) => /search/.test(c)), "no text search for PRs");
+  });
+  it("own batch is picked by repo and issues: a foreign concurrent batch is never adopted (#4952)", () => {
+    const mine = { repo: "o/Sandbox", argv: ["orchestrate", "10", "11", "--repo", "o/Sandbox"] };
+    assert.ok(isOwnBatch(mine, "o/sandbox", [10, 11]));
+    assert.ok(!isOwnBatch({ repo: "o/real", argv: ["orchestrate", "10", "11"] }, "o/sandbox", [10, 11]));
+    assert.ok(!isOwnBatch({ repo: "o/sandbox", argv: ["orchestrate", "7"] }, "o/sandbox", [10, 11]));
+    assert.ok(!isOwnBatch(null, "o/sandbox", [10]));
+  });
   it("unknown fixture language fails at setup", async () => {
     assert.equal((await runSelftestLive({ repo: "o/r", lang: "ruby", io: {} })).step, "setup");
   });

package/bin/lean/selftest-nightly.mjs

@@ -9,9 +9,9 @@ import { spawn } from "node:child_process";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { schedDir } from "./sched.mjs";
+import { neutralize, findMarked } from "./prodwatch-triage.mjs";
 
 export const NIGHTLY_MARKER = "<!-- FORGE:SELFTEST_NIGHTLY -->";
-const neutralize = (s) => String(s ?? "").replace(/</g, "&lt;").replace(/>/g, "&gt;");
 /** One last-run file per repo: two repos on one host never share (and so never skip) each other's schedule. */
 const stateFile = (repo) => `selftest-nightly-${String(repo).replace(/[^\w.-]+/g, "_")}.json`;
 
@@ -49,8 +49,12 @@ export async function reportNightly({ repo, io, now = () => Date.now(), result }
   const step = result?.step ?? "unknown";
   const title = `selftest --live failed at step ${step} (nightly)`;
   const body = `The nightly \`forgedock selftest --live\` failed at step **${neutralize(step)}**.\n\n${neutralize(String(result?.message ?? "").slice(0, 2000))}\n\nLast failure: ${new Date(now()).toISOString()}\n\n${NIGHTLY_MARKER}\n`;
-  const open = JSON.parse(String(await io.gh(["issue", "list", "-R", repo, "--state", "open", "--search", `"FORGE:SELFTEST_NIGHTLY" in:body`, "--limit", "30", "--json", "number,body"])))
-    .find((i) => String(i.body ?? "").includes(NIGHTLY_MARKER));
+  // #4952: the marker is public text, so only an issue authored by the actor running the harness is edited; an unreadable
+  // login trusts nobody (a new issue is filed instead).
+  let me = "";
+  try { me = String(await io.gh(["api", "user", "--jq", ".login"])).trim().toLowerCase(); } catch { /* untrusted */ }
+  const issues = JSON.parse(String(await io.gh(["issue", "list", "-R", repo, "--state", "open", "--search", `"FORGE:SELFTEST_NIGHTLY" in:body`, "--limit", "30", "--json", "number,body,author"])));
+  const open = findMarked(issues, NIGHTLY_MARKER, (i) => me && String(i.author?.login ?? "").toLowerCase() === me);
   if (open) {
     await io.gh(["issue", "edit", String(open.number), "-R", repo, "--title", title, "--body", body]);
     return { issue: open.number, action: "edit" };

package/bin/lean/selftest-nightly.test.mjs

@@ -8,7 +8,7 @@ import { nightlyConfig, nightlyDue, launchNightly, NIGHTLY_MARKER } from "./self
 
 const setup = (yaml) => { const cwd = tmpDir("nt-"), stateDir = tmpDir("ns-"); if (yaml) writeFileSync(join(cwd, "forge.yaml"), yaml); return { cwd, stateDir }; };
 const YAML = "selftest:\n  sandbox_repo: a/b\n  nightly_interval_hours: 24\n";
-const fakeIo = (open = []) => { const calls = []; return { calls, gh: async (a) => { calls.push(a); if (a[1] === "list") return JSON.stringify(open); if (a[1] === "create") return "https://github.com/o/r/issues/77\n"; return ""; } }; };
+const fakeIo = (open = []) => { const calls = []; return { calls, gh: async (a) => { calls.push(a); if (a[0] === "api") return "bot\n"; if (a[1] === "list") return JSON.stringify(open); if (a[1] === "create") return "https://github.com/o/r/issues/77\n"; return ""; } }; };
 
 test("nightlyConfig: unset, zero and other sections are off", () => {
   assert.equal(nightlyConfig(""), null);
@@ -42,7 +42,7 @@ test("failure files one issue naming the step; the second failure edits it", asy
   assert.match(create.join(" "), /step body/); assert.ok(create.join(" ").includes(NIGHTLY_MARKER));
   assert.ok(!create[create.indexOf("--body") + 1].includes("<!-- x"));
   t += 25 * 3600_000;
-  const io2 = fakeIo([{ number: 77, body: `x\n${NIGHTLY_MARKER}` }]);
+  const io2 = fakeIo([{ number: 77, body: `x\n${NIGHTLY_MARKER}`, author: { login: "Bot" } }]);
   const r2 = await nightlyDue({ ...s, repo: "o/r", io: io2, now: () => t, runLive: async () => ({ ok: false, step: "add", message: "m" }) });
   assert.deepEqual([r2.action, r2.issue], ["edit", 77]);
   assert.ok(!io2.calls.some((a) => a[1] === "create"));
@@ -82,3 +82,25 @@ test("launchNightly: detached, unref'd, and an async spawn error never escapes (
   assert.doesNotThrow(() => child.emit("error", Object.assign(new Error("spawn EAGAIN"), { code: "EAGAIN" })));
   assert.equal(launchNightly({ cli: "c", repo: "o/r", spawnFn: () => { throw new Error("EMFILE"); } }), null);
 });
+
+test("the dedupe never edits a marker-bearing issue authored by someone else (#4952)", async () => {
+  const s = setup(YAML), io = fakeIo([{ number: 5, body: `evil\n${NIGHTLY_MARKER}`, author: { login: "mallory" } }, { number: 6, body: `no marker`, author: { login: "bot" } }]);
+  const r = await nightlyDue({ ...s, repo: "o/r", io, runLive: async () => ({ ok: false, step: "add", message: "m" }) });
+  assert.deepEqual([r.action, r.issue], ["create", 77]);
+  assert.ok(!io.calls.some((a) => a[1] === "edit"));
+  assert.ok(io.calls.find((a) => a[1] === "list").join(" ").includes("number,body,author"));
+});
+
+test("an unreadable login trusts nobody: a new issue is filed", async () => {
+  const s = setup(YAML), io = fakeIo([{ number: 5, body: NIGHTLY_MARKER, author: { login: "bot" } }]), gh = io.gh;
+  io.gh = async (a) => { if (a[0] === "api") throw new Error("no token user"); return gh(a); };
+  const r = await nightlyDue({ ...s, repo: "o/r", io, runLive: async () => ({ ok: false, step: "add", message: "m" }) });
+  assert.equal(r.action, "create");
+});
+
+test("a marker quoted mid-body by the trusted author is not a dedupe hit (shared trailer rule)", async () => {
+  const s = setup(YAML), io = fakeIo([{ number: 5, body: `quote: ${NIGHTLY_MARKER}\nmore text`, author: { login: "bot" } }]);
+  const r = await nightlyDue({ ...s, repo: "o/r", io, runLive: async () => ({ ok: false, step: "add", message: "m" }) });
+  assert.equal(r.action, "create");
+  assert.ok(!io.calls.some((a) => a[1] === "edit"));
+});

package/bin/lean/selftest-static.mjs

@@ -18,15 +18,20 @@ export function listSources(dir) {
   return out.sort();
 }
 
-/** @returns {{cmd:string, fields:string[], line:number}[]} the literal gh `--json` field lists of `text` (cmd e.g. "issue view"). */
+/**
+ * The `gh issue|pr view|list --json <fields>` uses of `text`. A literal field list is `{cmd, fields, line}`; a list the
+ * scan cannot read (a variable or template, e.g. `"--json", fields`) is `{cmd, fields: [], unchecked: true, line}`.
+ * @returns {{cmd:string, fields:string[], line:number, unchecked?:true}[]} cmd e.g. "issue view"
+ */
 export function extractJsonFields(text) {
-  const out = [];
-  const re = /\[([^[\]]*?)["'`]--json["'`]\s*,\s*["'`]([A-Za-z0-9_,]+)["'`]/g;
+  const out = [], re = /["'`]--json["'`]\s*,\s*/g;
   for (let m; (m = re.exec(text));) {
-    const words = [...m[1].matchAll(/["'`]([a-z-]+)["'`]/g)].map((x) => x[1]);
-    const i = words.findIndex((w, k) => (w === "issue" || w === "pr") && (words[k + 1] === "view" || words[k + 1] === "list"));
-    if (i < 0) continue;
-    out.push({ cmd: `${words[i]} ${words[i + 1]}`, fields: m[2].split(",").filter(Boolean), line: text.slice(0, m.index).split("\n").length });
+    const before = text.slice(Math.max(0, m.index - 300), m.index), seg = before.slice(before.lastIndexOf(";") + 1);
+    const cmds = [...seg.matchAll(/["'`](issue|pr)["'`]\s*,\s*["'`](view|list)["'`]/g)];
+    if (!cmds.length) continue;
+    const c = cmds.at(-1), lit = /^["'`]([A-Za-z0-9_,]+)["'`]\s*[,\])]/.exec(text.slice(m.index + m[0].length));
+    const row = { cmd: `${c[1]} ${c[2]}`, fields: lit ? lit[1].split(",").filter(Boolean) : [], line: text.slice(0, m.index).split("\n").length };
+    out.push(lit ? row : { ...row, unchecked: true });
   }
   return out;
 }
@@ -47,14 +52,15 @@ export function parseHelpFields(help) {
 
 /**
  * @param {{root:string, io:{gh:(args:string[])=>string, read?:(p:string)=>string, list?:(d:string)=>string[]}}} a
- * @returns {{ok:boolean, checked:number, missing:{file:string,line:number,cmd:string,field:string}[], errors:string[]}}
+ * @returns {{ok:boolean, checked:number, unchecked:{file:string,line:number,cmd:string}[], missing:{file:string,line:number,cmd:string,field:string}[], errors:string[]}}
  */
 export function staticCheck({ root, io }) {
   const read = io.read ?? ((p) => readFileSync(p, "utf8")), list = io.list ?? listSources;
-  const known = new Map(), errors = [], missing = [];
+  const known = new Map(), errors = [], missing = [], unchecked = [];
   let checked = 0;
   for (const file of list(root)) {
     for (const u of extractJsonFields(read(file))) {
+      if (u.unchecked) { unchecked.push({ file, line: u.line, cmd: u.cmd }); continue; } // reported, never silently skipped (#4952)
       if (!known.has(u.cmd)) {
         let fields = [];
         try { fields = parseHelpFields(io.gh([...u.cmd.split(" "), "--help"])); } catch (e) { errors.push(`gh ${u.cmd} --help failed: ${e.message}`); }
@@ -67,7 +73,7 @@ export function staticCheck({ root, io }) {
       for (const field of u.fields) if (!set.has(field)) missing.push({ file, line: u.line, cmd: u.cmd, field });
     }
   }
-  return { ok: !missing.length && !errors.length, checked, missing, errors };
+  return { ok: !missing.length && !errors.length, checked, unchecked, missing, errors };
 }
 
 export const realGh = (args) => execFileSync("gh", args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 });
@@ -76,6 +82,7 @@ export function formatStatic(r) {
   return [
     ...r.missing.map((m) => `${m.file}:${m.line}: gh ${m.cmd} --json field "${m.field}" is not offered by this gh`),
     ...r.errors,
-    `static check: ${r.ok ? "ok" : "FAILED"} (${r.checked} field lists checked)`,
+    ...(r.unchecked ?? []).map((u) => `${u.file}:${u.line}: gh ${u.cmd} --json field list is not a string literal: unchecked`),
+    `static check: ${r.ok ? "ok" : "FAILED"} (${r.checked} field lists checked, ${(r.unchecked ?? []).length} unchecked)`,
   ].join("\n");
 }

package/bin/lean/selftest-static.test.mjs

@@ -29,4 +29,12 @@ describe("selftest static check (#4865)", () => {
     const r = staticCheck({ root: "/bin", io: { gh: () => { throw new Error("no gh"); }, list: () => ["/a.mjs"], read: () => 'x(["pr", "view", "--json", "title"])' } });
     assert.equal(r.ok, false);
   });
+  it("reports a non-literal --json list as unchecked, not as checked (#4952)", () => {
+    const t = 'x();\ngh(["issue", "view", String(n), ...(repo ? ["-R", repo] : []), "--json", fields]);\ngh(["pr", "list", "--json", `a,${f}`]);';
+    assert.deepEqual(extractJsonFields(t), [{ cmd: "issue view", fields: [], line: 2, unchecked: true }, { cmd: "pr list", fields: [], line: 3, unchecked: true }]);
+    const r = staticCheck({ root: "/bin", io: { gh, list: () => ["/bin/a.mjs"], read: () => t } });
+    assert.equal(r.ok, true); assert.equal(r.checked, 0); assert.equal(r.unchecked.length, 2);
+    assert.match(formatStatic(r), /a\.mjs:2: gh issue view --json field list is not a string literal: unchecked/);
+    assert.match(formatStatic(r), /0 field lists checked, 2 unchecked/);
+  });
 });