forgedock 1.43.0 → 1.44.0

package/bin/lean/schedd.mjs

@@ -172,6 +172,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     const run = Promise.resolve().then(() => runBatch({ argv, cwd, env, sink, keepBreaker, signal: ac.signal })).then((result) => {
       const report = ent.dir ? openBatch(root, ent.id).readReport()?.report ?? null : null;
       emit({ type: "done", result, report });
+      if (ent.id) rehostLater(ent.id); // #4917
     }, (e) => {
       const message = e?.message ?? String(e);
       if (e?.code === "batch.stopped") { log(`schedd: batch ${ent.id ?? argv.join(" ")} stopped (lanes keep running; resumed by the next daemon)`); if (!started) reject(new Error(message)); return; }
@@ -180,6 +181,28 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     runs.set(run, ac);
   });
 
+  // #4917: a batch that ended with a deferred issue still waiting (no report) is re-hosted once its retry-at is near, without a restart
+  const rehostTimers = new Set();
+  const rehostLater = (id) => {
+    if (closing) return;
+    let b, st;
+    try { b = openBatch(root, id); if (b.readReport() || b.readCancel() || !b.pendingIssues().length) return; st = b.store.load(); } catch { return; }
+    if (st?.breaker && !st.breaker.transport) return; // a tripped breaker holds the batch for an operator --resume (a re-host would stop again at once)
+    const at = Math.min(...b.pendingIssues().map((n) => Number(st?.notBefore?.[n]) || Infinity));
+    if (!Number.isFinite(at)) return; // nothing waits on the clock: a deliberate hold stays for --resume
+    if (at <= Date.now()) return; // the run that just ended already saw this retry-at pass and did not admit: re-hosting would loop every second
+    const t = setTimeout(() => {
+      rehostTimers.delete(t);
+      if (closing || hosted.has(id) || starting.has(id)) return;
+      log(`schedd: re-hosting batch ${id} (pending issues waiting on a retry)`);
+      const started = host({ argv: ["--resume", id], cwd: process.cwd(), env: {}, keepBreaker: true });
+      starting.set(id, started);
+      started.catch((e) => log(`schedd: re-host ${id} failed: ${e?.message ?? e}`)).finally(() => starting.delete(id));
+    }, Math.min(60_000, Math.max(1000, at - Date.now())));
+    t.unref?.();
+    rehostTimers.add(t);
+  };
+
   /** #4775: wait for a batch start at most `admitMs`; the start itself goes on (a second host() would duplicate the batch). */
   const bound = (p, what) => {
     let t;
@@ -204,7 +227,15 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
   };
 
   const handle = async (req, send) => {
-    if (req.op === "status") return send({ type: "status", pid, lastTick, batches: statusBatches() }, true);
+    if (req.op === "status") {
+      // #4917: a batch dir that still holds pending issues but is not hosted (finalised early, or never resumed)
+      const unhosted = [];
+      for (const id of listBatches(root)) {
+        if (hosted.has(id) || starting.has(id)) continue;
+        try { const pending = openBatch(root, id).pendingIssues(); if (pending.length) unhosted.push({ id, pending }); } catch { /* advisory */ }
+      }
+      return send({ type: "status", pid, lastTick, batches: statusBatches(), unhosted }, true);
+    }
     if (req.op === "set-parallel") {
       const b = openBatch(root, req.id);
       if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
@@ -215,8 +246,8 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       const b = openBatch(root, req.id);
       if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
       const { added, skipped, requeued = [] } = b.addIssues(req.issues ?? []);
-      // #4687: a re-queued skipped issue reopens a finished, unhosted batch like --requeue does
-      const reopen = requeued.length > 0 && !hosted.has(req.id) && !starting.has(req.id);
+      // #4687/#5000/#4917: a re-queued skipped issue or a newly added one reopens a finished, unhosted batch like --requeue does
+      const reopen = (added.length > 0 || requeued.length > 0 || b.unrunAdds().length > 0 || b.unrunRequeues().length > 0) && !hosted.has(req.id) && !starting.has(req.id);
       if (reopen) await hostBounded({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} }, null, `batch ${req.id}`);
       return send({ type: "ok", added, skipped, requeued, hosted: hosted.has(req.id), reopened: reopen }, true);
     }
@@ -225,7 +256,8 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       const b = openBatch(root, req.id);
       if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
       const { requeued, refused } = b.requeue(req.issues ?? [], { reason: req.reason, by: req.by });
-      const reopen = requeued.length > 0 && !hosted.has(req.id) && !starting.has(req.id);
+      // #4917: a refused repeat requeue still reopens when an earlier entry was never applied
+      const reopen = (requeued.length > 0 || b.unrunRequeues().length > 0) && !hosted.has(req.id) && !starting.has(req.id);
       if (reopen) await hostBounded({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} }, null, `batch ${req.id}`);
       return send({ type: "ok", requeued, refused, hosted: hosted.has(req.id), reopened: reopen }, true);
     }
@@ -306,6 +338,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     clearInterval(lockTimer);
     clearInterval(lagTimer);
     clearInterval(beatTimer);
+    for (const t of rehostTimers) clearTimeout(t);
     if (dueTimer) clearInterval(dueTimer);
     for (const c of conns) c.destroy();
     const stopping = [...runs.keys()];
@@ -448,7 +481,7 @@ export function defaultSpawn({ dir = schedDir(), script = SCHEDD_SCRIPT, env = p
   mkdirSync(dir, { recursive: true });
   const out = openSync(scheddPaths(dir).log, "a");
   try {
-    const child = spawn(process.execPath, [script, "schedd"], { env, detached: true, stdio: ["ignore", out, out] });
+    const child = spawn(process.execPath, [script, "schedd"], { env, detached: true, windowsHide: true, stdio: ["ignore", out, out] });
     child.on?.("error", () => { /* the connect retry reports it */ });
     child.unref?.();
   } finally { closeSync(out); }

package/bin/lean/schedd.test.mjs

@@ -249,6 +249,55 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
     } finally { await h.cleanup(); }
   });
 
+  it("#5000: add to a finished unhosted batch reopens it through the daemon and runs the new issue", async () => {
+    const h = host();
+    try {
+      await h.daemon();
+      const client = { ...h.client, io: { gh: async (a) => (a[0] === "issue" ? JSON.stringify({ state: "OPEN", labels: [] }) : "o/r\n") } };
+      await withRoot(h, () => capture(async () => {
+        writeFileSync(h.release, "");
+        await orchestrateFromCli(["1", "--parallel", "2", "--repo", "o/r", "--no-prefilter"], h.client);
+        const [id] = listBatches(h.root);
+        const b = openBatch(h.root, id);
+        const res = await orchestrateFromCli(["--add", id, "6"], client);
+        assert.deepEqual(res.added, [6]);
+        assert.equal(res.reopened, true, "the finished batch is hosted again");
+        await until(() => h.launched().includes(6));
+        await until(() => b.store.load()?.results?.[6]?.outcome === "merged");
+        assert.deepEqual(listBatches(h.root), [id], "no second batch");
+      }));
+    } finally { await h.cleanup(); }
+  });
+
+  it("#4917: a held (pending) issue is flagged unhosted by status once its report is gone; --requeue re-hosts it and says so", async () => {
+    const h = host();
+    try {
+      await h.daemon();
+      const client = { ...h.client, io: { gh: async (a) => (a[0] === "issue" ? JSON.stringify({ state: "OPEN", labels: [] }) : "o/r\n") } };
+      await withRoot(h, () => capture(async () => {
+        const run = orchestrateFromCli(["1", "2", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client);
+        await until(() => h.launched().length === 1 && listBatches(h.root).length === 1);
+        const [id] = listBatches(h.root);
+        const b = openBatch(h.root, id);
+        b.holdIssues([2]);
+        writeFileSync(h.release, "");
+        const res = await run;
+        assert.equal(res.rows.find((r) => r.issue === 2).outcome, "held");
+        assert.deepEqual(b.pendingIssues(), [2]);
+        b.clearReport(); // a batch finalised before the fix, or one never resumed
+        const st = await request(await connectDaemon({ dir: h.sched, spawnDaemon: null }), { op: "status" });
+        assert.deepEqual(st.unhosted, [{ id, pending: [2] }]);
+        // finalised by an older version: a report plus unapplied requeue entries still reopens, and the CLI says so
+        b.releaseIssues([2]);
+        b.store.save({ ...b.store.load(), results: { ...b.store.load().results, 1: { issue: 1, outcome: "needs-human", pr: null, cost_usd: 0, minutes: 0 } } });
+        b.writeReport({ rows: [{ issue: 1, outcome: "needs-human" }] });
+        const rq = await capture(() => orchestrateFromCli(["--requeue", id, "1"], client));
+        assert.deepEqual(rq.res.requeued, [1]);
+        assert.ok(rq.lines.some((l) => l.includes("reopened and hosted by the schedd")), rq.lines.join("\n"));
+      }));
+    } finally { await h.cleanup(); }
+  });
+
   it("#4376: --cancel stops one batch (its lane torn down, the rest held, report written); the other batch and the daemon run on", async () => {
     const h = host();
     try {
@@ -350,6 +399,66 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
     } finally { await h.cleanup(); }
   });
 
+  it("#4917: a batch that ends unreported with a deferred issue waiting is re-hosted by the running daemon once its retry-at is near", async () => {
+    const dir = tmpDir("schedd-rehost-"), root = join(dir, "batches"), id = "b20261031-000000-4917";
+    const calls = [];
+    const runBatch = async ({ argv, sink }) => {
+      calls.push(argv);
+      const b = openBatch(root, id);
+      if (argv[0] !== "--resume") {
+        b.writePlan({ daemon: true, repo: "o/r" });
+        b.store.save({ v: 1, order: [1], results: {}, running: {}, pending: [1], notBefore: { 1: Date.now() + 1200 } });
+      } else b.store.save({ ...b.store.load(), pending: [], results: { 1: { issue: 1, outcome: "merged" } } });
+      sink.batch({ id, dir: b.dir });
+      return {};
+    };
+    const d = await startDaemon({ dir, root, runBatch, log: () => {}, resumeOnStart: false });
+    try {
+      await request(await connectDaemon({ dir, spawnDaemon: null }), { op: "submit", argv: ["1"], cwd: dir, env: {} });
+      await until(() => calls.length === 2, 8000);
+      assert.deepEqual(calls[1], ["--resume", id]);
+    } finally { await d.close(); }
+  });
+
+  it("#4917: a resume that leaves the waiting issue unadmitted (breaker tripped, retry-at passed) is not re-hosted again", async () => {
+    const dir = tmpDir("schedd-rehost-loop-"), root = join(dir, "batches"), id = "b20261031-000000-4918";
+    const calls = [];
+    const runBatch = async ({ argv, sink }) => {
+      calls.push(argv);
+      const b = openBatch(root, id);
+      // first run: deferred issue 1 waits 300 ms; every resume ends at once (a tripped breaker), leaving it pending with a past retry-at
+      if (argv[0] !== "--resume") { b.writePlan({ daemon: true, repo: "o/r" }); b.store.save({ v: 1, order: [1], results: {}, running: {}, pending: [1], notBefore: { 1: Date.now() + 300 } }); }
+      sink.batch({ id, dir: b.dir });
+      return {};
+    };
+    const d = await startDaemon({ dir, root, runBatch, log: () => {}, resumeOnStart: false });
+    try {
+      await request(await connectDaemon({ dir, spawnDaemon: null }), { op: "submit", argv: ["1"], cwd: dir, env: {} });
+      await until(() => calls.length >= 2, 8000);
+      await new Promise((r) => setTimeout(r, 3500));
+      assert.equal(calls.length, 2, `re-hosted ${calls.length - 1} times; a past retry-at the resume did not act on must not loop`);
+    } finally { await d.close(); }
+  });
+
+  it("#4917: a batch whose breaker tripped (not transport) is not re-hosted by the timer", async () => {
+    const dir = tmpDir("schedd-rehost-brk-"), root = join(dir, "batches"), id = "b20261031-000000-4919";
+    const calls = [];
+    const runBatch = async ({ argv, sink }) => {
+      calls.push(argv);
+      const b = openBatch(root, id);
+      b.writePlan({ daemon: true, repo: "o/r" });
+      b.store.save({ v: 1, order: [1], results: {}, running: {}, pending: [1], notBefore: { 1: Date.now() + 300 }, breaker: { at: Date.now(), errors: 3 } });
+      sink.batch({ id, dir: b.dir });
+      return {};
+    };
+    const d = await startDaemon({ dir, root, runBatch, log: () => {}, resumeOnStart: false });
+    try {
+      await request(await connectDaemon({ dir, spawnDaemon: null }), { op: "submit", argv: ["1"], cwd: dir, env: {} });
+      await new Promise((r) => setTimeout(r, 2000));
+      assert.equal(calls.length, 1, "a tripped breaker holds the batch for an operator --resume");
+    } finally { await d.close(); }
+  });
+
   it("#4454: close() stops every hosted batch loop before releasing the lock; no batch admits a lane after close()", async () => {
     const dir = tmpDir("schedd-stop-"), root = join(dir, "batches"), lock = scheddPaths(dir).lock;
     const started = { a: [], b: [] }, stoppedWithLock = {};

package/bin/lean/selftest-live.mjs

@@ -10,10 +10,14 @@ import { fileURLToPath } from "node:url";
 import { spawn, spawnSync } from "node:child_process";
 import { batchRoot, listBatches, openBatch } from "./batch.mjs";
 import { staticCheck, formatStatic, realGh } from "./selftest-static.mjs";
+import { pollUntil } from "./retry.mjs";
 import { postMarker } from "./selftest-gate.mjs";
 import { reportNightly } from "./selftest-nightly.mjs";
 import { parseForgeYaml } from "../forge-utils.mjs";
 
+const BATCH_WAIT_TRIES = 60, BATCH_WAIT_MS = 1000;
+const DEFAULT_TIMEOUT_MS = 45 * 60 * 1000, DEFAULT_POLL_MS = 5000, GH_MAX_BUFFER = 64 * 1024 * 1024;
+
 export class StepError extends Error {
   constructor(step, message) { super(message); this.step = step; }
 }
@@ -33,20 +37,31 @@ export const FIXTURES = {
   },
 };
 
+/**
+ * The fixture with every path and title made unique to this run (#4952): a merged run leaves its files on the sandbox
+ * default branch, and the next run's "the test fails on the current code" premise must still hold.
+ */
+export function uniqueFixture(fx, lang, runId) {
+  const re = lang === "python" ? /`((?:test_)?[ab]\.py)`/g : /`src\/([ab](?:\.test)?\.js)`/g;
+  const dir = (m, f) => (lang === "python" ? `\`${runId}/${f}\`` : `\`src/${runId}/${f}\``);
+  const one = (i) => ({ title: `${i.title} (${runId})`, body: i.body.replace(re, dir) });
+  return { files: fx.files, a: one(fx.a), b: one(fx.b) };
+}
+
+/** Whether a batch plan is the one this harness started: the sandbox repo and every issue it passed (#4952). */
+export const isOwnBatch = (plan, repo, issues) =>
+  String(plan?.repo ?? "").toLowerCase() === repo.toLowerCase() && issues.every((n) => (plan?.argv ?? []).map(String).includes(String(n)));
+
 /** The issue body with its FORGE:STATE annotation removed. */
 export const stripState = (body) => String(body ?? "").replace(/<!--\s*FORGE:STATE[\s\S]*?-->/g, "").trim();
 
 const num = (out) => { const m = /\/issues\/(\d+)/.exec(String(out)) ?? /^(\d+)$/m.exec(String(out)); if (!m) throw new Error(`no issue number in: ${String(out).slice(0, 120)}`); return Number(m[1]); };
 
 async function waitFor(ctx, step, what, cond, onTick) {
-  const { io, timeoutMs } = ctx, t0 = io.now();
-  for (;;) {
-    if (onTick) await onTick();
-    const v = await cond();
-    if (v) return v;
-    if (io.now() - t0 > timeoutMs) fail(step, `timed out waiting for ${what}`);
-    await io.sleep(ctx.pollMs);
-  }
+  const { io, timeoutMs, pollMs } = ctx;
+  const v = await pollUntil(async () => { if (onTick) await onTick(); return cond(); }, { timeoutMs, pollMs, sleep: io.sleep, now: io.now });
+  if (!v) fail(step, `timed out waiting for ${what}`);
+  return v;
 }
 
 /** Step "body": every issue body still holds its creation text (#4860). */
@@ -59,16 +74,16 @@ async function checkBodies(ctx, step = "body") {
 
 export const STEPS = [
   { name: "create", run: async (ctx) => {
-    const { io, repo, fx } = ctx;
+    const { io, repo } = ctx, fx = ctx.fx = uniqueFixture(ctx.fx, ctx.lang, ctx.runId);
     for (const [path, content] of Object.entries(fx.files)) {
       await io.gh(["api", "-X", "PUT", `repos/${repo}/contents/${path}`, "-f", `message=selftest fixture ${path}`, "-f", `content=${Buffer.from(content).toString("base64")}`]).catch(() => { /* already present */ });
     }
-    const mk = async (t, body) => num(await io.gh(["issue", "create", "-R", repo, "--title", t, "--body", body]));
+    // Tracked the moment it exists: if B's create fails, A is still cleaned up.
+    const mk = async (t, body) => { const n = num(await io.gh(["issue", "create", "-R", repo, "--title", t, "--body", body])); ctx.created.push(n); return n; };
     const a = await mk(fx.a.title, fx.a.body);
     const b = await mk(fx.b.title, `${fx.b.body}\n\n**Depends on**: #${a}`);
     ctx.issues = { A: a, B: b };
     ctx.bodies = { [a]: fx.a.body, [b]: fx.b.body };
-    ctx.created.push(a, b);
   } },
   { name: "add", run: async (ctx) => {
     const { io, repo, issues } = ctx;
@@ -84,12 +99,13 @@ export const STEPS = [
     const { io, issues } = ctx;
     const pid = async () => (await io.orchestrate.state(ctx.run.id))?.running?.[issues.A]?.pid ?? null;
     const before = await waitFor(ctx, "resume", `a live lane for #${issues.A}`, pid);
+    // The batch runs inside schedd: killing the client stops nothing, so the batch itself is cancelled (#4952).
     await ctx.run.kill();
+    await io.orchestrate.stop(ctx.run.id);
+    await waitFor(ctx, "resume", `the lane of #${issues.A} (pid ${before}) to stop after the cancel`, async () => ((await pid()) ? null : { stopped: true }));
     ctx.run = await io.orchestrate.resume(ctx.run.id);
-    await io.sleep(ctx.pollMs);
-    const st = await io.orchestrate.state(ctx.run.id), after = st?.running?.[issues.A]?.pid ?? null;
-    const done = st?.results?.[issues.A];
-    if (!done && after !== before) fail("resume", `#${issues.A} was launched again after --resume (pid ${before} -> ${after}), not re-adopted`);
+    const st = await waitFor(ctx, "resume", `the batch to pick #${issues.A} up again after --resume`, async () => { const s = await io.orchestrate.state(ctx.run.id); return s?.results?.[issues.A] || s?.running?.[issues.A]?.pid ? s : null; });
+    if (!st.results?.[issues.A] && st.running?.[issues.A]?.pid === before) fail("resume", `#${issues.A} still has its pre-cancel lane (pid ${before}) after --resume: the batch was never stopped`);
     await checkBodies(ctx, "body");
   } },
   { name: "merge", run: async (ctx) => {
@@ -127,26 +143,31 @@ export const STEPS = [
  * Run the steps in order; the first failure stops them, cleanup always runs.
  * @returns {{ok:boolean, step:string|null, message:string|null, notes:string[], cleanupErrors:string[]}}
  */
-export async function runSelftestLive({ repo, lang = "node", io, timeoutMs = 45 * 60 * 1000, pollMs = 5000 }) {
+export async function runSelftestLive({ repo, lang = "node", io, timeoutMs = DEFAULT_TIMEOUT_MS, pollMs = DEFAULT_POLL_MS }) {
   const fx = FIXTURES[lang];
   if (!fx) return { ok: false, step: "setup", message: `unknown fixture language ${lang} (node|python)`, notes: [], cleanupErrors: [] };
-  const ctx = { io, repo, fx, timeoutMs, pollMs, issues: {}, bodies: {}, created: [], notes: [], run: null };
+  const ctx = { io, repo, fx, lang, runId: `r${io.now?.().toString(36) ?? "0"}`, timeoutMs, pollMs, issues: {}, bodies: {}, created: [], notes: [], run: null };
   let out = { ok: true, step: null, message: null };
   for (const s of STEPS) {
     try { await s.run(ctx); } catch (e) { out = { ok: false, step: e.step ?? s.name, message: String(e.message ?? e) }; break; }
   }
   const cleanupErrors = [];
   const tryC = async (what, f) => { try { await f(); } catch (e) { cleanupErrors.push(`${what}: ${String(e.message ?? e).slice(0, 120)}`); } };
-  if (ctx.run) await tryC("stop orchestrate", () => ctx.run.kill());
+  if (ctx.run) {
+    await tryC("kill orchestrate client", () => ctx.run.kill());
+    await tryC("stop batch", () => io.orchestrate.stop(ctx.run.id));
+  }
   for (const n of ctx.created) {
     await tryC(`close #${n}`, () => io.gh(["issue", "close", String(n), "-R", repo]));
-    await tryC(`delete branches of #${n}`, async () => {
-      const refs = String(await io.gh(["api", `repos/${repo}/git/matching-refs/heads/forge/issue-${n}`, "--jq", ".[].ref"])).split("\n").filter(Boolean);
-      for (const ref of refs) await io.gh(["api", "-X", "DELETE", `repos/${repo}/git/${ref}`]);
-    });
-    await tryC(`close PRs of #${n}`, async () => {
-      const prs = String(await io.gh(["pr", "list", "-R", repo, "--search", `#${n} in:title,body`, "--state", "open", "--json", "number", "--jq", ".[].number"])).split("\n").filter(Boolean);
-      for (const p of prs) await io.gh(["pr", "close", p, "-R", repo, "--delete-branch"]);
+    // The prefix search over-matches (issue-5 lists issue-50): keep only this issue's own branches, then find PRs by head.
+    const own = new RegExp(`^refs/heads/forge/issue-${n}(?![0-9])`);
+    const lines = async (args) => String(await io.gh(args)).split("\n").filter(Boolean);
+    await tryC(`clean up branches of #${n}`, async () => {
+      const refs = (await lines(["api", `repos/${repo}/git/matching-refs/heads/forge/issue-${n}`, "--jq", ".[].ref"])).filter((r) => own.test(r));
+      for (const ref of refs) {
+        for (const p of await lines(["pr", "list", "-R", repo, "--head", ref.replace(/^refs\/heads\//, ""), "--state", "open", "--json", "number", "--jq", ".[].number"])) await io.gh(["pr", "close", p, "-R", repo]);
+        await io.gh(["api", "-X", "DELETE", `repos/${repo}/git/${ref}`]);
+      }
     });
   }
   return { ...out, notes: ctx.notes, cleanupErrors };
@@ -158,29 +179,33 @@ const CLI = join(dirname(fileURLToPath(import.meta.url)), "..", "forgedock.mjs")
 export function realIo(env = process.env, { pool = false } = {}) {
   let self;
   const root = batchRoot(env);
-  const detached = (args) => { const p = spawn(process.execPath, [CLI, ...args], { detached: true, stdio: "ignore", env }); p.unref(); return { kill: async () => { try { process.kill(-p.pid, "SIGTERM"); } catch { /* gone */ } } }; };
-  const waitBatch = async (before) => { for (let i = 0; i < 60; i++) { const id = listBatches(root).find((b) => !before.has(b)); if (id) return id; await new Promise((r) => setTimeout(r, 1000)); } throw new Error("orchestrate did not create a batch within 60s"); };
+  const detached = (args) => { const p = spawn(process.execPath, [CLI, ...args], { detached: true, windowsHide: true, stdio: "ignore", env }); p.unref(); return { kill: async () => { try { process.kill(-p.pid, "SIGTERM"); } catch { /* gone */ } } }; };
+  // The batch of THIS start: new on disk, for the sandbox repo, and naming the issue we passed. A concurrent real batch is never adopted (#4952).
+  const ownBatch = (id, repo, issues) => { try { return isOwnBatch(openBatch(root, id).readPlan(), repo, issues); } catch { return false; } };
+  const waitBatch = async (before, repo, issues) => { const id = await pollUntil(() => listBatches(root).find((b) => !before.has(b) && ownBatch(b, repo, issues)), { timeoutMs: BATCH_WAIT_TRIES * BATCH_WAIT_MS, pollMs: BATCH_WAIT_MS }); if (id) return id; throw new Error(`orchestrate did not create a batch for ${repo} ${issues.join(",")} within ${BATCH_WAIT_TRIES * BATCH_WAIT_MS / 1000}s`); };
   return (self = {
     now: () => Date.now(), sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
-    gh: async (args) => { const r = spawnSync("gh", args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); if (r.status !== 0) throw new Error(r.stderr.trim() || `gh exited ${r.status}`); return r.stdout; },
+    gh: async (args) => { const r = spawnSync("gh", args, { encoding: "utf8", maxBuffer: GH_MAX_BUFFER }); if (r.status !== 0) throw new Error(r.stderr.trim() || `gh exited ${r.status}`); return r.stdout; },
     cli: async (args) => { const r = spawnSync(process.execPath, [CLI, ...args], { encoding: "utf8", env }); return { code: r.status ?? 1, stdout: r.stdout, stderr: r.stderr }; },
     cliVersion: async () => JSON.parse(readFileSync(join(dirname(CLI), "..", "package.json"), "utf8")).version,
     orchestrate: {
       start: async (repo, issues, parallel) => {
         const before = new Set(listBatches(root)), h = detached(["orchestrate", ...issues.map(String), "--parallel", String(parallel), "--repo", repo]);
-        return { id: await waitBatch(before), kill: h.kill };
+        return { id: await waitBatch(before, repo, issues), kill: h.kill };
       },
+      // The batch lives in schedd, not in the client: `--cancel` is what stops it (its lanes tear down, the rest is held for --resume).
+      stop: async (id) => { const r = await self.cli(["orchestrate", "--cancel", id]); if (r.code !== 0) throw new Error(`orchestrate --cancel ${id} exited ${r.code}: ${String(r.stderr || r.stdout).slice(0, 160)}`); },
       resume: async (id) => ({ id, ...detached(["orchestrate", "--resume", id]) }),
       state: async (id) => openBatch(root, id).store.load(),
     },
     // Step 6: one lane through the pool. The issue's FORGE:STATE records the placement and the lane's FORGE:TELEMETRY rows
     // carry the engine's forge_version (the sandbox's, not the CLI's: the #4818 stale-image defect).
     remote: pool ? { run: async ({ repo, issue, timeoutMs, pollMs }) => {
-      const run = await self.orchestrate.start(repo, [issue], 1), t0 = Date.now();
+      const run = await self.orchestrate.start(repo, [issue], 1);
       let row = null;
       try {
-        while (!row && Date.now() - t0 < timeoutMs) { row = (await self.orchestrate.state(run.id))?.results?.[issue] ?? null; if (!row) await self.sleep(pollMs); }
-      } finally { await run.kill(); }
+        row = await pollUntil(async () => (await self.orchestrate.state(run.id))?.results?.[issue] ?? null, { timeoutMs, pollMs, sleep: self.sleep, now: self.now });
+      } finally { await run.kill(); await self.orchestrate.stop(run.id).catch(() => {}); }
       if (!row) throw new Error(`#${issue} produced no result within ${Math.round(timeoutMs / 1000)}s`);
       const v = JSON.parse(await self.gh(["issue", "view", String(issue), "-R", repo, "--json", "body,comments"]));
       const text = `${v.body}\n${v.comments.map((c) => c.body).join("\n")}`;

package/bin/lean/selftest-live.test.mjs

@@ -5,11 +5,11 @@ import assert from "node:assert/strict";
 import { writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { tmpDir } from "../tests/_tmp.mjs";
-import { runSelftestLive, stripState, selftestFromCli, STEPS } from "./selftest-live.mjs";
+import { runSelftestLive, stripState, selftestFromCli, STEPS, FIXTURES, uniqueFixture, isOwnBatch } from "./selftest-live.mjs";
 
 /** A fake sandbox world. `d` toggles a defect: add4859 (--add skips), wipe4860 (body wiped), double (resume relaunches), orderBA, noRemoteAuth. */
 function world(d = {}) {
-  const w = { issues: {}, closed: [], calls: [], killed: 0, pid: 100, results: {}, order: [], deps: {}, running: {}, clock: 0, pool: !!d.pool };
+  const w = { issues: {}, closed: [], calls: [], killed: 0, stops: [], pid: 100, results: {}, order: [], deps: {}, running: {}, clock: 0, pool: !!d.pool };
   const gh = async (a) => {
     w.calls.push(a.join(" "));
     if (a[0] === "api" && a[1] === "-X" && a[2] === "PUT") return "{}";
@@ -36,7 +36,9 @@ function world(d = {}) {
     },
     orchestrate: {
       start: async (repo, issues) => { w.order.push(...issues); w.running[issues[0]] = { pid: ++w.pid }; w.a = issues[0]; return { id: "b1", kill: async () => { w.killed++; } }; },
-      resume: async (id) => { w.resumedAt = w.tick ?? 0; if (d.double) w.running[w.a] = { pid: ++w.pid }; return { id, kill: async () => { w.killed++; } }; },
+      // stop = `orchestrate --cancel`: the lane is torn down (d.nostop: the defect where only the client dies and the lane lives on)
+      stop: async (id) => { w.stops.push(id); if (!d.nostop) delete w.running[w.a]; },
+      resume: async (id) => { w.resumedAt = w.tick ?? 0; w.running[w.a] = { pid: d.nostop ? w.running[w.a]?.pid ?? ++w.pid : ++w.pid }; return { id, kill: async () => { w.killed++; } }; },
       state: async () => ({ order: w.order, deps: w.deps, running: w.running, results: w.results }),
     },
     remote: w.pool ? { run: async ({ issue }) => { w.remoteIssue = issue; return { where: d.local ? "local" : "remote", authenticated: !d.noRemoteAuth, engineVersion: d.oldEngine ? "1.37.0" : "1.39.0" }; } } : null,
@@ -73,9 +75,14 @@ describe("selftest --live steps (#4865)", () => {
     const { r } = await run({ wipe4860: true });
     assert.equal(r.ok, false); assert.equal(r.step, "body"); assert.match(r.message, /lost its body text/);
   });
-  it("a lane launched twice after --resume fails the resume step", async () => {
-    const { r } = await run({ double: true });
-    assert.equal(r.step, "resume"); assert.match(r.message, /launched again/);
+  it("resume really stops the batch (not just the client) and cleanup stops it again (#4952)", async () => {
+    const { r, w } = await run({});
+    assert.equal(r.ok, true, JSON.stringify(r));
+    assert.deepEqual(w.stops, ["b1", "b1"], "once in the resume step, once in cleanup");
+  });
+  it("a stop that leaves the lane running fails the resume step instead of passing vacuously (#4952)", async () => {
+    const { r } = await run({ nostop: true }, { timeoutMs: 20000 });
+    assert.equal(r.step, "resume"); assert.match(r.message, /to stop after the cancel/);
   });
   it("B merging before A fails the merge step", async () => {
     const { r } = await run({ orderBA: true });
@@ -89,6 +96,41 @@ describe("selftest --live steps (#4865)", () => {
     const ok = await run({ pool: true });
     assert.equal(ok.w.remoteIssue, 12); assert.deepEqual(ok.w.closed, ["10", "11", "12"]);
   });
+  it("fixtures are run-unique, so a merged run leaves nothing the next run trips on (#4952)", async () => {
+    const x = uniqueFixture(FIXTURES.node, "node", "r1"), y = uniqueFixture(FIXTURES.python, "python", "r1");
+    assert.match(x.a.body, /`src\/r1\/a\.js`.*`src\/r1\/a\.test\.js`/); assert.match(x.a.title, /\(r1\)/);
+    assert.match(y.b.body, /`r1\/b\.py`.*`r1\/test_b\.py`/);
+    const { w } = await run({});
+    assert.match(w.issues[10].body, /src\/r[0-9a-z]+\/a\.js/);
+  });
+  it("the created issue is tracked at once: A is closed when B's create fails (#4952)", async () => {
+    const { w, io } = world({}), gh = io.gh; let creates = 0;
+    io.gh = async (a) => { if (a[0] === "issue" && a[1] === "create" && ++creates === 2) throw new Error("rate limited"); return gh(a); };
+    const r = await runSelftestLive({ repo: "o/sandbox", io, pollMs: 1, timeoutMs: 60000 });
+    assert.equal(r.step, "create"); assert.deepEqual(w.closed, ["10"]);
+  });
+  it("cleanup matches branches exactly and finds PRs by the head branch it created (#4952)", async () => {
+    const { w, io } = world({ add4859: true }), gh = io.gh, log = [];
+    io.gh = async (a) => {
+      if (a[0] === "api" && a[1].includes("matching-refs/heads/forge/issue-10")) return "refs/heads/forge/issue-10\nrefs/heads/forge/issue-100\nrefs/heads/forge/issue-10-x\n";
+      if (a[0] === "api" && a[2] === "DELETE") { log.push(`del ${a[3].split("git/")[1]}`); return ""; }
+      if (a[0] === "api") return "";
+      if (a[0] === "pr" && a[1] === "list") { log.push(`list ${a[a.indexOf("--head") + 1]}`); return a.includes("forge/issue-10") ? "7\n" : ""; }
+      if (a[0] === "pr" && a[1] === "close") { log.push(`close ${a[2]}`); return ""; }
+      return gh(a);
+    };
+    const r = await runSelftestLive({ repo: "o/sandbox", io, pollMs: 1, timeoutMs: 60000 });
+    assert.deepEqual(r.cleanupErrors, []);
+    assert.deepEqual(log, ["list forge/issue-10", "close 7", "del refs/heads/forge/issue-10", "list forge/issue-10-x", "del refs/heads/forge/issue-10-x"]);
+    assert.ok(!w.calls.some((c) => /search/.test(c)), "no text search for PRs");
+  });
+  it("own batch is picked by repo and issues: a foreign concurrent batch is never adopted (#4952)", () => {
+    const mine = { repo: "o/Sandbox", argv: ["orchestrate", "10", "11", "--repo", "o/Sandbox"] };
+    assert.ok(isOwnBatch(mine, "o/sandbox", [10, 11]));
+    assert.ok(!isOwnBatch({ repo: "o/real", argv: ["orchestrate", "10", "11"] }, "o/sandbox", [10, 11]));
+    assert.ok(!isOwnBatch({ repo: "o/sandbox", argv: ["orchestrate", "7"] }, "o/sandbox", [10, 11]));
+    assert.ok(!isOwnBatch(null, "o/sandbox", [10]));
+  });
   it("unknown fixture language fails at setup", async () => {
     assert.equal((await runSelftestLive({ repo: "o/r", lang: "ruby", io: {} })).step, "setup");
   });

package/bin/lean/selftest-nightly.mjs

@@ -9,9 +9,9 @@ import { spawn } from "node:child_process";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { schedDir } from "./sched.mjs";
+import { neutralize, findMarked } from "./prodwatch-triage.mjs";
 
 export const NIGHTLY_MARKER = "<!-- FORGE:SELFTEST_NIGHTLY -->";
-const neutralize = (s) => String(s ?? "").replace(/</g, "&lt;").replace(/>/g, "&gt;");
 /** One last-run file per repo: two repos on one host never share (and so never skip) each other's schedule. */
 const stateFile = (repo) => `selftest-nightly-${String(repo).replace(/[^\w.-]+/g, "_")}.json`;
 
@@ -37,7 +37,7 @@ function writeLast(dir, repo, ts) {
  */
 export function launchNightly({ cli, repo, cwd = process.cwd(), spawnFn = spawn }) {
   try {
-    const c = spawnFn(process.execPath, [cli, "selftest", "--live", "--nightly-report", repo], { cwd, detached: true, stdio: "ignore" });
+    const c = spawnFn(process.execPath, [cli, "selftest", "--live", "--nightly-report", repo], { cwd, detached: true, windowsHide: true, stdio: "ignore" });
     c.on("error", () => {});
     c.unref();
     return c;
@@ -49,8 +49,12 @@ export async function reportNightly({ repo, io, now = () => Date.now(), result }
   const step = result?.step ?? "unknown";
   const title = `selftest --live failed at step ${step} (nightly)`;
   const body = `The nightly \`forgedock selftest --live\` failed at step **${neutralize(step)}**.\n\n${neutralize(String(result?.message ?? "").slice(0, 2000))}\n\nLast failure: ${new Date(now()).toISOString()}\n\n${NIGHTLY_MARKER}\n`;
-  const open = JSON.parse(String(await io.gh(["issue", "list", "-R", repo, "--state", "open", "--search", `"FORGE:SELFTEST_NIGHTLY" in:body`, "--limit", "30", "--json", "number,body"])))
-    .find((i) => String(i.body ?? "").includes(NIGHTLY_MARKER));
+  // #4952: the marker is public text, so only an issue authored by the actor running the harness is edited; an unreadable
+  // login trusts nobody (a new issue is filed instead).
+  let me = "";
+  try { me = String(await io.gh(["api", "user", "--jq", ".login"])).trim().toLowerCase(); } catch { /* untrusted */ }
+  const issues = JSON.parse(String(await io.gh(["issue", "list", "-R", repo, "--state", "open", "--search", `"FORGE:SELFTEST_NIGHTLY" in:body`, "--limit", "30", "--json", "number,body,author"])));
+  const open = findMarked(issues, NIGHTLY_MARKER, (i) => me && String(i.author?.login ?? "").toLowerCase() === me);
   if (open) {
     await io.gh(["issue", "edit", String(open.number), "-R", repo, "--title", title, "--body", body]);
     return { issue: open.number, action: "edit" };

package/bin/lean/selftest-nightly.test.mjs

@@ -8,7 +8,7 @@ import { nightlyConfig, nightlyDue, launchNightly, NIGHTLY_MARKER } from "./self
 
 const setup = (yaml) => { const cwd = tmpDir("nt-"), stateDir = tmpDir("ns-"); if (yaml) writeFileSync(join(cwd, "forge.yaml"), yaml); return { cwd, stateDir }; };
 const YAML = "selftest:\n  sandbox_repo: a/b\n  nightly_interval_hours: 24\n";
-const fakeIo = (open = []) => { const calls = []; return { calls, gh: async (a) => { calls.push(a); if (a[1] === "list") return JSON.stringify(open); if (a[1] === "create") return "https://github.com/o/r/issues/77\n"; return ""; } }; };
+const fakeIo = (open = []) => { const calls = []; return { calls, gh: async (a) => { calls.push(a); if (a[0] === "api") return "bot\n"; if (a[1] === "list") return JSON.stringify(open); if (a[1] === "create") return "https://github.com/o/r/issues/77\n"; return ""; } }; };
 
 test("nightlyConfig: unset, zero and other sections are off", () => {
   assert.equal(nightlyConfig(""), null);
@@ -42,7 +42,7 @@ test("failure files one issue naming the step; the second failure edits it", asy
   assert.match(create.join(" "), /step body/); assert.ok(create.join(" ").includes(NIGHTLY_MARKER));
   assert.ok(!create[create.indexOf("--body") + 1].includes("<!-- x"));
   t += 25 * 3600_000;
-  const io2 = fakeIo([{ number: 77, body: `x\n${NIGHTLY_MARKER}` }]);
+  const io2 = fakeIo([{ number: 77, body: `x\n${NIGHTLY_MARKER}`, author: { login: "Bot" } }]);
   const r2 = await nightlyDue({ ...s, repo: "o/r", io: io2, now: () => t, runLive: async () => ({ ok: false, step: "add", message: "m" }) });
   assert.deepEqual([r2.action, r2.issue], ["edit", 77]);
   assert.ok(!io2.calls.some((a) => a[1] === "create"));
@@ -82,3 +82,25 @@ test("launchNightly: detached, unref'd, and an async spawn error never escapes (
   assert.doesNotThrow(() => child.emit("error", Object.assign(new Error("spawn EAGAIN"), { code: "EAGAIN" })));
   assert.equal(launchNightly({ cli: "c", repo: "o/r", spawnFn: () => { throw new Error("EMFILE"); } }), null);
 });
+
+test("the dedupe never edits a marker-bearing issue authored by someone else (#4952)", async () => {
+  const s = setup(YAML), io = fakeIo([{ number: 5, body: `evil\n${NIGHTLY_MARKER}`, author: { login: "mallory" } }, { number: 6, body: `no marker`, author: { login: "bot" } }]);
+  const r = await nightlyDue({ ...s, repo: "o/r", io, runLive: async () => ({ ok: false, step: "add", message: "m" }) });
+  assert.deepEqual([r.action, r.issue], ["create", 77]);
+  assert.ok(!io.calls.some((a) => a[1] === "edit"));
+  assert.ok(io.calls.find((a) => a[1] === "list").join(" ").includes("number,body,author"));
+});
+
+test("an unreadable login trusts nobody: a new issue is filed", async () => {
+  const s = setup(YAML), io = fakeIo([{ number: 5, body: NIGHTLY_MARKER, author: { login: "bot" } }]), gh = io.gh;
+  io.gh = async (a) => { if (a[0] === "api") throw new Error("no token user"); return gh(a); };
+  const r = await nightlyDue({ ...s, repo: "o/r", io, runLive: async () => ({ ok: false, step: "add", message: "m" }) });
+  assert.equal(r.action, "create");
+});
+
+test("a marker quoted mid-body by the trusted author is not a dedupe hit (shared trailer rule)", async () => {
+  const s = setup(YAML), io = fakeIo([{ number: 5, body: `quote: ${NIGHTLY_MARKER}\nmore text`, author: { login: "bot" } }]);
+  const r = await nightlyDue({ ...s, repo: "o/r", io, runLive: async () => ({ ok: false, step: "add", message: "m" }) });
+  assert.equal(r.action, "create");
+  assert.ok(!io.calls.some((a) => a[1] === "edit"));
+});