forgedock 1.27.0 → 1.28.0

package/bin/lean/sched.mjs

@@ -0,0 +1,221 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Host lane registry (#4191): one host-wide cap on running `forgedock orchestrate` lanes across every batch and repo,
+ * and cross-batch priority admission. State is one file, `~/.forge/sched/registry.json` (or `FORGEDOCK_SCHED_DIR`):
+ * `{v, cap, weights, seq, slots:{key:{pid, lanePid?, batch, repo, issue, priority, at}}, queue:{key:{pid, batch, repo,
+ * issue, priority, seq, enqueuedAt}}}` with key `<batch>#<issue>`. Every mutation is a read-modify-write under an
+ * exclusive lock file (`registry.lock`, holding the owner pid; a dead owner's lock is taken over) and `writeAtomic`.
+ *
+ * A dispatcher offers the issues it could start now; `tryAcquire` grants a slot only while live slots < cap and the
+ * key ranks within the free slots of the queue in grant order (`grantOrder`): priority first, so a P0 offered last is
+ * granted next. Within a priority, slots are shared across repos by the registry's `weights` (#4193; `{"owner/repo": N}`,
+ * `forgedock sched weight`, unset = 1): the next entry comes from the repo with the smallest (live slots + entries
+ * ordered ahead) / weight, FIFO inside a repo; an entry without a repo groups by batch. Running lanes are never revoked.
+ * A queued entry dies with its dispatcher pid; a slot dies with its dispatcher unless its detached lane (`lanePid`)
+ * still runs, so an orphaned lane keeps counting until it exits or a `--resume` adopts it. Cap: the registry's own
+ * (`forgedock sched cap N`), else the caller's forge.yaml `orchestration.global_parallel`, else none. Fail-open: an
+ * unreadable registry or a lock not taken within `lockWaitMs` grants (`fallback: true`), since a stuck registry would
+ * stall every batch on the host.
+ */
+import { closeSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { writeAtomic } from "./batch.mjs";
+
+export const DEFAULT_PRIORITY = 2;
+export const LOCK_WAIT_MS = 2000;
+const EMPTY_LOCK_STALE_MS = 5000;
+
+export function schedDir(env = process.env) {
+  return env.FORGEDOCK_SCHED_DIR || join(homedir(), ".forge", "sched");
+}
+
+/** Priority from label names (or `{name}` objects): `priority:P<n>` or bare `P<n>`, lowest n wins; unlabelled = 2. */
+export function priorityFromLabels(labels) {
+  let best = null;
+  for (const l of labels ?? []) {
+    const m = /^(?:priority:)?P(\d+)$/i.exec(String(l?.name ?? l ?? "").trim());
+    if (m && (best === null || Number(m[1]) < best)) best = Number(m[1]);
+  }
+  return best ?? DEFAULT_PRIORITY;
+}
+
+export function pidAlive(pid) {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try { process.kill(pid, 0); return true; } catch (e) { return e?.code === "EPERM"; }
+}
+
+const posInt = (k) => (Number.isInteger(k) && k >= 1 ? k : null);
+const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+const byRank = ([ka, a], [kb, b]) => (a.priority - b.priority) || (a.seq - b.seq) || (ka < kb ? -1 : ka > kb ? 1 : 0);
+const groupOf = (k, e) => e?.repo ?? e?.batch ?? k;
+
+/**
+ * Queue entries `[key, entry]` in grant order: priority ascending, then a weighted fair share across repos within each
+ * priority (smallest load/weight first, load = live slots + entries already ordered; ties: higher weight, then FIFO).
+ */
+export function grantOrder(s) {
+  const weight = (g) => posInt(s.weights?.[g]) ?? 1;
+  const load = new Map();
+  for (const [k, e] of Object.entries(s.slots ?? {})) { const g = groupOf(k, e); load.set(g, (load.get(g) ?? 0) + 1); }
+  const levels = new Map(); // priority -> Map(group -> FIFO entries)
+  for (const ent of Object.entries(s.queue ?? {}).sort(byRank)) {
+    const [k, e] = ent;
+    if (!levels.has(e.priority)) levels.set(e.priority, new Map());
+    const groups = levels.get(e.priority), g = groupOf(k, e);
+    if (!groups.has(g)) groups.set(g, []);
+    groups.get(g).push(ent);
+  }
+  const out = [];
+  for (const p of [...levels.keys()].sort((a, b) => a - b)) {
+    const groups = levels.get(p);
+    while (groups.size) {
+      let best = null;
+      for (const [g, list] of groups) {
+        if (!best) { best = g; continue; }
+        const lg = (load.get(g) ?? 0) * weight(best), lb = (load.get(best) ?? 0) * weight(g);
+        if (lg < lb || (lg === lb && (weight(g) > weight(best) || (weight(g) === weight(best) && byRank(list[0], groups.get(best)[0]) < 0)))) best = g;
+      }
+      const list = groups.get(best);
+      out.push(list.shift());
+      load.set(best, (load.get(best) ?? 0) + 1);
+      if (!list.length) groups.delete(best);
+    }
+  }
+  return out;
+}
+
+/**
+ * Handle on the registry. `pid` is this dispatcher; `alive(pid)` is the liveness probe (injected in tests).
+ * @returns {{dir, tryAcquire, offer, withdraw, release, setLanePid, adoptSlot, cap, setCap, weights, setWeight, list}}
+ */
+export function openSched(dir = schedDir(), { now = () => Date.now(), pid = process.pid, alive = pidAlive, lockWaitMs = LOCK_WAIT_MS } = {}) {
+  const file = join(dir, "registry.json"), lockFile = join(dir, "registry.lock");
+
+  const lock = () => {
+    mkdirSync(dir, { recursive: true });
+    const end = Date.now() + lockWaitMs;
+    for (;;) {
+      try {
+        const fd = openSync(lockFile, "wx");
+        try { writeSync(fd, String(pid)); } finally { closeSync(fd); }
+        return;
+      } catch (e) { if (e?.code !== "EEXIST") throw e; }
+      let owner = NaN;
+      try { owner = parseInt(readFileSync(lockFile, "utf8"), 10); } catch { /* released meanwhile */ }
+      // An empty lock is one being created (open, then the pid is written): stale only once it is old.
+      let empty = false;
+      if (Number.isNaN(owner)) { try { empty = Date.now() - statSync(lockFile).mtimeMs < EMPTY_LOCK_STALE_MS; } catch { continue; } }
+      if (!empty && (Number.isNaN(owner) || !alive(owner))) {
+        // Take over a dead owner's lock: move it aside, and put it back if it turned out to be a fresh one.
+        const aside = `${lockFile}.stale-${pid}`;
+        try {
+          renameSync(lockFile, aside);
+          let got = NaN;
+          try { got = parseInt(readFileSync(aside, "utf8"), 10); } catch { /* gone */ }
+          if (!Number.isNaN(got) && got !== owner && alive(got)) { try { renameSync(aside, lockFile); } catch { /* best effort */ } }
+          else rmSync(aside, { force: true });
+        } catch { /* someone else took it over */ }
+        continue;
+      }
+      if (Date.now() >= end) throw new Error(`sched registry lock held by pid ${Number.isNaN(owner) ? "?" : owner}`);
+      pause(10);
+    }
+  };
+  const unlock = () => { try { rmSync(lockFile, { force: true }); } catch { /* best effort */ } };
+
+  const read = () => {
+    let o;
+    try { o = JSON.parse(readFileSync(file, "utf8")); } catch (e) {
+      if (e?.code !== "ENOENT") throw new Error(`sched registry ${file} unreadable: ${e?.message ?? e}`);
+      o = {};
+    }
+    const obj = (x) => (x && typeof x === "object" && !Array.isArray(x) ? x : {});
+    const weights = Object.fromEntries(Object.entries(obj(o?.weights)).filter(([, n]) => posInt(n)));
+    return { v: 1, cap: posInt(o?.cap), weights, seq: Number.isInteger(o?.seq) ? o.seq : 0, slots: obj(o?.slots), queue: obj(o?.queue) };
+  };
+  // Drop queued entries of dead dispatchers, and slots whose dispatcher and lane are both gone.
+  const reclaim = (s) => {
+    for (const [k, e] of Object.entries(s.queue)) if (!alive(e.pid)) delete s.queue[k];
+    for (const [k, e] of Object.entries(s.slots)) if (!alive(e.pid) && !(e.lanePid && alive(e.lanePid))) delete s.slots[k];
+  };
+  const mutate = (fn) => {
+    lock();
+    try {
+      const s = read();
+      reclaim(s);
+      const r = fn(s);
+      writeAtomic(file, JSON.stringify(s, null, 2) + "\n");
+      return r;
+    } finally { unlock(); }
+  };
+  const enqueue = (s, key, e) => {
+    if (s.slots[key]) return;
+    const prev = s.queue[key];
+    const priority = Number.isFinite(Number(e.priority)) ? Number(e.priority) : DEFAULT_PRIORITY;
+    s.queue[key] = prev
+      ? { ...prev, pid, priority }
+      : { pid, batch: e.batch ?? null, repo: e.repo ?? null, issue: e.issue ?? null, priority, seq: ++s.seq, enqueuedAt: now() };
+  };
+  const effCap = (s, callerCap) => s.cap ?? posInt(callerCap);
+
+  return {
+    dir,
+    /** Grant `key` a slot, queueing it when none is free for it now. @returns {{ok:boolean, fallback?:true, reason?:string}} */
+    tryAcquire: (key, entry = {}, callerCap = null) => {
+      try {
+        return mutate((s) => {
+          if (s.slots[key]) return { ok: true };
+          enqueue(s, key, entry);
+          const cap = effCap(s, callerCap);
+          if (cap !== null) {
+            const free = cap - Object.keys(s.slots).length;
+            const rank = grantOrder(s).findIndex(([k]) => k === key);
+            if (free <= 0) return { ok: false, reason: `host cap ${cap} reached` };
+            if (rank >= free) return { ok: false, reason: `queued behind ${rank} higher-ranked issue(s)` };
+          }
+          const q = s.queue[key];
+          delete s.queue[key];
+          s.slots[key] = { pid, batch: q.batch, repo: q.repo, issue: q.issue, priority: q.priority, at: now() };
+          return { ok: true };
+        });
+      } catch (e) { return { ok: true, fallback: true, reason: e?.message ?? String(e) }; }
+    },
+    /** Queue entries `[{key, batch, repo, issue, priority}]` (their FIFO position is kept on a re-offer). */
+    offer: (entries) => { try { mutate((s) => { for (const e of entries) enqueue(s, e.key, e); }); } catch { /* fail-open */ } },
+    withdraw: (keys) => { if (!keys.length) return; try { mutate((s) => { for (const k of keys) delete s.queue[k]; }); } catch { /* fail-open */ } },
+    release: (key) => { try { mutate((s) => { delete s.slots[key]; delete s.queue[key]; }); } catch { /* fail-open */ } },
+    setLanePid: (key, lanePid) => { try { mutate((s) => { if (s.slots[key] && posInt(lanePid)) s.slots[key].lanePid = lanePid; }); } catch { /* fail-open */ } },
+    /** Record a lane this dispatcher re-attached to: it runs already, so the cap does not gate it. */
+    adoptSlot: (key, entry = {}) => {
+      try {
+        mutate((s) => {
+          delete s.queue[key];
+          s.slots[key] = { pid, ...(posInt(entry.lanePid) ? { lanePid: entry.lanePid } : {}), batch: entry.batch ?? null, repo: entry.repo ?? null,
+            issue: entry.issue ?? null, priority: Number.isFinite(Number(entry.priority)) ? Number(entry.priority) : DEFAULT_PRIORITY, at: now() };
+        });
+      } catch { /* fail-open */ }
+    },
+    cap: () => read().cap,
+    /** Set the live host cap (a positive integer), or null to fall back to each dispatcher's config. Throws. */
+    setCap: (k) => {
+      if (k !== null && !posInt(k)) throw new Error("cap must be a positive integer");
+      return mutate((s) => { s.cap = k; return k; });
+    },
+    weights: () => read().weights,
+    /** Set the fair-share weight of `repo` (a positive integer), or null to reset it to 1. Throws. */
+    setWeight: (repo, n) => {
+      if (typeof repo !== "string" || !repo.trim()) throw new Error("repo must be a non-empty string");
+      if (n !== null && !posInt(n)) throw new Error("weight must be a positive integer");
+      return mutate((s) => { if (n === null) delete s.weights[repo]; else s.weights[repo] = n; return n; });
+    },
+    /** Live state after reclaim: `{cap, weights, slots:[{key,...}], queue:[{key,...}] in grant order}`. Throws when unreadable. */
+    list: () => mutate((s) => ({
+      cap: s.cap,
+      weights: { ...s.weights },
+      slots: Object.entries(s.slots).map(([key, e]) => ({ key, ...e })),
+      queue: grantOrder(s).map(([key, e]) => ({ key, ...e })),
+    })),
+  };
+}

package/bin/lean/sched.test.mjs

@@ -0,0 +1,310 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpDir } from "../tests/_tmp.mjs";
+import { grantOrder, openSched, priorityFromLabels } from "./sched.mjs";
+import { orchestrate } from "./orchestrate.mjs";
+import { schedFromCli } from "../engine-cli.mjs";
+
+const quiet = async (fn) => { const l = console.log; console.log = () => {}; try { return await fn(); } finally { console.log = l; } };
+const until = async (cond, what, ms = 10000) => {
+  const end = Date.now() + ms;
+  while (!cond()) { if (Date.now() > end) throw new Error(`timed out waiting for ${what}`); await new Promise((r) => setTimeout(r, 2)); }
+};
+// Injected pids and clock: `live` is the set of pids the registry sees as alive.
+const world = () => {
+  const live = new Set(), clock = { t: 1_000_000 };
+  const dir = tmpDir("sched-");
+  const handle = (pid, extra = {}) => { live.add(pid); return openSched(dir, { pid, alive: (p) => live.has(p), now: () => ++clock.t, ...extra }); };
+  return { dir, live, handle };
+};
+
+describe("host lane registry (#4191)", () => {
+  it("priorityFromLabels: priority:P<n> or bare P<n>, lowest wins, unlabelled = P2", () => {
+    assert.equal(priorityFromLabels(["priority:P0"]), 0);
+    assert.equal(priorityFromLabels([{ name: "P1" }, { name: "bug" }]), 1);
+    assert.equal(priorityFromLabels(["priority:P3", "p1", "Priority:P2"]), 1);
+    assert.equal(priorityFromLabels(["bug", "P", "priority:high", "XP0"]), 2);
+    assert.equal(priorityFromLabels([]), 2);
+    assert.equal(priorityFromLabels(undefined), 2);
+  });
+
+  it("grants within the cap in (priority, FIFO) order; a P0 queued last is granted next", () => {
+    const { handle } = world();
+    const a = handle(101), b = handle(102), c = handle(103);
+    const e = (batch, issue, priority) => ({ batch, repo: "o/r", issue, priority });
+    assert.equal(a.tryAcquire("A#1", e("A", 1, 2), 2).ok, true);
+    assert.equal(b.tryAcquire("B#1", e("B", 1, 2), 2).ok, true);
+    const r = b.tryAcquire("B#2", e("B", 2, 2), 2);
+    assert.equal(r.ok, false);
+    assert.match(r.reason, /host cap 2/);
+    a.offer([{ key: "A#2", ...e("A", 2, 3) }]);
+    c.offer([{ key: "C#9", ...e("C", 9, 0) }]); // P0, offered last
+    assert.deepEqual(a.list().queue.map((q) => q.key), ["C#9", "B#2", "A#2"]);
+    a.release("A#1");
+    assert.equal(b.tryAcquire("B#2", e("B", 2, 2), 2).ok, false, "the freed slot belongs to the P0");
+    assert.equal(c.tryAcquire("C#9", e("C", 9, 0), 2).ok, true);
+    b.release("B#1");
+    assert.equal(a.tryAcquire("A#2", e("A", 2, 3), 2).ok, false, "FIFO within priority: B#2 (P2) before A#2 (P3)");
+    assert.equal(b.tryAcquire("B#2", e("B", 2, 2), 2).ok, true);
+    assert.equal(a.list().slots.length, 2);
+    // A re-offer keeps the FIFO position.
+    a.offer([{ key: "A#2", ...e("A", 2, 3) }]);
+    assert.deepEqual(a.list().queue.map((q) => q.key), ["A#2"]);
+  });
+
+  it("dead dispatchers' queued entries and slots are reclaimed; a slot whose detached lane still runs is kept", () => {
+    const { handle, live } = world();
+    const a = handle(201), b = handle(202);
+    assert.equal(a.tryAcquire("A#1", { issue: 1 }, 2).ok, true);
+    assert.equal(a.tryAcquire("A#2", { issue: 2 }, 2).ok, true);
+    a.setLanePid("A#2", 9002);
+    live.add(9002);
+    a.offer([{ key: "A#3", issue: 3, priority: 0 }]);
+    assert.equal(b.tryAcquire("B#1", { issue: 1 }, 2).ok, false);
+    live.delete(201); // dispatcher A dies
+    assert.deepEqual(b.list().slots.map((s) => s.key), ["A#2"], "A#1 reclaimed; A#2's lane is still running");
+    assert.deepEqual(b.list().queue.map((q) => q.key), ["B#1"], "A's queued P0 is gone");
+    assert.equal(b.tryAcquire("B#1", { issue: 1 }, 2).ok, true);
+    live.delete(9002); // the orphaned lane exits
+    assert.deepEqual(b.list().slots.map((s) => s.key), ["B#1"]);
+    // A resume adopts a live lane without being gated by the cap.
+    const a2 = handle(203);
+    a2.adoptSlot("A#4", { issue: 4, lanePid: 9004 });
+    a2.adoptSlot("A#5", { issue: 5 });
+    assert.equal(a2.list().slots.length, 3);
+  });
+
+  it("forgedock sched cap changes the cap live; --clear falls back to the caller's config", async () => {
+    const { handle } = world();
+    const a = handle(301), b = handle(302);
+    assert.equal(a.tryAcquire("A#1", {}, 5).ok, true);
+    await quiet(() => schedFromCli(["cap", "1"], { sched: a }));
+    assert.equal(b.cap(), 1);
+    assert.equal(b.tryAcquire("B#1", {}, 5).ok, false, "the registry cap overrides the config cap");
+    const shown = await quiet(() => schedFromCli(["show"], { sched: b }));
+    assert.deepEqual([shown.cap, shown.slots.length, shown.queue.length], [1, 1, 1]);
+    await quiet(() => schedFromCli(["cap", "--clear"], { sched: a }));
+    assert.equal(b.tryAcquire("B#1", {}, 5).ok, true);
+    assert.equal(b.tryAcquire("B#2", {}, null).ok, true, "no cap anywhere: always granted");
+    for (const bad of [["cap"], ["cap", "0"], ["cap", "x"], ["cap", "1.5"], ["nope"]]) {
+      await assert.rejects(() => quiet(() => schedFromCli(bad, { sched: a })), /usage: forgedock sched/);
+    }
+  });
+
+  it("fails open: an unreadable registry or a held lock grants with fallback", () => {
+    const { dir, handle } = world();
+    writeFileSync(join(dir, "registry.json"), "{not json");
+    const a = handle(401);
+    const r = a.tryAcquire("A#1", {}, 1);
+    assert.deepEqual([r.ok, r.fallback], [true, true]);
+    assert.match(r.reason, /unreadable/);
+    const w2 = world();
+    const owner = w2.handle(402);
+    writeFileSync(join(w2.dir, "registry.lock"), "402"); // a live owner never releases it
+    const b = w2.handle(403, { lockWaitMs: 30 });
+    const r2 = b.tryAcquire("B#1", {}, 1);
+    assert.deepEqual([r2.ok, r2.fallback], [true, true]);
+    assert.match(r2.reason, /lock held by pid 402/);
+    // A dead owner's lock is taken over.
+    w2.live.delete(402);
+    const r3 = b.tryAcquire("B#1", {}, 1);
+    assert.deepEqual([r3.ok, r3.fallback], [true, undefined]);
+    assert.ok(owner);
+  });
+
+  it("14 dispatchers against one registry: running lanes ≤ the cap, each batch ≤ its K, a P0 submitted last starts next", async () => {
+    const { handle, live } = world();
+    const CAP = 4, K = 2, BATCHES = 14;
+    const inFlight = new Map(); // key -> resolve
+    const perBatch = new Map();
+    const starts = [];
+    let maxHost = 0, maxBatch = 0;
+    let autoRelease = false;
+    const runOneFor = (batch) => async (n) => {
+      const key = `${batch}#${n}`;
+      starts.push(key);
+      perBatch.set(batch, (perBatch.get(batch) ?? 0) + 1);
+      maxHost = Math.max(maxHost, inFlight.size + 1);
+      maxBatch = Math.max(maxBatch, perBatch.get(batch));
+      const r = await new Promise((res) => { inFlight.set(key, res); if (autoRelease) setTimeout(() => res(), 1); });
+      inFlight.delete(key);
+      perBatch.set(batch, perBatch.get(batch) - 1);
+      return r ?? { terminalReason: "merged", pr: n, cost_usd: 0 };
+    };
+    const dispatch = (i, issues, prio) => {
+      const batch = `b${i}`;
+      const registry = handle(5000 + i);
+      return orchestrate({ issues, parallel: K, readBody: async () => "", runOne: runOneFor(batch), pollMs: 2, batchId: batch,
+        sched: { registry, batch, repo: `o/r${i % 3}`, cap: CAP }, priorityOf: async (n) => prio(n) });
+    };
+    const reg = handle(4999);
+    const runs = [];
+    for (let i = 0; i < BATCHES; i++) runs.push(dispatch(i, [1, 2, 3], () => 2));
+    await until(() => inFlight.size === CAP && reg.list().queue.length === BATCHES * K - CAP, "the cap to fill and every batch to offer");
+    assert.equal(reg.list().slots.length, CAP);
+    // A P0 submitted last, in its own batch behind 24 queued P2s.
+    runs.push(dispatch(BATCHES, [7, 8], (n) => (n === 7 ? 0 : 3)));
+    await until(() => reg.list().queue.some((q) => q.key === `b${BATCHES}#7`), "the P0 to be queued");
+    assert.equal(reg.list().queue[0].key, `b${BATCHES}#7`);
+    const before = starts.length;
+    inFlight.get(starts[0])(); // one lane ends
+    await until(() => starts.length > before, "the next start");
+    assert.equal(starts[before], `b${BATCHES}#7`, "the freed slot went to the P0");
+    assert.equal(starts.length, before + 1);
+    autoRelease = true;
+    for (const res of [...inFlight.values()]) res();
+    const results = await Promise.all(runs);
+    assert.ok(results.flat().every((r) => r.outcome === "merged"), JSON.stringify(results));
+    assert.equal(starts.length, BATCHES * 3 + 2);
+    assert.ok(maxHost <= CAP, `host lanes peaked at ${maxHost}`);
+    assert.ok(maxBatch <= K, `a batch peaked at ${maxBatch}`);
+    const end = reg.list();
+    assert.deepEqual([end.slots.length, end.queue.length], [0, 0], "every slot released and offer withdrawn");
+    assert.ok(live.size > BATCHES);
+  });
+
+  it("a dead dispatcher's slots are reclaimed so a waiting batch proceeds; adopted lanes hold a slot until they settle", async () => {
+    const { handle, live } = world();
+    const ghost = handle(6001);
+    assert.equal(ghost.tryAcquire("g#1", { issue: 1 }, 1).ok, true);
+    const registry = handle(6002);
+    let ran = 0;
+    const p = orchestrate({ issues: [1], readBody: async () => "", runOne: async () => { ran++; return { terminalReason: "merged", cost_usd: 0 }; },
+      pollMs: 2, batchId: "b", sched: { registry, batch: "b", repo: "o/r", cap: 1 } });
+    await until(() => registry.list().queue.length === 1, "the offer");
+    assert.equal(ran, 0, "the host is full");
+    live.delete(6001); // the other dispatcher dies
+    const rows = await p;
+    assert.deepEqual([rows[0].outcome, ran], ["merged", 1]);
+    // --resume: a live lane recorded by the earlier dispatcher is adopted into the registry, and released once it settles.
+    const saved = { v: 1, order: [5], deps: { 5: [] }, results: {}, running: { 5: { startedAt: 0, pid: 777 } }, pending: [] };
+    let store = saved;
+    const calls = [];
+    const spy = { ...registry, adoptSlot: (k, e) => { calls.push(["adopt", k, e.lanePid]); registry.adoptSlot(k, e); calls.push(["slots", registry.list().slots.map((x) => x.key)]); },
+      release: (k) => { calls.push(["release", k]); registry.release(k); } };
+    live.add(777);
+    const q = await orchestrate({ issues: [], readBody: async () => "", runOne: async () => ({ terminalReason: "merged" }), adopt: async () => ({ terminalReason: "merged", cost_usd: 0 }),
+      store: { load: () => store, save: (s) => { store = s; } }, pollMs: 2, batchId: "r", sched: { registry: spy, batch: "r", repo: "o/r", cap: 1 } });
+    assert.equal(q[0].outcome, "merged");
+    assert.deepEqual(calls, [["adopt", "r#5", 777], ["slots", ["r#5"]], ["release", "r#5"]]);
+    assert.equal(registry.list().slots.length, 0);
+    assert.deepEqual(store.priorities, { 5: 2 });
+  });
+
+  it("an unreadable registry admits without the global cap and says so once", async () => {
+    const { dir, handle } = world();
+    writeFileSync(join(dir, "registry.json"), "garbage");
+    const rows = [];
+    const res = await orchestrate({ issues: [1, 2], parallel: 2, readBody: async () => "", runOne: async () => ({ terminalReason: "merged", cost_usd: 0 }),
+      pollMs: 2, batchId: "f", sched: { registry: handle(7001), batch: "f", repo: "o/r", cap: 1 }, onUpdate: (r) => rows.push(r) });
+    assert.deepEqual(res.map((r) => r.outcome), ["merged", "merged"]);
+    assert.equal(rows.filter((r) => r.outcome === "sched").length, 1);
+  });
+
+  it("without a registry, admission and state are unchanged (no priorities key, FIFO)", async () => {
+    let store = null;
+    const order = [];
+    await orchestrate({ issues: [3, 1, 2], parallel: 1, readBody: async () => "", runOne: async (n) => { order.push(n); return { terminalReason: "merged" }; },
+      priorityOf: async (n) => (n === 2 ? 0 : 2), store: { load: () => null, save: (s) => { store = s; } } });
+    assert.deepEqual(order, [3, 1, 2]);
+    assert.equal("priorities" in store, false);
+  });
+});
+
+describe("cross-repo fairness weights (#4193)", () => {
+  // Two repos, one dispatcher each, keep twice the cap queued at P2 (o/b offers its first batch before o/a). Each
+  // round the oldest running lane ends and every queued key retries, lowest-ranked first, so only the registry's
+  // grant order decides who starts.
+  const saturate = ({ weights = {}, cap = 6 } = {}) => {
+    const { handle } = world();
+    const disp = { "o/a": handle(501), "o/b": handle(502) };
+    for (const [r, n] of Object.entries(weights)) disp["o/a"].setWeight(r, n);
+    const next = { "o/a": 0, "o/b": 0 };
+    const offer = (repo, priority = 2) => {
+      const n = ++next[repo];
+      disp[repo].offer([{ key: `${repo}#${n}`, batch: repo, repo, issue: n, priority }]);
+      return `${repo}#${n}`;
+    };
+    for (const r of ["o/b", "o/a"]) for (let i = 0; i < cap * 2; i++) offer(r);
+    const pass = () => {
+      for (let granted = true; granted;) {
+        granted = false;
+        for (const q of disp["o/a"].list().queue.reverse()) {
+          if (disp[q.repo].tryAcquire(q.key, q, cap).ok) { granted = true; if (q.priority === 2) offer(q.repo); }
+        }
+      }
+    };
+    const slots = () => disp["o/a"].list().slots;
+    const running = () => { const c = { "o/a": 0, "o/b": 0 }; for (const s of slots()) c[s.repo]++; return c; };
+    const endOldest = () => { const s = slots().sort((x, y) => x.at - y.at)[0]; disp[s.repo].release(s.key); };
+    return { disp, offer, pass, slots, running, endOldest };
+  };
+
+  it("two repos weighted 2:1, both saturated at one priority: running lanes converge to 2:1", () => {
+    const w = saturate({ weights: { "o/a": 2 } });
+    w.pass();
+    assert.deepEqual(w.running(), { "o/a": 4, "o/b": 2 }, "the first fill already follows the weights, not who offered first");
+    for (let round = 0; round < 40; round++) {
+      w.endOldest();
+      w.pass();
+      assert.deepEqual(w.running(), { "o/a": 4, "o/b": 2 }, `round ${round}`);
+    }
+  });
+
+  it("default weights are equal: a repo that offered first does not take every slot", () => {
+    const w = saturate();
+    w.pass();
+    assert.deepEqual(w.running(), { "o/a": 3, "o/b": 3 });
+    for (let round = 0; round < 20; round++) { w.endOldest(); w.pass(); assert.deepEqual(w.running(), { "o/a": 3, "o/b": 3 }); }
+  });
+
+  it("a P0 from any repo still starts next, even from the repo already at its share", () => {
+    const w = saturate({ weights: { "o/a": 2 } });
+    w.pass();
+    const p0 = w.offer("o/b", 0);
+    assert.equal(w.disp["o/a"].list().queue[0].key, p0, "the P0 heads the queue ahead of the heavier repo's P2s");
+    assert.equal(w.disp["o/a"].tryAcquire("o/a#13", { repo: "o/a", priority: 2 }, 6).ok, false, "no slot free");
+    w.endOldest();
+    assert.equal(w.disp["o/a"].tryAcquire(w.disp["o/a"].list().queue[1].key, { repo: "o/a", priority: 2 }, 6).ok, false, "the freed slot belongs to the P0");
+    assert.equal(w.disp["o/b"].tryAcquire(p0, { repo: "o/b", priority: 0 }, 6).ok, true);
+    assert.ok(w.slots().some((s) => s.key === p0));
+  });
+
+  it("grantOrder on an injected queue: priority first, then load/weight across repos, FIFO inside a repo", () => {
+    const q = (repo, seq, priority = 2, batch = null) => ({ repo, batch, issue: seq, priority, seq });
+    const s = {
+      weights: {},
+      slots: { "x#1": { repo: "o/a", priority: 2 }, "x#2": { repo: "o/a", priority: 1 } },
+      queue: { a1: q("o/a", 1), a2: q("o/a", 2), a3: q("o/a", 3), b1: q("o/b", 4), b2: q("o/b", 5), c0: q("o/c", 9, 0) },
+    };
+    assert.deepEqual(grantOrder(s).map(([k]) => k), ["c0", "b1", "b2", "a1", "a2", "a3"]);
+    s.weights = { "o/a": 3 };
+    assert.deepEqual(grantOrder(s).map(([k]) => k), ["c0", "b1", "a1", "a2", "b2", "a3"], "ties in load/weight go to the heavier repo");
+    // One repo (or no repo, grouped by batch, else by key): plain FIFO, as before #4193.
+    const one = { weights: {}, slots: {}, queue: { k3: q(null, 3, 2, "B"), k1: q(null, 1, 2, "B"), k2: q(null, 2, 2, "B") } };
+    assert.deepEqual(grantOrder(one).map(([k]) => k), ["k1", "k2", "k3"]);
+    const none = { weights: {}, slots: {}, queue: { k3: q(null, 3), k1: q(null, 1), k2: q(null, 2) } };
+    assert.deepEqual(grantOrder(none).map(([k]) => k), ["k1", "k2", "k3"]);
+  });
+
+  it("forgedock sched weight sets and clears a repo's weight; show lists it; invalid input is rejected", async () => {
+    const { handle } = world();
+    const a = handle(601);
+    assert.deepEqual(a.weights(), {});
+    await quiet(() => schedFromCli(["weight", "o/a", "2"], { sched: a }));
+    assert.deepEqual(a.weights(), { "o/a": 2 });
+    const shown = await quiet(() => schedFromCli(["show"], { sched: a }));
+    assert.deepEqual(shown.weights, { "o/a": 2 });
+    await quiet(() => schedFromCli(["weight", "o/a", "--clear"], { sched: a }));
+    assert.deepEqual(a.weights(), {});
+    for (const bad of [["weight"], ["weight", "o/a"], ["weight", "o/a", "0"], ["weight", "o/a", "1.5"], ["weight", "--clear"]]) {
+      await assert.rejects(() => quiet(() => schedFromCli(bad, { sched: a })), /usage: forgedock sched/);
+    }
+    assert.throws(() => a.setWeight("o/a", 0), /positive integer/);
+    assert.throws(() => a.setWeight("", 2), /non-empty/);
+  });
+});