@edgehero/pi-dispatch 1.6.0 → 1.7.0

package/src/index.mjs

@@ -3,13 +3,16 @@ import { promisify } from "node:util";
 import { DelayedError, UnrecoverableError, Worker } from "bullmq";
 import { InfraRetry, runJob } from "./processor.mjs";
 import { targetFor } from "./run-history.mjs";
-import { budgetCapsFor, canonicalScope, concurrencyFor, makeInFlight } from "./scoped-limits.mjs";
+import { budgetCapsFor, canonicalScope, concurrencyFor, makeInFlight, scopeKeyPrefix } from "./scoped-limits.mjs";
 import { WAIT_AFTER_MAX_DEFAULT_MS, WAIT_INTERVAL_FLOOR_MS, afterMs, unreadableConditions, waitArmed, waitBackoffMs, waitLabel, waitProfileNames } from "./wait-for.mjs";
 import { makeWaitState } from "./wait-state.mjs";
 
 const exec = promisify(execFile);
 
 export const QUEUE = "pi-jobs";
+
+/** The key the host-wide in-flight count lives under. One machine, one counter, whatever the queue. */
+export const HOST_SLOT_KEY = "host";
 export const JOB_TIMEOUT_MS = 30 * 60 * 1000; // REQ-JOB-TIMEOUT-30M
 // The scope-busy re-check (issue #242): a held scope has no natural "until" (the holder may run to
 // JOB_TIMEOUT_MS), so a deferred job re-tests on a fixed cadence. 5s keeps the worst case trivial
@@ -60,7 +63,7 @@ const THROTTLE_FLOOR_MS = 11_000;
  * The overlay changes which values the spend caps take, never when they are checked -- reserveBudget still
  * runs inside runJob against the freshly passed caps (CONST-BUDGET-BEFORE-TOKENS).
  */
-export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
+export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), hostBound = null, checkLease = null, scopeLease = null, deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, checkTimeoutMs = () => 10_000, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
 	return async function processor(job, token, signal) {
 		// Scoped pause windows (REQ-SCOPED-PAUSE-WINDOWS): if this job's folder/repo is inside an active pause
 		// window, DEFER it to the window end via BullMQ's delayed set -- the job keeps its identity/dedup and
@@ -227,23 +230,63 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 					throw new DelayedError();
 				}
 
-				// Claimed BEFORE the check, not after: a second delivery for an already-held target is a free
-				// determinate refusal, and paying for a subprocess first inverts the free-before-costly rule
-				// this gate's own header invokes. Tier 1 already claims in this order.
-				const claim = await waitState.claim(job.id, { dedupId, untilMs: nowMs + waitBackoffMs(intervalMs(), held), isLive: deps?.isJobLive });
-				if (claim.heldBy) {
-					return await refuseWait("wait-superseded", "wait_superseded", { heldBy: claim.heldBy }, "Another delivery for this target is already waiting on the same conditions. Not run.");
-				}
-				if (claim.retry) {
-					deps?.log?.("wait_supersede_unverified", { jobId: job.id, heldBy: claim.holder ?? null, delayMs: SUPERSEDE_RECHECK_MS });
-					await job.moveToDelayed(nowMs + SUPERSEDE_RECHECK_MS, token);
-					throw new DelayedError();
-				}
-
-				await waitState.noteThrottle(job.id, { denied: false }); // granted: the run of denials ends here
+				// Declared outside the try below because the branches AFTER it read both.
 				let verdict = null;
 				let checked = null;
+				// A SECOND LAYER BENEATH THE FIRST, never a replacement (issue #57). The in-process map above
+				// stays exactly as it was and remains the correct per-host duty-cycle bound -- slots x timeout
+				// is the most wall-clock THIS worker spends answering questions instead of running jobs. What it
+				// cannot bound is the fleet: a held job's wakes land on any host, so `PI_WAIT_CHECK_SLOTS`
+				// silently multiplied by host count, and the one symptom the bound has gets QUIETER as you
+				// scale out, because multiplication produces fewer denials per host.
+				//
+				// Null when no peer could exist, so a single-host deployment issues no command at all.
+				let fleetSlot = null;
+				if (checkLease) {
+					// The TTL is DERIVED from what the lease actually guards: the gate holds it across every profile
+					// in turn, each bounded by `PI_WAIT_CHECK_TIMEOUT_MS`, so it is one timeout per PROFILE plus one
+					// for the overhead between them. Deriving it from the SLOT COUNT instead -- an unrelated
+					// quantity -- made a three-profile job at the shipped defaults hold 30s against a 20s lease,
+					// so the slot expired mid-check and another host took it while this one was still using it.
+					fleetSlot = await checkLease.acquire(job.id, { slots: checkSlotCount(), ttlMs: (profiles.length + 1) * checkTimeoutMs() });
+					if (!fleetSlot) {
+						// The same outcome as a local denial and the same remedy, so the same cadence and the same
+						// event -- with one conditional field, which is what keeps an unarmed deployment's log line
+						// byte-identical.
+						checkSlots.release(WAIT_CHECK_KEY);
+						const denials = await waitState.noteThrottle(job.id, { denied: true });
+						if (denials === THROTTLE_ALARM) deps?.log?.("wait_capacity_exceeded", { jobId: job.id, denials, slots: checkSlotCount(), where: "fleet", hint: "raise PI_WAIT_CHECK_SLOTS or PI_CONCURRENCY, lengthen PI_WAIT_INTERVAL_MS, or hold fewer jobs" });
+						await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: nowMs + maxWaitMs() });
+						const wait = Math.max(THROTTLE_FLOOR_MS, Math.floor(waitBackoffMs(intervalMs(), held) / 4));
+						const delay = wait + Math.floor(wait * 0.1 * random());
+						deps?.log?.("wait_check_throttled", { jobId: job.id, delayMs: delay, slots: checkSlotCount(), where: "fleet" });
+						await job.moveToDelayed(nowMs + delay, token);
+						throw new DelayedError();
+					}
+				}
+				// THE LEASE IS HELD FROM THE `tryAcquire` ABOVE, so every exit from here down must release it.
+				// The try opens here and not at the check loop, which is where it used to open: the supersede
+				// claim sits between the two, and BOTH of its exits leave -- one returns `wait-superseded`,
+				// the other re-defers and throws -- so a claim that refused or could not be verified walked
+				// out holding the slot. At the shipped default of one slot that wedged every wait check on
+				// the worker until it restarted, and the symptom was silent in the worst way: held jobs kept
+				// throttling and eventually recorded `wait-expired` with `max-wait-unchecked`, which blames
+				// the deployment's capacity for a slot this gate leaked.
 				try {
+					// Claimed BEFORE the check, not after: a second delivery for an already-held target is a free
+					// determinate refusal, and paying for a subprocess first inverts the free-before-costly rule
+					// this gate's own header invokes. Tier 1 already claims in this order.
+					const claim = await waitState.claim(job.id, { dedupId, untilMs: nowMs + waitBackoffMs(intervalMs(), held), isLive: deps?.isJobLive });
+					if (claim.heldBy) {
+						return await refuseWait("wait-superseded", "wait_superseded", { heldBy: claim.heldBy }, "Another delivery for this target is already waiting on the same conditions. Not run.");
+					}
+					if (claim.retry) {
+						deps?.log?.("wait_supersede_unverified", { jobId: job.id, heldBy: claim.holder ?? null, delayMs: SUPERSEDE_RECHECK_MS });
+						await job.moveToDelayed(nowMs + SUPERSEDE_RECHECK_MS, token);
+						throw new DelayedError();
+					}
+
+					await waitState.noteThrottle(job.id, { denied: false }); // granted: the run of denials ends here
 					// Sequential, in the operator's writing order: the resolver's reason applies unchanged --
 					// naming the first condition that did not clear is what makes a held row readable, and a
 					// parallel fan-out would blame whichever lost the race on any given wake.
@@ -253,6 +296,7 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 						if (verdict?.profileUnknown || verdict?.verdict !== "go") break;
 					}
 				} finally {
+					await fleetSlot?.release?.();
 					checkSlots.release(WAIT_CHECK_KEY);
 				}
 
@@ -369,19 +413,78 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 		// occurrence at pickup and promotes it on time alone, so a slow run overlaps its own successor
 		// (measured: 301ms of live container overlap through this very processor) unless this gate holds.
 		// Infinity-limited scopes still acquire, so release stays uniform for every scoped job.
+		// THE HOST-WIDE SLOT (issue #57), taken before the scope slot and released in the same finally.
+		//
+		// It exists only when this worker drains a second, host-affine queue. BullMQ's concurrency is per
+		// Worker, so two queues at `PI_CONCURRENCY` would run twice the containers -- and that knob bounds a
+		// MACHINE (its RAM, its share of the provider's concurrent-stream budget), not a queue. Deferral
+		// rather than refusal, at the scope gate's own cadence and for its reason: a full host is transient
+		// state, never a verdict about the job (CONST-RETRY-INFRA-ONLY).
+		//
+		// Before the scope acquire, so a job that cannot run on this machine at all never takes a folder
+		// mutex it would immediately have to give back, and so the two releases nest rather than interleave.
+		let hostHeld = false;
+		if (hostBound) {
+			if (!hostBound.slots.tryAcquire(HOST_SLOT_KEY, hostBound.limit())) {
+				deps?.log?.("host_busy_deferred", { jobId: job.id, delayMs: SCOPE_BUSY_RECHECK_MS });
+				await job.moveToDelayed(nowMs + SCOPE_BUSY_RECHECK_MS, token);
+				throw new DelayedError();
+			}
+			hostHeld = true;
+		}
+
 		const limits = scopedLimits();
 		const scope = canonicalScope(job.data);
 		let held = false;
+		let scopeSlot = null;
 		if (scope) {
-			if (!inFlight.tryAcquire(scope, concurrencyFor(job.data, limits))) {
+			const ceiling = concurrencyFor(job.data, limits);
+			if (!inFlight.tryAcquire(scope, ceiling)) {
 				// Optional-chained: makeProcessor gives `deps` no default and bare wirings pass deps: {}.
 				// The scope itself stays out of the log line (no-pii-in-logs -- a local scope is a full
 				// host path); the delayed count and the job id are what an operator needs to see it.
 				deps?.log?.("scope_busy_deferred", { jobId: job.id, kind: job.data?.kind === "local" ? "local" : "forge", delayMs: SCOPE_BUSY_RECHECK_MS });
+				// The host slot goes back before we defer: `makeInFlight().release` is not idempotent, so a slot
+				// held across a deferral would be a slot this machine never gets back.
+				if (hostHeld) {
+					hostBound.slots.release(HOST_SLOT_KEY);
+					hostHeld = false;
+				}
 				await job.moveToDelayed(nowMs + SCOPE_BUSY_RECHECK_MS, token);
 				throw new DelayedError();
 			}
 			held = true;
+
+			// THE FLEET-WIDE HALF of a scoped ceiling (issue #57). A `scoped-limits.json` row's day/week/month
+			// caps are already atomic INCRs on shared keys; its `concurrent` was a per-process Map, so it
+			// multiplied by host count -- a MONEY bound silently widened by the operator's deployment shape,
+			// which `INT-SCOPED-LIMITS-FILE-CONTRACT` calls out as the failure its version rule exists for.
+			//
+			// LOCAL scopes deliberately never claim, and the reason is not economy. The key is a hash of a
+			// PATH STRING, which carries no identity: `/srv/site` on two machines is, in the common case, two
+			// different repositories that share a layout convention. A shared claim keyed on that would
+			// serialise two genuinely independent working trees and break exactly the deployments this feature
+			// exists to enable. Local folders are answered by ROUTING instead -- a folder exists on one host,
+			// so its in-process mutex already spans everything it needs to.
+			//
+			// And an unlimited forge scope never claims either: `concurrencyFor` returns Infinity with no
+			// matching row, so a deployment with no scoped-limits file issues no command at all.
+			if (scopeLease && job.data?.kind !== "local" && Number.isFinite(ceiling)) {
+				scopeSlot = await scopeLease.acquire(job.id, { slots: ceiling, keyArgs: [scopeKeyPrefix(scope).slice("budget:s:".length)] });
+				if (!scopeSlot) {
+					// The local slot goes back BEFORE we defer: `makeInFlight().release` is not idempotent, so a
+					// slot held across a deferral is a slot this host never gets back.
+					inFlight.release(scope);
+					held = false;
+					if (hostHeld) {
+						hostBound.slots.release(HOST_SLOT_KEY);
+						hostHeld = false;
+					}
+					deps?.log?.("scope_busy_deferred", { jobId: job.id, kind: job.data?.kind === "local" ? "local" : "forge", delayMs: SCOPE_BUSY_RECHECK_MS, where: "fleet" });
+					await job.moveToDelayed(nowMs + SCOPE_BUSY_RECHECK_MS, token);
+					throw new DelayedError();
+				}
+			}
 		}
 
 		let startedAt;
@@ -414,6 +517,12 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 				inFlight.release(scope);
 				held = false;
 			}
+			if (hostHeld) {
+				hostBound.slots.release(HOST_SLOT_KEY);
+				hostHeld = false;
+			}
+			void scopeSlot?.release?.();
+			scopeSlot = null;
 			clearTimeout(timer);
 			throw error;
 		}
@@ -508,62 +617,107 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 			// Release FIRST and never throw (release clamps at zero by construction): a throw here would
 			// mask the job's real error, and a missed release wedges the scope until a worker restart.
 			if (held) inFlight.release(scope);
+			if (hostHeld) hostBound.slots.release(HOST_SLOT_KEY);
+			// AWAITED, not fire-and-forget. Two reasons, and the second is the one that bites: an unawaited
+			// DEL is dropped by `shutdown`'s `process.exit(0)`, stranding the claim for its whole TTL on a
+			// restart -- and the next same-scope job would otherwise race the release, be denied, and sit out a
+			// full re-check interval while the slot it wanted went free behind it. The finally is already inside
+			// an async function, and `release` never throws.
+			await scopeSlot?.release?.();
 			clearTimeout(timer);
 			signal.removeEventListener("abort", onAbort);
 		}
 	};
 }
 
-export function createWorker({ connection, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, scopedLimits, inFlight, waitState, afterMaxMs, checkSlots, checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, extraClosers = [] }) {
-	let worker; // referenced by cancelJob/applyConcurrency before assignment; only called later, so the TDZ is fine
-	const processor = makeProcessor({
-		cancelJob: (id, reason) => worker.cancelJob(id, reason),
-		stopContainer: (name) => exec("docker", ["stop", "-t", "5", name]),
-		redis,
-		getSettings,
-		// Late-bound over `worker`: an overlay concurrency change re-binds the live slot count at the next
-		// job start. Guarded so only an integer that actually differs touches the property.
-		applyConcurrency: (n) => {
-			if (Number.isInteger(n) && worker.concurrency !== n) worker.concurrency = n;
-		},
-		pauseUntil,
-		// Undefined pass-throughs take makeProcessor's own defaults (no limits; a fresh per-processor
-		// in-flight map -- one per worker process, which under DES-CONCURRENCY-3's one-worker-per-daemon
-		// shape means one per daemon).
-		scopedLimits,
-		inFlight,
-		// Issue #230. Undefined pass-throughs take makeProcessor's own defaults (a wait state over the same
-		// redis client, and the shared 30-day `after` ceiling), so a bare wiring behaves like a wired one.
-		waitState,
-		afterMaxMs,
-		// Issue #230, the polled tier. `concurrencyNow` reads the LIVE slot count rather than the boot value,
-		// because the overlay can lower it through `dispatch_set` and a check must never take the last free
-		// slot from a paid job. Late-bound over `worker` exactly as `applyConcurrency` is, and for the same
-		// reason: the value it needs does not exist until the Worker is constructed.
-		checkSlots,
-		checkSlotCount,
-		concurrencyNow: concurrencyNow ?? (() => worker?.concurrency ?? concurrency),
-		intervalMs,
-		maxWaitMs,
-		maxChecks,
-		maxFaults,
-		deps,
-		recordRun,
-	});
-
-	worker = new Worker(QUEUE, processor, {
-		// maxRetriesPerRequest: null is REQUIRED for BullMQ's blocking connections, or it throws.
-		connection: { ...connection, maxRetriesPerRequest: null },
-		concurrency,
-		maxStalledCount: 0, // a stalled paid job FAILS, never silently re-runs (verified live)
-		...(limiter ? { limiter } : {}),
-	});
+export function createWorker({ connection, name, hostQueue = null, checkLease = null, scopeLease = null, checkTimeoutMs, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, scopedLimits, inFlight = makeInFlight(), waitState, afterMaxMs, checkSlots = makeInFlight(), checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, hostSlots = makeInFlight(), extraClosers = [] }) {
+	// One Worker per queue name (issue #57). A host-affine job -- one whose folder, secret resolver or wait
+	// check lives on THIS machine -- is enqueued to `pi-jobs@<name>` rather than filtered for at pickup,
+	// because BullMQ has no selective pop and the put-it-back alternative does not work: promotion out of
+	// the delayed set is gated on each worker's own `Date.now()`, so the fastest clock wins every hop and a
+	// job that had to reach another host might never get there.
+	const names = hostQueue ? [QUEUE, hostQueue] : [QUEUE];
+	const workers = [];
+
+	// THE HOST-WIDE BOUND, and the reason it has to exist at all. `PI_CONCURRENCY` bounds a HOST -- its RAM
+	// and its share of the provider's concurrent-stream budget (DES-CONCURRENCY-3) -- but BullMQ's own
+	// concurrency is per Worker, so two Workers at 3 would run six containers. This semaphore restores the
+	// bound as a property of the machine. Process memory is still the correct store, for this entry's own
+	// unchanged reason: it counts THIS host's containers, and the boot reaper clears survivors before
+	// draining. Armed only when a host queue exists, so a single-host deployment builds one Worker and
+	// never reaches the acquire.
+	const hostBound = hostQueue ? { slots: hostSlots, limit: () => liveConcurrency() } : null;
+	const liveConcurrency = () => workers[0]?.concurrency ?? concurrency;
+
+	for (const queueName of names) {
+		let worker; // referenced by cancelJob/applyConcurrency before assignment; only called later, so the TDZ is fine
+		const processor = makeProcessor({
+			// Bound to THIS worker: a job on the host queue is cancelled by the worker draining that queue,
+			// and the shared handle could not reach it.
+			cancelJob: (id, reason) => worker.cancelJob(id, reason),
+			stopContainer: (name) => exec("docker", ["stop", "-t", "5", name]),
+			redis,
+			getSettings,
+			// Late-bound over EVERY worker: an overlay concurrency change re-binds the live slot count at the
+			// next job start, and with two queues both have to move or the host bound and the queue bounds
+			// stop agreeing. Guarded so only an integer that actually differs touches the property.
+			applyConcurrency: (n) => {
+				if (!Number.isInteger(n)) return;
+				for (const w of workers) if (w.concurrency !== n) w.concurrency = n;
+			},
+			pauseUntil,
+			// SHARED across both workers, and that sharing is the point rather than an optimisation: the
+			// folder mutex, the per-scope ceiling and the wait-check lease all bound the HOST, so two
+			// independent maps would double every one of them exactly as two Workers double concurrency.
+			scopedLimits,
+			inFlight,
+			hostBound,
+			scopeLease,
+			// Issue #230. Undefined pass-throughs take makeProcessor's own defaults (a wait state over the same
+			// redis client, and the shared 30-day `after` ceiling), so a bare wiring behaves like a wired one.
+			waitState,
+			afterMaxMs,
+			// Issue #230, the polled tier. `concurrencyNow` reads the LIVE slot count rather than the boot value,
+			// because the overlay can lower it through `dispatch_set` and a check must never take the last free
+			// slot from a paid job.
+			checkSlots,
+			checkLease,
+			checkSlotCount,
+			checkTimeoutMs,
+			concurrencyNow: concurrencyNow ?? liveConcurrency,
+			intervalMs,
+			maxWaitMs,
+			maxChecks,
+			maxFaults,
+			deps,
+			recordRun,
+		});
+
+		worker = new Worker(queueName, processor, {
+			// maxRetriesPerRequest: null is REQUIRED for BullMQ's blocking connections, or it throws.
+			connection: { ...connection, maxRetriesPerRequest: null },
+			concurrency,
+			maxStalledCount: 0, // a stalled paid job FAILS, never silently re-runs (verified live)
+			// Issue #57. Conditional, so a bare createWorker builds a byte-identical options object -- and because
+			// bullmq's own matcher accepts both the named and unnamed client-name spellings, naming costs nothing.
+			...(name ? { name } : {}),
+			...(limiter ? { limiter } : {}),
+		});
+		workers.push(worker);
+	}
+
+	const primary = workers[0];
+	// The host-queue worker, for the caller that must register listeners on both. Attached rather than
+	// returned as a pair so every existing caller keeps receiving exactly what it received before.
+	primary.hostWorker = workers[1] ?? null;
 
 	const shutdown = async () => {
 		// Abort active jobs (=> docker stop via onAbort), then close. Without the cancel,
-		// worker.close() would wait up to 30 minutes for the container.
-		await Promise.resolve(worker.cancelAllJobs?.("shutdown")).catch(() => {});
-		await worker.close();
+		// worker.close() would wait up to 30 minutes for the container. ONE shutdown for every queue: two
+		// registrations would mean two `process.exit(0)` racing, and the second worker's containers would
+		// outlive the handler that was meant to stop them.
+		for (const w of workers) await Promise.resolve(w.cancelAllJobs?.("shutdown")).catch(() => {});
+		for (const w of workers) await w.close().catch(() => {});
 		// Close auxiliary resources (e.g. a cron scheduler) after the worker drains. Per-item catch
 		// so one failing or absent closer never strands the others or blocks exit -- matches the
 		// swallow posture on cancelAllJobs above.
@@ -577,5 +731,5 @@ export function createWorker({ connection, concurrency, getSettings, redis, deps
 	// worker still aborts in-flight jobs and docker-stops their containers rather than orphaning them.
 	if (process.platform === "win32") process.once("SIGBREAK", shutdown);
 
-	return worker;
+	return primary;
 }

package/src/queue.mjs

@@ -8,10 +8,67 @@ import { PR_CLOSE_ACTIONS } from "./triggers.mjs";
 const PR_CLOSE_WORDS = new Set(Object.values(PR_CLOSE_ACTIONS));
 
 export const QUEUE = "pi-jobs";
+
+/**
+ * The queue a HOST-AFFINE job goes to (issue #57): work only one machine can do, because the folder, the
+ * secret resolver or the wait-check script lives there.
+ *
+ * `@` is the separator because it is outside the worker-name charset (`[A-Za-z0-9._-]`), so
+ * `pi-jobs@<name>` decomposes unambiguously and a name can never contain one. A SUFFIX rather than a
+ * prefix so `KEYS bull:pi-jobs*` still shows an operator the whole deployment.
+ *
+ * Deliberately a separate queue rather than a field the pickup gate filters on. BullMQ has no selective
+ * pop, so filtering would mean taking a job and putting it back -- and promotion out of the delayed set is
+ * gated on each worker's OWN `Date.now()` in two places, so the host whose clock runs fastest wins every
+ * hop deterministically. A job that had to reach a different host might never get there, and jitter cannot
+ * fix it: it randomises WHEN the wake is, not WHO wins it.
+ */
+export const hostQueueName = (worker) => `${QUEUE}@${worker}`;
+
+/**
+ * Every queue name this deployment drains: the shared one, plus one per named host (issue #57).
+ *
+ * Derived from the REGISTRY rather than from configuration, because the reader is usually the admin or
+ * the CLI, which know their own host at best and the fleet not at all. A deployment with no named worker
+ * that declared NO name yields exactly `[QUEUE]`, so every existing caller is unchanged. Note that this
+ * is derived from `routes`, not from a row existing: every worker publishes a row, named or not.
+ *
+ * This exists because a host queue that no reader knows about is worse than no host queue: the panel
+ * would show zero schedulers while cron ran, and `pi-dispatch pause` would stop half a deployment while
+ * reporting success -- the silent no-op its own comment already warns about for a mistyped name.
+ */
+export function fleetQueueNames(hosts) {
+	const seen = new Set();
+	for (const h of hosts ?? []) {
+		// A host has a queue only when it DECLARED a name. Every worker publishes a registry row -- that is
+		// what lets an unnamed fleet be seen at all -- but an undeclared one drains only the shared queue,
+		// so deriving queue names from every row would invent `pi-jobs@<hostname>` for a queue nothing
+		// reads: pausing it would create a real key for a phantom, and the counts would be a queue that can
+		// never have jobs.
+		if (h?.routes !== true && h?.routes !== "true") continue;
+		const name = h?.name;
+		// VALIDATED, because this is peer-written data crossing a trust boundary. `hostQueueName`'s own
+		// contract leans on the charset -- `@` is the separator precisely because a name cannot contain one
+		// -- and nothing else re-checks it. A name with a `:` makes `new Queue` throw, which would take the
+		// kill switch out entirely; one with an `@` would not decompose.
+		if (typeof name !== "string" || !WORKER_NAME_RE.test(name)) continue;
+		seen.add(name);
+	}
+	// Deduped: two rows naming one host would double-count its jobs in a summed status.
+	return [QUEUE, ...[...seen].sort().map(hostQueueName)];
+}
+
+/** The name charset, duplicated from `config.mjs` deliberately: this module imports nothing. */
+const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
+
 export { chainedJobId, localJobId, deliveryJobId, gitlabDeliveryJobId, forgeDeliveryJobId };
 
-export function makeQueue(connection) {
-	return new Queue(QUEUE, { connection });
+/**
+ * A queue handle. `name` defaults to the shared queue, so every existing caller is unchanged and a
+ * single-host deployment never names anything else.
+ */
+export function makeQueue(connection, { name = QUEUE } = {}) {
+	return new Queue(name, { connection });
 }
 
 /**
@@ -226,3 +283,74 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
 	});
 	return jobId;
 }
+
+/**
+ * Every host queue that EXISTS, read from BullMQ's own keyspace rather than from the host registry.
+ *
+ * The registry answers "who is alive", and for a kill switch that is the wrong question. A host whose
+ * registry writes fail for ninety seconds loses its row while its BullMQ worker -- a separate connection,
+ * built with `maxRetriesPerRequest: null` precisely to ride out blips -- keeps draining. Pausing "the
+ * fleet" would then miss it and report success. The same gap opens for the ~15s before a booting worker's
+ * first beat lands, and on every `service restart`, since a clean shutdown DELs the row.
+ *
+ * Worse is the direction with no recovery path: pause while a host is live durably pauses its queue, and a
+ * later resume while that host is DOWN enumerates nothing for it. The queue stays paused permanently, and
+ * no surface can see it, because every surface was reading the registry too.
+ *
+ * A queue's meta key is durable and outlives its worker, so this asks the only authority that cannot go
+ * stale: the queues themselves. It also restores what `host-registry.mjs` claims about itself -- delete the
+ * whole `host:*` keyspace and nothing decides differently -- which the registry-derived kill switch had
+ * quietly made false.
+ *
+ * SCAN, not KEYS, and it is why this is NOT on the panel's per-tick path: it is for the rare command where
+ * being wrong costs money, not for a reader that runs every second. Fails open to `[]`, so an unreadable
+ * keyspace degrades to the registry's answer rather than refusing.
+ */
+export async function discoverHostQueues(redis, { timeoutMs = 2_000, count = 500 } = {}) {
+	const prefix = `bull:${QUEUE}@`;
+	const names = new Set();
+	try {
+		const deadline = Date.now() + timeoutMs;
+		let cursor = "0";
+		do {
+			// BOUNDED, because BullMQ's connections carry `maxRetriesPerRequest: null` and a command against
+			// an unreachable server therefore QUEUES FOREVER rather than rejecting -- so an unguarded await
+			// here would hang the kill switch instead of failing it open. The same trap the registry's
+			// `bounded` exists for.
+			//
+			// CLEARED on the way out, and NOT `unref`'d. Leaving it pending held the event loop open for the
+			// rest of the budget after the work was done, so `pi-dispatch pause` sat for two seconds having
+			// already paused everything; unref'ing instead would stop it firing when the hang is the last
+			// thing on the loop, which is the one case it exists for.
+			let timer;
+			const [next, keys] = await Promise.race([
+				redis.scan(cursor, "MATCH", `${prefix}*:meta`, "COUNT", count),
+				new Promise((_, reject) => {
+					timer = setTimeout(() => reject(new Error("scan timed out")), Math.max(1, deadline - Date.now()));
+				}),
+			]).finally(() => clearTimeout(timer));
+			cursor = next;
+			for (const key of keys ?? []) {
+				const name = String(key).slice(prefix.length, -":meta".length);
+				// Validated like every other peer-derived name: a key an operator hand-created could hold
+				// anything, and `new Queue` throws on a `:`, which would take the kill switch out entirely.
+				if (WORKER_NAME_RE.test(name)) names.add(name);
+			}
+		} while (cursor !== "0" && Date.now() < deadline);
+	} catch {
+		// Fail open: the registry's answer alone is still better than refusing to pause.
+		return [];
+	}
+	return [...names].sort().map(hostQueueName);
+}
+
+/**
+ * The union of what is LIVE (the registry) and what EXISTS (the keyspace), which is the set a kill switch
+ * must act on: a live host with no queue yet has nothing to pause, and a dead host's queue still holds
+ * jobs and still has a paused flag somebody has to be able to clear.
+ */
+export function unionQueueNames(fromRegistry, fromKeyspace) {
+	const seen = new Set([...(fromRegistry ?? []), ...(fromKeyspace ?? [])]);
+	seen.delete(QUEUE);
+	return [QUEUE, ...[...seen].sort()];
+}