@edgehero/pi-dispatch 1.6.1 → 1.8.0

package/src/host-registry.mjs

@@ -0,0 +1,279 @@
+/**
+ * The `host:` keyspace: which workers are alive, and what each one is (issue #57).
+ *
+ * Every gap in the multi-host issue wants the same missing fact -- WHICH HOSTS EXIST AND WHAT CAN EACH
+ * ONE DO -- so one structure serves all of them rather than each growing its own. This module is that
+ * structure and nothing else. It carries no configuration and no authority: every row is one host's
+ * SELF-DESCRIPTION, published by that host, and no reader may write another's.
+ *
+ *   host:live            SET of worker names -- an index the READER prunes, never a source of truth
+ *   host:h:<name>        HASH of that host's self-description, PEXPIRE'd on every beat
+ *
+ * WHY THIS IS NOT THE REDIS STATE `OQ-008` AND `DES-CONCURRENCY-3` REFUSED. That refusal is about a
+ * claim whose truth-maker lives on the host while the claim lives in Redis -- an in-flight count
+ * asserting a container the boot reaper had just killed, with no way for the two to notice they
+ * disagree. Here the truth-maker IS the host process, and the refresh IS the claim: when the process
+ * dies the claim stops being renewed and expires on its own. There is no reaper to contradict and no
+ * second authority to drift from.
+ *
+ * The sharper test, and the one to apply to anything added here later: DELETE THE WHOLE `host:*`
+ * KEYSPACE WHILE THE FLEET RUNS, and every host must behave exactly as it does today. That holds
+ * because nothing here is consulted to decide anything a single-host deployment decides differently --
+ * a reader that cannot read treats absence as "no peers", which is v1.6.1's behaviour. A Redis-side
+ * toggle fails that test: deleting it loses the operator's edit. Anything that would fail it does not
+ * belong in this keyspace.
+ *
+ * THE CONTENT RULE, which is a contract and not a style note: names, integers and digests. Never a
+ * path, never a URL with credentials, never a repository name, never operator free text. This is
+ * `targetFor`'s `local:<basename>` discipline applied to a Valkey value, and it earns its strictness
+ * from the reader rather than the writer -- the panel is where an operator screenshots, and doctor
+ * prints these rows. A value that must be carried but cannot satisfy the rule is HASHED before it gets
+ * here (`scopeKeyPrefix`'s idiom), never abbreviated.
+ */
+
+/** The index. A SET cannot expire its members, so the leak is handled by the reader, as `wait:held` does. */
+export const HOST_SET = "host:live";
+
+/** One host's row. `h:` is a sub-namespace letter under one prefix, as `budget:` uses `w:`/`m:`/`t:`/`s:`. */
+export const hostKey = (name) => `host:h:${name}`;
+
+/** How often a live worker republishes. Cheap: three writes, off every job path. */
+export const HOST_BEAT_MS = 15_000;
+
+/**
+ * How long a row outlives its last beat. SIX missed beats, not one or two: a heartbeat competes with the
+ * event loop of a process whose whole job is spawning containers, so a single late beat is ordinary and
+ * must not evict a healthy host. Ninety seconds is short enough that a crashed host stops being counted
+ * within one panel refresh cycle of an operator noticing anything at all.
+ */
+export const HOST_TTL_MS = 90_000;
+
+/**
+ * Build the registry accessor. `redis` is the client the budget and the wait state already share.
+ *
+ * EVERY METHOD FAILS OPEN, and the direction is the design: a registry fault must be able to cost a
+ * panel row and must never be able to invent a refusal. Nothing here is on the paid path, so there is no
+ * case where throwing would be more correct than continuing.
+ */
+/**
+ * How long any single registry command may take before it is treated as a failure.
+ *
+ * THIS BOUND IS THE WHOLE FAIL-OPEN MECHANISM, and getting it wrong is subtle enough to be worth the
+ * paragraph. `makeRedisClient` sets `maxRetriesPerRequest: null`, which BullMQ's blocking connections
+ * require and which means a command issued against an unreachable server does NOT reject -- it QUEUES,
+ * forever. So a `try/catch` around it catches nothing: the failure mode of this client is a HANG, and a
+ * hang is not an exception. Every `await` in this module therefore goes through `bounded`, which converts
+ * "never answers" into "answered no" so the catch below can do its job.
+ */
+const REGISTRY_OP_TIMEOUT_MS = 2_000;
+
+/** Reject rather than wait forever. See REGISTRY_OP_TIMEOUT_MS for why this is not optional here. */
+function bounded(promise, ms = REGISTRY_OP_TIMEOUT_MS) {
+	return new Promise((resolve, reject) => {
+		// NOT unref'd, deliberately. An unref'd timer does not fire when nothing else is holding the event
+		// loop, so the bound would silently stop existing in exactly the situation it is for -- a shutdown
+		// where the hung command is the last thing running. The cost is that a pending call can delay exit
+		// by at most this timeout, which is the point: bounded, and short.
+		const t = setTimeout(() => reject(new Error("registry timeout")), ms);
+		Promise.resolve(promise).then(
+			(v) => (clearTimeout(t), resolve(v)),
+			(e) => (clearTimeout(t), reject(e)),
+		);
+	});
+}
+
+export function makeHostRegistry({ redis, name, now = () => Date.now(), ttlMs = HOST_TTL_MS, log = () => {}, timeoutMs = REGISTRY_OP_TIMEOUT_MS }) {
+	let timer = null;
+	let facts = {};
+	// Set the moment `close` is entered, so a beat already in flight cannot re-HSET the row AFTER the DEL
+	// and recreate the very ghost `close` exists to remove.
+	let closed = false;
+	// Logged once per TRANSITION rather than once per beat: at four beats a minute, a Valkey outage would
+	// otherwise drown the log it is meant to serve. `notePackageKey` sets this precedent and states it.
+	let reachable = true;
+
+	const write = async (fields) => {
+		const key = hostKey(name);
+		const flat = [];
+		// A fact may be a VALUE or a THUNK. A thunk is what lets a fact that changes without a restart --
+		// the cron fingerprint after a live triggers-file edit, the live concurrency after an overlay
+		// change -- be re-read on every beat rather than frozen at the call that started the heartbeat.
+		// Without it a peer would compare against what this host believed at boot, and two hosts would see
+		// each other's fingerprint oscillate on the beat period after any edit.
+		for (const [k, v] of Object.entries(fields)) {
+			let resolved;
+			try {
+				resolved = typeof v === "function" ? v() : v;
+			} catch {
+				resolved = ""; // a fact that cannot be computed is absent, never a reason to skip the beat
+			}
+			const value = String(resolved ?? "");
+			// THE CONTENT RULE, ENFORCED HERE rather than asserted in a test. A test can only check the fields
+			// it happens to publish itself, so it could never catch the day someone adds `logsDir` at a call
+			// site. This can: a value that is path-shaped is DROPPED and named, loudly, once.
+			//
+			// The rule is mechanical rather than a blanket ban on `/`, because one legitimate value contains
+			// one: an IANA zone is `Europe/Amsterdam`. What no admissible value has is a filesystem ROOT -- a
+			// leading separator, a backslash, or a drive letter.
+			if (/^[/\\]|\\|^[A-Za-z]:[/\\]/.test(value)) {
+				log("host_registry_field_refused", { host: name, field: k, reason: "path-shaped" });
+				continue;
+			}
+			flat.push(k, value);
+		}
+		await bounded(redis.hset(key, ...flat), timeoutMs);
+		// PEXPIRE ON EVERY BEAT, which REVERSES this project's stated TTL rule ("set the TTL only when the
+		// key is first created, so a long window cannot push its expiry forward" -- budget.mjs). The
+		// reversal is correct because the OBJECT is different, and the distinction is worth keeping:
+		//
+		//   a COUNTER whose TTL refreshes on every increment stops being a window, which is why the budget
+		//   keys set it once and why a busy day must not postpone its own reset;
+		//
+		//   a LEASE's expiry IS the liveness claim, so refreshing it is not a leak, it is the mechanism.
+		//
+		// The in-repo precedent is lease-shaped and two files away: `wait-state.hold` re-PEXPIREs the
+		// supersede lease `wait:key:<dedupId>` -- and only after checking the lease is still ours, which is
+		// the same ownership check this key gets for free by being named after its only writer.
+		await bounded(redis.pexpire(key, ttlMs), timeoutMs);
+		await bounded(redis.sadd(HOST_SET, name), timeoutMs);
+	};
+
+	// The beat currently in flight, so `close` can DRAIN before it deletes. A `closed` flag checked at the
+	// top of `beat` is not enough on its own: a beat that had already passed that check would land its
+	// HSET after the DEL and recreate the very ghost `close` exists to remove.
+	let inFlight = null;
+
+	const beat = async (fields = facts) => {
+		if (closed) return;
+		facts = { ...facts, ...fields };
+		try {
+			inFlight = write({ ...facts, name, beatAt: now() });
+			await inFlight;
+			if (!reachable) {
+				reachable = true;
+				log("host_registry_restored", { host: name });
+			}
+		} catch (err) {
+			if (reachable) {
+				reachable = false;
+				log("host_registry_unreachable", { host: name, reason: err?.message });
+			}
+		} finally {
+			inFlight = null;
+		}
+	};
+
+	return {
+		/** This worker's own name, so a caller never re-derives it from config and gets a different answer. */
+		self: () => name,
+
+		/**
+		 * Flush this host's current facts NOW, for a caller that must be visible before it reads peers.
+		 *
+		 * Takes NO fields, deliberately. It used to merge them, and that quietly destroyed the thunk
+		 * mechanism it depends on: a caller passing `fpCron` as a computed STRING replaced the closure the
+		 * heartbeat installed, so every later beat published a frozen value and two hosts' fingerprints
+		 * could drift apart again. A caller that wants a fact re-read on every beat installs a thunk once,
+		 * at `start`; a caller that wants it published NOW calls this.
+		 */
+		async publish() {
+			await beat();
+		},
+
+		/**
+		 * Every live host EXCEPT this one. Separate from `readLiveHosts` because "who else is there" is the
+		 * question every caller actually has, and excluding self at the one place stops each of them doing it
+		 * differently -- and stops a stale self-row, written by a previous process of this same host, being
+		 * read as a peer that disagrees with the process that is running now.
+		 */
+		async livePeers() {
+			const res = await readLiveHosts(redis, { now, timeoutMs });
+			if (res.unreachable) return res;
+			return { hosts: res.hosts.filter((h) => h.name !== name) };
+		},
+
+		/**
+		 * Start beating. ONE `setInterval` -- the first in `worker/src`, every other timer here being a
+		 * `setTimeout` -- and `.unref()`'d so it can never hold the process open, which is the posture the
+		 * three `fs.watch` watchers already take. `stop` is registered as an extraCloser beside the runtime
+		 * queue, so a clean shutdown clears it before `process.exit`.
+		 */
+		async start(fields = {}, { intervalMs = HOST_BEAT_MS } = {}) {
+			if (closed || timer) return; // a second start would leak the first interval
+			await beat({ ...fields, startedAt: now() });
+			if (closed) return; // close() landed while the first beat was in flight
+			timer = setInterval(() => void beat(), intervalMs);
+			timer.unref?.();
+		},
+
+		/**
+		 * Leave. A clean shutdown DELETES the row rather than letting it expire, so the TTL only ever
+		 * covers a crash and a rolling restart does not leave a ghost peer behind for ninety seconds.
+		 * `wait-state.release` takes the same posture for the same reason.
+		 */
+		async close() {
+			if (closed) return; // idempotent, and the flag is also what stops an in-flight beat resurrecting the row
+			closed = true;
+			if (timer) clearInterval(timer);
+			timer = null;
+			// Drain before deleting, bounded like everything else here: an unbounded wait on a beat that is
+			// itself hung would be the shutdown hang this module's timeout exists to prevent.
+			await bounded(inFlight ?? Promise.resolve(), timeoutMs).catch(() => {});
+			try {
+				await bounded(redis.del(hostKey(name)), timeoutMs);
+				await bounded(redis.srem(HOST_SET, name), timeoutMs);
+			} catch {
+				// Fail open: the TTL is the backstop, and a ghost row costs a panel line, never a decision.
+			}
+		},
+	};
+}
+
+/**
+ * Every live host's row, with the index pruned as it is read.
+ *
+ * THE INDEX LEAK IS HANDLED HERE, BY THE READER, and that is a decision rather than an oversight: a SET
+ * cannot expire its members, so a host that dies leaves one behind. The alternative -- no index, and a
+ * `SCAN host:h:*` instead -- was measured and refused for the `wait:held` keyspace on exactly this
+ * shape: the panel reads every second, and a keyspace scan walks HARDEST on deployments with nothing to
+ * show, because the early exit never fires. So the index exists, a member whose hash is gone is stale by
+ * definition, and the reader removes it in passing.
+ *
+ * `{ unreachable }` and `[]` are DIFFERENT ANSWERS and must never be collapsed by a caller, even where
+ * one treats them alike: "there are no other hosts" and "I could not find out" differ, and a panel that
+ * renders the second as the first tells an operator their fleet is gone when Valkey merely blinked.
+ */
+export async function readLiveHosts(redis, { now = () => Date.now(), timeoutMs = REGISTRY_OP_TIMEOUT_MS } = {}) {
+	let names;
+	try {
+		names = await bounded(redis.smembers(HOST_SET), timeoutMs);
+		// Inside the try, not after it: a client that answers with something non-iterable would otherwise
+		// throw straight out of a function whose contract is that every method fails open.
+		if (names !== null && names !== undefined && !Array.isArray(names)) throw new Error("smembers did not return a list");
+	} catch (err) {
+		return { unreachable: err?.message ?? "registry unreadable" };
+	}
+	const hosts = [];
+	for (const member of names ?? []) {
+		try {
+			const row = await bounded(redis.hgetall(hostKey(member)), timeoutMs);
+			if (!row || Object.keys(row).length === 0) {
+				await bounded(redis.srem(HOST_SET, member), timeoutMs).catch(() => {});
+				continue;
+			}
+			const beatAt = typeof row.beatAt === "string" && row.beatAt.trim() !== "" ? Number(row.beatAt) : NaN;
+			hosts.push({
+				...row,
+				name: row.name || member,
+				// Derived rather than stored, so the panel can say "stale 2m" about a row that still lives.
+				// A row whose clock is AHEAD of ours reads as 0 rather than negative: the difference is the
+				// skew between two machines, which is its own signal and not this field's to report.
+				staleMs: Number.isFinite(beatAt) ? Math.max(0, now() - beatAt) : null,
+			});
+		} catch {
+			// One unreadable row degrades that row, never the listing.
+		}
+	}
+	hosts.sort((a, b) => String(a.name).localeCompare(String(b.name)));
+	return { hosts };
+}

package/src/image-preflight.mjs

@@ -70,7 +70,7 @@ export function makeImagePreflight({ image, spawnFn = spawn }) {
 		// become the push race it exists to avoid, with nothing in the run record saying so.
 		const probe = await runDocker(spawnFn, ["image", "inspect", `--format={{.Id}}${FIELD_SEP}${PI_VERSION_TEMPLATE}${FIELD_SEP}${FORGES_TEMPLATE}${FIELD_SEP}${CAPABILITIES_TEMPLATE}`, wanted], true);
 		if (probe.code === 0) {
-			const [piVersion, forges, capabilities] = parseLabels(probe.stdout);
+			const [piVersion, forges, capabilities, imageDigest] = parseLabels(probe.stdout);
 			const kind = job?.kind;
 			// Absent label => ALLOW. The polarity matters and is the opposite of what "declare your
 			// capabilities" suggests: every operator-built image predating this label (OQ-012) declares
@@ -96,7 +96,7 @@ export function makeImagePreflight({ image, spawnFn = spawn }) {
 			if (job?.command !== undefined && !(capabilities ?? []).includes("commands")) {
 				return { commandUnsupported: wanted, declared: capabilities ?? [] };
 			}
-			return { ok: true, image: wanted, piVersion };
+			return { ok: true, image: wanted, piVersion, imageDigest };
 		}
 		if ((await runDocker(spawnFn, ["info"])).code === 0) return { missing: wanted };
 		return { unavailable: wanted };
@@ -135,10 +135,15 @@ function parseLabels(stdout) {
 	// the safe answer on every field. On `capabilities` "safe" means the caller refuses a replica job,
 	// which is the same direction a genuinely unlabelled image goes.
 	const parts = String(stdout ?? "").trim().split(FIELD_SEP);
+	// Field 0 is `{{.Id}}`, the image's own digest. It has been fetched on every job since this format
+	// string had four fields and was thrown away until issue #57, which needs it to answer "are these two
+	// hosts running the same image?" -- a question `OQ-012` records as unanswerable and which turns out to
+	// cost nothing to answer, because the inspect that would have asked it already runs.
+	const imageDigest = label(parts[0]);
 	const piVersion = label(parts[1]);
 	// A label that is present but parses to nothing usable is treated as ABSENT rather than as an empty
 	// list -- on `forges` the latter would refuse every job on an image whose label was merely malformed.
-	return [piVersion, list(parts[2]), list(parts[3])];
+	return [piVersion, list(parts[2]), list(parts[3]), imageDigest];
 }
 
 /** One comma-separated label value as a non-empty array, or `null` when it declares nothing usable. */

package/src/index.mjs

@@ -3,13 +3,16 @@ import { promisify } from "node:util";
 import { DelayedError, UnrecoverableError, Worker } from "bullmq";
 import { InfraRetry, runJob } from "./processor.mjs";
 import { targetFor } from "./run-history.mjs";
-import { budgetCapsFor, canonicalScope, concurrencyFor, makeInFlight } from "./scoped-limits.mjs";
+import { budgetCapsFor, canonicalScope, concurrencyFor, makeInFlight, scopeKeyPrefix } from "./scoped-limits.mjs";
 import { WAIT_AFTER_MAX_DEFAULT_MS, WAIT_INTERVAL_FLOOR_MS, afterMs, unreadableConditions, waitArmed, waitBackoffMs, waitLabel, waitProfileNames } from "./wait-for.mjs";
 import { makeWaitState } from "./wait-state.mjs";
 
 const exec = promisify(execFile);
 
 export const QUEUE = "pi-jobs";
+
+/** The key the host-wide in-flight count lives under. One machine, one counter, whatever the queue. */
+export const HOST_SLOT_KEY = "host";
 export const JOB_TIMEOUT_MS = 30 * 60 * 1000; // REQ-JOB-TIMEOUT-30M
 // The scope-busy re-check (issue #242): a held scope has no natural "until" (the holder may run to
 // JOB_TIMEOUT_MS), so a deferred job re-tests on a fixed cadence. 5s keeps the worst case trivial
@@ -60,7 +63,7 @@ const THROTTLE_FLOOR_MS = 11_000;
  * The overlay changes which values the spend caps take, never when they are checked -- reserveBudget still
  * runs inside runJob against the freshly passed caps (CONST-BUDGET-BEFORE-TOKENS).
  */
-export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
+export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), hostBound = null, checkLease = null, scopeLease = null, deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, checkTimeoutMs = () => 10_000, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
 	return async function processor(job, token, signal) {
 		// Scoped pause windows (REQ-SCOPED-PAUSE-WINDOWS): if this job's folder/repo is inside an active pause
 		// window, DEFER it to the window end via BullMQ's delayed set -- the job keeps its identity/dedup and
@@ -230,6 +233,37 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 				// Declared outside the try below because the branches AFTER it read both.
 				let verdict = null;
 				let checked = null;
+				// A SECOND LAYER BENEATH THE FIRST, never a replacement (issue #57). The in-process map above
+				// stays exactly as it was and remains the correct per-host duty-cycle bound -- slots x timeout
+				// is the most wall-clock THIS worker spends answering questions instead of running jobs. What it
+				// cannot bound is the fleet: a held job's wakes land on any host, so `PI_WAIT_CHECK_SLOTS`
+				// silently multiplied by host count, and the one symptom the bound has gets QUIETER as you
+				// scale out, because multiplication produces fewer denials per host.
+				//
+				// Null when no peer could exist, so a single-host deployment issues no command at all.
+				let fleetSlot = null;
+				if (checkLease) {
+					// The TTL is DERIVED from what the lease actually guards: the gate holds it across every profile
+					// in turn, each bounded by `PI_WAIT_CHECK_TIMEOUT_MS`, so it is one timeout per PROFILE plus one
+					// for the overhead between them. Deriving it from the SLOT COUNT instead -- an unrelated
+					// quantity -- made a three-profile job at the shipped defaults hold 30s against a 20s lease,
+					// so the slot expired mid-check and another host took it while this one was still using it.
+					fleetSlot = await checkLease.acquire(job.id, { slots: checkSlotCount(), ttlMs: (profiles.length + 1) * checkTimeoutMs() });
+					if (!fleetSlot) {
+						// The same outcome as a local denial and the same remedy, so the same cadence and the same
+						// event -- with one conditional field, which is what keeps an unarmed deployment's log line
+						// byte-identical.
+						checkSlots.release(WAIT_CHECK_KEY);
+						const denials = await waitState.noteThrottle(job.id, { denied: true });
+						if (denials === THROTTLE_ALARM) deps?.log?.("wait_capacity_exceeded", { jobId: job.id, denials, slots: checkSlotCount(), where: "fleet", hint: "raise PI_WAIT_CHECK_SLOTS or PI_CONCURRENCY, lengthen PI_WAIT_INTERVAL_MS, or hold fewer jobs" });
+						await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: nowMs + maxWaitMs() });
+						const wait = Math.max(THROTTLE_FLOOR_MS, Math.floor(waitBackoffMs(intervalMs(), held) / 4));
+						const delay = wait + Math.floor(wait * 0.1 * random());
+						deps?.log?.("wait_check_throttled", { jobId: job.id, delayMs: delay, slots: checkSlotCount(), where: "fleet" });
+						await job.moveToDelayed(nowMs + delay, token);
+						throw new DelayedError();
+					}
+				}
 				// THE LEASE IS HELD FROM THE `tryAcquire` ABOVE, so every exit from here down must release it.
 				// The try opens here and not at the check loop, which is where it used to open: the supersede
 				// claim sits between the two, and BOTH of its exits leave -- one returns `wait-superseded`,
@@ -262,6 +296,7 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 						if (verdict?.profileUnknown || verdict?.verdict !== "go") break;
 					}
 				} finally {
+					await fleetSlot?.release?.();
 					checkSlots.release(WAIT_CHECK_KEY);
 				}
 
@@ -378,19 +413,78 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 		// occurrence at pickup and promotes it on time alone, so a slow run overlaps its own successor
 		// (measured: 301ms of live container overlap through this very processor) unless this gate holds.
 		// Infinity-limited scopes still acquire, so release stays uniform for every scoped job.
+		// THE HOST-WIDE SLOT (issue #57), taken before the scope slot and released in the same finally.
+		//
+		// It exists only when this worker drains a second, host-affine queue. BullMQ's concurrency is per
+		// Worker, so two queues at `PI_CONCURRENCY` would run twice the containers -- and that knob bounds a
+		// MACHINE (its RAM, its share of the provider's concurrent-stream budget), not a queue. Deferral
+		// rather than refusal, at the scope gate's own cadence and for its reason: a full host is transient
+		// state, never a verdict about the job (CONST-RETRY-INFRA-ONLY).
+		//
+		// Before the scope acquire, so a job that cannot run on this machine at all never takes a folder
+		// mutex it would immediately have to give back, and so the two releases nest rather than interleave.
+		let hostHeld = false;
+		if (hostBound) {
+			if (!hostBound.slots.tryAcquire(HOST_SLOT_KEY, hostBound.limit())) {
+				deps?.log?.("host_busy_deferred", { jobId: job.id, delayMs: SCOPE_BUSY_RECHECK_MS });
+				await job.moveToDelayed(nowMs + SCOPE_BUSY_RECHECK_MS, token);
+				throw new DelayedError();
+			}
+			hostHeld = true;
+		}
+
 		const limits = scopedLimits();
 		const scope = canonicalScope(job.data);
 		let held = false;
+		let scopeSlot = null;
 		if (scope) {
-			if (!inFlight.tryAcquire(scope, concurrencyFor(job.data, limits))) {
+			const ceiling = concurrencyFor(job.data, limits);
+			if (!inFlight.tryAcquire(scope, ceiling)) {
 				// Optional-chained: makeProcessor gives `deps` no default and bare wirings pass deps: {}.
 				// The scope itself stays out of the log line (no-pii-in-logs -- a local scope is a full
 				// host path); the delayed count and the job id are what an operator needs to see it.
 				deps?.log?.("scope_busy_deferred", { jobId: job.id, kind: job.data?.kind === "local" ? "local" : "forge", delayMs: SCOPE_BUSY_RECHECK_MS });
+				// The host slot goes back before we defer: `makeInFlight().release` is not idempotent, so a slot
+				// held across a deferral would be a slot this machine never gets back.
+				if (hostHeld) {
+					hostBound.slots.release(HOST_SLOT_KEY);
+					hostHeld = false;
+				}
 				await job.moveToDelayed(nowMs + SCOPE_BUSY_RECHECK_MS, token);
 				throw new DelayedError();
 			}
 			held = true;
+
+			// THE FLEET-WIDE HALF of a scoped ceiling (issue #57). A `scoped-limits.json` row's day/week/month
+			// caps are already atomic INCRs on shared keys; its `concurrent` was a per-process Map, so it
+			// multiplied by host count -- a MONEY bound silently widened by the operator's deployment shape,
+			// which `INT-SCOPED-LIMITS-FILE-CONTRACT` calls out as the failure its version rule exists for.
+			//
+			// LOCAL scopes deliberately never claim, and the reason is not economy. The key is a hash of a
+			// PATH STRING, which carries no identity: `/srv/site` on two machines is, in the common case, two
+			// different repositories that share a layout convention. A shared claim keyed on that would
+			// serialise two genuinely independent working trees and break exactly the deployments this feature
+			// exists to enable. Local folders are answered by ROUTING instead -- a folder exists on one host,
+			// so its in-process mutex already spans everything it needs to.
+			//
+			// And an unlimited forge scope never claims either: `concurrencyFor` returns Infinity with no
+			// matching row, so a deployment with no scoped-limits file issues no command at all.
+			if (scopeLease && job.data?.kind !== "local" && Number.isFinite(ceiling)) {
+				scopeSlot = await scopeLease.acquire(job.id, { slots: ceiling, keyArgs: [scopeKeyPrefix(scope).slice("budget:s:".length)] });
+				if (!scopeSlot) {
+					// The local slot goes back BEFORE we defer: `makeInFlight().release` is not idempotent, so a
+					// slot held across a deferral is a slot this host never gets back.
+					inFlight.release(scope);
+					held = false;
+					if (hostHeld) {
+						hostBound.slots.release(HOST_SLOT_KEY);
+						hostHeld = false;
+					}
+					deps?.log?.("scope_busy_deferred", { jobId: job.id, kind: job.data?.kind === "local" ? "local" : "forge", delayMs: SCOPE_BUSY_RECHECK_MS, where: "fleet" });
+					await job.moveToDelayed(nowMs + SCOPE_BUSY_RECHECK_MS, token);
+					throw new DelayedError();
+				}
+			}
 		}
 
 		let startedAt;
@@ -423,6 +517,12 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 				inFlight.release(scope);
 				held = false;
 			}
+			if (hostHeld) {
+				hostBound.slots.release(HOST_SLOT_KEY);
+				hostHeld = false;
+			}
+			void scopeSlot?.release?.();
+			scopeSlot = null;
 			clearTimeout(timer);
 			throw error;
 		}
@@ -517,62 +617,107 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 			// Release FIRST and never throw (release clamps at zero by construction): a throw here would
 			// mask the job's real error, and a missed release wedges the scope until a worker restart.
 			if (held) inFlight.release(scope);
+			if (hostHeld) hostBound.slots.release(HOST_SLOT_KEY);
+			// AWAITED, not fire-and-forget. Two reasons, and the second is the one that bites: an unawaited
+			// DEL is dropped by `shutdown`'s `process.exit(0)`, stranding the claim for its whole TTL on a
+			// restart -- and the next same-scope job would otherwise race the release, be denied, and sit out a
+			// full re-check interval while the slot it wanted went free behind it. The finally is already inside
+			// an async function, and `release` never throws.
+			await scopeSlot?.release?.();
 			clearTimeout(timer);
 			signal.removeEventListener("abort", onAbort);
 		}
 	};
 }
 
-export function createWorker({ connection, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, scopedLimits, inFlight, waitState, afterMaxMs, checkSlots, checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, extraClosers = [] }) {
-	let worker; // referenced by cancelJob/applyConcurrency before assignment; only called later, so the TDZ is fine
-	const processor = makeProcessor({
-		cancelJob: (id, reason) => worker.cancelJob(id, reason),
-		stopContainer: (name) => exec("docker", ["stop", "-t", "5", name]),
-		redis,
-		getSettings,
-		// Late-bound over `worker`: an overlay concurrency change re-binds the live slot count at the next
-		// job start. Guarded so only an integer that actually differs touches the property.
-		applyConcurrency: (n) => {
-			if (Number.isInteger(n) && worker.concurrency !== n) worker.concurrency = n;
-		},
-		pauseUntil,
-		// Undefined pass-throughs take makeProcessor's own defaults (no limits; a fresh per-processor
-		// in-flight map -- one per worker process, which under DES-CONCURRENCY-3's one-worker-per-daemon
-		// shape means one per daemon).
-		scopedLimits,
-		inFlight,
-		// Issue #230. Undefined pass-throughs take makeProcessor's own defaults (a wait state over the same
-		// redis client, and the shared 30-day `after` ceiling), so a bare wiring behaves like a wired one.
-		waitState,
-		afterMaxMs,
-		// Issue #230, the polled tier. `concurrencyNow` reads the LIVE slot count rather than the boot value,
-		// because the overlay can lower it through `dispatch_set` and a check must never take the last free
-		// slot from a paid job. Late-bound over `worker` exactly as `applyConcurrency` is, and for the same
-		// reason: the value it needs does not exist until the Worker is constructed.
-		checkSlots,
-		checkSlotCount,
-		concurrencyNow: concurrencyNow ?? (() => worker?.concurrency ?? concurrency),
-		intervalMs,
-		maxWaitMs,
-		maxChecks,
-		maxFaults,
-		deps,
-		recordRun,
-	});
-
-	worker = new Worker(QUEUE, processor, {
-		// maxRetriesPerRequest: null is REQUIRED for BullMQ's blocking connections, or it throws.
-		connection: { ...connection, maxRetriesPerRequest: null },
-		concurrency,
-		maxStalledCount: 0, // a stalled paid job FAILS, never silently re-runs (verified live)
-		...(limiter ? { limiter } : {}),
-	});
+export function createWorker({ connection, name, hostQueue = null, checkLease = null, scopeLease = null, checkTimeoutMs, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, scopedLimits, inFlight = makeInFlight(), waitState, afterMaxMs, checkSlots = makeInFlight(), checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, hostSlots = makeInFlight(), extraClosers = [] }) {
+	// One Worker per queue name (issue #57). A host-affine job -- one whose folder, secret resolver or wait
+	// check lives on THIS machine -- is enqueued to `pi-jobs@<name>` rather than filtered for at pickup,
+	// because BullMQ has no selective pop and the put-it-back alternative does not work: promotion out of
+	// the delayed set is gated on each worker's own `Date.now()`, so the fastest clock wins every hop and a
+	// job that had to reach another host might never get there.
+	const names = hostQueue ? [QUEUE, hostQueue] : [QUEUE];
+	const workers = [];
+
+	// THE HOST-WIDE BOUND, and the reason it has to exist at all. `PI_CONCURRENCY` bounds a HOST -- its RAM
+	// and its share of the provider's concurrent-stream budget (DES-CONCURRENCY-3) -- but BullMQ's own
+	// concurrency is per Worker, so two Workers at 3 would run six containers. This semaphore restores the
+	// bound as a property of the machine. Process memory is still the correct store, for this entry's own
+	// unchanged reason: it counts THIS host's containers, and the boot reaper clears survivors before
+	// draining. Armed only when a host queue exists, so a single-host deployment builds one Worker and
+	// never reaches the acquire.
+	const hostBound = hostQueue ? { slots: hostSlots, limit: () => liveConcurrency() } : null;
+	const liveConcurrency = () => workers[0]?.concurrency ?? concurrency;
+
+	for (const queueName of names) {
+		let worker; // referenced by cancelJob/applyConcurrency before assignment; only called later, so the TDZ is fine
+		const processor = makeProcessor({
+			// Bound to THIS worker: a job on the host queue is cancelled by the worker draining that queue,
+			// and the shared handle could not reach it.
+			cancelJob: (id, reason) => worker.cancelJob(id, reason),
+			stopContainer: (name) => exec("docker", ["stop", "-t", "5", name]),
+			redis,
+			getSettings,
+			// Late-bound over EVERY worker: an overlay concurrency change re-binds the live slot count at the
+			// next job start, and with two queues both have to move or the host bound and the queue bounds
+			// stop agreeing. Guarded so only an integer that actually differs touches the property.
+			applyConcurrency: (n) => {
+				if (!Number.isInteger(n)) return;
+				for (const w of workers) if (w.concurrency !== n) w.concurrency = n;
+			},
+			pauseUntil,
+			// SHARED across both workers, and that sharing is the point rather than an optimisation: the
+			// folder mutex, the per-scope ceiling and the wait-check lease all bound the HOST, so two
+			// independent maps would double every one of them exactly as two Workers double concurrency.
+			scopedLimits,
+			inFlight,
+			hostBound,
+			scopeLease,
+			// Issue #230. Undefined pass-throughs take makeProcessor's own defaults (a wait state over the same
+			// redis client, and the shared 30-day `after` ceiling), so a bare wiring behaves like a wired one.
+			waitState,
+			afterMaxMs,
+			// Issue #230, the polled tier. `concurrencyNow` reads the LIVE slot count rather than the boot value,
+			// because the overlay can lower it through `dispatch_set` and a check must never take the last free
+			// slot from a paid job.
+			checkSlots,
+			checkLease,
+			checkSlotCount,
+			checkTimeoutMs,
+			concurrencyNow: concurrencyNow ?? liveConcurrency,
+			intervalMs,
+			maxWaitMs,
+			maxChecks,
+			maxFaults,
+			deps,
+			recordRun,
+		});
+
+		worker = new Worker(queueName, processor, {
+			// maxRetriesPerRequest: null is REQUIRED for BullMQ's blocking connections, or it throws.
+			connection: { ...connection, maxRetriesPerRequest: null },
+			concurrency,
+			maxStalledCount: 0, // a stalled paid job FAILS, never silently re-runs (verified live)
+			// Issue #57. Conditional, so a bare createWorker builds a byte-identical options object -- and because
+			// bullmq's own matcher accepts both the named and unnamed client-name spellings, naming costs nothing.
+			...(name ? { name } : {}),
+			...(limiter ? { limiter } : {}),
+		});
+		workers.push(worker);
+	}
+
+	const primary = workers[0];
+	// The host-queue worker, for the caller that must register listeners on both. Attached rather than
+	// returned as a pair so every existing caller keeps receiving exactly what it received before.
+	primary.hostWorker = workers[1] ?? null;
 
 	const shutdown = async () => {
 		// Abort active jobs (=> docker stop via onAbort), then close. Without the cancel,
-		// worker.close() would wait up to 30 minutes for the container.
-		await Promise.resolve(worker.cancelAllJobs?.("shutdown")).catch(() => {});
-		await worker.close();
+		// worker.close() would wait up to 30 minutes for the container. ONE shutdown for every queue: two
+		// registrations would mean two `process.exit(0)` racing, and the second worker's containers would
+		// outlive the handler that was meant to stop them.
+		for (const w of workers) await Promise.resolve(w.cancelAllJobs?.("shutdown")).catch(() => {});
+		for (const w of workers) await w.close().catch(() => {});
 		// Close auxiliary resources (e.g. a cron scheduler) after the worker drains. Per-item catch
 		// so one failing or absent closer never strands the others or blocks exit -- matches the
 		// swallow posture on cancelAllJobs above.
@@ -586,5 +731,5 @@ export function createWorker({ connection, concurrency, getSettings, redis, deps
 	// worker still aborts in-flight jobs and docker-stops their containers rather than orphaning them.
 	if (process.platform === "win32") process.once("SIGBREAK", shutdown);
 
-	return worker;
+	return primary;
 }