@edgehero/pi-dispatch 1.6.1 → 1.7.0

package/src/index.mjs

@@ -3,13 +3,16 @@ import { promisify } from "node:util";
 import { DelayedError, UnrecoverableError, Worker } from "bullmq";
 import { InfraRetry, runJob } from "./processor.mjs";
 import { targetFor } from "./run-history.mjs";
-import { budgetCapsFor, canonicalScope, concurrencyFor, makeInFlight } from "./scoped-limits.mjs";
+import { budgetCapsFor, canonicalScope, concurrencyFor, makeInFlight, scopeKeyPrefix } from "./scoped-limits.mjs";
 import { WAIT_AFTER_MAX_DEFAULT_MS, WAIT_INTERVAL_FLOOR_MS, afterMs, unreadableConditions, waitArmed, waitBackoffMs, waitLabel, waitProfileNames } from "./wait-for.mjs";
 import { makeWaitState } from "./wait-state.mjs";
 
 const exec = promisify(execFile);
 
 export const QUEUE = "pi-jobs";
+
+/** The key the host-wide in-flight count lives under. One machine, one counter, whatever the queue. */
+export const HOST_SLOT_KEY = "host";
 export const JOB_TIMEOUT_MS = 30 * 60 * 1000; // REQ-JOB-TIMEOUT-30M
 // The scope-busy re-check (issue #242): a held scope has no natural "until" (the holder may run to
 // JOB_TIMEOUT_MS), so a deferred job re-tests on a fixed cadence. 5s keeps the worst case trivial
@@ -60,7 +63,7 @@ const THROTTLE_FLOOR_MS = 11_000;
  * The overlay changes which values the spend caps take, never when they are checked -- reserveBudget still
  * runs inside runJob against the freshly passed caps (CONST-BUDGET-BEFORE-TOKENS).
  */
-export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
+export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), hostBound = null, checkLease = null, scopeLease = null, deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, checkTimeoutMs = () => 10_000, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
 	return async function processor(job, token, signal) {
 		// Scoped pause windows (REQ-SCOPED-PAUSE-WINDOWS): if this job's folder/repo is inside an active pause
 		// window, DEFER it to the window end via BullMQ's delayed set -- the job keeps its identity/dedup and
@@ -230,6 +233,37 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 				// Declared outside the try below because the branches AFTER it read both.
 				let verdict = null;
 				let checked = null;
+				// A SECOND LAYER BENEATH THE FIRST, never a replacement (issue #57). The in-process map above
+				// stays exactly as it was and remains the correct per-host duty-cycle bound -- slots x timeout
+				// is the most wall-clock THIS worker spends answering questions instead of running jobs. What it
+				// cannot bound is the fleet: a held job's wakes land on any host, so `PI_WAIT_CHECK_SLOTS`
+				// silently multiplied by host count, and the one symptom the bound has gets QUIETER as you
+				// scale out, because multiplication produces fewer denials per host.
+				//
+				// Null when no peer could exist, so a single-host deployment issues no command at all.
+				let fleetSlot = null;
+				if (checkLease) {
+					// The TTL is DERIVED from what the lease actually guards: the gate holds it across every profile
+					// in turn, each bounded by `PI_WAIT_CHECK_TIMEOUT_MS`, so it is one timeout per PROFILE plus one
+					// for the overhead between them. Deriving it from the SLOT COUNT instead -- an unrelated
+					// quantity -- made a three-profile job at the shipped defaults hold 30s against a 20s lease,
+					// so the slot expired mid-check and another host took it while this one was still using it.
+					fleetSlot = await checkLease.acquire(job.id, { slots: checkSlotCount(), ttlMs: (profiles.length + 1) * checkTimeoutMs() });
+					if (!fleetSlot) {
+						// The same outcome as a local denial and the same remedy, so the same cadence and the same
+						// event -- with one conditional field, which is what keeps an unarmed deployment's log line
+						// byte-identical.
+						checkSlots.release(WAIT_CHECK_KEY);
+						const denials = await waitState.noteThrottle(job.id, { denied: true });
+						if (denials === THROTTLE_ALARM) deps?.log?.("wait_capacity_exceeded", { jobId: job.id, denials, slots: checkSlotCount(), where: "fleet", hint: "raise PI_WAIT_CHECK_SLOTS or PI_CONCURRENCY, lengthen PI_WAIT_INTERVAL_MS, or hold fewer jobs" });
+						await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: nowMs + maxWaitMs() });
+						const wait = Math.max(THROTTLE_FLOOR_MS, Math.floor(waitBackoffMs(intervalMs(), held) / 4));
+						const delay = wait + Math.floor(wait * 0.1 * random());
+						deps?.log?.("wait_check_throttled", { jobId: job.id, delayMs: delay, slots: checkSlotCount(), where: "fleet" });
+						await job.moveToDelayed(nowMs + delay, token);
+						throw new DelayedError();
+					}
+				}
 				// THE LEASE IS HELD FROM THE `tryAcquire` ABOVE, so every exit from here down must release it.
 				// The try opens here and not at the check loop, which is where it used to open: the supersede
 				// claim sits between the two, and BOTH of its exits leave -- one returns `wait-superseded`,
@@ -262,6 +296,7 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 						if (verdict?.profileUnknown || verdict?.verdict !== "go") break;
 					}
 				} finally {
+					await fleetSlot?.release?.();
 					checkSlots.release(WAIT_CHECK_KEY);
 				}
 
@@ -378,19 +413,78 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 		// occurrence at pickup and promotes it on time alone, so a slow run overlaps its own successor
 		// (measured: 301ms of live container overlap through this very processor) unless this gate holds.
 		// Infinity-limited scopes still acquire, so release stays uniform for every scoped job.
+		// THE HOST-WIDE SLOT (issue #57), taken before the scope slot and released in the same finally.
+		//
+		// It exists only when this worker drains a second, host-affine queue. BullMQ's concurrency is per
+		// Worker, so two queues at `PI_CONCURRENCY` would run twice the containers -- and that knob bounds a
+		// MACHINE (its RAM, its share of the provider's concurrent-stream budget), not a queue. Deferral
+		// rather than refusal, at the scope gate's own cadence and for its reason: a full host is transient
+		// state, never a verdict about the job (CONST-RETRY-INFRA-ONLY).
+		//
+		// Before the scope acquire, so a job that cannot run on this machine at all never takes a folder
+		// mutex it would immediately have to give back, and so the two releases nest rather than interleave.
+		let hostHeld = false;
+		if (hostBound) {
+			if (!hostBound.slots.tryAcquire(HOST_SLOT_KEY, hostBound.limit())) {
+				deps?.log?.("host_busy_deferred", { jobId: job.id, delayMs: SCOPE_BUSY_RECHECK_MS });
+				await job.moveToDelayed(nowMs + SCOPE_BUSY_RECHECK_MS, token);
+				throw new DelayedError();
+			}
+			hostHeld = true;
+		}
+
 		const limits = scopedLimits();
 		const scope = canonicalScope(job.data);
 		let held = false;
+		let scopeSlot = null;
 		if (scope) {
-			if (!inFlight.tryAcquire(scope, concurrencyFor(job.data, limits))) {
+			const ceiling = concurrencyFor(job.data, limits);
+			if (!inFlight.tryAcquire(scope, ceiling)) {
 				// Optional-chained: makeProcessor gives `deps` no default and bare wirings pass deps: {}.
 				// The scope itself stays out of the log line (no-pii-in-logs -- a local scope is a full
 				// host path); the delayed count and the job id are what an operator needs to see it.
 				deps?.log?.("scope_busy_deferred", { jobId: job.id, kind: job.data?.kind === "local" ? "local" : "forge", delayMs: SCOPE_BUSY_RECHECK_MS });
+				// The host slot goes back before we defer: `makeInFlight().release` is not idempotent, so a slot
+				// held across a deferral would be a slot this machine never gets back.
+				if (hostHeld) {
+					hostBound.slots.release(HOST_SLOT_KEY);
+					hostHeld = false;
+				}
 				await job.moveToDelayed(nowMs + SCOPE_BUSY_RECHECK_MS, token);
 				throw new DelayedError();
 			}
 			held = true;
+
+			// THE FLEET-WIDE HALF of a scoped ceiling (issue #57). A `scoped-limits.json` row's day/week/month
+			// caps are already atomic INCRs on shared keys; its `concurrent` was a per-process Map, so it
+			// multiplied by host count -- a MONEY bound silently widened by the operator's deployment shape,
+			// which `INT-SCOPED-LIMITS-FILE-CONTRACT` calls out as the failure its version rule exists for.
+			//
+			// LOCAL scopes deliberately never claim, and the reason is not economy. The key is a hash of a
+			// PATH STRING, which carries no identity: `/srv/site` on two machines is, in the common case, two
+			// different repositories that share a layout convention. A shared claim keyed on that would
+			// serialise two genuinely independent working trees and break exactly the deployments this feature
+			// exists to enable. Local folders are answered by ROUTING instead -- a folder exists on one host,
+			// so its in-process mutex already spans everything it needs to.
+			//
+			// And an unlimited forge scope never claims either: `concurrencyFor` returns Infinity with no
+			// matching row, so a deployment with no scoped-limits file issues no command at all.
+			if (scopeLease && job.data?.kind !== "local" && Number.isFinite(ceiling)) {
+				scopeSlot = await scopeLease.acquire(job.id, { slots: ceiling, keyArgs: [scopeKeyPrefix(scope).slice("budget:s:".length)] });
+				if (!scopeSlot) {
+					// The local slot goes back BEFORE we defer: `makeInFlight().release` is not idempotent, so a
+					// slot held across a deferral is a slot this host never gets back.
+					inFlight.release(scope);
+					held = false;
+					if (hostHeld) {
+						hostBound.slots.release(HOST_SLOT_KEY);
+						hostHeld = false;
+					}
+					deps?.log?.("scope_busy_deferred", { jobId: job.id, kind: job.data?.kind === "local" ? "local" : "forge", delayMs: SCOPE_BUSY_RECHECK_MS, where: "fleet" });
+					await job.moveToDelayed(nowMs + SCOPE_BUSY_RECHECK_MS, token);
+					throw new DelayedError();
+				}
+			}
 		}
 
 		let startedAt;
@@ -423,6 +517,12 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 				inFlight.release(scope);
 				held = false;
 			}
+			if (hostHeld) {
+				hostBound.slots.release(HOST_SLOT_KEY);
+				hostHeld = false;
+			}
+			void scopeSlot?.release?.();
+			scopeSlot = null;
 			clearTimeout(timer);
 			throw error;
 		}
@@ -517,62 +617,107 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 			// Release FIRST and never throw (release clamps at zero by construction): a throw here would
 			// mask the job's real error, and a missed release wedges the scope until a worker restart.
 			if (held) inFlight.release(scope);
+			if (hostHeld) hostBound.slots.release(HOST_SLOT_KEY);
+			// AWAITED, not fire-and-forget. Two reasons, and the second is the one that bites: an unawaited
+			// DEL is dropped by `shutdown`'s `process.exit(0)`, stranding the claim for its whole TTL on a
+			// restart -- and the next same-scope job would otherwise race the release, be denied, and sit out a
+			// full re-check interval while the slot it wanted went free behind it. The finally is already inside
+			// an async function, and `release` never throws.
+			await scopeSlot?.release?.();
 			clearTimeout(timer);
 			signal.removeEventListener("abort", onAbort);
 		}
 	};
 }
 
-export function createWorker({ connection, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, scopedLimits, inFlight, waitState, afterMaxMs, checkSlots, checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, extraClosers = [] }) {
-	let worker; // referenced by cancelJob/applyConcurrency before assignment; only called later, so the TDZ is fine
-	const processor = makeProcessor({
-		cancelJob: (id, reason) => worker.cancelJob(id, reason),
-		stopContainer: (name) => exec("docker", ["stop", "-t", "5", name]),
-		redis,
-		getSettings,
-		// Late-bound over `worker`: an overlay concurrency change re-binds the live slot count at the next
-		// job start. Guarded so only an integer that actually differs touches the property.
-		applyConcurrency: (n) => {
-			if (Number.isInteger(n) && worker.concurrency !== n) worker.concurrency = n;
-		},
-		pauseUntil,
-		// Undefined pass-throughs take makeProcessor's own defaults (no limits; a fresh per-processor
-		// in-flight map -- one per worker process, which under DES-CONCURRENCY-3's one-worker-per-daemon
-		// shape means one per daemon).
-		scopedLimits,
-		inFlight,
-		// Issue #230. Undefined pass-throughs take makeProcessor's own defaults (a wait state over the same
-		// redis client, and the shared 30-day `after` ceiling), so a bare wiring behaves like a wired one.
-		waitState,
-		afterMaxMs,
-		// Issue #230, the polled tier. `concurrencyNow` reads the LIVE slot count rather than the boot value,
-		// because the overlay can lower it through `dispatch_set` and a check must never take the last free
-		// slot from a paid job. Late-bound over `worker` exactly as `applyConcurrency` is, and for the same
-		// reason: the value it needs does not exist until the Worker is constructed.
-		checkSlots,
-		checkSlotCount,
-		concurrencyNow: concurrencyNow ?? (() => worker?.concurrency ?? concurrency),
-		intervalMs,
-		maxWaitMs,
-		maxChecks,
-		maxFaults,
-		deps,
-		recordRun,
-	});
-
-	worker = new Worker(QUEUE, processor, {
-		// maxRetriesPerRequest: null is REQUIRED for BullMQ's blocking connections, or it throws.
-		connection: { ...connection, maxRetriesPerRequest: null },
-		concurrency,
-		maxStalledCount: 0, // a stalled paid job FAILS, never silently re-runs (verified live)
-		...(limiter ? { limiter } : {}),
-	});
+export function createWorker({ connection, name, hostQueue = null, checkLease = null, scopeLease = null, checkTimeoutMs, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, scopedLimits, inFlight = makeInFlight(), waitState, afterMaxMs, checkSlots = makeInFlight(), checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, hostSlots = makeInFlight(), extraClosers = [] }) {
+	// One Worker per queue name (issue #57). A host-affine job -- one whose folder, secret resolver or wait
+	// check lives on THIS machine -- is enqueued to `pi-jobs@<name>` rather than filtered for at pickup,
+	// because BullMQ has no selective pop and the put-it-back alternative does not work: promotion out of
+	// the delayed set is gated on each worker's own `Date.now()`, so the fastest clock wins every hop and a
+	// job that had to reach another host might never get there.
+	const names = hostQueue ? [QUEUE, hostQueue] : [QUEUE];
+	const workers = [];
+
+	// THE HOST-WIDE BOUND, and the reason it has to exist at all. `PI_CONCURRENCY` bounds a HOST -- its RAM
+	// and its share of the provider's concurrent-stream budget (DES-CONCURRENCY-3) -- but BullMQ's own
+	// concurrency is per Worker, so two Workers at 3 would run six containers. This semaphore restores the
+	// bound as a property of the machine. Process memory is still the correct store, for this entry's own
+	// unchanged reason: it counts THIS host's containers, and the boot reaper clears survivors before
+	// draining. Armed only when a host queue exists, so a single-host deployment builds one Worker and
+	// never reaches the acquire.
+	const hostBound = hostQueue ? { slots: hostSlots, limit: () => liveConcurrency() } : null;
+	const liveConcurrency = () => workers[0]?.concurrency ?? concurrency;
+
+	for (const queueName of names) {
+		let worker; // referenced by cancelJob/applyConcurrency before assignment; only called later, so the TDZ is fine
+		const processor = makeProcessor({
+			// Bound to THIS worker: a job on the host queue is cancelled by the worker draining that queue,
+			// and the shared handle could not reach it.
+			cancelJob: (id, reason) => worker.cancelJob(id, reason),
+			stopContainer: (name) => exec("docker", ["stop", "-t", "5", name]),
+			redis,
+			getSettings,
+			// Late-bound over EVERY worker: an overlay concurrency change re-binds the live slot count at the
+			// next job start, and with two queues both have to move or the host bound and the queue bounds
+			// stop agreeing. Guarded so only an integer that actually differs touches the property.
+			applyConcurrency: (n) => {
+				if (!Number.isInteger(n)) return;
+				for (const w of workers) if (w.concurrency !== n) w.concurrency = n;
+			},
+			pauseUntil,
+			// SHARED across both workers, and that sharing is the point rather than an optimisation: the
+			// folder mutex, the per-scope ceiling and the wait-check lease all bound the HOST, so two
+			// independent maps would double every one of them exactly as two Workers double concurrency.
+			scopedLimits,
+			inFlight,
+			hostBound,
+			scopeLease,
+			// Issue #230. Undefined pass-throughs take makeProcessor's own defaults (a wait state over the same
+			// redis client, and the shared 30-day `after` ceiling), so a bare wiring behaves like a wired one.
+			waitState,
+			afterMaxMs,
+			// Issue #230, the polled tier. `concurrencyNow` reads the LIVE slot count rather than the boot value,
+			// because the overlay can lower it through `dispatch_set` and a check must never take the last free
+			// slot from a paid job.
+			checkSlots,
+			checkLease,
+			checkSlotCount,
+			checkTimeoutMs,
+			concurrencyNow: concurrencyNow ?? liveConcurrency,
+			intervalMs,
+			maxWaitMs,
+			maxChecks,
+			maxFaults,
+			deps,
+			recordRun,
+		});
+
+		worker = new Worker(queueName, processor, {
+			// maxRetriesPerRequest: null is REQUIRED for BullMQ's blocking connections, or it throws.
+			connection: { ...connection, maxRetriesPerRequest: null },
+			concurrency,
+			maxStalledCount: 0, // a stalled paid job FAILS, never silently re-runs (verified live)
+			// Issue #57. Conditional, so a bare createWorker builds a byte-identical options object -- and because
+			// bullmq's own matcher accepts both the named and unnamed client-name spellings, naming costs nothing.
+			...(name ? { name } : {}),
+			...(limiter ? { limiter } : {}),
+		});
+		workers.push(worker);
+	}
+
+	const primary = workers[0];
+	// The host-queue worker, for the caller that must register listeners on both. Attached rather than
+	// returned as a pair so every existing caller keeps receiving exactly what it received before.
+	primary.hostWorker = workers[1] ?? null;
 
 	const shutdown = async () => {
 		// Abort active jobs (=> docker stop via onAbort), then close. Without the cancel,
-		// worker.close() would wait up to 30 minutes for the container.
-		await Promise.resolve(worker.cancelAllJobs?.("shutdown")).catch(() => {});
-		await worker.close();
+		// worker.close() would wait up to 30 minutes for the container. ONE shutdown for every queue: two
+		// registrations would mean two `process.exit(0)` racing, and the second worker's containers would
+		// outlive the handler that was meant to stop them.
+		for (const w of workers) await Promise.resolve(w.cancelAllJobs?.("shutdown")).catch(() => {});
+		for (const w of workers) await w.close().catch(() => {});
 		// Close auxiliary resources (e.g. a cron scheduler) after the worker drains. Per-item catch
 		// so one failing or absent closer never strands the others or blocks exit -- matches the
 		// swallow posture on cancelAllJobs above.
@@ -586,5 +731,5 @@ export function createWorker({ connection, concurrency, getSettings, redis, deps
 	// worker still aborts in-flight jobs and docker-stops their containers rather than orphaning them.
 	if (process.platform === "win32") process.once("SIGBREAK", shutdown);
 
-	return worker;
+	return primary;
 }

package/src/queue.mjs

@@ -8,10 +8,67 @@ import { PR_CLOSE_ACTIONS } from "./triggers.mjs";
 const PR_CLOSE_WORDS = new Set(Object.values(PR_CLOSE_ACTIONS));
 
 export const QUEUE = "pi-jobs";
+
+/**
+ * The queue a HOST-AFFINE job goes to (issue #57): work only one machine can do, because the folder, the
+ * secret resolver or the wait-check script lives there.
+ *
+ * `@` is the separator because it is outside the worker-name charset (`[A-Za-z0-9._-]`), so
+ * `pi-jobs@<name>` decomposes unambiguously and a name can never contain one. A SUFFIX rather than a
+ * prefix so `KEYS bull:pi-jobs*` still shows an operator the whole deployment.
+ *
+ * Deliberately a separate queue rather than a field the pickup gate filters on. BullMQ has no selective
+ * pop, so filtering would mean taking a job and putting it back -- and promotion out of the delayed set is
+ * gated on each worker's OWN `Date.now()` in two places, so the host whose clock runs fastest wins every
+ * hop deterministically. A job that had to reach a different host might never get there, and jitter cannot
+ * fix it: it randomises WHEN the wake is, not WHO wins it.
+ */
+export const hostQueueName = (worker) => `${QUEUE}@${worker}`;
+
+/**
+ * Every queue name this deployment drains: the shared one, plus one per named host (issue #57).
+ *
+ * Derived from the REGISTRY rather than from configuration, because the reader is usually the admin or
+ * the CLI, which know their own host at best and the fleet not at all. A deployment with no named worker
+ * that declared NO name yields exactly `[QUEUE]`, so every existing caller is unchanged. Note that this
+ * is derived from `routes`, not from a row existing: every worker publishes a row, named or not.
+ *
+ * This exists because a host queue that no reader knows about is worse than no host queue: the panel
+ * would show zero schedulers while cron ran, and `pi-dispatch pause` would stop half a deployment while
+ * reporting success -- the silent no-op its own comment already warns about for a mistyped name.
+ */
+export function fleetQueueNames(hosts) {
+	const seen = new Set();
+	for (const h of hosts ?? []) {
+		// A host has a queue only when it DECLARED a name. Every worker publishes a registry row -- that is
+		// what lets an unnamed fleet be seen at all -- but an undeclared one drains only the shared queue,
+		// so deriving queue names from every row would invent `pi-jobs@<hostname>` for a queue nothing
+		// reads: pausing it would create a real key for a phantom, and the counts would be a queue that can
+		// never have jobs.
+		if (h?.routes !== true && h?.routes !== "true") continue;
+		const name = h?.name;
+		// VALIDATED, because this is peer-written data crossing a trust boundary. `hostQueueName`'s own
+		// contract leans on the charset -- `@` is the separator precisely because a name cannot contain one
+		// -- and nothing else re-checks it. A name with a `:` makes `new Queue` throw, which would take the
+		// kill switch out entirely; one with an `@` would not decompose.
+		if (typeof name !== "string" || !WORKER_NAME_RE.test(name)) continue;
+		seen.add(name);
+	}
+	// Deduped: two rows naming one host would double-count its jobs in a summed status.
+	return [QUEUE, ...[...seen].sort().map(hostQueueName)];
+}
+
+/** The name charset, duplicated from `config.mjs` deliberately: this module imports nothing. */
+const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
+
 export { chainedJobId, localJobId, deliveryJobId, gitlabDeliveryJobId, forgeDeliveryJobId };
 
-export function makeQueue(connection) {
-	return new Queue(QUEUE, { connection });
+/**
+ * A queue handle. `name` defaults to the shared queue, so every existing caller is unchanged and a
+ * single-host deployment never names anything else.
+ */
+export function makeQueue(connection, { name = QUEUE } = {}) {
+	return new Queue(name, { connection });
 }
 
 /**
@@ -226,3 +283,74 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
 	});
 	return jobId;
 }
+
+/**
+ * Every host queue that EXISTS, read from BullMQ's own keyspace rather than from the host registry.
+ *
+ * The registry answers "who is alive", and for a kill switch that is the wrong question. A host whose
+ * registry writes fail for ninety seconds loses its row while its BullMQ worker -- a separate connection,
+ * built with `maxRetriesPerRequest: null` precisely to ride out blips -- keeps draining. Pausing "the
+ * fleet" would then miss it and report success. The same gap opens for the ~15s before a booting worker's
+ * first beat lands, and on every `service restart`, since a clean shutdown DELs the row.
+ *
+ * Worse is the direction with no recovery path: pause while a host is live durably pauses its queue, and a
+ * later resume while that host is DOWN enumerates nothing for it. The queue stays paused permanently, and
+ * no surface can see it, because every surface was reading the registry too.
+ *
+ * A queue's meta key is durable and outlives its worker, so this asks the only authority that cannot go
+ * stale: the queues themselves. It also restores what `host-registry.mjs` claims about itself -- delete the
+ * whole `host:*` keyspace and nothing decides differently -- which the registry-derived kill switch had
+ * quietly made false.
+ *
+ * SCAN, not KEYS, and it is why this is NOT on the panel's per-tick path: it is for the rare command where
+ * being wrong costs money, not for a reader that runs every second. Fails open to `[]`, so an unreadable
+ * keyspace degrades to the registry's answer rather than refusing.
+ */
+export async function discoverHostQueues(redis, { timeoutMs = 2_000, count = 500 } = {}) {
+	const prefix = `bull:${QUEUE}@`;
+	const names = new Set();
+	try {
+		const deadline = Date.now() + timeoutMs;
+		let cursor = "0";
+		do {
+			// BOUNDED, because BullMQ's connections carry `maxRetriesPerRequest: null` and a command against
+			// an unreachable server therefore QUEUES FOREVER rather than rejecting -- so an unguarded await
+			// here would hang the kill switch instead of failing it open. The same trap the registry's
+			// `bounded` exists for.
+			//
+			// CLEARED on the way out, and NOT `unref`'d. Leaving it pending held the event loop open for the
+			// rest of the budget after the work was done, so `pi-dispatch pause` sat for two seconds having
+			// already paused everything; unref'ing instead would stop it firing when the hang is the last
+			// thing on the loop, which is the one case it exists for.
+			let timer;
+			const [next, keys] = await Promise.race([
+				redis.scan(cursor, "MATCH", `${prefix}*:meta`, "COUNT", count),
+				new Promise((_, reject) => {
+					timer = setTimeout(() => reject(new Error("scan timed out")), Math.max(1, deadline - Date.now()));
+				}),
+			]).finally(() => clearTimeout(timer));
+			cursor = next;
+			for (const key of keys ?? []) {
+				const name = String(key).slice(prefix.length, -":meta".length);
+				// Validated like every other peer-derived name: a key an operator hand-created could hold
+				// anything, and `new Queue` throws on a `:`, which would take the kill switch out entirely.
+				if (WORKER_NAME_RE.test(name)) names.add(name);
+			}
+		} while (cursor !== "0" && Date.now() < deadline);
+	} catch {
+		// Fail open: the registry's answer alone is still better than refusing to pause.
+		return [];
+	}
+	return [...names].sort().map(hostQueueName);
+}
+
+/**
+ * The union of what is LIVE (the registry) and what EXISTS (the keyspace), which is the set a kill switch
+ * must act on: a live host with no queue yet has nothing to pause, and a dead host's queue still holds
+ * jobs and still has a paused flag somebody has to be able to clear.
+ */
+export function unionQueueNames(fromRegistry, fromKeyspace) {
+	const seen = new Set([...(fromRegistry ?? []), ...(fromKeyspace ?? [])]);
+	seen.delete(QUEUE);
+	return [QUEUE, ...[...seen].sort()];
+}

package/src/run-history.mjs

@@ -378,7 +378,7 @@ function rebuildUsage(u) {
  * default to `null` when the outcome does not carry them, so the record shape is stable whether or not
  * the source reports those fields.
  */
-export function buildRecord({ job, result, error, startedAt, endedAt }) {
+export function buildRecord({ job, result, error, startedAt, endedAt, host = null }) {
 	const data = job.data ?? {};
 	const kind = data.kind ?? job.name;
 	const source = result ?? error ?? {};
@@ -441,6 +441,28 @@ export function buildRecord({ job, result, error, startedAt, endedAt }) {
 		// BRANCH NAME ARE DELIBERATELY ABSENT: this record's PII-free-by-construction property rests on it
 		// holding no attacker-chosen string, and a branch name is exactly that.
 		session: source.session ?? null,
+		// Which machine ran this (issue #57). Additive, nullable, an explicit literal, TAIL position: every
+		// prior addition took the tail, and "field order is the serialisation order" is this record's
+		// contract, so the tail is the only placement that leaves twenty-four existing positions untouched.
+		// Unconditional rather than a conditional spread, on the same contract sentence and on the
+		// `tokens`/`usage`/`session` precedent that null-with-the-key-present is this record's normal case.
+		//
+		// ADMISSIBILITY, which has to engage this record's own sentences rather than sidestep them. The
+		// PII-free-by-construction property rests on holding NO ATTACKER-CHOSEN STRING, and `host`
+		// satisfies that absolutely: there is no path from a webhook payload, an issue body, a branch name
+		// or a folder to this value. It is fixed once at boot from one environment variable against a
+		// charset that excludes `/` and `\`, so it cannot even be path-shaped -- which is the property
+		// `targetFor` drops a local folder to its basename to get.
+		//
+		// What it is NOT is anonymous, and that is worth writing down rather than glossing. The default is
+		// `os.hostname()`, and on a personal machine a hostname is often a person's name. The honest word
+		// is OPERATOR-DISCLOSED: the operator names their own machines, this record is written to their own
+		// disk, that name is already on every packet the machine sends, and `PI_WORKER_NAME` is the
+		// documented answer for anyone who wants something else here.
+		//
+		// Passed in rather than read from a module-level value, so `buildRecord` stays pure and every
+		// existing caller keeps getting `null` without knowing this field exists.
+		host,
 	};
 }
 

package/src/schedules.mjs

@@ -23,7 +23,7 @@ import { parseTriggers } from "./triggers.mjs";
  * valid deployment. `readFileSync`/`existsSync` are injectable so tests exercise the full path with no
  * real filesystem.
  */
-export function loadSchedules(config, { readFileSync = fsReadFileSync, existsSync = fsExistsSync } = {}) {
+export function loadSchedules(config, { readFileSync = fsReadFileSync, existsSync = fsExistsSync, fleet = false } = {}) {
 	const path = config.triggersFile;
 	if (path === null || path === undefined) return []; // cron disabled
 
@@ -33,14 +33,77 @@ export function loadSchedules(config, { readFileSync = fsReadFileSync, existsSyn
 
 	const triggers = parseTriggers(readFileSync(path, "utf8"), path);
 
-	return triggers.filter((t) => t.on.type === "cron").map((t) => normalizeCronSchedule(t, path, existsSync));
+	return triggers.filter((t) => t.on.type === "cron").map((t) => normalizeCronSchedule(t, path, existsSync, fleet));
 }
 
-function normalizeCronSchedule({ on, run }, path, existsSync) {
+/**
+ * The cron set AS AUTHORED, before any placement decision (issue #57).
+ *
+ * This is the object two hosts have to agree about, and it is deliberately not `loadSchedules`'s output.
+ * That function resolves PLACEMENT -- it replaces every trigger whose folder is on another machine with
+ * a stub -- so its result differs per host BY CONSTRUCTION. Fingerprinting it would make every correctly
+ * configured fleet refuse itself forever: mini1 owns `/a`, mini2 owns `/b`, their sets never match, and
+ * neither ever reconciles again. What they share is the FILE, so the file is what gets hashed.
+ *
+ * Pure and fs-free apart from the read: no `existsSync`, because existence is exactly the question that
+ * makes two honest hosts differ.
+ */
+export function authoredCron(config, { readFileSync = fsReadFileSync, existsSync = fsExistsSync } = {}) {
+	const path = config.triggersFile;
+	if (path === null || path === undefined) return null; // cron disabled: no opinion at all (see cronFingerprint)
+	if (!existsSync(path)) return null;
+	try {
+		return parseTriggers(readFileSync(path, "utf8"), path)
+			.filter((t) => t.on.type === "cron")
+			.map((t) => ({ schedulerId: t.on.id, pattern: t.on.pattern, run: t.run }));
+	} catch {
+		// A file this host cannot parse is not an opinion about what should be scheduled. It refuses boot
+		// elsewhere and keeps last-good on reload; here it must not become a fingerprint that disagrees
+		// with every peer.
+		return null;
+	}
+}
+
+/**
+ * Split a schedule set into the triggers THIS host serves and the ones it does not (issue #57).
+ *
+ * `loadSchedules` already refused everything a pure validator could refuse and everything the filesystem
+ * could answer for a trigger this host owns. What is left is the third question, and it is the one Gap 2
+ * is about: a folder that is not here is not necessarily a mistake, it may simply be another machine's.
+ *
+ * Exported so the split is testable without an fs, and so a caller can report what it will not be running.
+ */
+export function servedSchedules(schedules) {
+	const served = [];
+	const unserved = [];
+	for (const s of schedules) (s.unserved ? unserved : served).push(s);
+	return { served, unserved };
+}
+
+function normalizeCronSchedule({ on, run }, path, existsSync, fleet) {
 	// The pure validator already guaranteed a non-empty, `:`-free, charset-valid, unique id and a
 	// well-formed pattern; folder existence is the one fs-dependent check it deferred to here.
+	//
+	// ON A FLEET THAT IS THE WRONG QUESTION. `INT-TRIGGERS-FILE-CONTRACT` splits this as "type here,
+	// reality where it can be known", and issue #57 adds a third level: PLACEMENT, where the fleet is
+	// known. A folder that is absent on THIS machine may simply belong to another one, and refusing the
+	// worker's boot for it takes every unrelated trigger -- every forge job, every other folder -- offline
+	// with it. That is the sentence #57's own acceptance forbids.
+	//
+	// `fleet` is `PI_WORKER_NAME` being DECLARED, deliberately, and not a registry read. Two reasons, and
+	// both are failures I would otherwise have shipped. A registry read here would make a fleet-wide
+	// restart into a fleet-wide boot refusal, because every host would come up seeing no peers yet. And it
+	// would have to happen after the Valkey client exists, which is BELOW the four destructive boot sweeps
+	// -- so a single-host deployment with one typo'd folder would reap containers, prune history and delete
+	// sandboxes on every restart before refusing. Declaring a name is the operator saying "this is a
+	// fleet", it is known before anything runs, and it keeps a single-host deployment byte-identical.
 	if (!existsSync(run.folder)) {
-		throw configError(`cron trigger "${on.id}": run.folder does not exist: ${run.folder} (${path})`);
+		if (!fleet) {
+			throw configError(`cron trigger "${on.id}": run.folder does not exist: ${run.folder} (${path})`);
+		}
+		// Not mine. Its skillsDir is not my business either: `isAbsolute` is OS-dependent, and judging
+		// another host's path on my platform is the exact mistake the shared validator refuses to make.
+		return { schedulerId: on.id, unserved: "folder-absent" };
 	}
 
 	// `run.skillsDir` gets the same treatment, and for the same reason (REQ-PER-TRIGGER-SKILLS): the pure