@edgehero/pi-dispatch 1.6.1 → 1.8.0

package/src/start.mjs

@@ -1,10 +1,10 @@
 import { execFile } from "node:child_process";
-import { watch } from "node:fs";
+import { readFileSync, watch } from "node:fs";
 import { dirname, basename, join } from "node:path";
 import { promisify } from "node:util";
 import { configError, loadConfig } from "./config.mjs";
 import { makeRedisClient, parseConnection } from "./connection.mjs";
-import { reconcile, reloadSchedules } from "./cron.mjs";
+import { reconcileGated, reloadSchedules } from "./cron.mjs";
 import { makeGitHubAuth } from "./get-token.mjs";
 import { makeGitHubHost } from "./github-host.mjs";
 import { makeGitLabAuth } from "./gitlab-auth.mjs";
@@ -14,8 +14,12 @@ import { makeForgejoHost } from "./forgejo-host.mjs";
 import { makeAzureAuth } from "./azure-auth.mjs";
 import { makeAzureHost } from "./azure-host.mjs";
 import { makeEgressPreflight } from "./egress.mjs";
+import { checkSlotKey, makeFleetLease, makeScopeClaimSweeper, scopeSlotKey } from "./fleet-lease.mjs";
+import { capabilityTokens, serializeCaps } from "./capabilities.mjs";
+import { cronFingerprint } from "./fingerprint.mjs";
+import { makeHostRegistry } from "./host-registry.mjs";
 import { makeImagePreflight } from "./image-preflight.mjs";
-import { createWorker } from "./index.mjs";
+import { createWorker, JOB_TIMEOUT_MS } from "./index.mjs";
 import { makeCollectChain } from "./outbox.mjs";
 import { containerPackagePaths, readStageManifest } from "./packages.mjs";
 import { makeCleanup, makeForgePreparers, makePrepareWorkspace } from "./prepare.mjs";
@@ -24,19 +28,45 @@ import { makeSandboxReaper } from "./sandbox-store.mjs";
 import { makeSessionStore } from "./session-store.mjs";
 import { makeCheckOnceSpent, makeCheckWaitSkew, makeDisarmOnce } from "./triggers-file.mjs";
 import { loadPauseWindows, pauseUntilMs } from "./pause-windows.mjs";
-import { loadScopedLimits } from "./scoped-limits.mjs";
+import { loadScopedLimits, scopeKeyPrefix } from "./scoped-limits.mjs";
 import { makeWaitChecker } from "./wait-check.mjs";
 import { makeWaitState } from "./wait-state.mjs";
-import { makeQueue } from "./queue.mjs";
+import { hostQueueName, makeQueue } from "./queue.mjs";
 import { makeRunContainer } from "./run-container.mjs";
 import { makeSecretsResolver } from "./secrets.mjs";
-import { buildRecord, makeFindPreviousRun, makeLogReaper, makeLogSink, makeRecordWriter } from "./run-history.mjs";
+import { buildRecord, makeFindPreviousRun, makeLogReaper, makeLogSink, makeRecordWriter, sanitizeJobId } from "./run-history.mjs";
+import { makeRunMirror } from "./run-mirror.mjs";
 import { effectiveSettings, readOverlay } from "./runtime-settings.mjs";
-import { loadSchedules } from "./schedules.mjs";
+import { authoredCron, loadSchedules, servedSchedules } from "./schedules.mjs";
 import { makeStallGuard } from "./scheduler-stall-guard.mjs";
 
 const exec = promisify(execFile);
 
+/** How long boot will wait for `docker image inspect` before shipping without a digest. */
+const BOOT_IMAGE_TIMEOUT_MS = 5_000;
+
+/**
+ * How long a fleet-wide scope claim lives. `JOB_TIMEOUT_MS` plus slack: a container cannot outlive that
+ * ceiling, so the claim cannot expire underneath a live job -- which is what makes a refresh unnecessary
+ * rather than merely unimplemented. DERIVED from that constant rather than written as a number, so the
+ * coupling maintains itself if the timeout ever moves.
+ */
+const SCOPE_CLAIM_TTL_MS = JOB_TIMEOUT_MS + 5 * 60 * 1000;
+
+/**
+ * This worker's own package version, for the registry row (issue #57): a rolling upgrade should be
+ * visible as a fact about the fleet rather than as a diff someone has to run. Read once, and never
+ * fatal -- an unreadable manifest costs a blank field, not a boot. npm always ships `package.json`
+ * whatever `files` says, so this resolves from an installed package as well as from a checkout.
+ */
+const WORKER_VERSION = (() => {
+	try {
+		return JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version ?? "";
+	} catch {
+		return "";
+	}
+})();
+
 /**
  * Boot-time reaper: clear stray `pi-job-*` containers a previous worker crash left behind, before
  * the new worker starts draining. A leaked container keeps spending, so it must go before any new
@@ -59,7 +89,7 @@ const exec = promisify(execFile);
  * `reloadSchedules`. Best-effort and unref'd so it never blocks shutdown; a platform without `fs.watch`
  * logs and the worker keeps its boot-time schedulers.
  */
-function watchTriggersFile(config, queue, log) {
+function watchTriggersFile(config, queue, log, ref, registry, tz, fleet) {
 	const path = config.triggersFile;
 	const dir = dirname(path) || ".";
 	const file = basename(path);
@@ -68,7 +98,7 @@ function watchTriggersFile(config, queue, log) {
 		watch(dir, (_event, changed) => {
 			if (changed && changed !== file) return; // only our file (a null name -> reload to be safe)
 			clearTimeout(timer);
-			timer = setTimeout(() => void reloadSchedules(config, queue, { log }), 150);
+			timer = setTimeout(() => void reloadSchedules(config, queue, { log, ref, registry, tz, fleet }), 150);
 		}).unref?.();
 		log("triggers_watching", { path });
 	} catch (err) {
@@ -170,8 +200,17 @@ export function makeReaper({ log }) {
 					log("reaped_network", { network: net });
 				} catch {} // still in use, or already gone -- either way not this boot's problem
 			}
+			// Whether the enumeration HAPPENED, which the scope-claim sweep depends on: it may only delete a
+			// claim naming this host once this host has actually established that it holds no containers.
+			return { reaped: true };
 		} catch (err) {
+			// The `docker ps` is inside this try, so this path CANNOT establish that this host holds no
+			// containers -- whether it failed before listing anything or after reaping some and then losing
+			// the daemon. Either way the claim "I hold nothing" is unproven, and sweeping on it would free
+			// slots for containers that may STILL BE RUNNING, letting another host start more alongside
+			// them: a money overrun rather than a tidy-up. Conservative in the only safe direction.
 			log("reaper_skipped", { reason: err?.message });
+			return { reaped: false };
 		}
 	};
 }
@@ -200,11 +239,14 @@ export async function startWorker(
 		makeReaper: makeReaperFn = makeReaper,
 		makeLogSink: makeLogSinkFn = makeLogSink,
 		makeRecordWriter: makeRecordWriterFn = makeRecordWriter,
+		makeRunMirror: makeRunMirrorFn = makeRunMirror,
 		makeLogReaper: makeLogReaperFn = makeLogReaper,
 		makeSandboxReaper: makeSandboxReaperFn = makeSandboxReaper,
 		makeRunContainer: makeRunContainerFn = makeRunContainer,
 		makeSecretsResolver: makeSecretsResolverFn = makeSecretsResolver,
 		makeImagePreflight: makeImagePreflightFn = makeImagePreflight,
+		makeScopeClaimSweeper: makeScopeClaimSweeperFn = makeScopeClaimSweeper,
+		makeHostRegistry: makeHostRegistryFn = makeHostRegistry,
 		makeEgressPreflight: makeEgressPreflightFn = makeEgressPreflight,
 		makeGitLabAuth: makeGitLabAuthFn = makeGitLabAuth,
 		makeGitLabHost: makeGitLabHostFn = makeGitLabHost,
@@ -215,12 +257,23 @@ export async function startWorker(
 	} = {},
 ) {
 	const config = loadConfig(env);
-	const log = (event, fields = {}) => process.stdout.write(`${JSON.stringify({ event, ...fields })}\n`);
+	// `host` sits AFTER the spread, so it is authoritative rather than overridable (issue #57). No call
+	// site can know better than this closure which process wrote a line, and one that passed `host` would
+	// be lying by construction -- verified: none does. This is also why the stamp lives ONLY here. Every
+	// other module takes `log` injected, and two tests pin the KEY SET of the fields object handed to an
+	// injected log (`run_record_failed`, `wait_check`); a `host` added at any call site would break them,
+	// while one added inside this closure cannot reach them.
+	const log = (event, fields = {}) => process.stdout.write(`${JSON.stringify({ event, ...fields, host: config.workerName })}\n`);
 
 	// DES-CRON-VIA-BULLMQ-SCHEDULER: load and validate the triggers file with the operator present and
 	// before any Valkey contact, so a misconfigured schedule refuses startup loudly (configError) rather
 	// than upserting a broken scheduler. [] means cron disabled (no PI_TRIGGERS_FILE, or no cron triggers).
-	const schedules = loadSchedules(config);
+	// A mutable ref, like its `pauseWindows` and `scopedLimits` siblings and for a reason this one only
+	// acquired with issue #57: the heartbeat fingerprints what this host CURRENTLY believes should be
+	// scheduled, and a `const` frozen at boot would make it publish the pre-edit set forever -- so two
+	// hosts would see each other's fingerprint oscillate on the beat period, refusing or agreeing
+	// depending on which half of a beat a reload happened to land in.
+	const schedules = { current: loadSchedules(config, { fleet: config.workerNameDeclared }) };
 
 	// REQ-SCOPED-PAUSE-WINDOWS: load + validate the pause-windows file with the operator present and before any
 	// Valkey contact, so a malformed file refuses startup (configError) rather than silently disabling scoped
@@ -290,8 +343,10 @@ export async function startWorker(
 
 	// Clear strays left by a previous crash before the worker starts draining. Best-effort: the reaper
 	// swallows its own docker errors; this guard keeps any reaper failure from blocking boot.
+	// Whether the container reaper actually ENUMERATED, which the scope-claim sweep below depends on.
+	let reaped = false;
 	try {
-		await makeReaperFn({ log })();
+		reaped = (await makeReaperFn({ log })())?.reaped === true;
 	} catch (err) {
 		log("reaper_skipped", { reason: err?.message });
 	}
@@ -324,12 +379,42 @@ export async function startWorker(
 	// One raw Redis client, shared by the budget (via the worker) and the scheduler stall guard, so it is
 	// hoisted out of the createWorkerFn arg object.
 	const redis = makeRedisClient(config.valkeyUrl);
+
+	// This host's own stale scope claims, gated on the reaper having having enumerated: the
+	// reaper is what establishes that this machine holds no `pi-job-*` containers, so a claim naming this
+	// host is a claim for a container that no longer exists. Deleting it is not a second source of truth --
+	// it is the SAME source writing down what it just established, which is what answers `OQ-008` here.
+	// Best-effort and double-wrapped like every other boot sweep: an OPTIMISATION over the TTL, never the
+	// mechanism, so a fault costs one TTL of a stale claim and never a boot.
+	try {
+		if (config.workerNameDeclared)
+			await makeScopeClaimSweeperFn({ redis, workerName: config.workerName, limits: scopedLimits.current.map((r) => ({ concurrent: r.concurrent, hash: scopeKeyPrefix(r.scope).slice("budget:s:".length) })), log })({ reaped });
+	} catch (err) {
+		log("scope_claims_sweep_skipped", { reason: err?.message });
+	}
+
 	// The persistent runtime queue: the stall guard tears schedulers down through it, AND the outbox
 	// collector enqueues chained children onto it -- the same pi-jobs queue, so one handle serves both.
 	// Non-failFast: a long-lived handle rides out a Valkey blip. Registered as an extraCloser so shutdown
 	// drains it after the worker.
 	const runtimeQueue = makeQueue(parseConnection(config.valkeyUrl));
 
+	// THE HOST QUEUE (issue #57): work only this machine can do, because the folder lives here.
+	//
+	// Armed by the operator DECLARING a name, not by a peer appearing. Two reasons, and the first is
+	// decisive: which queue a job is enqueued to is a routing decision made by whoever enqueues it, so it
+	// cannot be allowed to flip underneath a running deployment when a second host happens to register --
+	// a cron scheduler upserted on one queue and pruned from another is exactly the mutual teardown this
+	// issue exists to stop. And a second BullMQ Worker is a second blocking connection, which a single-host
+	// deployment should not pay for silently. Declaring a name IS the multi-host declaration; `doctor`
+	// warns when peers exist and nobody has made it.
+	const hostQueue = config.workerNameDeclared ? hostQueueName(config.workerName) : null;
+	// The long-lived handle the cron watcher reloads through. Its own when a host queue is armed, so a
+	// live triggers-file edit lands on the same queue the boot reconcile used; otherwise the shared
+	// runtime queue, exactly as before. Registered as an extraCloser only when it is a NEW handle --
+	// closing `runtimeQueue` twice would be closing another owner's connection.
+	const cronQueue = hostQueue ? makeQueue(parseConnection(config.valkeyUrl), { name: hostQueue }) : runtimeQueue;
+
 	// REQ-LOCAL-JOB-VISIBILITY durable run history, all host-side. The raw `.log` sink is gated on
 	// captureJobLogs (raw container output is user-authored data, opt-in per no-pii-in-logs); the id-only
 	// `.json` record via recordRun is ALWAYS on, so every run leaves a stable, non-PII trace regardless.
@@ -364,8 +449,22 @@ export async function startWorker(
 	// that can neither disarm nor pre-spend-check.
 	const onceTriggersFile = env.PI_TRIGGERS_FILE ?? join(process.cwd(), "triggers.json");
 	const disarmOnce = makeDisarmOnce({ triggersPath: onceTriggersFile, log });
+	// The fleet-visible copy of the run history (issue #57, Gap 3). Armed only on a deployment that declared
+	// a worker name: an unnamed one is a single host, its own files ARE the whole history, and a mirror
+	// would be bytes nothing reads. That is also what keeps a single-host deployment byte-identical, since
+	// no job then issues a single extra Valkey command.
+	const runMirror = config.workerNameDeclared ? makeRunMirrorFn({ redis, retentionDays: config.logRetentionDays, log }) : null;
 	const recordRun = ({ job, result, error, startedAt, endedAt }) => {
-		writeRecord(buildRecord({ job, result, error, startedAt, endedAt }));
+		// The `host` is stamped HERE rather than inside the processor, which is what keeps every one of its
+		// four `recordRun` call sites byte-unchanged and `buildRecord` a pure function of its arguments.
+		const record = buildRecord({ job, result, error, startedAt, endedAt, host: config.workerName });
+		writeRecord(record);
+		// STRICTLY AFTER the file, and deliberately not awaited. After, because a crash between the two must
+		// leave a record with no fleet row rather than a fleet row with no record -- the mirror is a VIEW,
+		// and a view that can outlive its source is a second source of truth. Not awaited, because this is
+		// the job's own completion path: a slow Valkey may cost a row in a panel and must never hold up a
+		// job that has already finished and already been written to disk. `mirror` never rejects.
+		void runMirror?.mirror(record, sanitizeJobId(record.jobId));
 		// Strictly AFTER the durable record: "fired" means "produced a run record", and the crash
 		// direction this ordering buys is the chosen one -- an armed one-shot with a record, never a
 		// disarm before writeRecord RETURNED. Returned, not succeeded: the record writer swallows fs
@@ -404,10 +503,13 @@ export async function startWorker(
 	const bootConcurrency = bootSettings.invalid ? config.concurrency : bootSettings.concurrency;
 
 	// INT-OUTBOX-CONTRACT chain collector: the host-side reader of a completed local parent's /outbox. It
-	// enqueues chained children onto runtimeQueue via enqueueLocalJob -- the same pi-jobs queue the stall
-	// guard tears down through. Never throws, so a chain fault cannot flip a completed parent
+	// enqueues chained children onto the CRON queue via enqueueLocalJob -- this host's own when one is
+	// armed, since a chained child continues the working tree this machine just used. Never throws, so a chain fault cannot flip a completed parent
 	// (CONST-RETRY-INFRA-ONLY). The processor calls it as the sole COMPLETED-path chain step.
-	const collectChain = makeCollectChain({ queue: runtimeQueue, config, log });
+	// Onto the HOST queue when one is armed. A chained child is same-folder and local-parent-only
+	// (`OQ-009`), so the working tree it needs is the one this machine just used: routing it anywhere
+	// else would enqueue a job only this host can run onto a queue every host drains.
+	const collectChain = makeCollectChain({ queue: cronQueue, config, log });
 
 	// REQ-GLOBAL-PI-OVERLAY staged packages: read the operator's stage manifest at EACH job start, like
 	// getSettings above and the pause-window ref below.
@@ -455,13 +557,83 @@ export async function startWorker(
 	if (config.globalPiDir && !readStageManifest({ globalPiDir: config.globalPiDir })) log("packages_manifest_absent", { overlay: config.globalPiDir });
 	getPackagePaths();
 
+	// Issue #57. Published before the worker starts draining, so a peer that boots a moment later sees this
+	// host rather than an empty fleet. The image identity rides the SAME preflight the job path uses -- one
+	// inspect implementation, one format string -- called once here with an empty job, which resolves the
+	// deployment default and trips none of the per-job label gates.
+	//
+	// The boot line and the registry may cache this where the GATE may not, and the distinction is the whole
+	// argument: a gate that caches gives a WRONG DECISION when an operator builds or removes an image
+	// mid-day, which is why `imagePreflight` is deliberately not memoised below. A heartbeat that caches
+	// gives a STALE ROW, and nothing reads a row to decide anything.
+	// ONE preflight instance, constructed once and shared: `start-wiring.test.mjs` pins that, and the
+	// reason is the module's own -- the tag the preflight checked has to be the tag `docker run` is
+	// handed, and two constructions are two chances for that to stop being true.
+	const imagePreflight = makeImagePreflightFn({ image: config.jobImage });
+	// BOUNDED, because `.catch()` cannot rescue a promise that never settles: `runDocker` resolves only on
+	// the child's `close` or `error` and has no timeout of its own, so a wedged daemon would hang boot
+	// here. This read is a nicety -- a digest for the boot line and the registry -- and a nicety may
+	// never be able to stop a worker starting. The per-JOB preflight keeps its unbounded wait, where a
+	// wedged daemon is the job's problem and the 30-minute job timeout already covers it.
+	const bootImage = await Promise.race([
+		imagePreflight({}).catch(() => ({})),
+		new Promise((resolve) => setTimeout(() => resolve({}), BOOT_IMAGE_TIMEOUT_MS).unref?.()),
+	]);
+	// Resolved once: `Intl` is not free, and this value cannot change without a restart.
+	const hostTz = Intl.DateTimeFormat().resolvedOptions().timeZone ?? "";
+	const registry = makeHostRegistryFn({ redis, name: config.workerName, log });
+	// NOT awaited, and that is load-bearing rather than an optimisation. `makeRedisClient` sets
+	// `maxRetriesPerRequest: null` -- required for BullMQ's blocking connections -- which means a command
+	// issued against an unreachable server QUEUES FOREVER instead of rejecting. Awaiting the first beat
+	// would therefore hang boot indefinitely on a deployment whose Valkey is down, turning a telemetry
+	// keyspace into a boot dependency. The registry is never on a decision path, so a worker that comes
+	// up before its own row does is correct: the row appears when Valkey does.
+	void registry.start({
+		version: WORKER_VERSION,
+		image: config.jobImage,
+		imageDigest: bootImage.imageDigest ?? "",
+		piVersion: bootImage.piVersion ?? "",
+		// A thunk, because the spec says this row carries the LIVE slot count and the overlay can lower it
+		// mid-run through `dispatch_set`. A literal here would publish the boot value forever.
+		concurrency: () => worker?.concurrency ?? bootConcurrency,
+		pid: process.pid,
+		// Whether this host DRAINS a queue of its own. Every worker publishes a row; only a host that declared
+		// a name has somewhere for routed work to go, and a reader must not invent a queue for one that has not.
+		routes: config.workerNameDeclared,
+		// What this host can serve that another might not (issue #57, `OQ-032`): the secret and wait profiles
+		// it has declared. NAMES only, never the resolver paths behind them -- a path is PII on Windows and
+		// operator topology everywhere, and the receiver only needs to know WHICH host, not what it runs.
+		//
+		// Recomputed per beat rather than frozen at boot, for the reason the digest is: a host that gains a
+		// profile on restart must start attracting that work within one beat, and one that loses it must stop.
+		caps: () => serializeCaps(capabilityTokens(config)),
+		// The host's IANA zone, because a cron PATTERN carries none: `triggers.json` has no `tz` field and
+		// BullMQ hands the pattern to cron-parser with no zone, so it resolves in each worker's LOCAL time.
+		// On one host that is exactly what an operator means; on two in different zones the same pattern is
+		// two different instants, and nothing anywhere says so. Published now so a later slice can refuse.
+		tz: hostTz,
+		// The cron fingerprint rides every beat from the LIVE ref, so a peer always compares against what
+		// this host believes now rather than what it believed at boot. `null` means abstain: cron disabled
+		// here is no opinion at all, and such a host must never be able to disagree with one that has one.
+		fpCron: () => cronFingerprint(authoredCron(config), { tz: hostTz }) ?? "",
+		cronCount: () => schedules.current.length,
+	});
+
+
 	const worker = createWorkerFn({
 		connection: parseConnection(config.valkeyUrl),
+		hostQueue,
+		// Names the BullMQ Worker, which makes `getWorkers()` rows tell hosts apart -- bullmq appends
+		// `:w:<name>` to the client name and `moveToActive` stamps `processedBy` onto each active job's
+		// hash, so per-job host attribution arrives for free. A NICETY on top of the registry and never the
+		// source of truth: that call rests on CLIENT SETNAME, which bullmq's own doc-comment says some
+		// providers do not support, and a host list that silently empties cannot be what a decision reads.
+		name: config.workerName,
 		concurrency: bootConcurrency,
 		getSettings,
 		redis,
 		recordRun,
-		extraClosers: [runtimeQueue],
+		extraClosers: [runtimeQueue, registry, ...(cronQueue === runtimeQueue ? [] : [cronQueue])],
 		// REQ-SCOPED-PAUSE-WINDOWS: the processor defers a job whose folder/repo is inside an active window.
 		// Reads the live-reloaded ref, so an operator edit takes effect on the next job without a restart.
 		pauseUntil: (job, now) => pauseUntilMs(pauseWindows.current, job, now),
@@ -475,7 +647,28 @@ export async function startWorker(
 		waitState: makeWaitState({ redis }),
 		// The polled tier's bounds, read per pickup from config so they are one value with one home. The
 		// slot count is a CEILING the gate clamps against the live concurrency, never the final number.
+		// The fleet-wide half of the wait-check bound (issue #57), armed on the same predicate as the host
+		// queue: declaring a name is declaring a fleet. Its TTL is DERIVED rather than guessed -- the gate
+		// holds the lease across every profile in turn, each bounded by the check timeout, so one timeout
+		// per profile plus one for the overhead between them.
+		// The fleet-wide half of a scoped `concurrent` ceiling. Its TTL is DERIVED rather than guessed, and
+		// derived is what makes a heartbeat unnecessary: `JOB_TIMEOUT_MS` is a hard 30-minute ceiling on how
+		// long any container can run, so a TTL above it cannot expire underneath a live job -- which is the
+		// failure that would matter, because it would let another host start a second container on a scope
+		// the operator limited to one. Nothing refreshes this claim, deliberately: a refresher would be a
+		// second thing to get wrong for a window that cannot be reached.
+		scopeLease: hostQueue ? makeFleetLease({ redis, holderPrefix: config.workerName, keyFor: scopeSlotKey, ttlMs: SCOPE_CLAIM_TTL_MS, log }) : null,
+		checkLease: hostQueue
+			? makeFleetLease({
+					redis,
+					holderPrefix: config.workerName,
+					keyFor: checkSlotKey,
+					ttlMs: config.waitCheckTimeoutMs, // a floor; the gate passes the real one, derived from the profile count
+					log,
+				})
+			: null,
 		checkSlotCount: () => config.waitCheckSlots,
+		checkTimeoutMs: () => config.waitCheckTimeoutMs,
 		intervalMs: () => config.waitIntervalMs,
 		maxWaitMs: () => config.waitMaxMs,
 		maxChecks: () => config.waitMaxChecks,
@@ -523,7 +716,7 @@ export async function startWorker(
 			// cache would be wrong in both directions -- an operator who builds the image mid-day would stay refused,
 			// one who removes it would stay admitted. Contrast the staged-package manifest, correctly read once at
 			// boot because it is deploy-time state under a :ro mount; the host's image set is not.
-			imagePreflight: makeImagePreflightFn({ image: config.jobImage }),
+			imagePreflight,
 			// REQ-EGRESS-ALLOWLIST, and built here for the same reason the image preflight is: one deployment
 			// value, one place, so the gate that checks the proxy and the runner that attaches to its network
 			// cannot disagree about which proxy is meant. Nothing is memoised here either -- an operator who
@@ -636,10 +829,13 @@ export async function startWorker(
 	// job-image-missing), never
 	// user content. Included only when present so success lines stay clean; a shutdown-aborted job logs
 	// { outcome: "policy", reason: "worker-abort" }, making a restart-dropped job visible.
-	worker.on("completed", (job, result) =>
+	// BOTH workers, or a cron job on the host queue produces no `job_completed` line at all -- and
+	// REQ-LOCAL-JOB-VISIBILITY's whole point is that a missing line is what tells a human a run did nothing.
+	const allWorkers = [worker, ...(worker.hostWorker ? [worker.hostWorker] : [])];
+	for (const w of allWorkers) w.on("completed", (job, result) =>
 		log("job_completed", { jobId: job?.id, outcome: result?.outcome, ...(result?.reason ? { reason: result.reason } : {}) }),
 	);
-	worker.on("failed", (job, err) =>
+	for (const w of allWorkers) w.on("failed", (job, err) =>
 		log("job_failed", { jobId: job?.id, attempt: job?.attemptsMade, reason: String(err?.message ?? err).slice(0, 120) }),
 	);
 
@@ -649,31 +845,45 @@ export async function startWorker(
 	const guard = makeStallGuard({
 		redis,
 		threshold: config.schedulerStallMax,
-		removeJobScheduler: (id) => runtimeQueue.removeJobScheduler(id),
+		// The queue the schedulers were actually INSTALLED on. Torn down from `runtimeQueue` on a named
+		// host, this money backstop -- a wedged scheduled run is re-paid on every stall -- silently no-ops.
+		removeJobScheduler: (id) => cronQueue.removeJobScheduler(id),
 		log,
 	});
-	worker.on("stalled", (jobId) => void guard.onStalled(jobId));
+	for (const w of allWorkers) w.on("stalled", (jobId) => void guard.onStalled(jobId));
 
 	// DES-CRON-VIA-BULLMQ-SCHEDULER: install the schedule set (and prune orphans) before announcing the
 	// worker is up, so schedules_installed always precedes worker_started. An empty set skips the reconcile
 	// queue entirely -- no getJobSchedulers Redis hit -- but still logs {0,0} so the operator sees cron is off.
-	if (schedules.length > 0) {
-		const rq = makeQueue(parseConnection(config.valkeyUrl, { failFast: true }));
+	// What this host will NOT be running, said once at boot and per trigger. A folder that belongs to
+	// another machine is ordinary on a fleet; a folder that belongs to NO machine is a trigger that will
+	// silently never fire, which is the silent no-op this project refuses -- and which `doctor` is the
+	// right place to catch, because it can ask the registry and this cannot.
+	const { served, unserved } = servedSchedules(schedules.current);
+	for (const s of unserved) log("schedule_unserved", { schedulerId: s.schedulerId, reason: s.unserved });
+
+	if (served.length > 0) {
+		// Onto the HOST queue when one is armed. That makes Gap 1 structural rather than merely gated: a
+		// host queue's resident schedulers are only ever that host's, so `reconcile`'s "resident minus my
+		// config" is correct again by construction and two hosts can no longer prune each other at all. The
+		// fingerprint gate stays, because it still catches the divergence itself -- including a timezone
+		// disagreement, which no queue split can detect.
+		const rq = makeQueue(parseConnection(config.valkeyUrl, { failFast: true }), { ...(hostQueue ? { name: hostQueue } : {}) });
 		try {
-			const r = await reconcile(rq, schedules, { log });
-			log("schedules_installed", { installed: r.installed, removed: r.removed });
+			const r = await reconcileGated(rq, served, { registry, log, tz: hostTz, authored: authoredCron(config) });
+			log("schedules_installed", { installed: r.installed, removed: r.removed, ...(unserved.length > 0 && { unserved: unserved.length }) });
 		} finally {
 			await rq.close().catch(() => {});
 		}
 	} else {
-		log("schedules_installed", { installed: 0, removed: 0 });
+		log("schedules_installed", { installed: 0, removed: 0, ...(unserved.length > 0 && { unserved: unserved.length }) });
 	}
 
 	// DES-CRON-VIA-BULLMQ-SCHEDULER live edit (OQ-008): watch the triggers file and re-reconcile schedulers
 	// on change, so an operator's add/edit/delete of a cron trigger takes effect without a worker restart.
 	// Only when a triggers file is configured; best-effort + unref'd; a bad edit keeps the running schedulers.
 	if (config.triggersFile) {
-		watchTriggersFile(config, runtimeQueue, log);
+		watchTriggersFile(config, cronQueue, log, schedules, registry, hostTz, config.workerNameDeclared);
 	}
 
 	// REQ-SCOPED-PAUSE-WINDOWS live edit: watch the pause-windows file and hot-swap the in-memory windows, so
@@ -689,6 +899,8 @@ export async function startWorker(
 
 	log("worker_started", {
 		queue: "pi-jobs",
+		host: config.workerName, // issue #57; `log` stamps it on every line, and the boot line names it where an operator looks first
+		imageDigest: bootImage.imageDigest ?? null, // two hosts on two builds of one tag used to emit byte-identical boot lines
 		concurrency: bootConcurrency, // the slot count the Worker is actually constructed with (overlay may raise/lower it)
 		dailyCap: config.dailyCap,
 		weeklyCap: config.weeklyCap, // null when the weekly window is disabled