@edgehero/pi-dispatch 1.6.1 → 1.8.0

package/src/doctor.mjs

@@ -49,7 +49,7 @@ import { homedir, tmpdir } from "node:os";
 import { dirname, join, delimiter } from "node:path";
 import { fileURLToPath } from "node:url";
 import { spawn as nodeSpawn } from "node:child_process";
-import { defaultSandboxDir, globalExtensionsEnabled } from "./config.mjs";
+import { defaultSandboxDir, defaultWorkerName, globalExtensionsEnabled } from "./config.mjs";
 import { canonicalScope, parseScopedLimits } from "./scoped-limits.mjs";
 import { WAIT_AFTER_MAX_DEFAULT_MS, afterInstantMs, parseWaitProfiles } from "./wait-for.mjs";
 import { isForgeKind } from "./forges.mjs";
@@ -89,6 +89,7 @@ export async function runDoctor(env = process.env, deps = {}) {
 		out = (s) => process.stdout.write(s),
 		spawn = nodeSpawn,
 		probeValkey = defaultProbeValkey,
+		readHosts = defaultReadHosts,
 		fileExists = existsSync,
 		nodeVersion = process.versions.node,
 		// --fix (REQ-DEPLOYMENT-BOOTSTRAP): offer to run the exact fixes doctor already prints. The prompt
@@ -111,7 +112,7 @@ export async function runDoctor(env = process.env, deps = {}) {
 		platform = process.platform,
 		home = homedir(),
 	} = deps;
-	const seams = { cwd, out, spawn, probeValkey, fileExists, nodeVersion, mkdir, chmod, rm, agentDir, platform, home };
+	const seams = { cwd, out, spawn, probeValkey, readHosts, fileExists, nodeVersion, mkdir, chmod, rm, agentDir, platform, home };
 
 	let checks = await collectChecks(env, seams);
 	let failed = render(checks, out);
@@ -213,7 +214,7 @@ export async function defaultPromptFn(question, { input = process.stdin, output
  * a comment.
  */
 export async function collectChecks(env, seams) {
-	const { cwd, spawn, probeValkey, fileExists, nodeVersion, platform, agentDir = agentDirFrom(env) } = seams;
+	const { cwd, spawn, probeValkey, readHosts = defaultReadHosts, fileExists, nodeVersion, platform, agentDir = agentDirFrom(env) } = seams;
 
 	const jobImage = env.PI_JOB_IMAGE ?? "pi-job:latest";
 	const valkeyUrl = env.VALKEY_URL ?? "redis://127.0.0.1:6379";
@@ -829,6 +830,85 @@ export async function collectChecks(env, seams) {
 			: {}),
 	});
 
+	// --- the fleet (issue #57) -------------------------------------------------------------------------
+	//
+	// Every line here is gated on a peer actually existing, so a single-host deployment's output is
+	// byte-identical. And every one is a WARN rather than a failure, with one exception noted below: this
+	// command runs on ONE machine and must not refuse a deployment for a condition that machine cannot fix.
+	// This host's own image id, read through the same seam every other docker probe here uses. Only when
+	// the image is actually present -- an absent one is already reported above, and a second line saying
+	// its digest is unknown would be noise on a fault the operator has been told about.
+	const fleet = await readHosts(valkeyUrl);
+	const peers = (fleet.hosts ?? []).filter((h) => h.name !== workerNameOf(env));
+	// Read only when there is a peer to compare against. Every line below is gated on a peer existing, and
+	// the SUBPROCESS has to be too: otherwise every `doctor` run on every single-host deployment spawns an
+	// extra docker call whose answer nothing reads.
+	const imageDigest =
+		peers.length > 0 && imageCode === 0
+			? (await runCmdCapture(spawn, "docker", ["image", "inspect", "--format={{.Id}}", jobImage])).output.trim() || null
+			: null;
+	if (peers.length > 0) {
+		const mine = workerNameOf(env);
+		checks.push({ ok: true, label: `Fleet: ${peers.length + 1} worker${peers.length === 0 ? "" : "s"} (${[mine, ...peers.map((h) => h.name)].sort().join(", ")})` });
+
+		// The one thing that is silently WRONG rather than merely undeclared. Without a declared name this
+		// host enqueues its own folder work to the SHARED queue, where a peer that has no such folder can
+		// pop it -- so the routing that makes a fleet safe is simply off, and nothing else says so.
+		if (!env.PI_WORKER_NAME) {
+			checks.push({
+				ok: false,
+				warn: true,
+				label: "This worker has peers but no PI_WORKER_NAME, so host routing is OFF here",
+				fix: "set PI_WORKER_NAME in this host's .env and restart: without it, this host's folder work is enqueued where any host can pop it, and its records carry a hostname it never chose",
+			});
+		}
+
+		// Two hosts on two builds of one tag is the failure Gap 6 names: same flow, different behaviour,
+		// undebuggable. A WARN and never a failure, because `{{.Id}}` is the LOCAL image id -- two
+		// independent builds of one Dockerfile differ, and under docker's containerd image store it is the
+		// manifest digest rather than the config digest, so a mixed-store fleet disagrees about identical
+		// content. Suspicious, never wrong.
+		const digests = new Set(peers.map((h) => h.imageDigest).filter(Boolean));
+		if (digests.size > 0 && imageDigest && !digests.has(imageDigest)) {
+			checks.push({
+				ok: false,
+				warn: true,
+				label: `Job image digest differs from ${peers.length === 1 ? "the other host" : "other hosts"}`,
+				fix: "rebuild or re-pull so every host runs the same image; digests are identical only when both hosts pulled one tag from one registry, so two local builds differ legitimately",
+			});
+		}
+
+		// A cron PATTERN carries no timezone and resolves in each worker's LOCAL time, so one pattern is two
+		// different instants on two hosts in two zones -- and the cron gate refuses that divergence rather
+		// than letting it drift, which is why this reads as an explanation for a refusal an operator has
+		// probably already met.
+		const zones = new Set([Intl.DateTimeFormat().resolvedOptions().timeZone, ...peers.map((h) => h.tz).filter(Boolean)]);
+		if (zones.size > 1) {
+			checks.push({
+				ok: false,
+				warn: true,
+				label: `Hosts disagree about the timezone (${[...zones].sort().join(", ")}), so one cron pattern is two different instants`,
+				fix: "set the same TZ on every host: a cron trigger carries no timezone of its own, so cron reconcile refuses while they disagree",
+			});
+		}
+
+		// Clocks. The registry's own heartbeats are the measurement, and skew matters here beyond tidiness:
+		// every hold clock, every TTL and the UTC day boundary the budget windows key on are read against
+		// whichever host is looking.
+		const skewed = peers.filter((h) => Number.isFinite(h.staleMs) && h.staleMs > 5 * 60_000);
+		if (skewed.length > 0) {
+			checks.push({
+				ok: false,
+				warn: true,
+				label: `${skewed.length} host row${skewed.length === 1 ? " is" : "s are"} stale by more than five minutes (${skewed.map((h) => h.name).join(", ")})`,
+				fix: "check that those workers are running and that the clocks agree -- a stale row is either a dead worker or a skewed clock, and both matter",
+			});
+		}
+	} else if (fleet.unreachable) {
+		// Said, rather than silently absent: "no peers" and "could not ask" are different facts.
+		checks.push({ ok: true, label: `Fleet: could not read the host registry (${fleet.unreachable})` });
+	}
+
 	const keys = PROVIDER_KEYS[provider] ?? [`${provider.toUpperCase()}_API_KEY`];
 	let keyOk = keys.some((k) => (env[k] ?? "").trim().length > 0);
 	let keyNote = "";
@@ -2212,6 +2292,36 @@ function parseGhTokenScopes(output) {
  * error handler is attached, so a down Valkey is reported as one ✗ line — not the ioredis stack traces
  * a BullMQ Queue's internal client would dump. Reuses `parseConnection`'s fail-fast options (cli.mjs:88).
  */
+/**
+ * The fleet's registry rows (issue #57), through a fail-fast client that is always disconnected.
+ *
+ * A SEAM rather than a direct import so the multi-host checks are testable with no Valkey at all, which
+ * is the posture every other network-touching check here already takes. Never throws: a fleet this
+ * command cannot see is a fleet it says nothing about, not a doctor that fails.
+ */
+/** This host's name as the worker computes it, so doctor and the worker cannot disagree about who "I" am. */
+function workerNameOf(env) {
+	return env.PI_WORKER_NAME || defaultWorkerName();
+}
+
+async function defaultReadHosts(url) {
+	try {
+		const { Redis } = await import("ioredis");
+		const { parseConnection } = await import("./connection.mjs");
+		const { readLiveHosts } = await import("./host-registry.mjs");
+		const client = new Redis({ ...parseConnection(url, { failFast: true }), lazyConnect: true });
+		client.on("error", () => {});
+		try {
+			await client.connect();
+			return await readLiveHosts(client);
+		} finally {
+			client.disconnect();
+		}
+	} catch (err) {
+		return { unreachable: err?.message ?? "registry unreadable" };
+	}
+}
+
 async function defaultProbeValkey(url) {
 	const { Redis } = await import("ioredis");
 	const { parseConnection } = await import("./connection.mjs");

package/src/fingerprint.mjs

@@ -0,0 +1,78 @@
+/**
+ * Canonical fingerprints of configuration, so two hosts can find out whether they agree (issue #57).
+ *
+ * Pure, and importing nothing but `node:crypto`: a fingerprint must be computable in a tier-1 test with
+ * no queue, no Valkey and no filesystem, exactly as `parseTriggers` is.
+ *
+ * WHY A HASH RATHER THAN THE VALUE. Two of these travel through the host registry, whose content rule
+ * refuses paths outright (`INT-HOST-REGISTRY-CONTRACT`) -- and a cron schedule set legitimately contains
+ * `run.folder`, `run.task` and secret NAMES. Hashing is what makes them admissible: the registry carries
+ * proof of agreement rather than the thing agreed on. This is the inverse of `scopeKeyPrefix`'s argument,
+ * which hashes a scope because it may contain `:` and `/`; here the reason is disclosure, not syntax.
+ */
+
+import { createHash } from "node:crypto";
+
+/**
+ * A stable 16-hex digest of any JSON-able value.
+ *
+ * Object keys are sorted RECURSIVELY, and that is load-bearing rather than tidy. `normalizeCronSchedule`
+ * builds its `data` object as a literal, so its key ORDER is a property of the worker's source: two hosts
+ * mid-upgrade would otherwise canonicalise the same file differently and refuse each other for the whole
+ * rollout. Sorting removes the spurious disagreement while leaving the real one -- a genuinely new field
+ * still changes the hash, which is correct, because the stored repeatable's data really did change.
+ *
+ * Sixteen hex, the `scopeKeyPrefix` and `localJobId` idiom, because this is compared and displayed rather
+ * than used as a security boundary.
+ */
+export function fingerprint(value) {
+	return createHash("sha256").update(canonical(value)).digest("hex").slice(0, 16);
+}
+
+function canonical(value) {
+	if (value === null || typeof value !== "object") return JSON.stringify(value ?? null);
+	if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
+	const keys = Object.keys(value).sort();
+	return `{${keys.map((k) => `${JSON.stringify(k)}:${canonical(value[k])}`).join(",")}}`;
+}
+
+/**
+ * The fingerprint of a worker's cron schedule set, or `null` when this worker has no opinion.
+ *
+ * ABSTAIN VERSUS OPINE IS THE SUBTLE PART, and conflating the two would leave the bug this gate exists to
+ * close. `loadSchedules` returns `[]` for two different states:
+ *
+ *   - `PI_TRIGGERS_FILE` unset, which means CRON IS DISABLED on this host. Such a worker has no view of
+ *     what should be scheduled, so it must never be able to disagree with one that does. It ABSTAINS.
+ *   - a triggers file that is present and declares zero cron entries. That is an OPINION -- "there should
+ *     be no schedulers" -- and it is this bug's purest form: today, deleting the last cron trigger on one
+ *     host prunes the whole fleet's schedulers through the file-watch path.
+ *
+ * So `null` in means abstain (`null` out); an empty ARRAY is a real fingerprint.
+ *
+ * The `tz` rides the hash because a cron PATTERN carries no timezone: `triggers.json` has no `tz` field on
+ * a cron entry, and BullMQ hands the pattern to cron-parser with no zone, so it resolves in each worker's
+ * LOCAL system time. On one host that is exactly what an operator means; on two hosts in different zones
+ * the same pattern is two different instants, with nothing anywhere saying so. Including the zone makes
+ * that a disagreement the gate can see. (`pause-windows.json` has carried an explicit `tz` since it
+ * shipped and is already fleet-correct; the asymmetry is why this one needs stating.)
+ *
+ * Fingerprinted over the NORMALIZED schedules rather than the file's bytes, deliberately. Bytes diverge on
+ * whitespace, on key order, and on every webhook trigger the worker does not own -- so two hosts differing
+ * only in a `label` rule would freeze cron forever over a difference that cannot affect it. The normalized
+ * set diverges exactly when the reconcile INPUTS diverge, which is the property that makes reconcile
+ * idempotent in the first place.
+ */
+export function cronFingerprint(schedules, { tz } = {}) {
+	// Takes the AUTHORED shape (`schedules.mjs -> authoredCron`), never the placement-resolved one.
+	if (schedules === null || schedules === undefined) return null;
+	return fingerprint({
+		tz: tz ?? "",
+		// EVERY field of an authored entry, projected explicitly. An earlier shape listed the keys of a
+		// NORMALIZED schedule (`name`, `data`, `opts`), which are all `undefined` on an authored one -- so
+		// the hash saw only the id and the pattern, and an operator changing `run.folder`, `run.flow` or
+		// `run.image` on one host and not the other passed the gate silently. That broke this module's own
+		// stated invariant, that the set diverges exactly when the reconcile inputs diverge.
+		schedules: schedules.map((s) => ({ schedulerId: s.schedulerId, pattern: s.pattern, run: s.run })),
+	});
+}

package/src/fleet-lease.mjs

@@ -0,0 +1,179 @@
+/**
+ * Fleet-wide leases for the two bounds that stopped meaning what they say when a second host appeared
+ * (issue #57).
+ *
+ * `PI_WAIT_CHECK_SLOTS` and a `scoped-limits.json` row's `concurrent` are both enforced by
+ * `makeInFlight()`, an in-process `Map`. One worker per docker daemon made that correct; two hosts make
+ * it a bound that MULTIPLIES BY THE OPERATOR'S DEPLOYMENT SHAPE, which is not a bound. Four hosts with
+ * `PI_WAIT_CHECK_SLOTS=1` run four concurrent checks against one Jira; four hosts with
+ * `{"scope":"acme/web","concurrent":1}` run four paid containers on one repository.
+ *
+ * WHAT THIS OWES `OQ-008` AND `DES-CONCURRENCY-3`. Both refuse a Redis-held in-flight count, in terms:
+ * "a Redis-held count would survive a crash WRONGLY -- a claim for a container the reaper just killed,
+ * demanding TTL/heartbeat machinery, a second source of truth about what is running". Every clause of
+ * that is about a CONTAINER, and the two leases here answer it differently:
+ *
+ *   The CHECK lease claims no container. It claims a subprocess this same process spawned, bounded by
+ *   `PI_WAIT_CHECK_TIMEOUT_MS`, holding no folder and spending no money. Three properties invert. What a
+ *   stale claim costs: a container claim is a folder mutex nobody holds and a job that never runs, while
+ *   a check claim is one check deferred by at most the TTL. What contradicts it: the reaper is a second
+ *   source of truth about containers and runs at boot with authority, while nothing enumerates,
+ *   inspects or reaps a check. And how long it can be wrong: a container has no natural expiry, while a
+ *   check has a hard, configured, small timeout, so the TTL is DERIVED rather than guessed.
+ *
+ *   The SCOPE claim really is for a container, so the refusal lands squarely -- and the answer is the
+ *   boot reaper. It establishes, at boot, that this host holds no `pi-job-*` containers, so a claim
+ *   whose value names this host is a claim for a container that no longer exists, and deleting it is not
+ *   a second source of truth: it is the SAME source writing down what it just established. Making the
+ *   reaper the claim's owner removes the contradiction rather than arguing around it.
+ *
+ * N INDEPENDENT KEYS, NEVER ONE COUNTER. A counter with one TTL loses every claim when it expires and
+ * leaks a permanent `+1` on a crash; N `SET NX PX` keys mean a lost release costs exactly one slot for
+ * exactly the TTL and never the whole semaphore. `wait:key:<dedupId>` is the in-repo precedent.
+ *
+ * And a property the in-process map does not have: RELEASE IS IDEMPOTENT here, because it deletes only a
+ * key whose value is still ours. `makeInFlight().release` clamps at zero but a double release on a
+ * `concurrent: 2` scope frees the other holder's slot.
+ */
+
+/** Slot keys. Both live under a prefix an operator can see whole with one `KEYS`. */
+export const checkSlotKey = (i) => `wait:check:${i}`;
+export const scopeSlotKey = (hash, i) => `slot:s:${hash}:${i}`;
+
+/**
+ * Build a lease over `slots` numbered keys.
+ *
+ * `holder` is `<workerName>#<jobId>`: the worker-name charset excludes `#`, so the value decomposes and
+ * the boot sweep can recognise its own claims without a second index to keep in step.
+ *
+ * FAIL OPEN, and in the GRANTING direction. A Valkey fault must never be able to wedge every wait in a
+ * deployment or stop every scoped job; the in-process bound is still there underneath, so failing open
+ * degrades the fleet bound to the per-host one, which is exactly the behaviour before this existed.
+ */
+export function makeFleetLease({ redis, holderPrefix, keyFor, ttlMs, now = () => Date.now(), log = () => {}, timeoutMs = 2_000 }) {
+	// Bounded for `host-registry.mjs`'s reason, which applies to every module sharing this client:
+	// `maxRetriesPerRequest: null` makes a command against an unreachable server QUEUE rather than reject,
+	// so a try/catch around it catches nothing and an outage would hang the gate rather than fail it.
+	const bounded = (p) =>
+		new Promise((resolve, reject) => {
+			const t = setTimeout(() => reject(new Error("lease timeout")), timeoutMs);
+			Promise.resolve(p).then(
+				(v) => (clearTimeout(t), resolve(v)),
+				(e) => (clearTimeout(t), reject(e)),
+			);
+		});
+
+	return {
+		/**
+		 * Take one of `slots`, or `null` when they are all held. Returns a handle whose `release` and
+		 * `refresh` act only on the key this call actually won.
+		 *
+		 * Probing starts at `hash(id) mod slots` rather than at 0. Without the rotation every host tries
+		 * index 0 first, so a host can sit behind a busy slot while a free one exists two along -- a
+		 * starvation that looks exactly like the capacity shortage the bound is meant to report.
+		 */
+		async acquire(id, { slots, keyArgs = [], ttlMs: perCall } = {}) {
+			if (!Number.isFinite(slots) || slots < 1) return { ok: true, release: async () => {}, refresh: async () => true };
+			const holder = `${holderPrefix}#${id}`;
+			const start = Math.abs(hashCode(String(id))) % slots;
+			for (let n = 0; n < slots; n++) {
+				const key = keyFor(...keyArgs, (start + n) % slots);
+				try {
+					const won = await bounded(redis.set(key, holder, "PX", perCall ?? ttlMs, "NX"));
+					if (!won) continue;
+					return {
+						ok: true,
+						key,
+						async release() {
+							try {
+								// Release-if-MINE, which is what makes this idempotent where the in-process map is
+								// not: a double release cannot free another holder's slot, because the second call
+								// finds a value that is no longer ours.
+								if ((await bounded(redis.get(key))) === holder) await bounded(redis.del(key));
+							} catch {
+								// The TTL is the backstop. A lost release costs one slot for one TTL.
+							}
+						},
+						async refresh(nextMs = ttlMs) {
+							try {
+								if ((await bounded(redis.get(key))) !== holder) return false;
+								await bounded(redis.pexpire(key, nextMs));
+								return true;
+							} catch {
+								return true; // a blip is not a reason to believe we lost a slot we hold
+							}
+						},
+					};
+				} catch (err) {
+					// Granting is the safe direction: the in-process bound is still underneath, so this
+					// degrades the fleet-wide ceiling to the per-host one rather than to nothing.
+					log("fleet_lease_unavailable", { reason: err?.message });
+					return { ok: true, degraded: true, release: async () => {}, refresh: async () => true };
+				}
+			}
+			return null;
+		},
+	};
+}
+
+/** A small, stable, non-cryptographic spread for the slot rotation. Not a key, so not a digest. */
+function hashCode(s) {
+	let h = 0;
+	for (let i = 0; i < s.length; i++) h = (h * 31 + s.charCodeAt(i)) | 0;
+	return h;
+}
+
+/**
+ * Delete every scope slot this host still claims, at boot, right after the container reaper.
+ *
+ * THIS IS THE ANSWER TO `OQ-008`, and its correctness rests entirely on one precondition: the reaper
+ * must have ENUMERATED. `makeReaper` catches its own `docker ps` failure and logs `reaper_skipped`, and
+ * on that path nothing was listed and nothing reaped -- so this host has NOT established that it holds
+ * no containers, and its claims may be for containers that are still running. Sweeping then would free
+ * slots for another host to start more alongside them, which is a money overrun rather than a tidy-up.
+ * `makeInFlight`'s own escape ("a state where no NEW container can start either") does not transfer,
+ * because the sweep frees slots for a DIFFERENT machine.
+ *
+ * Driven by config rather than by a scan: `scoped-limits.json` enumerates every scope that can carry a
+ * claim and `concurrent` bounds the index, so this is `sum(concurrent)` GETs -- typically under twenty.
+ * No `KEYS`, no `SCAN`, and no index set to leak.
+ */
+export function makeScopeClaimSweeper({ redis, workerName, limits, log = () => {}, timeoutMs = 2_000 }) {
+	return async function sweep({ reaped }) {
+		if (!reaped) {
+			log("scope_claims_sweep_skipped", { reason: "reaper-skipped" });
+			return { swept: 0, skipped: true };
+		}
+		const prefix = `${workerName}#`;
+		let swept = 0;
+		for (const row of limits ?? []) {
+			const n = Number(row?.concurrent);
+			if (!Number.isFinite(n) || n < 1 || !row?.hash) continue;
+			for (let i = 0; i < n; i++) {
+				const key = scopeSlotKey(row.hash, i);
+				try {
+					const held = await withTimeout(redis.get(key), timeoutMs);
+					if (typeof held === "string" && held.startsWith(prefix)) {
+						await withTimeout(redis.del(key), timeoutMs);
+						swept++;
+					}
+				} catch {
+					// Best-effort by contract: this is an OPTIMISATION over the TTL, never the mechanism, so a
+					// fault here costs at most one TTL of a stale claim and must never block boot.
+				}
+			}
+		}
+		if (swept > 0) log("scope_claims_swept", { count: swept });
+		return { swept, skipped: false };
+	};
+}
+
+function withTimeout(p, ms) {
+	return new Promise((resolve, reject) => {
+		const t = setTimeout(() => reject(new Error("lease timeout")), ms);
+		Promise.resolve(p).then(
+			(v) => (clearTimeout(t), resolve(v)),
+			(e) => (clearTimeout(t), reject(e)),
+		);
+	});
+}