@edgehero/pi-dispatch-receiver 1.4.0 → 1.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@edgehero/pi-dispatch-receiver",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "type": "module",
   "description": "Webhook receiver for pi-dispatch: the always-on edge that verifies GitHub, GitLab, Forgejo and Azure DevOps deliveries and enqueues (at most) one job per event for the worker.",
   "keywords": [

package/src/receiver.mjs

@@ -134,7 +134,11 @@ export function parseSubset(payload) {
  * undefined, i.e. never drops the harness's own comments. Absent property therefore means no route; the
  * failure of a forgotten property is a 404 an operator sees, never a paid recursion they get billed for.
  */
-export function makeReceiver({ queue, selfId, cfg, log, gitlab = null, forgejo = null, azure = null, resolveAuthority }) {
+export function makeReceiver({ queue, selfId, cfg, log, gitlab = null, forgejo = null, azure = null, resolveAuthority, router = null }) {
+	// Which queue a delivery is enqueued onto (issue #57, `OQ-032`). The default is the shared queue for
+	// EVERY job, which is what this receiver did before multi-host existed -- so a deployment that wires no
+	// router, and every test that constructs one without it, is byte-identical.
+	const routeTo = router ? (kind, job) => router.queueFor(kind, job) : () => queue;
 	// Built only when the deployment serves GitHub. Construction is not free of the secret either: the
 	// `new Webhooks({ secret })` inside makeVerifiedHandler throws "options.secret required" on an absent
 	// one, so not building the arm is what lets a github-free deployment legitimately have no secret --
@@ -143,7 +147,7 @@ export function makeReceiver({ queue, selfId, cfg, log, gitlab = null, forgejo =
 	// `resolveAuthority` is the GITHUB closer resolver (issue #231), riding top-level beside `selfId`
 	// because github's dependencies always have -- the other forges bundle theirs in per-forge objects.
 	// Optional, because only a close delivery an armed close rule matches ever consults it.
-	const github = cfg.servesGithub ? makeGitHubHandler({ queue, selfId, cfg, log, resolveAuthority }) : null;
+	const github = cfg.servesGithub ? makeGitHubHandler({ queue, routeTo, selfId, cfg, log, resolveAuthority }) : null;
 
 	// A TABLE, built once, rather than one `if` per forge. Two forges made that a single branch; four make
 	// it a chain, and a chain is where one arm quietly ends up checked after the fallthrough. A path present
@@ -151,9 +155,9 @@ export function makeReceiver({ queue, selfId, cfg, log, gitlab = null, forgejo =
 	// falls through to GitHub, which is what keeps `/` working -- and a configured-off GitHub answers the
 	// same 404 from the fallthrough itself, below.
 	const routes = {
-		"/gitlab": gitlab ? makeGitLabHandler({ queue, cfg, log, ...gitlab }) : null,
-		"/forgejo": forgejo ? makeForgejoHandler({ queue, cfg, log, ...forgejo }) : null,
-		"/azure": azure ? makeAzureHandler({ queue, cfg, log, ...azure }) : null,
+		"/gitlab": gitlab ? makeGitLabHandler({ queue, routeTo, cfg, log, ...gitlab }) : null,
+		"/forgejo": forgejo ? makeForgejoHandler({ queue, routeTo, cfg, log, ...forgejo }) : null,
+		"/azure": azure ? makeAzureHandler({ queue, routeTo, cfg, log, ...azure }) : null,
 	};
 
 	return async function receiverHandler(req, res) {
@@ -223,7 +227,7 @@ async function fanout(job, enqueue) {
  * uses, so a lookup is never spent on a delivery the gate then ignores -- every label, comment, PR and
  * review delivery, and every close nothing wants, stays payload-only and byte-identical to before.
  */
-function makeGitHubHandler({ queue, selfId, cfg, log, resolveAuthority }) {
+function makeGitHubHandler({ queue, routeTo, selfId, cfg, log, resolveAuthority }) {
 	return makeVerifiedHandler({ secret: cfg.webhookSecret }, async ({ rawBody, event, delivery }, res) => {
 		let subset;
 		try {
@@ -264,7 +268,7 @@ function makeGitHubHandler({ queue, selfId, cfg, log, resolveAuthority }) {
 		// Fanout (REQ-REPLICA-RUNS) lives in `fanout` above; the 202/503 decision stays here, where it always was.
 		let replicas;
 		try {
-			replicas = await fanout(result.job, (j) => enqueueGitHubJob(queue, j));
+			replicas = await fanout(result.job, async (j) => await enqueueGitHubJob(await routeTo("github", j), j));
 		} catch (err) {
 			// Own try/catch so a Valkey-down enqueue is a 503 (retryable), not verify's outer 500.
 			log?.({ event: "enqueue_failed", delivery, reason: err?.message });
@@ -289,7 +293,7 @@ function makeGitHubHandler({ queue, selfId, cfg, log, resolveAuthority }) {
  * The lookup runs only for events that could still fire -- after verification, and after the payload has
  * been projected -- so an unauthenticated flood cannot make this project call GitLab at all.
  */
-function makeGitLabHandler({ queue, cfg, log, mode, secret, selfId, resolveAuthority, now }) {
+function makeGitLabHandler({ routeTo, queue, cfg, log, mode, secret, selfId, resolveAuthority, now }) {
 	return makeGitLabVerifiedHandler({ mode, secret, ...(now ? { now } : {}) }, async ({ rawBody, delivery }, res) => {
 		let subset;
 		try {
@@ -314,7 +318,7 @@ function makeGitLabHandler({ queue, cfg, log, mode, secret, selfId, resolveAutho
 
 		let replicas;
 		try {
-			replicas = await fanout(result.job, (j) => enqueueGitLabJob(queue, j));
+			replicas = await fanout(result.job, async (j) => await enqueueGitLabJob(await routeTo("gitlab", j), j));
 		} catch (err) {
 			log?.({ event: "enqueue_failed", delivery, reason: err?.message });
 			return respond(res, 503, { error: "enqueue-failed" }); // GitLab redelivers; dedup by webhook-id coalesces
@@ -341,7 +345,7 @@ function makeGitLabHandler({ queue, cfg, log, mode, secret, selfId, resolveAutho
  * per construction, and serving two sources from one would mean either sharing a secret between forges or
  * letting the request choose which one it was checked against.
  */
-function makeForgejoHandler({ queue, cfg, log, secret, selfId, resolveAuthority }) {
+function makeForgejoHandler({ routeTo, queue, cfg, log, secret, selfId, resolveAuthority }) {
 	return makeVerifiedHandler({ secret }, async ({ rawBody, event, delivery }, res) => {
 		let subset;
 		try {
@@ -366,7 +370,7 @@ function makeForgejoHandler({ queue, cfg, log, secret, selfId, resolveAuthority
 
 		let replicas;
 		try {
-			replicas = await fanout(result.job, (j) => enqueueForgeJob(queue, "forgejo", j));
+			replicas = await fanout(result.job, async (j) => await enqueueForgeJob(await routeTo("forgejo", j), "forgejo", j));
 		} catch (err) {
 			log?.({ event: "enqueue_failed", delivery, reason: err?.message });
 			return respond(res, 503, { error: "enqueue-failed" }); // Forgejo redelivers; dedup by GUID coalesces
@@ -391,7 +395,7 @@ function makeForgejoHandler({ queue, cfg, log, secret, selfId, resolveAuthority
  * `"Display Name <email>"`. Both the resolver and the bot-loop guard handle both forms -- see
  * filter-azure.mjs, where the ordering constraint is stated in full.
  */
-function makeAzureHandler({ queue, cfg, log, mode, secret, headerName, selfId, resolveAuthority }) {
+function makeAzureHandler({ routeTo, queue, cfg, log, mode, secret, headerName, selfId, resolveAuthority }) {
 	return makeAzureVerifiedHandler({ mode, secret, headerName }, async ({ rawBody }, res) => {
 		let subset;
 		try {
@@ -424,7 +428,7 @@ function makeAzureHandler({ queue, cfg, log, mode, secret, headerName, selfId, r
 
 		let replicas;
 		try {
-			replicas = await fanout(result.job, (j) => enqueueForgeJob(queue, "azure", j));
+			replicas = await fanout(result.job, async (j) => await enqueueForgeJob(await routeTo("azure", j), "azure", j));
 		} catch (err) {
 			log?.({ event: "enqueue_failed", delivery, reason: err?.message });
 			return respond(res, 503, { error: "enqueue-failed" });

package/src/route.mjs

@@ -0,0 +1,126 @@
+/**
+ * Which queue a forge delivery is enqueued onto, when the deployment runs on more than one machine
+ * (issue #57, `OQ-032`).
+ *
+ * The receiver is the only process that can make this decision. Routing has to happen at ENQUEUE -- a
+ * delayed job is promoted on each worker's own clock, so the fastest clock wins every attempt and a job
+ * cannot reliably be handed from a host that will not serve it to one that will -- and forge deliveries are
+ * enqueued here. `capabilities.mjs` owns the RULE and is shared with the worker so the two cannot drift;
+ * this module owns the plumbing: one cached registry read, and a pool of queue handles.
+ *
+ * EVERYTHING HERE FAILS OPEN ONTO THE SHARED QUEUE, which is what this receiver did before any of it
+ * existed. An unreadable registry, a timed-out read, a malformed row, a name that is not routable: every
+ * one of them returns the shared queue, so the worst this can do is fail to improve on a coin flip. A
+ * webhook handler is the wrong place to invent a new way to drop work.
+ */
+
+import { readLiveHosts } from "@edgehero/pi-dispatch/host-registry";
+import { makeQueue, hostQueueName } from "@edgehero/pi-dispatch/queue";
+import { parseConnection, makeRedisClient } from "@edgehero/pi-dispatch/connection";
+import { forgeDeliveryJobId } from "@edgehero/pi-dispatch/job-id";
+import { jobNeeds, routeForgeJob } from "@edgehero/pi-dispatch/capabilities";
+
+/**
+ * How long a registry read is reused.
+ *
+ * The registry beats every 15s, so anything below that mostly re-reads a value that cannot have changed.
+ * Five seconds is well inside one beat and bounds the extra load a busy receiver puts on Valkey at one read
+ * per five seconds rather than one per delivery -- and a delivery burst, which is exactly when this must
+ * not add latency, is served entirely from cache.
+ *
+ * Staleness costs nothing that freshness would have saved: a host that appears within the window is missed
+ * and its work goes to the shared queue, which is today's behaviour, and a host that vanishes is already
+ * guarded by `ROUTE_FRESH_MS` on the row's own beat timestamp.
+ */
+const HOSTS_TTL_MS = 5_000;
+
+/** Bound on the registry read itself. A routing hint is never worth making a webhook wait. */
+const READ_TIMEOUT_MS = 1_500;
+
+/**
+ * @returns {{queueFor: (kind: string, job: object) => Promise<object>, close: () => Promise<void>}}
+ */
+export function makeForgeRouter({
+	valkeyUrl,
+	shared,
+	log = () => {},
+	makeQueueFn = makeQueue,
+	parseConnectionFn = parseConnection,
+	redisFn = makeRedisClient,
+	readLiveHostsFn = readLiveHosts,
+	now = () => Date.now(),
+	ttlMs = HOSTS_TTL_MS,
+} = {}) {
+	const pool = new Map(); // queue name -> Queue, opened lazily and only for hosts actually routed to
+	let redis = null;
+	let cached = { at: -Infinity, hosts: [] };
+	let inFlight = null;
+
+	// One read at a time. Without this a burst of deliveries arriving on a cold cache each start their own
+	// registry read, which is the moment the deployment can least afford N of them.
+	const hosts = async () => {
+		if (now() - cached.at < ttlMs) return cached.hosts;
+		if (inFlight) return await inFlight;
+		inFlight = (async () => {
+			try {
+				if (!redis) {
+					redis = redisFn(valkeyUrl);
+					// A receiver must never print ioredis reconnect noise into its own log: this client is an
+					// optimisation, and its failures are already handled by falling back to the shared queue.
+					redis.on?.("error", () => {});
+				}
+				const res = await readLiveHostsFn(redis, { timeoutMs: READ_TIMEOUT_MS });
+				cached = { at: now(), hosts: Array.isArray(res?.hosts) ? res.hosts : [] };
+			} catch {
+				// Cache the FAILURE too, so an unreachable Valkey costs one read per window rather than one
+				// per delivery. The empty list routes everything to the shared queue.
+				cached = { at: now(), hosts: [] };
+			} finally {
+				inFlight = null;
+			}
+			return cached.hosts;
+		})();
+		return await inFlight;
+	};
+
+	return {
+		async queueFor(kind, job) {
+			const needs = jobNeeds(job);
+			// The overwhelming majority of deliveries bind neither a secret profile nor a wait profile, and
+			// they must not pay even a cache lookup for a decision that cannot apply to them.
+			if (needs.length === 0) return shared;
+
+			let name = null;
+			try {
+				name = routeForgeJob({ hosts: await hosts(), needs, jobId: forgeDeliveryJobId(kind, job?.trigger?.deliveryId, job?.replica) });
+			} catch {
+				return shared; // an unroutable id, a bad row: the shared queue is always a correct answer
+			}
+			if (!name) return shared;
+
+			const queueName = hostQueueName(name);
+			if (!pool.has(queueName)) pool.set(queueName, makeQueueFn(parseConnectionFn(valkeyUrl), { name: queueName }));
+			// Logged because a routed job is the one case where "which host ran it" was DECIDED rather than
+			// observed, and an operator debugging a job that never started needs to know it was sent
+			// somewhere specific. Host names are deployment topology: this reaches the log, never a forge
+			// comment (`triggers.mjs` sets that rule for refusal messages and it holds here).
+			log({ event: "forge_job_routed", kind, host: name, needs: needs.join(",") });
+			return pool.get(queueName);
+		},
+		async close() {
+			for (const q of pool.values()) {
+				try {
+					await q.close();
+				} catch {
+					// best-effort teardown
+				}
+			}
+			pool.clear();
+			try {
+				redis?.disconnect?.();
+			} catch {
+				// best-effort teardown
+			}
+		},
+	};
+}

package/src/start.mjs

@@ -44,6 +44,7 @@ import { makeResolveForgejoAuthority } from "./forgejo-members.mjs";
 import { makeResolveAzureAuthority } from "./azure-members.mjs";
 import { makeResolveGitHubAuthority } from "./github-members.mjs";
 import { makeQueue } from "@edgehero/pi-dispatch/queue";
+import { makeForgeRouter } from "./route.mjs";
 import { parseConnection } from "@edgehero/pi-dispatch/connection";
 
 /**
@@ -55,6 +56,7 @@ export async function startReceiver(
 	{
 		makeAuth = makeGitHubAuth,
 		makeQueueFn = makeQueue,
+		makeForgeRouterFn = makeForgeRouter,
 		createServer = http.createServer,
 		resolveGitLabSelfId: resolveSelfIdFn = resolveGitLabSelfId,
 		makeResolveAuthority: makeResolveAuthorityFn = makeResolveAuthority,
@@ -111,6 +113,15 @@ export async function startReceiver(
 	// restart, not give up on a transient disconnect.
 	const queue = makeQueueFn(parseConnection(cfg.valkeyUrl));
 
+	// Multi-host routing for deliveries that bind a host-local resource (issue #57, `OQ-032`). A trigger
+	// naming `run.secretsProfile` or a `run.waitFor` profile can only run where that profile is declared, and
+	// which worker pops a shared-queue job is a coin flip -- so the same trigger succeeded or failed by
+	// chance, permanently, and read like a configuration error rather than a placement one.
+	//
+	// Every failure path inside the router returns the shared queue, so a receiver whose Valkey read fails,
+	// or whose deployment has no named hosts, behaves exactly as it did before this existed.
+	const router = makeForgeRouterFn({ valkeyUrl: cfg.valkeyUrl, shared: queue, log });
+
 	// The GitLab arm, when configured. Its identity resolution is HARD-FAIL for the same reason github's
 	// is: without a selfId the bot-loop guard cannot run, and a receiver that listens without it turns the
 	// harness's own status comment into another paid job.
@@ -158,7 +169,7 @@ export async function startReceiver(
 		};
 	}
 
-	const handler = makeReceiver({ queue, selfId, cfg, log, gitlab, forgejo, azure, resolveAuthority });
+	const handler = makeReceiver({ queue, router, selfId, cfg, log, gitlab, forgejo, azure, resolveAuthority });
 	const server = createServer(handler);
 	server.listen(cfg.port, cfg.bind, () =>
 		log({ event: "receiver_started", port: cfg.port, bind: cfg.bind, valkey: cfg.valkeyUrl }),
@@ -173,6 +184,7 @@ export async function startReceiver(
 			log({ event: "receiver_stopping", signal });
 			await new Promise((resolve) => server.close(resolve));
 			await queue.close();
+			await router.close();
 			process.exit(0);
 		};
 		process.once("SIGTERM", () => void shutdown("SIGTERM"));