@edgehero/pi-dispatch-receiver 4.3.1 → 5.1.0

package/src/receiver.mjs

@@ -19,6 +19,8 @@
  * session and binds no port.
  */
 
+import { makeReceiverObservations } from "@edgehero/pi-dispatch/receiver-status";
+
 import { makeVerifiedHandler } from "./verify.mjs";
 import { filter, wantsCloserAuthority } from "./filter.mjs";
 import { enqueueForgeJob, enqueueGitHubJob, enqueueGitLabJob } from "@edgehero/pi-dispatch/queue";
@@ -30,9 +32,18 @@ import { parseForgejoSubset } from "./forgejo-subset.mjs";
 import { filterAzure } from "./filter-azure.mjs";
 import { parseAzureSubset } from "./azure-subset.mjs";
 import { makeAzureVerifiedHandler } from "./verify-azure.mjs";
+import { trySteer } from "./steer.mjs";
+import { holdAfterDedup, holdBeforeLabel, wantsHoldAfterDedup } from "./held.mjs";
+import { makeWatchHandler } from "./watch.mjs";
+import { makeWatchStore } from "@edgehero/pi-dispatch/watch-store";
+import { enqueueWatchJob } from "@edgehero/pi-dispatch/queue";
 
 /** Write a small JSON body with an explicit status; a 204 carries no body. verify's `respond` is private. */
+const responseResults = new WeakMap();
 function respond(res, status, obj) {
+	// Only our own closed response words disambiguate 202 and 503; no request body is inspected.
+	if (status === 202) responseResults.set(res, obj?.status);
+	if (status === 503) responseResults.set(res, obj?.error === "enqueue-failed" ? "enqueue-failed" : "lookup-failed");
 	if (status === 204) {
 		res.writeHead(204);
 		res.end();
@@ -134,7 +145,14 @@ export function parseSubset(payload) {
  * undefined, i.e. never drops the harness's own comments. Absent property therefore means no route; the
  * failure of a forgotten property is a 404 an operator sees, never a paid recursion they get billed for.
  */
-export function makeReceiver({ queue, selfId, cfg, log, gitlab = null, forgejo = null, azure = null, resolveAuthority, router = null }) {
+export function makeReceiver({ queue, selfId, cfg, log, gitlab = null, forgejo = null, azure = null, resolveAuthority, router = null, steerRedis, now = Date.now, watchEnv = process.env, watchStore = null, enqueueWatch = enqueueWatchJob, observations = makeReceiverObservations({ now }) }) {
+	const watch = makeWatchHandler({ cfg, env: watchEnv, now, log: (event, fields) => log?.({ event, ...fields }), store: watchStore ?? Object.fromEntries(["ingest", "appendTrace", "recordVerdict", "noteIssueActivity"].map(method => [method, async (...args) => makeWatchStore(await queue.client, { now })[method](...args)])), enqueue: fields => enqueueWatch(queue, fields, { now }) });
+	// The Valkey client comment steering (issue #646) reads the live runs with and pushes into: by default the shared
+	// queue's own connection (`queue.client`, BullMQ's promise of its ioredis client), so steering opens no second
+	// connection and has no lifecycle of its own to close. Asked only for a job that carries `steerMode`, so a
+	// deployment with no steer-armed comment trigger never touches it. `steerRedis` is the seam.
+	const steerClient = typeof steerRedis === "function" ? steerRedis : () => queue?.client;
+	const steer = { steerClient, now, log };
 	// Which queue a delivery is enqueued onto (issue #57, `OQ-032`). The default is the shared queue for
 	// EVERY job, which is what this receiver did before multi-host existed -- so a deployment that wires no
 	// router, and every test that constructs one without it, is byte-identical.
@@ -147,7 +165,7 @@ export function makeReceiver({ queue, selfId, cfg, log, gitlab = null, forgejo =
 	// `resolveAuthority` is the GITHUB closer resolver (issue #231), riding top-level beside `selfId`
 	// because github's dependencies always have -- the other forges bundle theirs in per-forge objects.
 	// Optional, because only a close delivery an armed close rule matches ever consults it.
-	const github = cfg.servesGithub ? makeGitHubHandler({ queue, routeTo, selfId, cfg, log, resolveAuthority }) : null;
+	const github = cfg.servesGithub ? makeGitHubHandler({ queue, routeTo, selfId, cfg, log, resolveAuthority, steer }) : null;
 
 	// A TABLE, built once, rather than one `if` per forge. Two forges made that a single branch; four make
 	// it a chain, and a chain is where one arm quietly ends up checked after the fallthrough. A path present
@@ -155,13 +173,14 @@ export function makeReceiver({ queue, selfId, cfg, log, gitlab = null, forgejo =
 	// falls through to GitHub, which is what keeps `/` working -- and a configured-off GitHub answers the
 	// same 404 from the fallthrough itself, below.
 	const routes = {
-		"/gitlab": gitlab ? makeGitLabHandler({ queue, routeTo, cfg, log, ...gitlab }) : null,
-		"/forgejo": forgejo ? makeForgejoHandler({ queue, routeTo, cfg, log, ...forgejo }) : null,
-		"/azure": azure ? makeAzureHandler({ queue, routeTo, cfg, log, ...azure }) : null,
+		"/gitlab": gitlab ? makeGitLabHandler({ queue, routeTo, cfg, log, ...gitlab, steer }) : null,
+		"/forgejo": forgejo ? makeForgejoHandler({ queue, routeTo, cfg, log, ...forgejo, steer }) : null,
+		"/azure": azure ? makeAzureHandler({ queue, routeTo, cfg, log, ...azure, steer }) : null,
 	};
 
-	return async function receiverHandler(req, res) {
+	const route = async (req, res) => {
 		const path = pathOf(req.url);
+		if (path === "/watch" || path.startsWith("/watch/")) return watch(req, res);
 		if (Object.hasOwn(routes, path)) {
 			const handler = routes[path];
 			// Not a 401. An endpoint that answers "unauthorized" for a forge nobody configured is an endpoint
@@ -177,6 +196,24 @@ export function makeReceiver({ queue, selfId, cfg, log, gitlab = null, forgejo =
 		if (!github) return respond(res, 404, { error: "github webhooks are not configured" });
 		return await github(req, res);
 	};
+	const receiverHandler = async (req, res) => {
+		const path = pathOf(req.url);
+		if (path === "/watch" || path.startsWith("/watch/")) return route(req, res);
+		const forge = ({ "/gitlab": "gitlab", "/forgejo": "forgejo", "/azure": "azure" })[path] ?? "github";
+		const end = res.end;
+		let counted = false;
+		res.end = function (...args) {
+			const result = end.apply(this, args);
+			if (!counted) {
+				counted = true;
+				try { observations.observe(forge, res.statusCode, responseResults.get(res)); } catch {}
+			}
+			return result;
+		};
+		try { return await route(req, res); }
+		finally { res.end = end; responseResults.delete(res); }
+	};
+	return Object.assign(receiverHandler, { observations });
 }
 
 /** The path portion of a request URL, without query or trailing slash. Never throws on a malformed URL. */
@@ -198,9 +235,17 @@ function pathOf(url) {
  * Absent `replicas` is `1` and the call below is byte-identical to the single enqueue it replaced: no
  * `replica` key on the job, so the jobId, the dedup id and `data` are exactly what they were.
  *
- * PARTIAL FAILURE IS IDEMPOTENT BY CONSTRUCTION, which is why there is no compensating logic here: if
- * replica k throws, the caller's catch answers 503, the forge redelivers, replicas 1..k-1 dedup on their
- * own now-taken jobIds and k..n enqueue. The retry converges on exactly n jobs rather than n + (k-1).
+ * A PARTIAL FAILURE LEAVES REPLICAS 1..k-1 QUEUED, and nothing here takes them back. If replica k throws,
+ * the caller answers 503 (`answerEnqueueFailed`), and what happens next is the forge's (WHAT A 503 MEANS,
+ * below): on GitHub, GitLab and Forgejo nothing, so k-1 of the n jobs run and the rest never queue unless
+ * someone sends the delivery again. A GitHub or GitLab redelivery by hand, or Azure's own resend, keeps the
+ * delivery id, so replicas 1..k-1 dedup on their taken jobIds, k..n enqueue, and it converges on exactly n.
+ * A Forgejo replay is a NEW delivery id: inside the 10-minute semantic window the queued replicas coalesce
+ * on their `:r<i>` keys and it still converges on n, but after it all n queue again beside the k-1, so
+ * cancel those first. Compensating here was rejected: removing 1..k-1 needs the Valkey that just failed,
+ * and one of them may already be running. So the failure says how far it got instead: past one replica,
+ * the error that goes up carries `queued` (k-1, created or swallowed by the window) and `replicas` (n) for
+ * the `enqueue_failed` line. A single enqueue's error goes up untouched, so its line is byte-identical.
  *
  * `enqueue` is a callback because the four arms spell their enqueue differently (a named github/gitlab
  * wrapper, or `enqueueForgeJob` with an explicit kind); the fanout itself is forge-blind.
@@ -216,24 +261,103 @@ async function fanout(job, enqueue) {
 	let created = 0;
 	const deduplicated = [];
 	for (let i = 1; i <= replicas; i++) {
-		const r = await enqueue(replicas > 1 ? { ...job, replica: i } : job);
+		let r;
+		try {
+			r = await enqueue(replicas > 1 ? { ...job, replica: i } : job);
+		} catch (err) {
+			if (replicas === 1) throw err;
+			throw Object.assign(new Error(err?.message, { cause: err }), { queued: i - 1, replicas });
+		}
 		if (r && typeof r === "object" && r.deduplicated === true) deduplicated.push({ jobId: r.jobId, survivingJobId: r.survivingJobId });
 		else created += 1;
 	}
 	return { replicas, created, deduplicated };
 }
 
+/**
+ * WHAT A 503 MEANS FOR EACH FORGE. Every arm answers 503 when it cannot finish a delivery it would otherwise
+ * queue: an INDETERMINATE author lookup (and the GitHub arm's unwired resolver), or an enqueue that throws.
+ * The delivery is not queued, and the answer is kept because it is the true one: a 204 would read as a
+ * stranger correctly refused, and a 202 would say "queued". What happens to the delivery is the forge's:
+ *   - GitHub never resends a failed delivery on its own (docs.github.com, "Handling failed webhook
+ *     deliveries"). It is lost unless someone redelivers it from the hook's Recent Deliveries or the REST
+ *     API within 3 days; that keeps `X-GitHub-Delivery`, so it queues at most once.
+ *   - GitLab never resends a 5xx or a timeout either, and after 4 failures in a row it disables the hook
+ *     for a while (on gitlab.com, and a group hook anywhere; on a self-managed instance a project hook only
+ *     where the `auto_disabling_web_hooks` feature flag is on). A "Resend Request" (Recent events, 2 days;
+ *     the API's resend since 17.4) keeps the `Idempotency-Key` and is signed afresh, so it queues at most
+ *     once and the replay window holds.
+ *   - Forgejo makes one attempt (`services/webhook/deliver.go`). Its replay, from the UI, is a NEW hook task
+ *     with a NEW delivery id: inside the 10-minute semantic window it coalesces with what did queue, after
+ *     the window it runs again, which matters for a delivery Forgejo saw fail only because the answer came
+ *     after its 5 s `DELIVER_TIMEOUT`, when the job had in fact queued.
+ *   - Azure DevOps treats a 503 as transient and resends the notification up to 8 times over about 3
+ *     minutes. It resends the same notification, so it carries the same payload `id` and queues at most
+ *     once. Past that the subscription goes on probation: Azure retries the failed notification up to 7
+ *     more times over about 36 hours, and new events are lost meanwhile. So a delivery someone re-triggers
+ *     by hand in that time (a new event, a new `id`) can run twice: a probation retry that lands outside the
+ *     10-minute semantic window is not coalesced with it. A 500 is an enduring failure at once.
+ * The receiver logs each 503 with the delivery id an operator redelivers by, and the Connections page counts
+ * `lookup-failed` and `enqueue-failed` per forge and warns when either is above 0.
+ *
+ * This function is the enqueue-failure answer, shared by the four arms for the four-copies reason `fanout`
+ * gives. Their own try/catch exists so a Valkey-down enqueue is this 503 and not verify's outer 500, which
+ * Azure would count as an enduring failure. The `enqueue_failed` line names the delivery and the reason; past
+ * one replica it adds how many of the `replicas` were `queued` before the failure (`fanout`), which is what an
+ * operator needs before redelivering, and on Forgejo before replaying.
+ */
+function answerEnqueueFailed({ log, res, delivery, err }) {
+	// Past one replica `err` is `fanout`'s wrapper, and the reason is its cause's: a thrown non-Error has no
+	// `message`, so its line keeps leaving `reason` out instead of logging the wrapper's empty one.
+	const fanned = err?.replicas > 1;
+	const partial = fanned ? { queued: err.queued, replicas: err.replicas } : {};
+	log?.({ event: "enqueue_failed", delivery, reason: (fanned ? err.cause : err)?.message, ...partial });
+	return respond(res, 503, { error: "enqueue-failed" });
+}
+
+/**
+ * Comment steering (issue #646), shared by all four arms for the four-copies reason `fanout` gives. Runs after the gate
+ * and before any enqueue, and only for a job the filter marked `steerMode` (a phrase comment from an admitted author on
+ * a steer-armed trigger): every other delivery passes through untouched, so its job is byte-identical to before.
+ *
+ * Returns `{ handled: true }` when a live run took the comment, after answering `202 {status:"steered", runs}`; else
+ * `{ job }`, the job to enqueue exactly as today, with `steerMode` removed (`enqueueForgeJob` names its fields and
+ * would drop it anyway; removing it here keeps it off every seam a test or a router sees). The `steered` line names the
+ * delivery, the target in the arm's own grammar, the mode, and the runs, never the comment's text.
+ */
+async function steerOrEnqueue({ steer, res, kind, delivery, job, target }) {
+	if (job.steerMode === undefined) return { job };
+	const { steerMode: mode, ...rest } = job;
+	let redis = null;
+	try {
+		redis = (await steer.steerClient()) ?? null;
+	} catch {
+		redis = null; // trySteer logs `steer_lookup_failed` for a missing client and the comment is enqueued
+	}
+	const steered = await trySteer({ redis, job, kind, now: steer.now, log: (line) => steer.log?.(line) });
+	if (steered === null) return { job: rest };
+	// A redelivery of a delivery steered earlier (its run has ended since): answered, never enqueued. trySteer logged it.
+	if (steered.redelivered === true) {
+		respond(res, 202, { status: "steered", runs: 0 });
+		return { handled: true };
+	}
+	steer.log?.({ event: "steered", delivery, repo: job.repo, target, mode, runs: steered.steered, runIds: steered.runIds });
+	respond(res, 202, { status: "steered", runs: steered.steered });
+	return { handled: true };
+}
+
 /**
  * One log-and-answer for a fanout outcome, shared by all four arms so a weakened copy cannot hide (the
  * four-copies doctrine above). Issue #289's honesty rule: `enqueued` is logged only when something was
  * CREATED, with `replicas` meaning jobs that now exist; every semantic-window swallow gets its own
  * `deduplicated` line naming the surviving job's id (every field forge- or worker-minted, no payload
  * text); and a delivery that created NOTHING answers `202 {status:"deduplicated"}` -- still a 2xx,
- * because a non-2xx would trigger a redelivery storm for a delivery that was handled, but no longer
+ * because a non-2xx would record a delivery that was handled as failed in the forge's own history, count
+ * toward GitLab disabling the hook, and make Azure resend it, but no longer
  * "queued", because a receiver that logs the swallow while answering success on the wire would be the
  * honest-log/lying-wire split this project refuses. Each arm passes its own target grammar.
  */
-function respondEnqueueOutcome({ log, res, delivery, repo, target, flow, outcome }) {
+function respondEnqueueOutcome({ log, res, delivery, repo, target, flow, outcome, held = null }) {
 	for (const d of outcome.deduplicated) {
 		log?.({ event: "deduplicated", delivery, repo, target, flow, jobId: d.jobId, survivingJobId: d.survivingJobId });
 	}
@@ -241,9 +365,54 @@ function respondEnqueueOutcome({ log, res, delivery, repo, target, flow, outcome
 		log?.({ event: "enqueued", delivery, repo, target, flow, replicas: outcome.created });
 		return respond(res, 202, { status: "queued" });
 	}
+	// Issue #638: a swallow a parking run took as its held answer is not "nothing happened" either; the `held` lines
+	// were written by held.mjs, and the wire says so the same honest way.
+	if (held !== null) return respond(res, 202, { status: "held", runs: held.held });
 	return respond(res, 202, { status: "deduplicated" });
 }
 
+/** The Valkey client held replies use: the steer one (`steer.steerClient()`), null when there is none. */
+async function heldClientOf(steer) {
+	try {
+		return (await steer?.steerClient?.()) ?? null;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * The fanout outcome, answered, with held replies (issue #638) in between, shared by the four arms. Only a delivery
+ * `wantsHoldAfterDedup` names (a resume comment of an awaitInput trigger that the window swallowed) asks Valkey at all,
+ * so every other delivery's log lines and answer are exactly `respondEnqueueOutcome`'s as they were.
+ */
+async function answerEnqueueOutcome({ steer, kind, log, res, delivery, job, target, outcome }) {
+	let held = null;
+	if (wantsHoldAfterDedup(job, outcome)) {
+		held = await holdAfterDedup({ redis: await heldClientOf(steer), job, kind, outcome, now: steer?.now, log: (line) => log?.(line), delivery, target });
+	}
+	return respondEnqueueOutcome({ log, res, delivery, repo: job.repo, target, flow: job.flow, outcome, held });
+}
+
+/** The two target grammars the arms log in: `type#n` (github, forgejo) and `repo#n` / `repo!n` (gitlab, azure). */
+const hashTarget = (j) => `${j?.target?.type}#${j?.target?.number}`;
+const sepTarget = (j) => `${j?.repo}${j?.target?.type === "pull_request" ? "!" : "#"}${j?.target?.number}`;
+
+/**
+ * A filter drop that carries `heldCandidate` (issue #638): held under every awaitInput run live on its target, then
+ * answered `202 {status:"held", runs}` and "held". "enqueue" when a run parked on the target within the last minute
+ * (held.mjs): the caller enqueues the candidate as the parked reply it is. Null for every other drop, and for a
+ * candidate nothing took, and the caller answers its drop exactly as before. A missing client is today's behaviour.
+ */
+async function heldBeforeLabelOr({ steer, res, kind, delivery, result, targetOf }) {
+	if (result?.heldCandidate === undefined) return null;
+	const candidate = result.heldCandidate;
+	const held = await holdBeforeLabel({ redis: await heldClientOf(steer), candidate, kind, now: steer?.now, log: (line) => steer?.log?.(line), delivery, target: targetOf(candidate) });
+	if (held === null) return null;
+	if (held.enqueue === true) return "enqueue";
+	respond(res, 202, { status: "held", runs: held.held });
+	return "held";
+}
+
 /**
  * The GitHub arm. Returns `makeVerifiedHandler`'s handler directly, so it only ever sees an already-verified
  * request. `onVerified` owns parse, filter, enqueue, and response; a good signature is the sole
@@ -256,7 +425,7 @@ function respondEnqueueOutcome({ log, res, delivery, repo, target, flow, outcome
  * uses, so a lookup is never spent on a delivery the gate then ignores -- every label, comment, PR and
  * review delivery, and every close nothing wants, stays payload-only and byte-identical to before.
  */
-function makeGitHubHandler({ queue, routeTo, selfId, cfg, log, resolveAuthority }) {
+function makeGitHubHandler({ queue, routeTo, selfId, cfg, log, resolveAuthority, steer }) {
 	return makeVerifiedHandler({ secret: cfg.webhookSecret }, async ({ rawBody, event, delivery }, res) => {
 		let subset;
 		try {
@@ -270,7 +439,8 @@ function makeGitHubHandler({ queue, routeTo, selfId, cfg, log, resolveAuthority
 			if (!resolveAuthority) {
 				// Defensive only: unreachable in a wired receiver -- start.mjs hard-fails on github auth
 				// before `/` is ever mounted, and the resolver is built over that same auth object. Fail
-				// closed but RETRYABLE, because a wiring fault must not read on the wire as a stranger
+				// closed with a 503, a failed delivery in the hook's history that can be redelivered once
+				// the wiring is fixed, because a wiring fault must not read on the wire as a stranger
 				// being correctly refused.
 				return respond(res, 503, { error: "permission-lookup-failed" });
 			}
@@ -279,32 +449,45 @@ function makeGitHubHandler({ queue, routeTo, selfId, cfg, log, resolveAuthority
 				// The reason names the lookup, never the actor -- `sender.login` is personal data and exists
 				// here only to have been asked about (no-pii-in-logs).
 				log?.({ event: "github_permission_lookup_failed", delivery, reason: resolved.indeterminate });
-				// 503: GitHub redelivers, the GUID jobId coalesces the retry. GitHub's auto-redelivery is
-				// weaker than GitLab's, so a lookup outage outlasting the window can lose a webhook-only
-				// deployment's close -- the polling transport's closed source is the backstop where the
-				// poller runs, with its own bounded retry against the same lookup.
+				// 503, and GitHub never resends it on its own (WHAT A 503 MEANS, above `answerEnqueueFailed`):
+				// the close is lost unless someone redelivers it within 3 days, which keeps the GUID jobId and
+				// so queues it at most once. The polling transport's closed source is the backstop on a repo
+				// the poller already polls, with its own bounded retry against the same lookup; a
+				// webhook-only deployment has none.
 				return respond(res, 503, { error: "permission-lookup-failed" });
 			}
 			closerAuthorized = resolved.authorized;
 		}
 
-		const result = filter(event, subset, cfg, selfId, delivery, closerAuthorized);
+		let result = filter(event, subset, cfg, selfId, delivery, closerAuthorized);
 		if (!result.enqueue) {
-			log?.({ event: "dropped", delivery, reason: result.reason });
-			return respond(res, 204);
+			// Issue #638: a phrase-free reply that missed only the parked marker is held for a run parking on its target, or
+			// enqueued as a parked reply when a run just parked there; otherwise this is today's drop, byte for byte.
+			const rescued = await heldBeforeLabelOr({ steer, res, kind: "github", delivery, result, targetOf: hashTarget });
+			if (rescued === "held") return;
+			if (rescued === "enqueue") {
+				result = { enqueue: true, job: result.heldCandidate };
+			} else {
+				log?.({ event: "dropped", delivery, reason: result.reason });
+				return respond(res, 204);
+			}
 		}
 
+		const target = `${result.job.target.type}#${result.job.target.number}`;
+		const steered = await steerOrEnqueue({ steer, res, kind: "github", delivery, job: result.job, target });
+		if (steered.handled) return;
+		const job = steered.job;
+
 		// Fanout (REQ-REPLICA-RUNS) lives in `fanout` above; the 202/503 decision stays here, where it always was.
 		let outcome;
 		try {
-			outcome = await fanout(result.job, async (j) => await enqueueGitHubJob(await routeTo("github", j), j));
+			outcome = await fanout(job, async (j) => await enqueueGitHubJob(await routeTo("github", j), j));
 		} catch (err) {
-			// Own try/catch so a Valkey-down enqueue is a 503 (retryable), not verify's outer 500.
-			log?.({ event: "enqueue_failed", delivery, reason: err?.message });
-			return respond(res, 503, { error: "enqueue-failed" }); // GitHub redelivers; dedup by GUID coalesces
+			// Lost unless redelivered by hand within 3 days; the redelivery keeps the GUID (WHAT A 503 MEANS).
+			return answerEnqueueFailed({ log, res, delivery, err });
 		}
 
-		return respondEnqueueOutcome({ log, res, delivery, repo: result.job.repo, target: `${result.job.target.type}#${result.job.target.number}`, flow: result.job.flow, outcome });
+		return await answerEnqueueOutcome({ steer, kind: "github", log, res, delivery, job, target, outcome });
 	});
 }
 
@@ -315,13 +498,15 @@ function makeGitHubHandler({ queue, routeTo, selfId, cfg, log, resolveAuthority
  *
  * That resolution is a network call, and its three outcomes are deliberately not two:
  *   - a determinate verdict (authorized or not) is handed to the gate, which decides;
- *   - an INDETERMINATE lookup is a 503, so GitLab redelivers and the stable `webhook-id` dedups the
- *     retry. Answering 204 would drop real work during an outage, and it would look on the wire exactly
- *     like a stranger being correctly refused.
+ *   - an INDETERMINATE lookup is a 503. GitLab never resends it on its own (WHAT A 503 MEANS, above
+ *     `answerEnqueueFailed`), so the delivery waits for a "Resend Request", which keeps the `webhook-id`
+ *     and so queues at most once. Answering 204 would drop real work during an outage behind a success
+ *     nobody would think to resend, and it would look on the wire exactly like a stranger being correctly
+ *     refused.
  * The lookup runs only for events that could still fire -- after verification, and after the payload has
  * been projected -- so an unauthenticated flood cannot make this project call GitLab at all.
  */
-function makeGitLabHandler({ routeTo, queue, cfg, log, mode, secret, selfId, resolveAuthority, now }) {
+function makeGitLabHandler({ routeTo, queue, cfg, log, mode, secret, selfId, resolveAuthority, now, steer }) {
 	return makeGitLabVerifiedHandler({ mode, secret, ...(now ? { now } : {}) }, async ({ rawBody, delivery }, res) => {
 		let subset;
 		try {
@@ -335,27 +520,41 @@ function makeGitLabHandler({ routeTo, queue, cfg, log, mode, secret, selfId, res
 			// The reason names the lookup, never the actor -- `user.username` is personal data and exists
 			// only to have been asked about (no-pii-in-logs).
 			log?.({ event: "gitlab_access_lookup_failed", delivery, reason: resolved.indeterminate });
+			// Lost unless resent from Recent events within 2 days (WHAT A 503 MEANS): GitLab never resends.
 			return respond(res, 503, { error: "access-lookup-failed" });
 		}
 
-		const result = filterGitLab(subset, cfg.triggers?.gitlab, cfg.triggers?.knownFlows, selfId, resolved.authorized, delivery);
+		let result = filterGitLab(subset, cfg.triggers?.gitlab, cfg.triggers?.knownFlows, selfId, resolved.authorized, delivery);
 		if (!result.enqueue) {
-			log?.({ event: "dropped", delivery, reason: result.reason });
-			return respond(res, 204);
+			// Issue #638: a phrase-free reply that missed only the parked marker is held for a run parking on its target, or
+			// enqueued as a parked reply when a run just parked there; otherwise this is today's drop, byte for byte.
+			const rescued = await heldBeforeLabelOr({ steer, res, kind: "gitlab", delivery, result, targetOf: sepTarget });
+			if (rescued === "held") return;
+			if (rescued === "enqueue") {
+				result = { enqueue: true, job: result.heldCandidate };
+			} else {
+				log?.({ event: "dropped", delivery, reason: result.reason });
+				return respond(res, 204);
+			}
 		}
 
+		// `!` for a merge request, `#` for an issue -- GitLab's own notation, and the same discrimination
+		// the semantic dedup key makes, because the two are separate number sequences.
+		const sep = result.job.target.type === "pull_request" ? "!" : "#";
+		const target = `${result.job.repo}${sep}${result.job.target.number}`;
+		const steered = await steerOrEnqueue({ steer, res, kind: "gitlab", delivery, job: result.job, target });
+		if (steered.handled) return;
+		const job = steered.job;
+
 		let outcome;
 		try {
-			outcome = await fanout(result.job, async (j) => await enqueueGitLabJob(await routeTo("gitlab", j), j));
+			outcome = await fanout(job, async (j) => await enqueueGitLabJob(await routeTo("gitlab", j), j));
 		} catch (err) {
-			log?.({ event: "enqueue_failed", delivery, reason: err?.message });
-			return respond(res, 503, { error: "enqueue-failed" }); // GitLab redelivers; dedup by webhook-id coalesces
+			// Lost unless resent by hand within 2 days; the resend keeps the webhook-id (WHAT A 503 MEANS).
+			return answerEnqueueFailed({ log, res, delivery, err });
 		}
 
-		// `!` for a merge request, `#` for an issue -- GitLab's own notation, and the same discrimination
-		// the semantic dedup key makes, because the two are separate number sequences.
-		const sep = result.job.target.type === "pull_request" ? "!" : "#";
-		return respondEnqueueOutcome({ log, res, delivery, repo: result.job.repo, target: `${result.job.repo}${sep}${result.job.target.number}`, flow: result.job.flow, outcome });
+		return await answerEnqueueOutcome({ steer, kind: "gitlab", log, res, delivery, job, target, outcome });
 	});
 }
 
@@ -367,12 +566,13 @@ function makeGitLabHandler({ routeTo, queue, cfg, log, mode, secret, selfId, res
  * Forgejo signs the raw body with HMAC-SHA256 and sends `X-Hub-Signature-256`, `X-GitHub-Delivery` and
  * `X-GitHub-Event`, which is byte-for-byte what `makeVerifiedHandler` already reads. So
  * CONST-HMAC-OVER-RAW-BODY is satisfied here by EXISTING code rather than new code, and
- * REQ-DEDUP-BY-DELIVERY-GUID transfers with no adaptation at all. What Forgejo needs is its own SECRET,
+ * REQ-DEDUP-BY-DELIVERY-GUID's mechanism transfers with no adaptation at all (what it cannot see is a replay,
+ * which Forgejo sends under a new GUID: WHAT A 503 MEANS). What Forgejo needs is its own SECRET,
  * which is why it is a separate handler and a separate path: `makeVerifiedHandler` closes over one secret
  * per construction, and serving two sources from one would mean either sharing a secret between forges or
  * letting the request choose which one it was checked against.
  */
-function makeForgejoHandler({ routeTo, queue, cfg, log, secret, selfId, resolveAuthority }) {
+function makeForgejoHandler({ routeTo, queue, cfg, log, secret, selfId, resolveAuthority, steer }) {
 	return makeVerifiedHandler({ secret }, async ({ rawBody, event, delivery }, res) => {
 		let subset;
 		try {
@@ -386,24 +586,38 @@ function makeForgejoHandler({ routeTo, queue, cfg, log, secret, selfId, resolveA
 			// The reason names the lookup, never the actor -- `sender.login` is personal data and exists here
 			// only to have been asked about (no-pii-in-logs).
 			log?.({ event: "forgejo_permission_lookup_failed", delivery, reason: resolved.indeterminate });
+			// Lost unless replayed from the hook's history (WHAT A 503 MEANS): Forgejo makes one attempt.
 			return respond(res, 503, { error: "permission-lookup-failed" });
 		}
 
-		const result = filterForgejo(event, subset, cfg.triggers?.forgejo, cfg.triggers?.knownFlows, selfId, resolved.authorized, delivery);
+		let result = filterForgejo(event, subset, cfg.triggers?.forgejo, cfg.triggers?.knownFlows, selfId, resolved.authorized, delivery);
 		if (!result.enqueue) {
-			log?.({ event: "dropped", delivery, reason: result.reason });
-			return respond(res, 204);
+			// Issue #638: a phrase-free reply that missed only the parked marker is held for a run parking on its target, or
+			// enqueued as a parked reply when a run just parked there; otherwise this is today's drop, byte for byte.
+			const rescued = await heldBeforeLabelOr({ steer, res, kind: "forgejo", delivery, result, targetOf: hashTarget });
+			if (rescued === "held") return;
+			if (rescued === "enqueue") {
+				result = { enqueue: true, job: result.heldCandidate };
+			} else {
+				log?.({ event: "dropped", delivery, reason: result.reason });
+				return respond(res, 204);
+			}
 		}
 
+		const target = `${result.job.target.type}#${result.job.target.number}`;
+		const steered = await steerOrEnqueue({ steer, res, kind: "forgejo", delivery, job: result.job, target });
+		if (steered.handled) return;
+		const job = steered.job;
+
 		let outcome;
 		try {
-			outcome = await fanout(result.job, async (j) => await enqueueForgeJob(await routeTo("forgejo", j), "forgejo", j));
+			outcome = await fanout(job, async (j) => await enqueueForgeJob(await routeTo("forgejo", j), "forgejo", j));
 		} catch (err) {
-			log?.({ event: "enqueue_failed", delivery, reason: err?.message });
-			return respond(res, 503, { error: "enqueue-failed" }); // Forgejo redelivers; dedup by GUID coalesces
+			// Lost unless replayed by hand, and a replay is a new GUID: only the semantic window coalesces it.
+			return answerEnqueueFailed({ log, res, delivery, err });
 		}
 
-		return respondEnqueueOutcome({ log, res, delivery, repo: result.job.repo, target: `${result.job.target.type}#${result.job.target.number}`, flow: result.job.flow, outcome });
+		return await answerEnqueueOutcome({ steer, kind: "forgejo", log, res, delivery, job, target, outcome });
 	});
 }
 
@@ -421,7 +635,7 @@ function makeForgejoHandler({ routeTo, queue, cfg, log, secret, selfId, resolveA
  * `"Display Name <email>"`. Both the resolver and the bot-loop guard handle both forms -- see
  * filter-azure.mjs, where the ordering constraint is stated in full.
  */
-function makeAzureHandler({ routeTo, queue, cfg, log, mode, secret, headerName, selfId, resolveAuthority }) {
+function makeAzureHandler({ routeTo, queue, cfg, log, mode, secret, headerName, selfId, resolveAuthority, steer }) {
 	return makeAzureVerifiedHandler({ mode, secret, headerName }, async ({ rawBody }, res) => {
 		let subset;
 		try {
@@ -443,26 +657,42 @@ function makeAzureHandler({ routeTo, queue, cfg, log, mode, secret, headerName,
 			// The reason names the lookup, never the actor -- an email address is personal data and exists
 			// here only to have been asked about (no-pii-in-logs).
 			log?.({ event: "azure_membership_lookup_failed", delivery, reason: resolved.indeterminate });
+			// Azure resends the same notification up to 8 times over about 3 minutes, so with the same `id`,
+			// then retries it on probation for about 36 hours (WHAT A 503 MEANS).
 			return respond(res, 503, { error: "membership-lookup-failed" });
 		}
 
-		const result = filterAzure(subset, cfg.triggers?.azure, cfg.triggers?.knownFlows, selfId, resolved.authorized, delivery);
+		let result = filterAzure(subset, cfg.triggers?.azure, cfg.triggers?.knownFlows, selfId, resolved.authorized, delivery);
 		if (!result.enqueue) {
-			log?.({ event: "dropped", delivery, reason: result.reason });
-			return respond(res, 204);
+			// Issue #638: a phrase-free reply that missed only the parked marker is held for a run parking on its target, or
+			// enqueued as a parked reply when a run just parked there; otherwise this is today's drop, byte for byte.
+			const rescued = await heldBeforeLabelOr({ steer, res, kind: "azure", delivery, result, targetOf: sepTarget });
+			if (rescued === "held") return;
+			if (rescued === "enqueue") {
+				result = { enqueue: true, job: result.heldCandidate };
+			} else {
+				log?.({ event: "dropped", delivery, reason: result.reason });
+				return respond(res, 204);
+			}
 		}
 
+		// `!` for a pull request, `#` for a work item -- Azure numbers them separately, and this is the same
+		// discrimination the semantic dedup key makes.
+		const sep = result.job.target.type === "pull_request" ? "!" : "#";
+		const target = `${result.job.repo}${sep}${result.job.target.number}`;
+		const steered = await steerOrEnqueue({ steer, res, kind: "azure", delivery, job: result.job, target });
+		if (steered.handled) return;
+		const job = steered.job;
+
 		let outcome;
 		try {
-			outcome = await fanout(result.job, async (j) => await enqueueForgeJob(await routeTo("azure", j), "azure", j));
+			outcome = await fanout(job, async (j) => await enqueueForgeJob(await routeTo("azure", j), "azure", j));
 		} catch (err) {
-			log?.({ event: "enqueue_failed", delivery, reason: err?.message });
-			return respond(res, 503, { error: "enqueue-failed" });
+			// Azure resends the same notification, so the same `id`, for about 3 minutes, then on probation
+			// for about 36 hours, losing new events meanwhile (WHAT A 503 MEANS).
+			return answerEnqueueFailed({ log, res, delivery, err });
 		}
 
-		// `!` for a pull request, `#` for a work item -- Azure numbers them separately, and this is the same
-		// discrimination the semantic dedup key makes.
-		const sep = result.job.target.type === "pull_request" ? "!" : "#";
-		return respondEnqueueOutcome({ log, res, delivery, repo: result.job.repo, target: `${result.job.repo}${sep}${result.job.target.number}`, flow: result.job.flow, outcome });
+		return await answerEnqueueOutcome({ steer, kind: "azure", log, res, delivery, job, target, outcome });
 	});
 }

package/src/start.mjs

@@ -30,9 +30,11 @@
  */
 
 import http from "node:http";
+import { workerName } from "@edgehero/pi-dispatch/config";
+import { makeReceiverPublisher, receiverBind } from "@edgehero/pi-dispatch/receiver-status";
 import { readFileSync, watch } from "node:fs";
 import { dirname, basename } from "node:path";
-import { loadReceiverConfig, triggersFilePath, reloadTriggers } from "./config.mjs";
+import { loadReceiverConfig, servedForges, triggersFilePath, reloadTriggers } from "./config.mjs";
 import { makeReceiver } from "./receiver.mjs";
 import { entryExitCode } from "./cli.mjs";
 import { installRejectionPrinter } from "@edgehero/pi-dispatch/exit-code";
@@ -52,6 +54,11 @@ import { WATCH_DEBOUNCE_MS, changedWhileArming, makeWatchCloser, readBeforeArmin
 import { githubFailureFields } from "@edgehero/pi-dispatch/octokit-log";
 import { retryIdentity } from "./boot-retry.mjs";
 
+const RECEIVER_VERSION = (() => {
+	try { return JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version ?? ""; }
+	catch { return ""; }
+})();
+
 /**
  * Boot the receiver. Collaborators are injected (defaulting to the real ones) so the whole wiring is
  * testable offline with no GitHub, no Valkey, and no socket. Returns the listening server.
@@ -67,6 +74,7 @@ export async function startReceiver(
 		makeQueueFn = makeQueue,
 		makeForgeRouterFn = makeForgeRouter,
 		createServer = http.createServer,
+		makeReceiverPublisherFn = makeReceiverPublisher,
 		resolveGitLabSelfId: resolveSelfIdFn = resolveGitLabSelfId,
 		makeResolveAuthority: makeResolveAuthorityFn = makeResolveAuthority,
 		resolveForgejoSelfId: resolveForgejoSelfIdFn = resolveForgejoSelfId,
@@ -238,11 +246,23 @@ export async function startReceiver(
 			};
 		}
 
-		const handler = makeReceiver({ queue, router, selfId, cfg, log, gitlab, forgejo, azure, resolveAuthority });
+		const handler = makeReceiver({ queue, router, selfId, cfg, log, gitlab, forgejo, azure, resolveAuthority, watchEnv: env, now });
 		const server = createServer(handler);
-		server.listen(cfg.port, cfg.bind, () =>
-			log({ event: "receiver_started", port: cfg.port, bind: cfg.bind, valkey: cfg.valkeyUrl }),
-		);
+		let receiverPublisher = null;
+		server.listen(cfg.port, cfg.bind, () => {
+			log({ event: "receiver_started", port: cfg.port, bind: cfg.bind, valkey: cfg.valkeyUrl });
+			// A socket, not configuration, proves where the process actually listens. Publication is optional:
+			// an unusable name or missing socket fact must never change the receiver's startup decision.
+			try {
+				const socket = server.address?.();
+				if (!socket || typeof socket !== "object" || !Number.isInteger(socket.port) || socket.port < 1 || socket.port > 65535) throw new Error("socket");
+				const name = workerName(env);
+				const publisher = makeReceiverPublisherFn({ client: () => queue.client, name, version: RECEIVER_VERSION, port: socket?.port, bind: receiverBind(socket?.address), forges: servedForges(cfg), observations: handler.observations, now, log: (event, fields) => log({ event, ...fields }) });
+				receiverPublisher = publisher;
+				closers.push(publisher);
+				publisher.start();
+			} catch { try { log({ event: "receiver_status_unavailable", reason: "identity-or-socket" }); } catch {} }
+		});
 
 		// The watch arms UNCONDITIONALLY now (issue #301). Armed only on the real entry, the lifecycle defect
 		// was muted under test rather than closed, and this file had no coverage that the watch arms at all --
@@ -264,6 +284,7 @@ export async function startReceiver(
 			const shutdown = async (signal) => {
 				log({ event: "receiver_stopping", signal });
 				await new Promise((resolve) => server.close(resolve));
+				try { receiverPublisher?.close(); } catch {}
 				await queue.close();
 				await router.close();
 				// The watch closer, and whatever joins it later. Per-item try, because a throw here would strand