@edgehero/pi-dispatch 1.10.2 → 2.0.0

package/src/index.mjs

@@ -1,6 +1,12 @@
 import { DelayedError, UnrecoverableError, Worker } from "bullmq";
+import { assertJudgedConnection, onValkeyError } from "./connection.mjs";
 import { jobContainerName } from "./backend-local.mjs";
-import { InfraRetry, runJob } from "./processor.mjs";
+import { scrubCredentials } from "./redact.mjs";
+import { BACKEND_NOT_REGISTERED } from "./backend-registry.mjs";
+import { CANCEL_ACK_TTL_MS, cancelAckKey, cancelReqKey } from "./cancel-state.mjs";
+import { InfraRetry, NETNS_KEEPER_CRASH_LOOP, NETNS_KEEPER_NOT_HOLDING, TERMINAL_COMMENTS, runJob } from "./processor.mjs";
+import { NETNS_KEEPER_YOUNG_HOLD_MAX_MS, netnsKeeperCrashLoopSentence, netnsKeeperLoopAgainSentence } from "./netns-keeper.mjs";
+import { PODMAN_RESTART_HOLD_EXPIRED, PODMAN_RESTART_HOLD_MAX_MS, PODMAN_RESTART_HOLD_RECHECK_MS } from "./runtime-observations.mjs";
 import { targetFor } from "./run-history.mjs";
 import { budgetCapsFor, canonicalScope, concurrencyFor, makeInFlight, scopeKeyPrefix } from "./scoped-limits.mjs";
 import { WAIT_AFTER_MAX_DEFAULT_MS, WAIT_INTERVAL_FLOOR_MS, afterMs, unreadableConditions, waitArmed, waitBackoffMs, waitLabel, waitProfileNames } from "./wait-for.mjs";
@@ -74,7 +80,7 @@ function boundAfterAbort(run, signal, job, log, graceMs = ABORT_GRACE_MS) {
 				if (settled) return;
 				settled = true;
 				log("stop_did_not_take", { job: job.id, graceMs });
-				resolve({ code: 137, aborted: true, turns: null, tokens: null, session: null, usage: null, context: null });
+				resolve({ code: 137, aborted: true, turns: null, tokens: null, session: null, usage: null, context: null, exitReason: null });
 			}, graceMs);
 			// A boot-blocking handle is not wanted here: the worker should be able to exit if everything else
 			// has finished, and this timer only matters while a job is still in flight.
@@ -106,7 +112,7 @@ function boundAfterAbort(run, signal, job, log, graceMs = ABORT_GRACE_MS) {
  * The overlay changes which values the spend caps take, never when they are checked -- reserveBudget still
  * runs inside runJob against the freshly passed caps (CONST-BUDGET-BEFORE-TOKENS).
  */
-export function makeProcessor({ cancelJob, stopContainer, containerName = (job) => jobContainerName(job.id), redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), hostBound = null, checkLease = null, scopeLease = null, deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, checkTimeoutMs = () => 10_000, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
+export function makeProcessor({ cancelJob, stopContainer, containerName = (job) => jobContainerName(job.id), redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), hostBound = null, checkLease = null, scopeLease = null, deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, cancelPollMs = 2_000, cancelStopBoundMs = CANCEL_STOP_BOUND_MS, hostName = "", now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, checkTimeoutMs = () => 10_000, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
 	return async function processor(job, token, signal) {
 		// Scoped pause windows (REQ-SCOPED-PAUSE-WINDOWS): if this job's folder/repo is inside an active pause
 		// window, DEFER it to the window end via BullMQ's delayed set -- the job keeps its identity/dedup and
@@ -533,14 +539,55 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 		let startedAt;
 		let name;
 		let timer;
+		let cancelPoll;
+		let cancelPolling = false;
+		// The poll tick in flight, KEPT (gate round 2 of PR #479), so the error handler can stop the poll and wait for it
+		// before it asks whether the job was cancelled: a flag says a tick runs, only its promise says when it is done.
+		let cancelTick = null;
+		// THE ONE STOP (gate rounds 2 to 4 of PR #479), used wherever this job's outcome is about to be decided: at the
+		// container's observed exit, when runJob returns, and in the error handler. `pollStopped` is set first and read by
+		// the tick synchronously right before it aborts and acknowledges, so after a stop no cancel is ever acknowledged;
+		// a tick already past that point has aborted the signal and writes its ack before the stop's await returns. So
+		// "acknowledged" and "the signal says operator-cancel" are the same fact by the time anything is decided.
+		//
+		// BOUNDED (gate round 4 of PR #479): the worker's Valkey client queues commands while offline, so a tick's read can
+		// hang for a whole outage, and an unbounded await held the job's outcome (its record, comment and slot) with it: a
+		// restart in that window lost the record. The bound is safe because the flag is set FIRST: a read still hanging
+		// returns without acknowledging, and a tick already past the flag check aborted the signal synchronously, so the
+		// decision sees the cancel even while its ack write waits. Said once per job when the bound is hit.
+		let pollStopped = false;
+		let stopBoundSaid = false;
+		const stopCancelPoll = async () => {
+			pollStopped = true;
+			clearInterval(cancelPoll);
+			if (!cancelTick) return;
+			let bound;
+			const hit = await Promise.race([
+				Promise.resolve(cancelTick).then(
+					() => false,
+					() => false,
+				),
+				new Promise((resolve) => {
+					bound = setTimeout(() => resolve(true), cancelStopBoundMs);
+					bound.unref?.();
+				}),
+			]);
+			clearTimeout(bound);
+			if (hit && !stopBoundSaid) {
+				stopBoundSaid = true;
+				deps?.log?.("cancel_poll_stop_bounded", { jobId: job.id, boundMs: cancelStopBoundMs });
+			}
+		};
 		let onAbort;
 		let venue;
 		try {
 			// Nothing between the acquire above and the main `try` below may throw unguarded: the releasing
 			// finally belongs to THAT try, so an unguarded throw here would leak the hold and wedge the
-			// scope until a worker restart. Nothing in this block CAN throw today (setTimeout and
-			// addEventListener on the bullmq-allocated controller are total at processor arity 3); the
-			// guard is structural, not observational.
+			// scope until a worker restart. Nothing in this block throws past its own guards (setTimeout,
+			// setInterval and addEventListener on the bullmq-allocated controller are total at processor arity
+			// 3, and the cancel poll's redis reads happen inside its own guarded ticks). `containerName` is the
+			// exception, and it is named: the registry's refusal of an unheld venue is caught at the call, and
+			// anything else it throws propagates to the catch below, which releases what was acquired.
 			startedAt = new Date().toISOString();
 			// The producer of the name both boot reapers sweep by substring. Built from the shared prefix
 			// rather than typed here, so a rename cannot land in the producer and not in the sweeps (#227).
@@ -558,7 +605,25 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			// the trigger's, which lives in `data`. Built once so the name and the stop cannot resolve
 			// different backends -- the whole reason the name moved onto the registry in the first place.
 			venue = { ...job.data, id: job.id };
-			name = containerName(venue);
+			// The registry REFUSES a venue it does not hold (#277), and that one refusal is caught here.
+			// Unguarded, it skipped `runJob` and every `recordRun` below, so a job naming a venue this worker
+			// never built -- a producer on a newer build that knows a venue this one does not -- left NO
+			// record, no refusal comment, and was retried as if the infrastructure had failed. Such a venue is
+			// never blessed here (the registry refuses a blessed name it does not hold, at boot), so `runJob`
+			// refuses it pre-spend -- as `backend-unblessed`, or by an earlier refusal that happens to win --
+			// and records it; nothing starts, so there is no container for this name to stop. A null name
+			// reaches the abort path's stop only if an abort lands in that window, and that stop fails into
+			// its own log line.
+			//
+			// ONLY that refusal, by its code. Any other throw is a REGISTERED venue's own `containerName`
+			// failing, a broken adapter: swallowing it would reserve the budget and start a container under a
+			// null name that the timeout could not stop and no reaper sweeps, so it propagates as before.
+			try {
+				name = containerName(venue);
+			} catch (err) {
+				if (err?.code !== BACKEND_NOT_REGISTERED) throw err;
+				name = null;
+			}
 			timer = setTimeout(() => {
 				// BullMQ has no per-job kill timer; this is ours. cancelJob raises the AbortSignal.
 				Promise.resolve(cancelJob(job.id, "job-timeout-30m")).catch(() => {});
@@ -580,12 +645,51 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// host. Losing the kill for one job is bad; losing the process is worse.
 				const note = deps.log ?? (() => {});
 				try {
-					Promise.resolve(stopContainer(name, venue)).catch((err) => note("stop_container_failed", { job: job.id, reason: err?.message }));
+					Promise.resolve(stopContainer(name, venue)).catch((err) => note("stop_container_failed", { job: job.id, reason: scrubCredentials(err?.message) }));
 				} catch (err) {
-					note("stop_container_failed", { job: job.id, reason: err?.message });
+					note("stop_container_failed", { job: job.id, reason: scrubCredentials(err?.message) });
 				}
 			};
 			signal.addEventListener("abort", onAbort, { once: true });
+
+			// The operator-cancel poll (issue #287). `cancelJob` is reachable only from THIS process, so the
+			// CLI and the panel leave a `cancel:req:<jobId>` key instead, and the worker that holds the job
+			// answers. Polled beside the kill timer rather than subscribed, for the reasons cancel-state.mjs
+			// records; one GET per 2s per active job is invisible against the 30s abort grace. Guarded on
+			// `redis.get` because bare test wirings pass a redis with only `incr`/`expire` -- they arm
+			// nothing and stay byte-identical. Each tick is re-entrancy-guarded and swallows redis faults:
+			// a blip may cost the ack, never the job.
+			if (typeof redis?.get === "function") {
+				cancelPoll = setInterval(() => {
+					if (cancelPolling) return;
+					cancelPolling = true;
+					cancelTick = (async () => {
+						try {
+							const req = await redis.get(cancelReqKey(job.id));
+							if (req === null || req === undefined) return;
+							// Stopped while this read was in flight: the outcome is being decided, so this request is left
+							// unanswered (its requester's timeout re-reads the job), never acknowledged after the fact.
+							if (pollStopped) return;
+							// Bound to this worker (the injection comment in createWorker): `false` means the job
+							// left the tracked map in this same instant, in which case the finally below clears
+							// this interval anyway and the request's TTL reaps the key.
+							const took = await Promise.resolve(cancelJob(job.id, "operator-cancel")).catch(() => false);
+							if (took === false) return;
+							// Ack first, then consume the request: losing the DEL to a blip costs one redundant
+							// re-read, losing the ack costs the operator a false "nobody answered".
+							await redis.set(cancelAckKey(job.id), hostName, "PX", CANCEL_ACK_TTL_MS).catch(() => {});
+							await redis.del(cancelReqKey(job.id)).catch(() => {});
+							clearInterval(cancelPoll);
+						} catch {
+							// Fail open: the next tick re-asks.
+						} finally {
+							cancelPolling = false;
+						}
+					})();
+				}, cancelPollMs);
+				// Like the abort-grace timer: this must never keep an otherwise-finished worker alive.
+				cancelPoll.unref?.();
+			}
 		} catch (error) {
 			// Release and CLEAR the flag: this throw never reaches the main finally below, but a shared
 			// scope must never be releasable twice -- a double release frees another holder's slot.
@@ -600,6 +704,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			void scopeSlot?.release?.();
 			scopeSlot = null;
 			clearTimeout(timer);
+			clearInterval(cancelPoll);
 			throw error;
 		}
 
@@ -669,7 +774,26 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// next boot reaper sweeps; what is NOT leaked is the slot, the lease and the reservation.
 				// The `stop_did_not_take` line is the loud half, because a host whose daemon ignores a stop
 				// is a fact an operator has to learn from somewhere.
-				runContainer: (ctx) => boundAfterAbort(deps.runContainer({ ...ctx, name, signal }), signal, job, deps.log ?? (() => {})),
+				// Issue #287: WHO aborted rides the result, read off the signal at the one seam both abort
+				// paths flow through (a real stop resolves through here, and boundAfterAbort's synthesised
+				// `{ code: 137, aborted: true }` does too). `cancelJob(id, reason)` becomes `signal.reason`,
+				// so the processor can classify an operator's cancel apart from the kill timer's without
+				// this file growing a second channel. Mapped only when `aborted` -- an unaborted result
+				// stays byte-identical -- and only a STRING rides: a non-string reason (AbortError objects,
+				// future bullmq surprises) collapses to null, which classifies as worker-abort, the
+				// conservative direction.
+				// Gate round 4 of PR #479: the poll STOPS at the container's observed exit, since after it there is nothing
+				// left for a cancel to stop. A cancel acknowledged before that moment raced the exit and lost nothing it was
+				// promised: the run is recorded as that cancel (the existing path, "partial work may exist") even when the
+				// container happened to exit on its own, because the operator was told the record would say operator-cancel.
+				runContainer: (ctx) =>
+					boundAfterAbort(deps.runContainer({ ...ctx, name, signal }), signal, job, deps.log ?? (() => {})).then(async (r) => {
+						await stopCancelPoll();
+						const raced = r && r.aborted !== true && signal.aborted === true && signal.reason === "operator-cancel";
+						if (raced) deps.log?.("cancel_acked_as_container_exited", { jobId: job.id, exitCode: r.code ?? null });
+						const run = raced ? { ...r, aborted: true } : r;
+						return run?.aborted ? { ...run, abortReason: typeof signal.reason === "string" ? signal.reason : null } : run;
+					}),
 				// REQ-TRIGGER-SECRETS. The resolver runs INSIDE the 30-minute kill timer armed above, so it has
 				// to be abortable for the same reason runContainer does: a resolver blocking on an unreachable
 				// vault would otherwise hold its slot until its own timeout, and an abort landing mid-resolution
@@ -711,9 +835,149 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// replacing wrapper would lose it in the same silence.
 				...(deps.checkOnceSpent ? { checkOnceSpent: (j, opts) => deps.checkOnceSpent(j, { ...opts, queueJobId: job.id }) } : {}),
 			});
-			recordRun({ job, result, startedAt, endedAt: new Date().toISOString() });
-			return result;
+			// Gate round 4 of PR #479: stopped again as runJob returns, before anything is recorded (a result with no
+			// container never passed the exit stop above). A cancel that lands after is never acknowledged, and its
+			// requester's re-read says the job finished. One acknowledged before, on a result that is not the cancel (a
+			// pre-spend refusal that finished after the ack, since a container would have seen the abort), is recorded as
+			// the cancel the operator was told of, the result it replaced named in the log.
+			await stopCancelPoll();
+			let outcome = result;
+			if (signal.aborted === true && signal.reason === "operator-cancel" && result?.reason !== "operator-cancel") {
+				deps.log?.("cancel_acked_before_result", { jobId: job.id, was: `${result?.outcome}/${result?.reason ?? ""}` });
+				outcome = { ...result, outcome: "policy", reason: "operator-cancel" };
+			}
+			recordRun({ job, result: outcome, startedAt, endedAt: new Date().toISOString() });
+			return outcome;
 		} catch (error) {
+			// Issue #448 (gate round 2 of PR #473): a local job held until rootful Podman's service restarts goes back to the
+			// delayed set WITHOUT spending an attempt, the pause gate's move, since a service held up past one retry backoff
+			// failed a job that heals by itself (measured). Bounded by the hold's own start, stored on the job because a
+			// deferral leaves `attemptsMade` alone and a worker restart must not reset it; past the bound it fails for good
+			// with its own reason token, whose comment names the restart. Nothing is recorded per deferral: the job never
+			// started, and one record a minute for an hour would bury the run history.
+			//
+			// Gate round 3: the hour counts only while the worker kept checking. A check that comes more than two recheck
+			// periods after the last one (the queue was paused, `pi-dispatch pause`, or no worker ran) starts the hold
+			// afresh, since the time between was no evidence that the service stayed up; the last check's time is stored
+			// beside the start. And the expired hold is RECORDED with its own token, the one its comment and the failure
+			// hook carry, never the `container-never-started` of the throw it ends.
+			// A CANCELLED JOB IS NEVER RUN AGAIN (gate of PR #479). The operator-cancel poll runs from pickup to the finally
+			// below, so a cancel can be acknowledged (the signal aborted with "operator-cancel") after the job was picked up
+			// and before this handler hands it back to the queue: a hold's `moveToDelayed`, or an `InfraRetry` BullMQ
+			// retries after its backoff. Either ran the job later, with the request key already consumed, so nothing
+			// cancelled it again. So every error this handler would hand back (every `InfraRetry`, the two holds' included,
+			// since both are one) is asked ONE question first, in ONE place, so the three paths cannot drift apart: was this
+			// job cancelled? A request not yet polled is read too, and acknowledged and consumed as the poll would. Either
+			// ends the job as the operator's cancel, the policy result the aborted-container path already returns, recorded,
+			// one fixed comment, never retried. A shutdown's abort is not a cancel: that job is held or retried as before,
+			// so it survives the restart.
+			// `readPending: false` asks only whether the poll ALREADY acknowledged a cancel (the signal), and leaves a request it
+			// had not read unanswered.
+			const operatorCancelled = async ({ readPending = true } = {}) => {
+				if (signal?.aborted === true) return signal.reason === "operator-cancel";
+				if (!readPending) return false;
+				if (typeof redis?.get !== "function") return false;
+				let req = null;
+				try {
+					req = await redis.get(cancelReqKey(job.id));
+				} catch {
+					return false;
+				}
+				if (req === null || req === undefined) return false;
+				await Promise.resolve(redis.set?.(cancelAckKey(job.id), hostName, "PX", CANCEL_ACK_TTL_MS)).catch(() => {});
+				await Promise.resolve(redis.del?.(cancelReqKey(job.id))).catch(() => {});
+				return true;
+			};
+			// A retry that already spent (a container that ran and exited as infra) keeps what it spent on the record, and
+			// says the processor's own operator-cancel sentence; one that never started says it was cancelled before it did.
+			// The session rides along as the aborted-container path keeps it (`mergeSession`, already applied to the throw).
+			// A NON-retryable error (gate round 3) is not known to have started nothing, so it says the processor's own
+			// operator-cancel sentence, keeps whatever `budgetReserved` it carried, and its own words go to the log, never lost.
+			const endCancelled = async ({ retryable }) => {
+				const spent = error?.budgetReserved === true;
+				const beforeStart = retryable && !spent;
+				const result = { outcome: "policy", reason: "operator-cancel", exitCode: spent ? (error.exitCode ?? null) : null, turns: spent ? (error.turns ?? null) : null, tokens: spent ? (error.tokens ?? null) : null, ...(spent && error.usage ? { usage: error.usage } : {}), provider: error?.provider ?? null, model: error?.model ?? null, session: error?.session ?? null, budgetReserved: retryable ? spent : (error?.budgetReserved ?? null) };
+				deps?.log?.("job_cancelled_instead_of_retry", { jobId: job.id, spent, retryable, ...(retryable ? {} : { failure: scrubCredentials(String(error?.message ?? error)).slice(0, 300) }) });
+				if (deps?.comment) await Promise.resolve(deps.comment(job.data, beforeStart ? CANCELLED_BEFORE_START_COMMENT : TERMINAL_COMMENTS["operator-cancel"])).catch(() => {});
+				recordRun({ job, result, startedAt, endedAt: new Date().toISOString() });
+				return result;
+			};
+			// STOP THE POLL BEFORE ASKING (gate round 2 of PR #479). The poll ran until the finally below, so a request that
+			// landed after the question was still acknowledged by it (ack written, request consumed, the CLI saying "cancel
+			// accepted") while the job went on to its hold or its retry and ran again. So the poll is stopped first and any
+			// tick already in flight awaited, and only then is the question asked: a cancel the poll took is seen on the
+			// signal, one it had not read is read here, and one that lands after is never acknowledged, so its requester
+			// truthfully says nothing was changed while the job is held or retried, and a second cancel removes it.
+			//
+			// EVERY ERROR, NOT ONLY A RETRY (gate round 3 of PR #479). On a non-retryable error the still-running poll could
+			// acknowledge a cancel, so the CLI said the record would say operator-cancel while it said failed. The poll is
+			// stopped the same way before the terminal path, and the question differs by one thing only. A cancel the poll
+			// ALREADY acknowledged (the signal) ends the job as operator-cancel: the requester was told that, so the record
+			// must agree, and the failure's own words go to the log. A request it had NOT read is left unanswered: the job
+			// has failed for its own reason, the record says so, and the requester's re-read then says the job finished
+			// before its worker read the cancel and nothing was changed. Consistent either way, never one said and the
+			// other recorded. A retryable error reads that request too, since the job would otherwise run again.
+			await stopCancelPoll();
+			const retryable = error instanceof InfraRetry;
+			if (await operatorCancelled({ readPending: retryable })) return await endCancelled({ retryable });
+			if (error?.holdUntilRestart === true) {
+				const heldAt = now();
+				const last = job.data?.podmanRestartHoldLastMs;
+				const resumed = !Number.isFinite(last) || heldAt - last > 2 * PODMAN_RESTART_HOLD_RECHECK_MS;
+				const since = !resumed && Number.isFinite(job.data?.podmanRestartHoldSinceMs) ? job.data.podmanRestartHoldSinceMs : heldAt;
+				if (heldAt - since < PODMAN_RESTART_HOLD_MAX_MS) {
+					await job.updateData({ ...job.data, podmanRestartHoldSinceMs: since, podmanRestartHoldLastMs: heldAt });
+					deps?.log?.("podman_restart_hold", { jobId: job.id, heldForMs: heldAt - since, delayMs: PODMAN_RESTART_HOLD_RECHECK_MS, reason: String(error.message).slice(0, 300) });
+					await job.moveToDelayed(heldAt + PODMAN_RESTART_HOLD_RECHECK_MS, token);
+					throw new DelayedError();
+				}
+				const expired = Object.assign(new UnrecoverableError(`held ${Math.round((heldAt - since) / 60_000)} min for rootful Podman's service to restart, and it did not: ${error.message}`), { reason: PODMAN_RESTART_HOLD_EXPIRED, provider: error.provider ?? null, model: error.model ?? null, budgetReserved: false });
+				recordRun({ job, error: expired, startedAt, endedAt: new Date().toISOString() });
+				throw expired;
+			}
+			// Issue #476: a job whose egress preflight found the rootless network keeper running on its own bridge but younger
+			// than the minimum age is HELD until it is old enough (one `moveToDelayed`, no attempt), the move the hold above
+			// makes, since a keeper merely young is not a keeper that does not hold: every joint start of the stack read it
+			// at under a second and spent a queued job's attempt (measured on 4.9.3). A crash loop is young at every start,
+			// so the hold is BOUNDED, and ends early on the evidence of one: the keeper the job waited on is gone, or another
+			// start has taken its place. Either ends in an ordinary retry (an attempt, as any keeper that does not hold
+			// costs) with its own reason token naming the loop. The hold's state is stored on the job, as the podman.service
+			// hold's is, and read only while the checks come within the bound of each other: a later one starts afresh.
+			const keeperHold = keeperHoldState(job, now());
+			if (error?.keeperYoung && typeof error.keeperYoung === "object") {
+				const young = error.keeperYoung;
+				const restarted = keeperHold.startedMs !== null && keeperHold.startedMs !== young.startedMs;
+				const heldMs = keeperHold.at - keeperHold.since;
+				if (!restarted && heldMs < NETNS_KEEPER_YOUNG_HOLD_MAX_MS) {
+					await job.updateData({ ...job.data, netnsKeeperHoldSinceMs: keeperHold.since, netnsKeeperHoldLastMs: keeperHold.at, netnsKeeperHoldStartedMs: young.startedMs });
+					deps?.log?.("netns_keeper_young_hold", { jobId: job.id, keeperAgeMs: young.ageMs, heldForMs: heldMs, delayMs: young.waitMs });
+					await job.moveToDelayed(keeperHold.at + young.waitMs, token);
+					throw new DelayedError();
+				}
+				const was = keeperHold.startedMs ?? young.startedMs;
+				await markKeeperLoop(job, was === young.startedMs ? [was] : [was, young.startedMs]);
+				const loop = new InfraRetry(netnsKeeperCrashLoopSentence({ was, now: young.startedMs, heldMs, remedy: error.keeperRemedy }), { reason: NETNS_KEEPER_CRASH_LOOP, provider: error.provider ?? null, model: error.model ?? null, budgetReserved: false });
+				recordRun({ job, error: loop, startedAt, endedAt: new Date().toISOString() });
+				throw loop;
+			}
+			if (error?.reason === NETNS_KEEPER_NOT_HOLDING && keeperHold.startedMs !== null) {
+				// The keeper this job was waiting on is no longer running on its bridge: it died young, the loop's other face.
+				await markKeeperLoop(job, [keeperHold.startedMs]);
+				const loop = new InfraRetry(netnsKeeperCrashLoopSentence({ was: keeperHold.startedMs, now: null, problem: error.keeperProblem ?? null, heldMs: keeperHold.at - keeperHold.since, remedy: error.keeperRemedy }), { reason: NETNS_KEEPER_CRASH_LOOP, provider: error.provider ?? null, model: error.model ?? null, budgetReserved: false });
+				recordRun({ job, error: loop, startedAt, endedAt: new Date().toISOString() });
+				throw loop;
+			}
+			// A LATER ATTEMPT OF A JOB THAT SAW THE LOOP (gate of PR #479). The loop's retry comes after the queue's 60 s
+			// backoff, when the hold's 30 s window has closed, and it meets a keeper that restarted out of order against the
+			// proxy, or none: measured, it failed as `netns-keeper-not-holding`, and that attempt's record and terminal
+			// comment replaced the loop's. The marker (the keeper starts seen) is not reset by the window, so any keeper
+			// failure of this job after it is still named the loop.
+			const loopSeen = job.data?.netnsKeeperLoopSeen;
+			if (error?.reason === NETNS_KEEPER_NOT_HOLDING && Array.isArray(loopSeen) && loopSeen.length > 0) {
+				const loop = new InfraRetry(netnsKeeperLoopAgainSentence({ seen: loopSeen, problem: error.keeperProblem ?? null, remedy: error.keeperRemedy }), { reason: NETNS_KEEPER_CRASH_LOOP, provider: error.provider ?? null, model: error.model ?? null, budgetReserved: false });
+				recordRun({ job, error: loop, startedAt, endedAt: new Date().toISOString() });
+				throw loop;
+			}
 			recordRun({ job, error, startedAt, endedAt: new Date().toISOString() });
 			if (error instanceof InfraRetry) throw error; // retryable: BullMQ retries per attempts
 			// A non-retryable, non-infra error (our bug) must not retry forever. UnrecoverableError
@@ -731,11 +995,46 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			// an async function, and `release` never throws.
 			await scopeSlot?.release?.();
 			clearTimeout(timer);
+			clearInterval(cancelPoll);
 			signal.removeEventListener("abort", onAbort);
 		}
 	};
 }
 
+/**
+ * How long the stop of a job's cancel poll waits for a tick already in flight (gate round 4 of PR #479). About one Valkey
+ * round trip under load, and far under the 30 s abort grace; past it a read still hanging cannot acknowledge anything.
+ */
+export const CANCEL_STOP_BOUND_MS = 1_000;
+
+/** The comment for a job the operator cancelled before it started, where it would have been held or retried (gate of PR #479). */
+export const CANCELLED_BEFORE_START_COMMENT = "Stopped: the operator cancelled this run before it started. Nothing was spent. Not retried.";
+
+/**
+ * Store the keeper starts a crash loop was seen at on the job (gate of PR #479), outside the hold's 30 s window, so a
+ * later attempt that fails on the keeper is still named the loop. Fail open: a lost marker costs only the name.
+ */
+async function markKeeperLoop(job, seen) {
+	try {
+		await job.updateData({ ...job.data, netnsKeeperLoopSeen: seen.filter((ms) => Number.isFinite(ms)) });
+	} catch {
+		// The loop is still named on this attempt; only a later attempt's name depends on the marker.
+	}
+}
+
+/**
+ * A job's young-keeper hold as stored on it (issue #476): `{ at, since, startedMs }`, `at` being now. The stored start
+ * and the keeper start it waited on count only while the last check was within `NETNS_KEEPER_YOUNG_HOLD_MAX_MS` of
+ * now; an older one (a retry after its backoff, a paused queue) is a fresh hold with no keeper start seen.
+ */
+export function keeperHoldState(job, at) {
+	const last = job?.data?.netnsKeeperHoldLastMs;
+	const current = Number.isFinite(last) && at - last >= 0 && at - last <= NETNS_KEEPER_YOUNG_HOLD_MAX_MS;
+	const since = current && Number.isFinite(job.data.netnsKeeperHoldSinceMs) ? job.data.netnsKeeperHoldSinceMs : at;
+	const startedMs = current && Number.isFinite(job.data.netnsKeeperHoldStartedMs) ? job.data.netnsKeeperHoldStartedMs : null;
+	return { at, since, startedMs };
+}
+
 export function createWorker({ connection, name, stopContainer, containerName, hostQueue = null, checkLease = null, scopeLease = null, checkTimeoutMs, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, scopedLimits, inFlight = makeInFlight(), waitState, afterMaxMs, checkSlots = makeInFlight(), checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, hostSlots = makeInFlight(), extraClosers = [] }) {
 	// One Worker per queue name (issue #57). A host-affine job -- one whose folder, secret resolver or wait
 	// check lives on THIS machine -- is enqueued to `pi-jobs@<name>` rather than filtered for at pickup,
@@ -768,6 +1067,10 @@ export function createWorker({ connection, name, stopContainer, containerName, h
 			// Bound to THIS worker: a job on the host queue is cancelled by the worker draining that queue,
 			// and the shared handle could not reach it.
 			cancelJob: (id, reason) => worker.cancelJob(id, reason),
+			// Issue #287: the name the cancel poll writes into its ack, so the operator's terminal can say
+			// WHICH host took the cancel. `name` is already the registry/client identity; "" for a
+			// deployment that never declared one, and the ack's reader prints it as such.
+			hostName: name ?? "",
 			// #227. INJECTED, not built here. This was a one-line `docker stop` literal, which meant the abort
 			// path -- the only thing that can end a runaway job -- was the one backend function unreachable
 			// from `startWorker`. The wiring now passes the registry's per-job stop, so the container is
@@ -811,6 +1114,8 @@ export function createWorker({ connection, name, stopContainer, containerName, h
 			recordRun,
 		});
 
+		// Issue #464: only a connection `parseConnection` built, which judges and pins the Valkey it dials.
+		assertJudgedConnection(connection);
 		worker = new Worker(queueName, processor, {
 			// maxRetriesPerRequest: null is REQUIRED for BullMQ's blocking connections, or it throws.
 			connection: { ...connection, maxRetriesPerRequest: null },
@@ -821,6 +1126,9 @@ export function createWorker({ connection, name, stopContainer, containerName, h
 			...(name ? { name } : {}),
 			...(limiter ? { limiter } : {}),
 		});
+		// Issue #468: an error of this worker is one line, its message, never BullMQ's console.error of the whole object
+		// (which carried a failed AUTH's password in `command.args` before connection.mjs scrubbed it).
+		onValkeyError(worker, `worker ${queueName}`);
 		workers.push(worker);
 	}
 
@@ -829,17 +1137,72 @@ export function createWorker({ connection, name, stopContainer, containerName, h
 	// returned as a pair so every existing caller keeps receiving exactly what it received before.
 	primary.hostWorker = workers[1] ?? null;
 
-	const shutdown = async () => {
+	const stop = async () => {
 		// Abort active jobs (=> docker stop via onAbort), then close. Without the cancel,
 		// worker.close() would wait up to 30 minutes for the container. ONE shutdown for every queue: two
 		// registrations would mean two `process.exit(0)` racing, and the second worker's containers would
 		// outlive the handler that was meant to stop them.
 		for (const w of workers) await Promise.resolve(w.cancelAllJobs?.("shutdown")).catch(() => {});
 		for (const w of workers) await w.close().catch(() => {});
-		// Close auxiliary resources (e.g. a cron scheduler) after the worker drains. Per-item catch
-		// so one failing or absent closer never strands the others or blocks exit -- matches the
-		// swallow posture on cancelAllJobs above.
-		await Promise.all(extraClosers.map((c) => Promise.resolve(c.close?.()).catch(() => {})));
+		// Close auxiliary resources (a cron scheduler, the live-edit file watchers) after the worker drains.
+		// Per-item catch so one failing or absent closer never strands the others or blocks exit -- matches
+		// the swallow posture on cancelAllJobs above. The try/catch is NOT redundant with the `.catch`:
+		// `Promise.resolve(x)` does not catch a SYNCHRONOUS throw from `x`, and `c.close` on a null entry
+		// throws before `Promise.resolve` is ever reached. Either would escape this callback, reject the whole
+		// shutdown and skip the `process.exit(0)` below. Jobs and containers are already stopped by then, so
+		// what a stranded loop leaks is the rest of the list: `registry.close()` is the DEL that keeps a
+		// stopped host from lingering as a ghost peer for its full TTL, and a ghost peer with a stale
+		// `fpCron` is what makes a later `reconcileGated` refuse a legitimate reconcile. The comment above
+		// promised this isolation before the code delivered it (issue #295). It bounds nothing, though: a
+		// closer that never settles still blocks exit, which no closer here does.
+		//
+		// Read LATE and deliberately: `start.mjs` pushes its live-edit watchers into this array AFTER handing
+		// it over, because they are armed after the boot reconcile. Anything here that snapshots or copies
+		// the array un-registers them in silence.
+		await Promise.all(
+			extraClosers.map((c) => {
+				try {
+					return Promise.resolve(c?.close?.()).catch(() => {});
+				} catch {
+					return Promise.resolve();
+				}
+			}),
+		);
+		// Release the shared ioredis client LAST (issue #300). Eight consumers ride it -- the budget, the
+		// wait state, the leases, the run mirror, the host registry, the stall guard, the scope-claim sweep
+		// -- and until here nothing in the product ever closed it; only the test harness did, reaching into
+		// the captured wiring, which was the tell.
+		//
+		// SEQUENCED AFTER the drain above, never inside it. The raw client has no `.close`, so pushing it
+		// into `extraClosers` is a silent no-op -- and the natural wrapper, `{ close: () => redis.disconnect() }`,
+		// is worse than nothing: drained CONCURRENTLY by the Promise.all, it takes the connection down beside
+		// `registry.close()`, and a recording server then received NO commands at all where this ordering
+		// delivers the registry's DEL and SREM -- the DEL being what keeps a stopped host from lingering as
+		// a ghost peer for its full TTL.
+		//
+		// `disconnect()`, not `quit()`, for a MEASURED reason rather than the plausible one. `quit()` answers
+		// OK in 0ms against a REFUSED port; the hang it can suffer is a server that accepts the TCP
+		// connection and never answers, where the client sits in status "connect" awaiting its ready check --
+		// independent of `maxRetriesPerRequest` and of `enableOfflineQueue`, both measured. `disconnect()`
+		// returns immediately in every case, and everything whose replies matter has already drained above.
+		// Guarded, because the wiring tests hand createWorker a bare `redis: {}`. And wrapped, on the loop's
+		// own rule stated above: a SYNCHRONOUS throw from this line would skip the `process.exit(0)` below
+		// and hang the stop. The real client's disconnect does not throw; a test-injected one is one edit
+		// away from doing so.
+		try {
+			redis?.disconnect?.();
+		} catch {
+			// A release that failed has already stopped mattering; the exit that follows is what a stop owes.
+		}
+	};
+	// The signal path is `stop()` then exit, and the split is issue #299's: a boot that refuses AFTER the
+	// Worker exists must be able to undo what it built WITHOUT exiting, because the refusal's own error --
+	// not a 0 -- has to reach `cli.mjs`'s entryExitCode, and with every handle released above the process
+	// drains to that code on its own. The closure keeps the name `shutdown` because the source pin in
+	// `wiring.test.mjs` reads the registration lines below, deliberately, rather than constructing a
+	// Worker to observe them.
+	const shutdown = async () => {
+		await stop();
 		process.exit(0);
 	};
 	process.once("SIGTERM", shutdown);
@@ -849,6 +1212,13 @@ export function createWorker({ connection, name, stopContainer, containerName, h
 	// worker still aborts in-flight jobs and docker-stops their containers rather than orphaning them.
 	if (process.platform === "win32") process.once("SIGBREAK", shutdown);
 
+	// Beside `hostWorker` and for the same reason it rides the return value rather than changing it:
+	// every existing caller keeps receiving exactly what it received before, and the one new caller --
+	// `startWorker`'s post-handoff catch (issue #299) -- reaches the teardown through the worker it was
+	// handed. `stop` is the shutdown minus the exit; it is safe to call more than once, because every
+	// step it takes is idempotent by that step's own contract.
+	primary.stop = stop;
+
 	return primary;
 }