@edgehero/pi-dispatch 1.10.3 → 2.1.0

package/src/processor.mjs

@@ -1,10 +1,40 @@
-import { DEFAULT_BACKEND, DOCKER_NEVER_STARTED_EXITS } from "./backends.mjs";
+import { DAEMON_APPLIES_BOUNDS, DEFAULT_BACKEND, DOCKER_ENDPOINT_LOCAL, DOCKER_NEVER_STARTED_EXITS, PODMAN_ADDS_NO_MOUNTS, PODMAN_BACKEND, PODMAN_BOUNDS_DELEGATED, PODMAN_CONF_WIDENS_JOB, PODMAN_SERVICE_LOCAL, RUNTIME_ADDS_NO_MOUNTS } from "./backends.mjs";
+import { resolveBackendName } from "./backend-registry.mjs";
 import { lstatSync } from "node:fs";
 import { checkTokenCap, recordTokenSpend, releaseBudget, reserveBudget } from "./budget.mjs";
 import { configError } from "./config.mjs";
 import { scopeKeyPrefix } from "./scoped-limits.mjs";
 import { DEFAULT_SECRETS_PROFILE, secretsArmed } from "./secrets.mjs";
 import { EXIT_COMPLETED, EXIT_INFRA, EXIT_POLICY } from "./exit-code.mjs";
+import { RUNNER_POLICY_REASONS } from "./run-history.mjs";
+import { DEFAULT_EGRESS_PROXY } from "./egress.mjs";
+
+/**
+ * The forge comment's reason for each observation a floor refusal missed (issues #278 and #345), keyed like
+ * `OBSERVATIONS` in `backends.mjs` and pinned to it. Fixed words only.
+ */
+export const OBSERVATION_COMMENT = Object.freeze({
+	[DOCKER_ENDPOINT_LOCAL]: "the docker CLI is not observed sending containers to a daemon on this host, so the job's credentials could cross a network the deployment does not own",
+	[DAEMON_APPLIES_BOUNDS]: "the container runtime is not observed applying a container's pid and memory bounds",
+	[RUNTIME_ADDS_NO_MOUNTS]: "the container runtime is not observed adding no mounts of its own to a job container",
+	// Issue #354: the podman venue's three, in the same fixed register. No path, no controller list and no service URL:
+	// those are the evidence, which goes to the operator's log only.
+	// Cause-neutral (issue #453): the observation misses both when a controller is not delegated and when no systemd user
+	// manager runs for the account (measured: podman info then still lists every controller). Which one is the evidence's
+	// to say, in the operator's log.
+	[PODMAN_BOUNDS_DELEGATED]: "the worker's rootless Podman is not observed applying a container's pid, memory and cpu bounds",
+	[PODMAN_ADDS_NO_MOUNTS]: "the worker's rootless Podman is not observed adding no mounts of its own to a job container",
+	[PODMAN_SERVICE_LOCAL]: "the podman CLI is not observed running containers on this host rather than through a remote service, so the job's credentials could cross a network the deployment does not own",
+});
+
+/**
+ * The forge comment's reason when a floor refusal names no observation this build has words for (issue #354): a
+ * venue other than `local` that named none, or named one outside `OBSERVATIONS`. Venue-neutral on purpose. The
+ * fallback used to be the docker endpoint's sentence, which is right only for `local`, whose preflight is the one
+ * that could ever refuse without naming what it missed; told to a job on another runtime it blames a CLI that job
+ * never touched. Not a key of `OBSERVATION_COMMENT`, which is pinned to the closed list one-for-one.
+ */
+export const OBSERVATION_COMMENT_UNNAMED = "the venue this job runs on did not confirm a guarantee the floor requires";
 
 /**
  * The job orchestration. Deliberately a pure-ish function over INJECTED side-effecting deps, so
@@ -13,6 +43,13 @@ import { EXIT_COMPLETED, EXIT_INFRA, EXIT_POLICY } from "./exit-code.mjs";
  * The order is the contract, and every step before `runContainer` must be free of provider spend:
  *
  *   0. refuse a job image this host does not have  -- INT-CONTAINER-RUNTIME-CONTRACT
+ *   0a. refuse a job no non-root uid can run on this daemon, or whose image cannot run as the worker's uid
+ *       (issue #341), and retry one whose job user could not be decided -- DES-JOB-USER-INFERRED-READ-BACK-ON-REQUEST
+ *   0b. REFUSE a deployment with no usable credential for the job's provider, which costs nothing to
+ *       ask and would otherwise be discovered with the budget already reserved -- CONST-BUDGET-BEFORE-TOKENS
+ *       (gates 0 and 0b are not the first two: a one-shot already spent, a skewed wait, an unblessed
+ *       backend and a backend floor this host is not observed to meet (#278, #345) are refused above them, and
+ *       this ladder has never listed those)
  *   1. REFUSE an armed `run.resume` with no session store to persist into (the one fail-CLOSED case)
  *                                                 -- REQ-RESUMABLE-SESSION
  *   2. mint a scoped token (GitHub jobs, and local jobs opted in via `github: true`)
@@ -33,6 +70,62 @@ import { EXIT_COMPLETED, EXIT_INFRA, EXIT_POLICY } from "./exit-code.mjs";
  * retries per `attempts`. The caller (the BullMQ processor) turns the thrown/returned distinction
  * into the queue's retry behaviour -- that is INT-RUNNER-EXIT-CODE-PROTOCOL.
  */
+
+// The post-spend terminal comments (issue #288). Every FREE refusal above the container already comments;
+// these are the paths where money was spent and the run still ended without the agent's own status step,
+// which used to tell the issue nothing (REQ-JOB-STATUS-COMMENTS' acceptance -- "exactly one completion or
+// failure comment" -- held only below the spend line). Keyed by the reason token so a new abort
+// classification adds a ROW here, never a re-plumb; sentences are FIXED and path-free, because for a
+// local job the adapter logs the full text into a persistent service log (the "this folder" discipline
+// at the scope-cap refusal). The completed path stays silent on purpose: exit 0 is where the AGENT'S own
+// status comment lives (the prompt contract instructs it, including for "I cannot fix this"), and exit 2
+// by construction means the agent was cut off before that step.
+// EXPORTED only so a test can hold every RUNNER_POLICY_REASONS member to a row here (issue #437 review).
+export const TERMINAL_COMMENTS = {
+	"worker-abort": "Stopped: the worker ended this run before it finished (the 30-minute job limit, or a worker shutdown). Partial work may exist. Not retried.",
+	"operator-cancel": "Stopped: the operator cancelled this run. Partial work may exist. Not retried.",
+	"runner-policy": "Stopped: the run ended inside the container before finishing (a turn or token budget, or an in-container configuration refusal). Partial work may exist. Not retried.",
+	// Issue #437. Names the cause but never the provider's own message, which may echo a key fragment. "Or
+	// access" because a 403 is as often a key that works but may not use this model or route as a bad key.
+	"provider-auth-refused": "Stopped: the AI provider refused this worker's credentials or access (an authentication or permission error). The operator needs to check the provider key and what it is allowed to use. Not retried.",
+};
+
+// Issue #341: the forge comments for a `job-user-unmappable` refusal, keyed by cause. Shorter than the operator
+// texts in job-user.mjs on purpose: a comment's reader may be an issue author, who can act on none of it.
+const JOB_USER_COMMENTS = Object.freeze({
+	default: "Refused: the worker host's container runtime cannot give this job a non-root user that can read and write its own files. Not run.",
+	"runtime-unreadable": "Refused: the worker host's container runtime answered in a form the worker cannot read, so which user this job may run as is unknown. Not run.",
+});
+
+/**
+ * The runtime a venue's own preflights talk to, named in a retry's words (issue #354). The message is the job_failed log
+ * line and BullMQ's failedReason, so `local`'s stays byte-identical ("docker unavailable, ..."), the native podman venue
+ * names podman (an operator reading "docker unavailable" on a host with no docker would chase the wrong daemon), and a
+ * venue this build has no runtime word for gets the neutral phrase rather than a guess. `Object.hasOwn`, so a venue
+ * named like a prototype key is not a runtime.
+ */
+const VENUE_RUNTIME = Object.freeze({ [DEFAULT_BACKEND]: "docker", [PODMAN_BACKEND]: "podman" });
+
+function runtimeUnavailable(venue, gate) {
+	return Object.hasOwn(VENUE_RUNTIME, venue) ? `${VENUE_RUNTIME[venue]} unavailable, ${gate} could not run` : `the container runtime is unavailable, ${gate} could not run`;
+}
+
+/**
+ * The remedy an egress-proxy refusal's comment names. The compose profile starts the proxy on the DOCKER daemon, which a
+ * job on the native podman venue cannot reach: rootless podman's `--internal` network reaches nothing on the host
+ * (measured, issue #354), so its proxy must run under the SAME rootless podman on a named bridge network, which is what
+ * docs/podman.md sets up. Every other venue names `pi-dispatch up` from the deployment folder, and ONLY that (issue #480,
+ * PR #488's review): it starts the proxy in any folder init made, and it knows the folder. A compose line printed from
+ * here could not: a folder /dispatch setup laid out needs its project and override, and without them `--profile egress
+ * up -d` also starts compose's unprofiled valkey under project `deploy`, a second Valkey on a fresh volume or a clash on
+ * its port. A proxy PI_EGRESS_PROXY names is the operator's own, which `up` never starts, so that one is theirs to start.
+ */
+function egressProxyFix(venue, proxy = DEFAULT_EGRESS_PROXY) {
+	if (venue === PODMAN_BACKEND) return "Start it under the worker account's own rootless podman, on a named bridge network (docs/podman.md)";
+	if (proxy !== DEFAULT_EGRESS_PROXY) return `PI_EGRESS_PROXY names your own proxy, which \`pi-dispatch up\` does not start: start ${proxy} yourself`;
+	return "Start it with `pi-dispatch up` from the deployment folder";
+}
+
 export async function runJob(job, deps) {
 	const {
 		redis,
@@ -58,9 +151,25 @@ export async function runJob(job, deps) {
 		// Issue #230. Admit-everything by default, like checkOnceSpent above and for its reason: an
 		// unwired seam must not refuse, and the wiring is what turns the check on.
 		checkWaitSkew = async () => ({ ok: true }),
+		// () => { ok } | { message }. Issue #310. Resolves this deployment's provider credential the way
+		// buildContainerEnv will, and answers whether it exists AT ALL, so an unconfigured provider refuses
+		// here rather than inside runContainer with the budget already reserved. Admit-everything by default,
+		// like the two above and for their reason. A PROBE, deliberately: it discards whatever it resolves and
+		// the real read happens where it always did, because threading a live credential through the processor
+		// would put it in scope for every log line and record between here and the container.
+		checkProviderCredential = () => ({ ok: true }),
 		// REQ-EGRESS-ALLOWLIST. Default admits everything, so a wiring that omits it behaves exactly as a
 		// deployment with no egress policy does -- which is also what the real factory returns when unarmed.
 		egressPreflight = async () => ({ ok: true }),
+		// Issue #278: a live read of something the backend's declaration holds only while observed (today, which
+		// docker endpoint the CLI resolves), judged against the deployment's floor. `{ ok }`, `{ refused, message }`
+		// or `{ unavailable }`. Defaults to ok, so a bare wiring gates nothing, like the two preflights above it.
+		observationPreflight = async () => ({ ok: true }),
+		// Issue #341: which uid this job's container runs as. `(job, { capabilities, observed }) =>` `{ user, home }`
+		// (`user` null = the image's own USER), `{ refused, cause }` or `{ unavailable, reason }`. The default runs
+		// every job as the image's user, exactly as before, so a wiring that omits it changes nothing. A non-refused answer
+		// may also carry `relabel: true` (issue #355), which reaches `runContainer` beside the user it was decided with.
+		jobUserPreflight = async () => ({ user: null, home: null }),
 		// (session, { piVersion, context }) => { promoted, reason, bytes }. Promotes this job's transcript back into
 		// the store, on a COMPLETED exit only. Never throws. The default is a no-op so a wiring that omits
 		// it behaves exactly as before -- no store, no promotion, no session in the record.
@@ -121,10 +230,15 @@ export async function runJob(job, deps) {
 		mintToken,
 		isDefaultBranchProtected, // (job, token) => boolean; same reason -- the forge is the job's, not the process's
 		prepareWorkspace, // (job, token) => { workspaceDir, jobDir }  (clone+materialise+prompt)
-		// runContainer({ job, token, prepared, secrets, name, signal }) => { code, aborted, turns, tokens, session, usage, context }.
+		// runContainer({ job, token, prepared, secrets, name, signal, user, home }) => { code, aborted, abortReason, turns, tokens, session, usage, context, exitReason }.
+		// `exitReason` (issue #437) is parseExitReason's closed-set label, read only inside the exit-2 branch.
+		// `user`/`home` are the job-user gate's answer (issue #341), null for the image's own USER.
 		// `secrets` is the resolved map from the gate above: values, already fetched, host-side. It MUST honour
 		// `signal`: stop the container on abort, and reject/exit promptly if `signal.aborted` is already
-		// true at entry (the timeout can fire during a slow prepare). The wiring injects name + signal.
+		// true at entry (the timeout can fire during a slow prepare). The wiring injects name + signal, and
+		// on an aborted result it also maps `signal.reason` onto `abortReason` (issue #287) so the
+		// classification below can tell an operator's cancel from the kill timer's; a bare wiring that
+		// never sets it classifies every abort as worker-abort, exactly as before.
 		runContainer,
 		cleanup, // (dirs) => void
 		comment, // (job, text) => void   (issue status; no-op for local jobs)
@@ -149,6 +263,18 @@ export async function runJob(job, deps) {
 	let prepared = null;
 	let reserved = false;
 	let scopedReserved = false;
+	// Set once `runContainer` has RESOLVED, which is the only moment a container is known to have run. The
+	// config classifier in the catch refunds both ledgers, and its whole justification is that nothing was
+	// spent; without a fact to test, that is a claim about where config throws happen to live today rather
+	// than a property of the code. A config-tagged throw raised after a paid run would otherwise refund a slot
+	// the container really spent AND tell the operator publicly that nothing was.
+	//
+	// AFTER the await and not before, deliberately: `runContainer` resolves the credential and assembles the
+	// env before it spawns anything, so its config throws (an unconfigured provider, an unknown forge kind)
+	// happen with no container started at all -- and those are exactly what the classifier exists to refund.
+	// A spawn fault that fails between is an InfraRetry carrying `container-never-started`, refunded by the
+	// arm below, which has a discriminator of its own.
+	let containerRan = false;
 
 	try {
 		// The one-shot pre-spend check (issue #231), FIRST on the ladder: one file read, cheaper than
@@ -213,6 +339,114 @@ export async function runJob(job, deps) {
 			return { outcome: "policy", reason: "backend-unblessed", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
 		}
 
+		// Issue #278. PI_BACKEND_FLOOR can ask for a guarantee a backend declares but EARNS only while something
+		// about this host is observed -- `local`'s credentialTransit, which holds only while the docker CLI sends
+		// containers to a daemon on this host. The read is repeated here, per job, because the CLI's context can
+		// change after boot and every later job's provider key and forge token would follow it. FREE and
+		// pre-spend: a refusal that will recur on every job must not cost each one a slot. Determinate is a
+		// RETURN (CONST-RETRY-INFRA-ONLY); a CLI that could not be asked for a transient reason is a throw,
+		// pre-reserve, so the refund is a no-op.
+		//
+		// AHEAD of the image and egress preflights, not beside them, because both of those talk to the daemon
+		// this gate is about: on a redirected CLI an unreachable daemon made the image inspect throw a retry, and
+		// a reachable one without the image refused as `job-image-missing` with a comment blaming the image --
+		// the right refusal lost behind the wrong one. Issue #345 adds `isolation` (the daemon observed applying a
+		// container's bounds) and `mountSet` (the runtime observed adding no mounts), read here from the same
+		// `docker info` the job user is decided from, cached per endpoint once it answers, so this read now contacts the
+		// daemon (after the endpoint check, which still refuses without it).
+		// One refusal, reached from two places (the venue's own answer below, or the job user decided after the image
+		// probe). Fixed text per cause class: the cause and anything the runtime said go to the operator's log, never a
+		// forge comment.
+		const refuseJobUserUnmappable = async (cause) => {
+			await comment(job, JOB_USER_COMMENTS[cause] ?? JOB_USER_COMMENTS.default);
+			log("refused_job_user_unmappable", { cause: cause ?? null });
+			return { outcome: "policy", reason: "job-user-unmappable", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
+		};
+		const observed = await observationPreflight(job);
+		if (observed?.refused) {
+			// Fixed text per observation: the endpoint (an internal host name or address) and the evidence go to the
+			// operator's log, never to a forge comment. "Not observed" rather than "not on this host", because a CLI that
+			// could not be asked for a determinate reason (no docker on PATH, a context that does not exist) refuses too.
+			// A refusal naming nothing keeps the endpoint's words on `local` only (#278); elsewhere it is the neutral
+			// sentence, and so is a name this build has no words for, on any venue (issue #354).
+			const onLocal = resolveBackendName(job, blessedBackends[0]) === DEFAULT_BACKEND;
+			const missed = Array.isArray(observed.observations) && observed.observations.length > 0 ? observed.observations : [onLocal ? DOCKER_ENDPOINT_LOCAL : null];
+			const why = missed.map((o) => (Object.hasOwn(OBSERVATION_COMMENT, o ?? "") ? OBSERVATION_COMMENT[o] : OBSERVATION_COMMENT_UNNAMED));
+			await comment(job, `Refused: this deployment's PI_BACKEND_FLOOR requires a guarantee this host is not observed to provide right now (${[...new Set(why)].join("; ")}). Not run.`);
+			log("refused_backend_floor_unobserved", { message: observed.message });
+			return {
+				outcome: "policy",
+				reason: "backend-floor-unobserved",
+				exitCode: null,
+				turns: null,
+				tokens: null,
+				provider: job.provider ?? null,
+				model: job.model ?? null,
+				budgetReserved: false,
+			};
+		}
+		if (observed?.unavailable) {
+			// The local venue's words stay what they were (they are its log line and BullMQ's failedReason); another venue
+			// is not docker's to name.
+			const localVenue = resolveBackendName(job, blessedBackends[0]) === DEFAULT_BACKEND;
+			// A host FILE that could not be read for a moment (issue #428) is named as that, never as a runtime outage.
+			const why = typeof observed.message === "string" && observed.message !== "" ? `a host file an observation the floor needs could not be read just now (${observed.message})` : localVenue ? "docker CLI or daemon unavailable, an observation the floor needs could not run" : "the container runtime or its CLI is unavailable, an observation the floor needs could not run";
+			throw new InfraRetry(why, { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
+		}
+
+		// A venue that already knows no uid can run a job there says so HERE, before the image preflight asks that same
+		// runtime (issue #354): behind it, a runtime that is not there is retried as unavailable and one that answers
+		// without the image is refused as `job-image-missing`, both the wrong fix. Only an answer no image can change
+		// comes this way; the per-image half (`anyUid`) stays below, after the probe that reads it.
+		if (observed?.jobUserRefused?.refused === "job-user-unmappable") return refuseJobUserUnmappable(observed.jobUserRefused.cause);
+		// Issue #428: the podman venue's account sets a containers.conf key that widens what every job reaches (the host's
+		// loopback services, the account's groups) and no argv takes back. A VENUE refusal, not a floor miss: with egress
+		// off no declared property covers what a job reaches on the host, so a floor would never ask on the deployments at
+		// risk. Determinate (the account's own files), so a RETURN (CONST-RETRY-INFRA-ONLY), here for the identity's reason:
+		// ahead of the image preflight and every spend. The file and key go to the operator's log; the comment is fixed,
+		// and says the configuration widens a job only when a key was FOUND: a chain that could not be read whole is
+		// refused for not being known, which is a different sentence. A read that failed for a moment (`transient`) is
+		// infrastructure, so it throws and is retried, pre-reserve, exactly like an unanswered observation.
+		if (observed?.podmanConfRefused?.transient) {
+			// `evidence` names the file (`<path> could not be read (<errno>)`), so the retry says which one: a containers.conf,
+			// or since issue #450 the /proc entry of the account's running rootless network. Issue #448: `rootful` is the local
+			// venue's, a containers.conf or unit file of rootful Podman's service on this host.
+			const what = observed.podmanConfRefused.rootful ? "rootful Podman's containers.conf or podman.service" : "the podman venue's containers.conf or running rootless network";
+			throw new InfraRetry(`${what} could not be read just now, so whether it widens a job is not known (${observed.podmanConfRefused.evidence ?? "no file named"})`, { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
+		}
+		if (observed?.podmanConfRefused?.retry) {
+			// Gate round 1 of PR #473: rootful Podman's service running since before its containers.conf changed (or a change
+			// time ahead of the clock) heals by itself, once the service restarts or idles out or the clock passes, so it is
+			// never a final `policy` outcome, pre-reserve. Gate round 2: a HOLD, not a retry (`PodmanRestartHold`), which the
+			// processor moves back to the delayed set without spending an attempt, for up to `PODMAN_RESTART_HOLD_MAX_MS`.
+			throw new PodmanRestartHold(`rootful Podman's service may still hold a containers.conf older than the files, so this job waits for it to restart (${observed.podmanConfRefused.evidence ?? "no file named"})`, { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
+		}
+		if (observed?.podmanConfRefused) {
+			// Issue #450: `live` is the account's RUNNING rootless network, not a file, so it has its own two sentences: one
+			// still carrying an option a removed key gave it (the fix is a restart, not a configuration change), and one
+			// whose process could not be read.
+			const { key, live, rootful } = observed.podmanConfRefused;
+			// Issue #448: `rootful` is the LOCAL venue on rootful Podman's Docker API service, with its own sentences: a key
+			// that reaches every local job, and a file it cannot read or does not decode. (A service older than its
+			// configuration is a retry, above, since gate round 1 of PR #473.)
+			await comment(
+				job,
+				rootful
+					? key
+						? "Refused: the worker host's Podman configuration adds to every job's container what this venue does not allow (variables, the Podman service's groups, looser limits and filters, or the programs that run it), so the operator must change it before local jobs run. Not run."
+						: "Refused: the worker host's Podman configuration could not be read in full, or is written in a form the worker does not decode, so whether it adds to a job's container what this venue does not allow is not known, and the operator must fix that before local jobs run. Not run."
+					: live
+						? key
+							? "Refused: the worker host's running Podman network still lets a job's container reach the host's own services, with an option from a configuration since changed, so the operator must restart that network before podman jobs run. Not run."
+							: "Refused: the worker host's running Podman network could not be read, so whether it lets a job's container reach more than this venue allows is not known, and the operator must fix that before podman jobs run. Not run."
+						: key
+							? "Refused: the worker host's Podman configuration lets a job's container reach more than this venue allows (the host's own services, the worker account's groups, or looser limits and filters than the worker sets), so the operator must change it before podman jobs run. Not run."
+							: "Refused: the worker host's Podman configuration could not be read in full, or is written in a form the worker does not decode, so whether it lets a job's container reach more than this venue allows is not known, and the operator must fix that before podman jobs run. Not run.",
+			);
+			log("refused_podman_conf_widens_job", { key: key ?? null, ...(live ? { live: true } : {}), ...(rootful ? { rootful: true } : {}), message: observed.podmanConfRefused.message });
+			return { outcome: "policy", reason: PODMAN_CONF_WIDENS_JOB, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
+		}
+
 		// The job image must exist on THIS host before anything else happens. Free, determinate and
 		// credential-less, so it precedes the mint, the clone and the reservation: a host that cannot run the
 		// image refuses without minting a credential it will not use, cloning a repo it will not read, or
@@ -287,14 +521,90 @@ export async function runJob(job, deps) {
 			log("refused_image_commands_unsupported", { image: img.commandUnsupported, declared: img.declared });
 			return { outcome: "policy", reason: "job-image-commands-unsupported", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
 		}
+		if (img.excludeToolsUnsupported) {
+			// The image is present and does not declare exclude-tools support (issue #291), so its runner
+			// predates run.excludeTools: it reads no PI_EXCLUDE_TOOLS, and the job would run with every
+			// tool the trigger says to remove -- a "read-only" trigger with a working editor and shell,
+			// recording a clean exit. That is a PERMISSION quietly not enforced, the silent fail-open this
+			// repo brands the worst outcome available, which is exactly why the host refuses before spend
+			// rather than letting the container fail open.
+			//
+			// Determinate, so a refusal rather than a retry, and pre-spend, because no version of this
+			// gets better by running. Like the command branch above, the message names the FIX rather
+			// than the label that noticed it.
+			await comment(
+				job,
+				`Refused: the job image "${img.excludeToolsUnsupported}" does not declare exclude-tools support (\`dev.pi-dispatch.capabilities\` ${img.declared.length > 0 ? `declares: ${img.declared.join(", ")}` : "is absent"}), so its runner would ignore \`run.excludeTools\` and run this trigger with every tool it says to remove. Rebuild the image from a version that has this feature. Not run.`,
+			);
+			log("refused_image_exclude_tools_unsupported", { image: img.excludeToolsUnsupported, declared: img.declared });
+			return { outcome: "policy", reason: "job-image-exclude-tools-unsupported", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
+		}
 		if (img.unavailable) {
 			// docker itself did not answer -- transient infra, NOT a determinate refusal. THROWN so BullMQ
 			// retries (CONST-RETRY-INFRA-ONLY). `container-never-started` is literally true here, and it reuses
 			// the refund path below: a no-op pre-reserve, and still honest if this gate ever moves.
 			// provider/model attribute even this pre-container death; no usage -- nothing ran to emit one.
-			throw new InfraRetry("docker unavailable, image preflight could not run", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
+			throw new InfraRetry(runtimeUnavailable(resolveBackendName(job, blessedBackends[0]), "image preflight"), { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
 		}
 
+		// Issue #341: WHO runs the container. On a daemon that enforces bind-mount ownership the job must run as the
+		// uid that owns its job dir and mounts, or it cannot read its own inputs; where no uid works (a rootless
+		// daemon, userns-remap, a root worker) nothing runs. After the image probe because the answer needs its
+		// `anyUid` capability, and still FREE: one cached `docker info` per endpoint, no mint, no clone, no reserve.
+		const jobUser = await jobUserPreflight(job, { capabilities: img.capabilities ?? [], observed });
+		if (jobUser?.refused === "job-user-unmappable") return refuseJobUserUnmappable(jobUser.cause);
+		if (jobUser?.refused === "job-image-any-uid-unsupported") {
+			// The image ref is operator config, the same PII class as the refusals above.
+			await comment(
+				job,
+				`Refused: the job image "${img.image}" does not declare \`anyUid\` (\`dev.pi-dispatch.capabilities\`), so it cannot run as this worker's own uid, which this host's container runtime requires. Rebuild the image from a version that has this feature. Not run.`,
+			);
+			log("refused_job_image_any_uid_unsupported", { image: img.image });
+			return { outcome: "policy", reason: "job-image-any-uid-unsupported", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
+		}
+		if (jobUser?.unavailable) {
+			// The reason is a fixed token (`daemon-unreachable`, `timeout`, `endpoint-unresolved`...), never CLI text.
+			log("job_user_unavailable", { reason: jobUser.reason ?? null });
+			throw new InfraRetry("the job user could not be decided", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
+		}
+
+		// Is there a credential to run this job with at all? FREE, determinate and I/O-light: a pure function
+		// of the job's provider, the worker env, and (only when the env has no key) one small readFileSync of
+		// pi's auth.json. So it goes here, with the other free gates, which is further than issue #310 asked
+		// for: ahead of the image probe, the mint, the clone, the token-cap read and both reserves. That is
+		// CONST-BUDGET-BEFORE-TOKENS in its own words, "every gate that costs nothing runs before every gate
+		// that costs something".
+		//
+		// AFTER the image probe and not before, which is the one ordering choice here that costs something:
+		// a `docker inspect` runs before a `readFileSync`. This file says twice that a missing image blocks
+		// EVERY job of EVERY kind on this host, so it is the fault an operator must fix first either way, and
+		// a missing credential is in exactly that class -- on a host with both, the reported reason should be
+		// the one that was already the rule. Everything the gate is actually FOR is still below it: the egress
+		// probe, the secret resolvers, the mint, the clone, the token-cap read and both reserves.
+		//
+		// It used to be discovered inside runContainer, where buildContainerEnv resolves the credential for
+		// real -- AFTER both reserves -- and the throw fell through this function's catch to a bare rethrow.
+		// The catch now classifies that too (below), so this gate is the cheap path and that is the backstop;
+		// neither alone is enough, because the backstop cannot un-mint a token or un-clone a repository.
+		//
+		// The refusal names no path. `credentialFromPiAuth`'s messages carry auth.json's location, `comment`
+		// posts publicly on the issue, and the reason an operator needs is the same either way: their
+		// deployment has no usable provider credential and `doctor` will say exactly which variable.
+		const credential = await checkProviderCredential(job);
+		if (!credential.ok) {
+			// The PROVIDER is named and the message is not. The provider is operator-authored config, already
+			// on the record and the mirror, and named freely by the sibling refusals (`backend-unblessed` names
+			// the backend, `job-image-missing` names the image), so withholding it would tell the operator less
+			// than is safe. The message is withheld from BOTH surfaces: `credentialFromPiAuth`'s refusals carry
+			// `auth.json`'s absolute path, which is an OS account name, and the scoped-budget refusal below
+			// keeps a host path out of its own log line citing no-pii-in-logs. `doctor` prints the variable and
+			// the path, on the operator's terminal, which is where that belongs.
+			await comment(job, `Refused: this deployment has no usable credential for the "${job.provider}" provider, so no container was started and nothing was spent. Ask the operator to run \`pi-dispatch doctor\`. Not run.`);
+			log("refused_provider_unconfigured", { provider: job.provider ?? null });
+			return { outcome: "policy", reason: "provider-unconfigured", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
+		}
+
+
 		// REQ-EGRESS-ALLOWLIST. The egress policy this deployment claims must be able to serve this job
 		// BEFORE the job costs anything. It is one `docker inspect` when the policy is armed and ZERO spawns
 		// when it is not, so a deployment without one pays nothing at all.
@@ -313,7 +623,7 @@ export async function runJob(job, deps) {
 		if (egress.proxyMissing || egress.proxyStopped) {
 			const proxy = egress.proxyMissing ?? egress.proxyStopped;
 			const state = egress.proxyMissing ? "is not on this host" : "is not running";
-			await comment(job, `Refused: this deployment runs jobs behind an egress policy and its allowlist proxy "${proxy}" ${state}, so the job could not reach the provider and would burn its budget slot proving it. Start it with \`docker compose -f deploy/docker-compose.yml --profile egress up -d\`, or set PI_EGRESS=0 to run without an egress policy. Not run.`);
+			await comment(job, `Refused: this deployment runs jobs behind an egress policy and its allowlist proxy "${proxy}" ${state}, so the job could not reach the provider and would burn its budget slot proving it. ${egressProxyFix(resolveBackendName(job, blessedBackends[0]), proxy)}, or set PI_EGRESS=0 to run without an egress policy. Not run.`);
 			// The proxy's NAME is operator-authored deployment config, never payload -- the same PII class as
 			// the image ref on the refusal above.
 			log(egress.proxyMissing ? "refused_egress_proxy_missing" : "refused_egress_proxy_stopped", { proxy });
@@ -328,11 +638,27 @@ export async function runJob(job, deps) {
 				budgetReserved: false, // refused before reserveBudget, so no job-count slot was consumed
 			};
 		}
+		if (egress.unavailable && egress.keeper) {
+			// Issue #458: the podman venue on Podman 4.x, proxy up, its rootless network keeper not holding. INFRA, not a
+			// refusal: the keeper is one `systemctl --user` away, and a retry after it holds runs the job. Pre-reserve, so
+			// nothing is refunded. Its OWN reason token (PR #463 round 2), so the run record, the failure hook and the
+			// terminal comment can name the keeper rather than a generic never-started container; the full sentence rides
+			// the error message and is logged whole here, where `job_failed` cuts it at 120 characters.
+			// Issue #476: a keeper whose only fault is its age is HELD for, not failed on. `makeProcessor` moves the job to
+			// the delayed set until the keeper is old enough, without an attempt, and names a crash loop if it keeps
+			// restarting; a path that does not know the hold still retries it, since it is an `InfraRetry`.
+			if (egress.young) throw new NetnsKeeperYoungHold(egress.keeper, { reason: NETNS_KEEPER_NOT_HOLDING, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false, young: egress.young, remedy: egress.remedy ?? null });
+			log("egress_keeper_not_holding", { proxy: egress.unavailable, reason: egress.keeper });
+			throw Object.assign(new InfraRetry(egress.keeper, { reason: NETNS_KEEPER_NOT_HOLDING, provider: job.provider ?? null, model: job.model ?? null }), { keeperProblem: egress.problem ?? null, keeperRemedy: egress.remedy ?? null });
+		}
 		if (egress.unavailable) {
 			// The daemon did not answer, so this is indeterminate rather than a refusal -- the same
 			// determinate/indeterminate split the image preflight draws one gate up, and thrown for the same
 			// reason. Pre-reserve, so the refund below is a no-op and still honest if this gate ever moves.
-			throw new InfraRetry("docker unavailable, egress preflight could not run", { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
+			// With a proxy STATE (issue #453, gate round 3): the daemon answered and the proxy is on its way somewhere
+			// (restarting, created, ...), so the words name the proxy and its state rather than blaming the runtime.
+			const said = typeof egress.state === "string" ? `egress proxy "${egress.unavailable}" is ${egress.state}, not running; the job is retried once, then failed` : runtimeUnavailable(resolveBackendName(job, blessedBackends[0]), "egress preflight");
+			throw new InfraRetry(said, { reason: "container-never-started", provider: job.provider ?? null, model: job.model ?? null });
 		}
 
 		// REQ-RESUMABLE-SESSION's one fail-CLOSED case. Everything else in that feature fails OPEN and
@@ -421,8 +747,8 @@ export async function runJob(job, deps) {
 		if (resolved.ambiguous) {
 			// Two sources declared one profile name. Neither wins, deliberately: runtime-settings documents the
 			// overlay's precedence as overlay > env, so inverting it here would leave two rules disagreeing about
-			// what an overlay is, while honouring it would let a settings file in a world-writable default
-			// directory redirect a profile the operator wrote in .env. This project already refuses ambiguity
+			// what an overlay is, while honouring it would let a settings file -- which PI_SETTINGS_FILE can put
+			// anywhere -- redirect a profile the operator wrote in .env. This project already refuses ambiguity
 			// rather than resolving it (PI_EGRESS: "a typo must never leave you believing you have a policy you
 			// do not"), and an operator who sees this fixes it in seconds.
 			await comment(job, "Refused: this trigger set `run.secrets`, and the resolver profile it names is declared twice on this worker host, once in the environment and once in the settings overlay. Neither wins, on purpose: the job would otherwise run against whichever one happened to be picked. Remove one of the two. Not run.");
@@ -430,16 +756,22 @@ export async function runJob(job, deps) {
 			return { outcome: "policy", reason: "secret-profile-ambiguous", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
 		}
 
-		if (resolved.reserved) {
-			// A key the worker itself writes, and one parseTriggers could not have caught: the provider
-			// credential's variable names depend on this job's resolved provider and on what this host has set,
+		if (resolved.reserved !== undefined) {
+			// A key the worker itself writes OR that pi reads for this job's provider, and one parseTriggers
+			// could not have caught: which variables the provider uses depends on this job's resolved provider,
 			// and PI_FORWARD_ENV is an operator env list. Both are deployment state, so this is the same
 			// load-time / pre-spend split run.resume makes against PI_SESSIONS_DIR.
 			//
-			// It matters most in the direction that is hardest to see: buildContainerEnv writes the provider
-			// credential BEFORE this feature's values, so a trigger binding ANTHROPIC_API_KEY would silently
-			// redirect which credential every job of that trigger spends.
-			await comment(job, `Refused: this trigger's \`run.secrets\` binds \`${resolved.reserved}\`, and the worker sets that variable itself for every job. The container would receive the worker's value rather than this trigger's, and the trigger would look like it worked. Rename it in the triggers file. Not run.`);
+			// TWO failures, opposite in direction, which is why the message names neither (issue #309):
+			//   - the worker WRITES the name: buildContainerEnv assigns the provider credential and
+			//     PI_FORWARD_ENV before this feature's values, so the trigger's value replaces the operator's
+			//     and every job of that trigger spends the trigger author's key;
+			//   - the worker does NOT write the name but pi READS it first: the OAuth token variable, and from
+			//     the 0.99.1 pin the bearer ANTHROPIC_AUTH_TOKEN (issue #509), are deliberately never written
+			//     (apiKeyVariable skips both), so a trigger binding one lands beside the operator's key and
+			//     outranks it in pi's own precedence.
+			// The old message asserted the first for both, which is exactly backwards for the second.
+			await comment(job, `Refused: this trigger's \`run.secrets\` binds \`${resolved.reserved}\`, which is a variable this deployment already uses for the job's own credentials. Whichever of the two values reached the container, one of them would be silently ignored. Rename it in the triggers file. Not run.`);
 			// The variable NAME only. It is the operator's own choice of name, not payload, and naming it is what
 			// makes the refusal actionable -- but the REFERENCE behind it never appears.
 			log("refused_secret_name_reserved", { kind: job.kind ?? null, name: resolved.reserved });
@@ -495,7 +827,8 @@ export async function runJob(job, deps) {
 			}
 		}
 
-		prepared = await prepareWorkspace(job, token, { piVersion }); // resolves SHA, clones, materialises .pi/, writes prompt
+		// `podmanStore` (issue #429) only where the venue's job user carried one: the podman store the container ran in.
+		prepared = await prepareWorkspace(job, token, { piVersion, jobUser: { user: jobUser?.user ?? null, home: jobUser?.home ?? null }, ...(typeof jobUser?.store === "string" ? { podmanStore: jobUser.store } : {}) }); // resolves SHA, clones, materialises .pi/, writes prompt
 
 		// A determinate prepare refusal -- sha-gone (the default branch advanced past the resolved tip),
 		// or a `pi-*` materialiser cap breach (the repo's .pi/ is too large to place in /job, issue #60)
@@ -568,6 +901,12 @@ export async function runJob(job, deps) {
 			// never a refusal it did not issue.
 			if (scopedReserved && scopedCaps) {
 				await releaseBudget(redis, { caps: scopedCaps.caps, now, keyPrefix: scopeKeyPrefix(scopedCaps.scope) });
+				// CLEARED, so the ledger's state and the flag agree. Nothing between here and the return can
+				// throw today (`comment` is non-throwing by construction and `log` is a write), but the catch
+				// below now releases on a whole CLASS of error rather than one reason, and a second release
+				// against this same key would take it to -1: `releaseBudget` is a floorless DECR. The invariant
+				// belongs where the release is, not in the guard of every future reader.
+				scopedReserved = false;
 			}
 			const w = budget.blockedWindow;
 			const win = budget.windows[w];
@@ -583,8 +922,10 @@ export async function runJob(job, deps) {
 			return { outcome: "policy", reason: budget.reason, exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: true }; // return => not retried
 		}
 
-		const { code, aborted, turns, tokens, session, usage, context } = await runContainer({ job, token, prepared, secrets });
-		log("container_exit", { exitCode: code, aborted });
+		// The user the gate above decided is the user that runs: one answer, never two call sites that agree.
+		const { code, aborted, abortReason, turns, tokens, session, usage, context, detached, exitReason } = await runContainer({ job, token, prepared, secrets, user: jobUser?.user ?? null, home: jobUser?.home ?? null, relabel: jobUser?.relabel === true });
+		containerRan = true;
+		log("container_exit", { exitCode: code, aborted, ...(detached === true ? { detached: true } : {}) });
 
 		// Record token spend post-run (the check-AFTER half of the lagging token cap). The container ran,
 		// so it spent real tokens on EVERY path that reaches here -- abort, completed, policy, AND the infra
@@ -597,13 +938,31 @@ export async function runJob(job, deps) {
 			await recordSpend(redis, tokensSpent, { now }).catch((err) => log("token_spend_error", { reason: err?.message }));
 		}
 
-		// A WORKER-initiated stop (30-min timeout via cancelJob, or graceful-shutdown docker stop) kills
-		// the container -> exit 143/137. That is our decision, not an infra fault: it is POLICY and must
-		// NOT retry, or a wedged job re-runs into a second PR / double spend. Keyed on the abort FLAG,
-		// not the code -- an unbidden 137 (kernel OOM) carries `aborted: false`, falls to the switch, and
-		// stays infra-retryable.
+		// Issue #345: `docker run` exited with a never-started code, but THIS attempt's container was found by its cidfile,
+		// still there, and was stopped and removed (run-container.mjs, measured on Podman with its API service killed
+		// mid-job). It DID start, so this is never refunded as never-started: it keeps its slot and retries as infrastructure,
+		// BEFORE the exit-code switch, where the same code would read as a free never-started exit.
+		if (detached === true) {
+			throw new InfraRetry(`the container outlived its docker run, exit ${code}`, { reason: "container-detached", exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session) });
+		}
+
+		// A WORKER-initiated stop (30-min timeout via cancelJob, graceful-shutdown docker stop, or an
+		// operator's cancel, issue #287) kills the container -> exit 143/137. That is our decision, not an
+		// infra fault: it is POLICY and must NOT retry, or a wedged job re-runs into a second PR / double
+		// spend. Keyed on the abort FLAG, not the code -- an unbidden 137 (kernel OOM) carries
+		// `aborted: false`, falls to the switch, and stays infra-retryable.
+		// WHO aborted is an exact-match on `abortReason` (the wiring maps it off `signal.reason`), and the
+		// match is deliberately closed: "job-timeout-30m", "shutdown", undefined and any future garbage all
+		// classify as worker-abort, so a pin bump that changes what rides the signal can widen nothing.
+		// `abortReason` itself never reaches the record -- buildRecord copies named fields only.
 		// exitCode/turns/tokens carry the container's own exit, turn count, and usage totals; budgetReserved true post-reserve.
-		if (aborted) return { outcome: "policy", reason: "worker-abort", exitCode: code, turns, tokens, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true };
+		if (aborted) {
+			const reason = abortReason === "operator-cancel" ? "operator-cancel" : "worker-abort";
+			// Awaited bare like every determinate refusal above: the adapter never throws by contract, and
+			// the one swallowed comment in this file (the catch's) justifies itself by its position.
+			await comment(job, TERMINAL_COMMENTS[reason]);
+			return { outcome: "policy", reason, exitCode: code, turns, tokens, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true };
+		}
 
 		switch (code) {
 			case EXIT_COMPLETED: {
@@ -638,11 +997,34 @@ export async function runJob(job, deps) {
 					chainRefused: chain.refused,
 				};
 			}
-			case EXIT_POLICY:
+			case EXIT_POLICY: {
 				// A policy exit still ran a paid container, so it carries the ledger like the completed
 				// branch does -- the spend is real whichever way the runner classified itself.
-				return { outcome: "policy", reason: "runner-policy", exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true };
+				// The comment is UNCONDITIONAL (issue #288 asked for "when the agent did not already comment
+				// its own refusal", and the discriminator already exists at the exit-code boundary): an agent
+				// that composed its own refusal exits 0 -- github-prompt.mjs instructs the status comment,
+				// including for "I cannot fix this" -- so exit 2 means it was cut off before that step.
+				// Residual: an agent that posted a status and THEN blew its turn budget yields one extra
+				// comment, bounded at one.
+				//
+				// The runner's exit-line reason is read for ONE purpose (issue #437): a provider that refused
+				// the credential needs an operator, not a wait, and "runner-policy" hid that behind budget
+				// wording. It picks a label INSIDE this branch and nothing else, which is why reading it is
+				// safe where reading it to classify would not be: `code` has already placed the job in the
+				// not-retried class, so a forged or stale line can at worst swap one not-retried label for
+				// another from the closed RUNNER_POLICY_REASONS set. The `code === 2` guard is redundant with
+				// the case label today and is kept so the label cannot follow this line if it is ever moved.
+				// Every other reason the runner gives still reads as runner-policy.
+				const reason = RUNNER_POLICY_REASONS.has(exitReason) && code === 2 ? exitReason : "runner-policy";
+				await comment(job, TERMINAL_COMMENTS[reason]);
+				return { outcome: "policy", reason, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true };
+			}
 			case EXIT_INFRA:
+				// NO comment on any infra throw, here or in the catch: an InfraRetry may be retried and
+				// recover, and a flaky daemon must not post three comments for one recovery. Once-ness for
+				// the whole infra class lives at the terminal seam -- start.mjs's failed listener, guarded on
+				// BullMQ's own finishedOn -- which also catches the stall-kill and wait-gate paths this
+				// function never sees (issue #288).
 				throw new InfraRetry(`infra failure, container exit ${code}`, { exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session) });
 			default:
 				// THE RUNTIME NEVER HANDED CONTROL TO THE RUNNER, in whatever integers this venue spells that
@@ -663,6 +1045,69 @@ export async function runJob(job, deps) {
 				throw new InfraRetry(`unknown container exit ${code}`, { exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session) });
 		}
 	} catch (e) {
+		// A CONFIG-tagged throw is a determinate policy refusal wearing an exception, and issue #310 is the
+		// bill for treating it as neither. `CONST-RETRY-INFRA-ONLY` says a determinate refusal RETURNS and only
+		// infrastructure THROWS; every other gate in this function obeys that, and this class did not, because
+		// nothing here read the tag that `cli.mjs` and `doctor` both already read.
+		//
+		// What it actually cost, corrected against the issue text: it was NOT retried. `index.mjs` wraps every
+		// non-InfraRetry throw in BullMQ's `UnrecoverableError`, so the job failed once. What it did cost is
+		// the reserve, kept and never refunded, on a fault an operator has to fix by hand -- so every later
+		// delivery took another slot out of the same daily cap and the same scope, and the record said
+		// `outcome: "failed"` with a null reason, which the panel paints red beside real infrastructure faults
+		// and insights buckets as a failure. A determinate refusal that reads as an outage is the diagnosis
+		// this project exists to make legible.
+		//
+		// Refunding is right BECAUSE no container started: identical to `container-never-started`, and the
+		// same both-or-neither pair, under the same `reserved`/`scopedReserved` guards so it still cannot
+		// double-release. The gate above catches the provider case for free, before the mint and the clone;
+		// this is the backstop for every other config throw that can still land here (an unknown forge kind in
+		// `buildContainerEnv`, a prepare-time refusal), which would otherwise keep the same slot silently.
+		// `!containerRan` is the discriminator, and it is what makes the refund and the sentence below TRUE
+		// rather than merely true today. Every config-tagged throw site in the worker is pre-container, so this
+		// changes nothing now; the day one is added after a paid run, that run keeps its slot and falls through
+		// to the untagged path instead of being refunded and publicly declared free.
+		if (e?.piDispatchConfig === true && !containerRan) {
+			// GUARDED, and the flags are the record of what actually happened. `releaseBudget` is a loop of
+			// DECRs over the active windows and can reject part-way (a read-only replica, a dropped
+			// connection), which would otherwise replace this determinate refusal with a Redis message: the
+			// operator would be told their queue is broken when their deployment is misconfigured, and the
+			// escaping error is untagged so it is not retried either. A refund that did not land must not be
+			// reported as one, so `budgetReserved` follows the ledger and not the intent.
+			let refunded = true;
+			try {
+				if (reserved) {
+					await releaseBudget(redis, { caps, now });
+					reserved = false;
+				}
+				if (scopedReserved && scopedCaps) {
+					await releaseBudget(redis, { caps: scopedCaps.caps, now, keyPrefix: scopeKeyPrefix(scopedCaps.scope) });
+					scopedReserved = false;
+				}
+			} catch (releaseError) {
+				refunded = false;
+				log("budget_release_failed", { at: "config-refused", code: releaseError?.code ?? null });
+			}
+			// A FIXED sentence, and NO message in the log either. Two different reasons, both load-bearing:
+			// `credentialFromPiAuth` puts `auth.json`'s location in its refusals and `prepare-local` puts the
+			// operator's folder in its own, which `buildRecord` reduces to a basename precisely because a host
+			// path carries an OS account name; and `branch.mjs`'s refusal interpolates a forge PAYLOAD field,
+			// which no-pii-in-logs forbids anywhere. The scoped-budget refusal above keeps a host path out of
+			// its log line for the first of those reasons, and this follows it. `doctor` is where the operator
+			// reads the specific variable and the specific path, which is what both sentences point at.
+			// SWALLOWED, and only here. Every other refusal in this function awaits `comment` bare, which is
+			// right: they are on the happy path of a determinate decision, and a forge that cannot be told is
+			// worth surfacing. This one is inside the catch, so a throw from `comment` would discard the
+			// classification that has ALREADY released the budget, and the job would escape as whatever the
+			// comment threw, with the ledger refunded and the record saying something else entirely. The
+			// shipped adapter never throws; this makes that a property of the arm rather than of the wiring.
+			await comment(job, "Refused: this deployment is misconfigured, so the job could not be started. Ask the operator to run `pi-dispatch doctor`. Not run.").catch(() => {});
+			log("refused_config", { kind: job.kind ?? null, refunded });
+			// budgetReserved reflects the LEDGER: false when the refund landed, true when it did not and the
+			// slot is still out there.
+			return { outcome: "policy", reason: "config-refused", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: !refunded }; // return => not retried
+		}
+
 		// A spawn fault (docker daemon down / binary missing) reserved a slot but never started a
 		// container, so nothing was spent -- give the slot back before the retry. Every other throw
 		// here (exit-1 infra, unknown exit) means the container ran and legitimately spent its slot,
@@ -684,6 +1129,14 @@ export async function runJob(job, deps) {
 	}
 }
 
+/**
+ * The reason a job retried for the podman venue's rootless network keeper carries (issue #458, PR #463 round 2): a
+ * fixed token, as every run-record reason is, and the key the terminal comment is chosen by.
+ */
+export const NETNS_KEEPER_NOT_HOLDING = "netns-keeper-not-holding";
+/** Issue #476: a job held on a young keeper that kept restarting, or stayed young past the hold's bound. */
+export const NETNS_KEEPER_CRASH_LOOP = "netns-keeper-crash-loop";
+
 /** Thrown for the retryable (infra) class only. The BullMQ processor lets this propagate to retry. */
 /**
  * The one `session` object the run record carries, from the host's intent and the container's report
@@ -768,4 +1221,31 @@ export class InfraRetry extends Error {
 	}
 }
 
+/**
+ * A local job held until rootful Podman's service restarts (issue #448, gate round 2 of PR #473). An `InfraRetry`, so
+ * any path that does not know it still retries rather than failing the job for good; `makeProcessor` knows it, and
+ * moves the job to the delayed set without spending an attempt until the hold has lasted `PODMAN_RESTART_HOLD_MAX_MS`.
+ */
+export class PodmanRestartHold extends InfraRetry {
+	constructor(message, options) {
+		super(message, options);
+		this.name = "PodmanRestartHold";
+		this.holdUntilRestart = true;
+	}
+}
+
+/**
+ * A job held on a rootless network keeper that is running on its own bridge but younger than the minimum age (issue
+ * #476). An `InfraRetry`, so any path that does not know it still retries; `makeProcessor` knows it, and moves the job to
+ * the delayed set for `young.waitMs` without spending an attempt, up to `NETNS_KEEPER_YOUNG_HOLD_MAX_MS`.
+ */
+export class NetnsKeeperYoungHold extends InfraRetry {
+	constructor(message, { young, remedy = null, ...options } = {}) {
+		super(message, options);
+		this.name = "NetnsKeeperYoungHold";
+		this.keeperYoung = young;
+		this.keeperRemedy = remedy;
+	}
+}
+
 export { EXIT_COMPLETED, EXIT_INFRA, EXIT_POLICY };