@edgehero/pi-dispatch 1.9.0 → 1.10.1

package/src/doctor.mjs

@@ -58,6 +58,7 @@ import { agentDirFrom, readHostPi } from "./host-pi.mjs";
 import { PACKAGES_SUBDIR, readStagedSkills, readStageManifest } from "./packages.mjs";
 import { copySkillTree } from "./copy-tree.mjs";
 import { SKILL_NAME_RE } from "./flow-gate.mjs";
+import { ABSENT, ASSERTED, PROPERTY_NAMES, declarationOf, floorShortfall, parseBackendFloor, parseBackendList, unarmedFloor } from "./backends.mjs";
 import { DEFAULT_EGRESS_PROXY, egressArmed } from "./egress.mjs";
 import { installedUnitPaths, readUnitSeam } from "./service.mjs";
 import { parseSecretProfiles } from "./secret-profiles.mjs";
@@ -469,6 +470,7 @@ export async function collectChecks(env, seams) {
 	// byte-identical output. Gated on docker and the image, because two of these checks run a container and
 	// the rest are noise on top of a down daemon.
 	checks.push(...(await egressChecks(env, seams, { dockerCode, imageCode, jobImage })));
+	checks.push(...backendChecks(env));
 
 	// The receiver itself, when the triggers file names ANY forge (issue #80). Only forge deliveries need
 	// the receiver at all, so a cron/local-only deployment gets no receiver noise here. WARNS rather than
@@ -2337,3 +2339,119 @@ async function defaultProbeValkey(url) {
 		client.disconnect();
 	}
 }
+
+/**
+ * WHERE this deployment's jobs run, and what that place actually guarantees (issue #227).
+ *
+ * THIS IS THE CHECK THAT MAKES THE DECLARATION ADMISSIBLE AT ALL. `CONST-EGRESS-POLICY-IN-THE-ARGV` says a
+ * control an operator BELIEVES in is worse than one they know is missing, because the belief displaces the
+ * credential bound that is really holding. A table of guarantees nothing ever prints is exactly such a
+ * belief. So the three words must stay TOLD APART on the way out, and told apart ON THE SCREEN rather than
+ * in a field nobody renders:
+ *
+ *   enforced -- ours, in this worker's own code, readable back from what it produced. Quiet.
+ *   asserted -- someone else's. Rendered as a WARNING, and it NAMES who is asserting it, because "not us"
+ *               without "them" leaves an operator nothing to go and check.
+ *   absent   -- not provided at all. A failure, since a deployment reaching it must know before a job does.
+ *
+ * `ok: false, warn: true` IS THE WARNING SHAPE, and it is the one thing to get right when editing here.
+ * `render` reads `c.ok` FIRST, so `ok: true, warn: true` renders as a plain pass and drops the `fix` line
+ * with it. An earlier draft used that shape and every asserted property printed as a green tick, which made
+ * this section say the opposite of what it exists to say. `warn` keeps the RUN green -- `render` only fails
+ * on `!ok && !warn` -- so an operator's CI is unaffected while the operator is actually told.
+ *
+ * A property a deployment switch gates is printed with the switch AND its position, never the bare
+ * capability word: `local` can enforce egress, and a `PI_EGRESS=0` deployment is not getting it. Those are
+ * two different sentences. `absent` OUTRANKS the gate, because a control that does not exist is a different
+ * fact from one that is merely unarmed, and "CAN be absent but the switch is off" would be both meaningless
+ * and green.
+ *
+ * Reads the environment directly, like every other check here, and parses through `backends.mjs` so doctor
+ * and the worker cannot disagree about what a floor says.
+ */
+export function backendChecks(env) {
+	const checks = [];
+	let backends;
+	let floor;
+	try {
+		backends = parseBackendList(env.PI_BACKENDS);
+		floor = parseBackendFloor(env.PI_BACKEND_FLOOR);
+	} catch (error) {
+		// The worker refuses to boot on this, so doctor must not soften it to a warning.
+		return [{ ok: false, label: `backend configuration does not parse: ${error.message}`, fix: "fix PI_BACKENDS / PI_BACKEND_FLOOR, then re-run doctor" }];
+	}
+
+	// The switch positions every `armedBy` in the table can name. A MAP rather than one boolean, because
+	// `armedBy` is a general field: hardcoding one variable name here would silently hide a second switch's
+	// off-position the day one is added, which is the defect `armedBy` exists to prevent.
+	const switches = {};
+	try {
+		switches.PI_EGRESS = egressArmed(env);
+	} catch (error) {
+		// NOT an abstention that falls through to the good case. A value doctor cannot parse is a value the
+		// worker refuses to boot on, and an earlier draft claimed in a comment that "its own check reports
+		// that" -- nothing did, so doctor printed every gated property as quietly enforced on a deployment
+		// that could not start.
+		checks.push({ ok: false, label: `PI_EGRESS does not parse, so what this deployment actually gets cannot be determined: ${error.message}`, fix: 'set PI_EGRESS to exactly "0" (off) or "1"/unset (on)' });
+	}
+
+	// The dispatch landed in slice 4, so the "nothing selects yet" qualifier came off -- and it came off HERE
+	// as well as in the code, because a stale caveat on the one surface that makes the table admissible is
+	// its own kind of false statement.
+	checks.push({ ok: true, label: `Jobs run on: ${backends.join(", ")}${backends.length > 1 ? ` (a trigger that names none runs on ${backends[0]}; run.backend selects)` : ""}` });
+
+	for (const name of backends) {
+		for (const property of PROPERTY_NAMES) {
+			const d = declarationOf(name, property);
+			if (!d) continue;
+			// FIRST, ahead of the gate: a control that does not exist is not a control that is unarmed.
+			if (d.word === ABSENT) {
+				checks.push({ ok: false, label: `${name}: ${property} is ABSENT -- ${d.question}`, fix: `this backend does not provide ${property}; a deployment that needs it must not run jobs on ${name}` });
+				continue;
+			}
+			if (d.armedBy && switches[d.armedBy] === undefined) {
+				// The switch did not parse. Say so rather than pick a side; the failure is already reported.
+				checks.push({ ok: false, warn: true, label: `${name}: ${property} depends on ${d.armedBy}, which does not parse -- cannot say whether this deployment gets it`, fix: `fix ${d.armedBy}, then re-run doctor` });
+				continue;
+			}
+			if (d.armedBy && switches[d.armedBy] === false) {
+				checks.push({ ok: false, warn: true, label: `${name}: ${property} CAN be ${d.word} here, but ${d.armedBy} is off, so this deployment is not getting it`, fix: `arm ${d.armedBy} to get it (${d.question})` });
+				continue;
+			}
+			if (d.word === ASSERTED) {
+				checks.push({ ok: false, warn: true, label: `${name}: ${property} is ASSERTED by ${d.assertedBy ?? "something outside this worker"}, not enforced by it`, fix: `not verifiable from here, so treat it as a claim rather than a control: ${d.question}` });
+				continue;
+			}
+			// enforced, and armed if it is gated at all. The good case, and it stays quiet.
+		}
+	}
+
+	// ALWAYS a line, including when no floor is set. `PI_BACKENDS_FLOOR` is a plausible one-character-off
+	// spelling of the real name, and nothing in this project warns on an unknown PI_* variable, so silence
+	// here would make a typo'd VARIABLE NAME look exactly like a floor that holds -- the same belief the
+	// strict parsing inside the string exists to prevent, arriving from outside the string.
+	const floorNames = Object.keys(floor);
+	if (floorNames.length === 0) {
+		checks.push({ ok: true, label: "PI_BACKEND_FLOOR is not set, so no minimum is required of any backend" });
+		return checks;
+	}
+
+	const misses = floorShortfall(backends, floor);
+	const unarmed = unarmedFloor(floor, switches);
+	// A floor whose every entry is `absent` parses, reads, and bounds NOTHING: `meets(have, absent)` is true
+	// for every value. It is the one READABLE word that reproduces the outcome `isDeclaration` refuses a
+	// typo for, so it is named rather than affirmed.
+	const bounding = floorNames.filter((p) => floor[p] !== ABSENT);
+	const spelled = floorNames.map((p) => `${p}=${floor[p]}`).join(", ");
+	if (misses.length > 0) {
+		checks.push({ ok: false, label: `PI_BACKEND_FLOOR is not met: ${misses.map((m) => `${m.backend}.${m.property} is ${m.have}`).join(", ")}`, fix: "raise the backend, lower PI_BACKEND_FLOOR, or drop the backend from PI_BACKENDS" });
+	} else if (unarmed.length > 0) {
+		checks.push({ ok: false, label: `PI_BACKEND_FLOOR asks for ${unarmed.map((u) => `${u.property}=${u.want}`).join(", ")}, which ${[...new Set(unarmed.map((u) => u.armedBy))].join(", ")} has switched off`, fix: "arm the switch, or lower that entry to `absent` if you did not mean to require it" });
+	} else if (bounding.length === 0) {
+		checks.push({ ok: false, warn: true, label: `PI_BACKEND_FLOOR (${spelled}) requires nothing: every entry asks for "absent", which every backend meets`, fix: "raise an entry to `asserted` or `enforced` for it to bound anything" });
+	} else {
+		checks.push({ ok: true, label: `PI_BACKEND_FLOOR holds (${spelled})` });
+	}
+
+	return checks;
+}

package/src/index.mjs

@@ -1,13 +1,11 @@
-import { execFile } from "node:child_process";
-import { promisify } from "node:util";
 import { DelayedError, UnrecoverableError, Worker } from "bullmq";
+import { jobContainerName } from "./backend-local.mjs";
 import { InfraRetry, runJob } from "./processor.mjs";
 import { targetFor } from "./run-history.mjs";
 import { budgetCapsFor, canonicalScope, concurrencyFor, makeInFlight, scopeKeyPrefix } from "./scoped-limits.mjs";
 import { WAIT_AFTER_MAX_DEFAULT_MS, WAIT_INTERVAL_FLOOR_MS, afterMs, unreadableConditions, waitArmed, waitBackoffMs, waitLabel, waitProfileNames } from "./wait-for.mjs";
 import { makeWaitState } from "./wait-state.mjs";
 
-const exec = promisify(execFile);
 
 export const QUEUE = "pi-jobs";
 
@@ -44,6 +42,50 @@ const THROTTLE_ALARM = 5;
 // the only evidence there is. A test pins the two apart.
 const THROTTLE_FLOOR_MS = 11_000;
 
+/**
+ * How long a container gets to actually die after the abort's `docker stop` before the worker stops waiting.
+ *
+ * `docker stop -t 5` is SIGTERM then an unignorable SIGKILL five seconds later, so a reachable daemon ends
+ * the container well inside this. The margin is for the daemon being slow, not for the container being
+ * stubborn -- a container cannot outlive SIGKILL.
+ */
+const ABORT_GRACE_MS = 30_000;
+
+/**
+ * Resolve `run` normally, but stop waiting once the abort has fired and the grace has passed.
+ *
+ * See the call site for why this exists. Returns the same `{ code: 137, aborted: true }` shape a killed
+ * container produces, so nothing downstream needs to know the difference -- the processor's abort
+ * classification, the run record and the refund all behave exactly as they do for a stop that worked.
+ */
+function boundAfterAbort(run, signal, job, log, graceMs = ABORT_GRACE_MS) {
+	if (!signal) return run;
+	return new Promise((resolve, reject) => {
+		let timer = null;
+		let settled = false;
+		const done = (fn) => (v) => {
+			if (settled) return;
+			settled = true;
+			clearTimeout(timer);
+			fn(v);
+		};
+		const onAbort = () => {
+			timer = setTimeout(() => {
+				if (settled) return;
+				settled = true;
+				log("stop_did_not_take", { job: job.id, graceMs });
+				resolve({ code: 137, aborted: true, turns: null, tokens: null, session: null, usage: null, context: null });
+			}, graceMs);
+			// A boot-blocking handle is not wanted here: the worker should be able to exit if everything else
+			// has finished, and this timer only matters while a job is still in flight.
+			timer.unref?.();
+		};
+		if (signal.aborted) onAbort();
+		else signal.addEventListener("abort", onAbort, { once: true });
+		run.then(done(resolve), done(reject));
+	});
+}
+
 /**
  * Build the BullMQ processor.
  *
@@ -53,7 +95,8 @@ const THROTTLE_FLOOR_MS = 11_000;
  * abort -- with no error. A test asserts the arity precisely because the failure is silent.
  *
  * Dependencies are injected so this is testable without a live queue: `cancelJob` (fired by the
- * timeout), `stopContainer` (fired by the abort), and the orchestration deps.
+ * timeout), `stopContainer` (fired by the abort, and INJECTED so the venue that built the container is
+ * the one that stops it), and the orchestration deps.
  *
  * Once per job, before runJob, it resolves the runtime-settings overlay via `getSettings`
  * (INT-CONFIG-OVERLAY-CONTRACT). A present-but-invalid overlay resolves to a POLICY refusal RETURNED
@@ -63,7 +106,7 @@ const THROTTLE_FLOOR_MS = 11_000;
  * The overlay changes which values the spend caps take, never when they are checked -- reserveBudget still
  * runs inside runJob against the freshly passed caps (CONST-BUDGET-BEFORE-TOKENS).
  */
-export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), hostBound = null, checkLease = null, scopeLease = null, deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, checkTimeoutMs = () => 10_000, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
+export function makeProcessor({ cancelJob, stopContainer, containerName = (job) => jobContainerName(job.id), redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), hostBound = null, checkLease = null, scopeLease = null, deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, checkTimeoutMs = () => 10_000, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
 	return async function processor(job, token, signal) {
 		// Scoped pause windows (REQ-SCOPED-PAUSE-WINDOWS): if this job's folder/repo is inside an active pause
 		// window, DEFER it to the window end via BullMQ's delayed set -- the job keeps its identity/dedup and
@@ -491,6 +534,7 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 		let name;
 		let timer;
 		let onAbort;
+		let venue;
 		try {
 			// Nothing between the acquire above and the main `try` below may throw unguarded: the releasing
 			// finally belongs to THAT try, so an unguarded throw here would leak the hold and wedge the
@@ -498,7 +542,23 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 			// addEventListener on the bullmq-allocated controller are total at processor arity 3); the
 			// guard is structural, not observational.
 			startedAt = new Date().toISOString();
-			name = `pi-job-${job.id}`;
+			// The producer of the name both boot reapers sweep by substring. Built from the shared prefix
+			// rather than typed here, so a rename cannot land in the producer and not in the sweeps (#227).
+			// From the VENUE that will build the container, not from the local adapter reached for directly:
+			// the abort stops this name, so the name and the stop have to come from the same backend. The
+			// default keeps every wiring that predates the seam building it exactly as before.
+			//
+			// `job.data`, NOT `job`. This function's `job` is the BullMQ WRAPPER -- its own keys are `id` and
+			// `data` -- so `job.backend` is always undefined and the registry's resolution would fall to
+			// `?? defaultName` for every job, silently, on the one path where the fail-closed throw can never
+			// fire because "names nothing" is exactly the case it permits. `runJob` is handed `effectiveJob`,
+			// a spread of `job.data`, which is why `runContainer` and the two preflights dispatch correctly
+			// while these two did not.
+			// One object carrying BOTH halves: the id is the BullMQ wrapper's (it always was) and the venue is
+			// the trigger's, which lives in `data`. Built once so the name and the stop cannot resolve
+			// different backends -- the whole reason the name moved onto the registry in the first place.
+			venue = { ...job.data, id: job.id };
+			name = containerName(venue);
 			timer = setTimeout(() => {
 				// BullMQ has no per-job kill timer; this is ours. cancelJob raises the AbortSignal.
 				Promise.resolve(cancelJob(job.id, "job-timeout-30m")).catch(() => {});
@@ -507,7 +567,23 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 			// Abort (timeout OR shutdown) => stop the container. docker stop sends SIGTERM then SIGKILL
 			// after the grace period; the runner exits and runContainer returns/throws.
 			onAbort = () => {
-				Promise.resolve(stopContainer(name)).catch(() => {});
+				// The JOB'S DATA goes with the name -- `job.data`, not `job`. A container name alone cannot say
+				// which runtime holds it once there is more than one venue, and this call is the only thing
+				// standing between a runaway job and the 30-minute bound (REQ-JOB-TIMEOUT-30M). Passing the
+				// BullMQ wrapper sent every abort to the DEFAULT venue: `docker stop` on a host that never
+				// held the container, rejecting into a log line while the real one kept running and kept
+				// spending, with the local reaper unable to see it either.
+				// The whole call is inside the try, not just its promise. `Promise.resolve(f())` evaluates `f()`
+				// FIRST, so a missing or throwing `stopContainer` raises synchronously, inside an
+				// AbortSignal listener, where it surfaces as an uncaughtException and takes the worker
+				// process down 30 minutes into a runaway job -- killing every other in-flight job on the
+				// host. Losing the kill for one job is bad; losing the process is worse.
+				const note = deps.log ?? (() => {});
+				try {
+					Promise.resolve(stopContainer(name, venue)).catch((err) => note("stop_container_failed", { job: job.id, reason: err?.message }));
+				} catch (err) {
+					note("stop_container_failed", { job: job.id, reason: err?.message });
+				}
 			};
 			signal.addEventListener("abort", onAbort, { once: true });
 		} catch (error) {
@@ -574,7 +650,26 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 				// life. Null when no row carries a money window for this scope.
 				scopedCaps: budgetCapsFor(job.data, limits),
 				...deps,
-				runContainer: (ctx) => deps.runContainer({ ...ctx, name, signal }),
+				// #227. BOUNDED AFTER THE ABORT, and this is what makes `abortable` an honest declaration.
+				//
+				// `makeRunContainer`'s promise settles ONLY on the docker child's `close` or `error`. Nothing
+				// else ends that await -- the signal is read at entry and captured at close, never passed to
+				// the spawn. So `stopContainer` is the sole kill channel, and if it does not take (an
+				// unreachable daemon, a wiring whose stop is a no-op) the container keeps running, `docker
+				// run` never exits, and the processor awaits FOREVER: an active job renewing its lock and
+				// holding its in-flight slot, its host slot, its scope lease and its budget reservation, with
+				// nothing in the log and nothing in the record. REQ-JOB-TIMEOUT-30M's Acceptance says "the
+				// slot is freed", and it was not.
+				//
+				// So once the abort has fired, the wait is bounded. On expiry this resolves the SAME shape a
+				// killed container returns -- `{ code: 137, aborted: true }`, which `run-container.mjs`
+				// already uses for "aborted before it could start" -- so the processor classifies it as
+				// POLICY and does not retry. That is deliberate: the container may still be running, and a
+				// retry would pay for a second one alongside it. What is leaked is the container, which the
+				// next boot reaper sweeps; what is NOT leaked is the slot, the lease and the reservation.
+				// The `stop_did_not_take` line is the loud half, because a host whose daemon ignores a stop
+				// is a fact an operator has to learn from somewhere.
+				runContainer: (ctx) => boundAfterAbort(deps.runContainer({ ...ctx, name, signal }), signal, job, deps.log ?? (() => {})),
 				// REQ-TRIGGER-SECRETS. The resolver runs INSIDE the 30-minute kill timer armed above, so it has
 				// to be abortable for the same reason runContainer does: a resolver blocking on an unreachable
 				// vault would otherwise hold its slot until its own timeout, and an abort landing mid-resolution
@@ -598,12 +693,23 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 				// job.data with no `.id` -- and the real wrapper is only in scope here, so inject it as
 				// `queueJobId`, mirroring the collectChain injection above. Omitted when unwired so a bare
 				// processor keeps runJob's plain (job, token) call.
-				...(deps.prepareWorkspace ? { prepareWorkspace: (j, t) => deps.prepareWorkspace(j, t, { queueJobId: job.id }) } : {}),
+				// The third argument is EXTENDED, never replaced. `runJob` calls this as
+				// `prepareWorkspace(job, token, { piVersion })`, so a wrapper passing only `{ queueJobId }`
+				// dropped it -- and `piVersion` defaults to `null`, which `readCanonical` treats as
+				// "never resume": `if (piVersion === null) return COLD("pi-version-changed")`. So EVERY
+				// `run.resume` cold-started, on every wired worker, while reporting success. The stamp
+				// `promoteSession` writes was correct the whole time; the comparison simply never happened.
+				// The processor tests inject `prepareWorkspace` directly and never see this wrapper, which is
+				// why nothing caught it (REQ-RESUMABLE-SESSION).
+				...(deps.prepareWorkspace ? { prepareWorkspace: (j, t, opts) => deps.prepareWorkspace(j, t, { ...opts, queueJobId: job.id }) } : {}),
 				// The one-shot pre-spend check (issue #231) needs the REAL BullMQ job's `.id` to excuse this
 				// delivery's own earlier attempt -- runJob's effectiveJob has no `.id`, prepareWorkspace's
 				// own injection above states why, and this one mirrors it. Omitted when unwired so a bare
 				// processor keeps runJob's admit-everything default.
-				...(deps.checkOnceSpent ? { checkOnceSpent: (j) => deps.checkOnceSpent(j, { queueJobId: job.id }) } : {}),
+				// Same extend-don't-replace shape as `prepareWorkspace` above, for its reason: `runJob` passes
+				// this one no options today, so there is nothing to lose yet -- and the day it does, a
+				// replacing wrapper would lose it in the same silence.
+				...(deps.checkOnceSpent ? { checkOnceSpent: (j, opts) => deps.checkOnceSpent(j, { ...opts, queueJobId: job.id }) } : {}),
 			});
 			recordRun({ job, result, startedAt, endedAt: new Date().toISOString() });
 			return result;
@@ -630,12 +736,19 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 	};
 }
 
-export function createWorker({ connection, name, hostQueue = null, checkLease = null, scopeLease = null, checkTimeoutMs, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, scopedLimits, inFlight = makeInFlight(), waitState, afterMaxMs, checkSlots = makeInFlight(), checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, hostSlots = makeInFlight(), extraClosers = [] }) {
+export function createWorker({ connection, name, stopContainer, containerName, hostQueue = null, checkLease = null, scopeLease = null, checkTimeoutMs, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, scopedLimits, inFlight = makeInFlight(), waitState, afterMaxMs, checkSlots = makeInFlight(), checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, hostSlots = makeInFlight(), extraClosers = [] }) {
 	// One Worker per queue name (issue #57). A host-affine job -- one whose folder, secret resolver or wait
 	// check lives on THIS machine -- is enqueued to `pi-jobs@<name>` rather than filtered for at pickup,
 	// because BullMQ has no selective pop and the put-it-back alternative does not work: promotion out of
 	// the delayed set is gated on each worker's own `Date.now()`, so the fastest clock wins every hop and a
 	// job that had to reach another host might never get there.
+	// #227. REFUSED AT CONSTRUCTION, because the alternative is discovering it 30 minutes into a runaway
+	// job. `stopContainer` is the only thing that enforces REQ-JOB-TIMEOUT-30M, and it is what the backend
+	// table declares as `abortable: enforced` -- a wiring that omits it would make that declaration false
+	// while every test that never aborts stayed green.
+	if (typeof stopContainer !== "function") {
+		throw new Error("createWorker: stopContainer is required -- it is the only thing that enforces the 30-minute job timeout");
+	}
 	const names = hostQueue ? [QUEUE, hostQueue] : [QUEUE];
 	const workers = [];
 
@@ -655,7 +768,12 @@ export function createWorker({ connection, name, hostQueue = null, checkLease =
 			// Bound to THIS worker: a job on the host queue is cancelled by the worker draining that queue,
 			// and the shared handle could not reach it.
 			cancelJob: (id, reason) => worker.cancelJob(id, reason),
-			stopContainer: (name) => exec("docker", ["stop", "-t", "5", name]),
+			// #227. INJECTED, not built here. This was a one-line `docker stop` literal, which meant the abort
+			// path -- the only thing that can end a runaway job -- was the one backend function unreachable
+			// from `startWorker`. The wiring now passes the registry's per-job stop, so the container is
+			// stopped by whatever venue built it.
+			stopContainer,
+			containerName,
 			redis,
 			getSettings,
 			// Late-bound over EVERY worker: an overlay concurrency change re-binds the live slot count at the
@@ -733,3 +851,9 @@ export function createWorker({ connection, name, hostQueue = null, checkLease =
 
 	return primary;
 }
+
+/**
+ * `boundAfterAbort` with a zero grace, for tests only. The real bound is 30 seconds, which no test can wait
+ * for, and the behaviour under test is what happens WHEN the grace expires -- not how long it is.
+ */
+export const __boundAfterAbortForTests = (run, signal, job, log) => boundAfterAbort(run, signal, job, log, 0);

package/src/outbox.mjs

@@ -171,6 +171,14 @@ export function makeCollectChain({ queue, enqueue = enqueueLocalJob, readFlowGat
 					// (INT-OUTBOX-CONTRACT's explicit-property-reads rule). Undefined stays undefined, so a parent with
 					// no image chains a child whose data is byte-identical to today's.
 					image: job.data?.image,
+					// #227, and INHERITED for the reason `image` directly above is: a chained child continues its
+					// parent's work, so it belongs in the venue the parent's trigger chose, not silently back on
+					// the deployment default. Written down rather than left to inference because this file's
+					// convention is that every inherit/don't-inherit decision here carries its reason -- and the
+					// distinction it turns on is the same one `secrets` is excluded by: a venue is toolchain, a
+					// resolved credential is a capability. The agent cannot choose it any more than it can choose
+					// its child's image: the value comes off `job.data`, never off the request file.
+					backend: job.data?.backend,
 					// The parent's injected skills follow the child, off validated JOB DATA and never off the
 					// request file (REQ-PER-TRIGGER-SKILLS). Same reason `image` does: a chained child runs the
 					// same operator's flows and, without them, would look up a skill that is not there, write a

package/src/packages.mjs

@@ -34,10 +34,13 @@ import { configError } from "./config.mjs";
 // and the admin block cannot drift between the stager and this validator (doctor.mjs sets the precedent
 // of importing from import-pi.mjs).
 import { ADMIN_RE, ENTRY_NAME_RE } from "./import-pi.mjs";
-// The container-side mount point is docker-run.mjs's fact -- IMPORTED, never re-typed, so the mount and the
-// packages root below cannot drift apart while both test suites stay green. docker-run.mjs is dependency-free
-// (it builds an argv array and nothing else), so this costs no cycle and no weight in the admin's bundle.
-import { CONTAINER_GLOBAL_PI_DIR } from "./docker-run.mjs";
+// The container-side mount point is container-spec.mjs's fact -- IMPORTED, never re-typed, so the mount and
+// the packages root below cannot drift apart while both test suites stay green. That module is a LEAF and
+// imports nothing, so this costs no cycle and no weight in the admin's bundle. It used to be imported from
+// docker-run.mjs, which held the same constant and made the same claim; issue #227 moved the container-side
+// half out, and this points at the half it actually wants -- a container path is not a Docker fact, and
+// docker-run.mjs now has an import edge of its own.
+import { CONTAINER_GLOBAL_PI_DIR } from "./container-spec.mjs";
 
 /** Staged packages live under `<globalPiDir>/packages/` -- a subdir of the overlay, not a new mount. */
 export const PACKAGES_SUBDIR = "packages";

package/src/processor.mjs

@@ -1,3 +1,4 @@
+import { DEFAULT_BACKEND, DOCKER_NEVER_STARTED_EXITS } from "./backends.mjs";
 import { lstatSync } from "node:fs";
 import { checkTokenCap, recordTokenSpend, releaseBudget, reserveBudget } from "./budget.mjs";
 import { configError } from "./config.mjs";
@@ -100,6 +101,19 @@ export async function runJob(job, deps) {
 		// would leave an INJECTED resolver running on every job, which is how a probe nobody wanted starts
 		// spawning a subprocess per delivery to learn nothing.
 		resolveSecrets = async (job) => ({ profileUnknown: job.secretsProfile ?? DEFAULT_SECRETS_PROFILE }),
+		// #227. Which backends this deployment BLESSED (PI_BACKENDS). Defaults to the one name every
+		// deployment already runs rather than to admit-everything: a wiring that says nothing blesses
+		// `local` only, so a job naming anything else is refused instead of running somewhere the operator
+		// never approved. That is the same fail-closed direction `resolveSecrets` above defaults in, and it
+		// is safe to default at all only because the gate below fires ONLY when a job names a backend --
+		// an unflagged job never consults this list.
+		blessedBackends = [DEFAULT_BACKEND],
+		// #227. The exit codes THIS JOB'S venue uses for "the runner never ran" -- a function of the job,
+		// not of the wiring, because which venue ran it is a per-job fact and the registry resolves it per
+		// job for every other backend function too. Docker's triple is the default because it is the only
+		// runtime this repo ships, so a wiring that omits this keeps today's behaviour exactly; an adapter
+		// that normalises to `container-never-started` itself returns an empty list.
+		neverStartedExits = () => DOCKER_NEVER_STARTED_EXITS,
 		// (job) => scoped short-lived token. Takes the JOB, not the repo: which forge mints -- and therefore
 		// which credential the container gets -- is a property of `job.kind`, and only the wiring knows the
 		// map. Called for forge-backed jobs and for local jobs opted in via `github: true`; unflagged local
@@ -180,6 +194,25 @@ export async function runJob(job, deps) {
 			}
 		}
 
+		// #227. WHERE this job wants to run, against what this deployment blessed. FREE, determinate and
+		// credential-less, so it precedes the image inspect below for that gate's own stated reason: a job
+		// that names a venue this host will not use must refuse before anything spawns, mints or clones.
+		//
+		// The LOADER already refused a name this build does not know; what it could not check is PI_BACKENDS,
+		// which is a per-host setting a reviewed file must not be refused over -- the same split
+		// `run.secretsProfile` draws between its charset check at load and `secret-profile-unknown` here.
+		//
+		// Enforced HERE and not only in the panel's picker, because `DES-PER-TRIGGER-SECRET-PROFILE` says the
+		// overlay is not the reviewed artifact: a tool-side allowlist bounds what an operator can pick, and
+		// this bounds what actually runs.
+		if (job.backend !== undefined && !blessedBackends.includes(job.backend)) {
+			await comment(job, `Refused: this trigger asks to run on the "${job.backend}" backend, which this deployment does not bless. Not run.`);
+			// The backend NAME is operator-authored config, never payload, so naming it is PII-safe -- the
+			// same class as the image ref below.
+			log("refused_backend_unblessed", { backend: job.backend, blessed: blessedBackends });
+			return { outcome: "policy", reason: "backend-unblessed", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
+		}
+
 		// The job image must exist on THIS host before anything else happens. Free, determinate and
 		// credential-less, so it precedes the mint, the clone and the reservation: a host that cannot run the
 		// image refuses without minting a credential it will not use, cloning a repo it will not read, or
@@ -611,18 +644,22 @@ export async function runJob(job, deps) {
 				return { outcome: "policy", reason: "runner-policy", exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true };
 			case EXIT_INFRA:
 				throw new InfraRetry(`infra failure, container exit ${code}`, { exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session) });
-			case 125: // `docker run` itself failed (unusable image reference, bad flag)
-			case 126: // the entrypoint exists but is not executable
-			case 127: // the entrypoint was not found
-				// In all three docker never handed control to the runner, so NOTHING was spent -- which is
-				// exactly what `container-never-started` means, and it reuses the refund below rather than
-				// keeping a slot the agent never used. These used to fall to `default:`, which kept the slot
-				// AND retried, burning a second one. The preflight above converts the KNOWABLE case (an absent
-				// image) into a pre-spend policy refusal; a 125 that survives it is a race (the image was
-				// removed between the inspect and the run) or a docker-side fault we did not foresee --
-				// genuinely infra, and now with a retry that costs nothing.
-				throw new InfraRetry(`docker could not start the container, exit ${code}`, { reason: "container-never-started", exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session) });
 			default:
+				// THE RUNTIME NEVER HANDED CONTROL TO THE RUNNER, in whatever integers this venue spells that
+				// (issue #227). For docker it is 125 (`docker run` itself failed), 126 (the entrypoint exists
+				// but is not executable) and 127 (the entrypoint was not found). Nothing was spent -- which is
+				// exactly what `container-never-started` means -- so this reuses the refund below rather than
+				// keeping a slot the agent never used. They used to fall to the unknown-exit branch, which
+				// kept the slot AND retried, burning a second one.
+				//
+				// ASKED OF THE BACKEND rather than hardcoded, because those integers are Docker's and they
+				// COLLIDE with the runner's own channel (`INT-RUNNER-EXIT-CODE-PROTOCOL`). Assuming them is
+				// silently wrong for any venue where 125 is a real runner exit, and the assumption was
+				// invisible while there was one runtime. An adapter declares its own set, or declares none
+				// and normalises to this outcome itself.
+				if ((neverStartedExits(job) ?? []).includes(code)) {
+					throw new InfraRetry(`the runtime could not start the container, exit ${code}`, { reason: "container-never-started", exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session) });
+				}
 				throw new InfraRetry(`unknown container exit ${code}`, { exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session) });
 		}
 	} catch (e) {

package/src/queue.mjs

@@ -78,7 +78,7 @@ export function makeQueue(connection, { name = QUEUE } = {}) {
  * removeOnComplete keeps the dedup window ~= the retention. Unlike webhooks, local jobs are not
  * redelivered, so a modest window is enough.
  */
-export async function enqueueLocalJob(queue, { folder, flow, task, command, provider, model, maxTurns, image, skillsDir, secrets, secretsProfile, chainDepth, parentJobId, jobId, now = new Date() }) {
+export async function enqueueLocalJob(queue, { folder, flow, task, command, provider, model, maxTurns, image, backend, skillsDir, secrets, secretsProfile, chainDepth, parentJobId, jobId, now = new Date() }) {
 	const minute = now.toISOString().slice(0, 16); // YYYY-MM-DDTHH:MM -- the dedup window
 	// A caller-supplied jobId (the outbox collector's retry-idempotent chainedJobId) wins; otherwise the
 	// minute-windowed localJobId is the dedup key. A command job (issue #189) fills the flow slot with
@@ -104,6 +104,12 @@ export async function enqueueLocalJob(queue, { folder, flow, task, command, prov
 		model,
 		maxTurns,
 		...(image !== undefined && { image }),
+		// #227. WHERE this job's container is built. Conditional like `image`, so a trigger that named no
+		// venue produces byte-identical job data -- and at JOB level, never inside `trigger`, for the reason
+		// `image` and `skillsDir` are: `trigger` is copied VERBATIM into `/job/event.json`, so a key added
+		// there becomes agent-visible input, and where the box was built is the worker's business rather
+		// than the agent's.
+		...(backend !== undefined && { backend }),
 		// The host directory of operator-authored skills this trigger injects (REQ-PER-TRIGGER-SKILLS).
 		// Conditional like `image`, so an unflagged job's data stays byte-identical, and at JOB level rather
 		// than inside `trigger` because a worker-host path is an execution knob, not a fact about the
@@ -191,7 +197,7 @@ export async function enqueueGitLabJob(queue, fields) {
  * window, replicas never coalesce against each other, and an unflagged job's dedup id is the same string it
  * has always been.
  */
-export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, command, trigger, provider, model, maxTurns, packages, image, skillsDir, instructions, resume, secrets, secretsProfile, waitFor, replica, replicas }) {
+export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, command, trigger, provider, model, maxTurns, packages, image, backend, skillsDir, instructions, resume, secrets, secretsProfile, waitFor, replica, replicas }) {
 	const jobId = forgeDeliveryJobId(kind, trigger?.deliveryId, replica);
 	// `packages` (whether to load the operator-staged pi packages) and `image` (which container image to run)
 	// come off the MATCHED trigger (INT-TRIGGERS-FILE-CONTRACT / REQ-GLOBAL-PI-OVERLAY) and land on `data`
@@ -219,6 +225,12 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
 		maxTurns,
 		...(packages !== undefined && { packages }),
 		...(image !== undefined && { image }),
+		// #227. WHERE this job's container is built. Conditional like `image`, so a trigger that named no
+		// venue produces byte-identical job data -- and at JOB level, never inside `trigger`, for the reason
+		// `image` and `skillsDir` are: `trigger` is copied VERBATIM into `/job/event.json`, so a key added
+		// there becomes agent-visible input, and where the box was built is the worker's business rather
+		// than the agent's.
+		...(backend !== undefined && { backend }),
 		// The host directory of operator-authored skills this trigger injects (REQ-PER-TRIGGER-SKILLS).
 		// Conditional like `image`, so an unflagged job's data stays byte-identical, and at JOB level rather
 		// than inside `trigger` because a worker-host path is an execution knob, not a fact about the

package/src/sandbox-cli.mjs

@@ -1,11 +1,30 @@
 import { spawn } from "node:child_process";
 import { parseArgs } from "node:util";
+import { DEFAULT_BACKEND, backendFor } from "./backends.mjs";
 import { loadConfig } from "./config.mjs";
 import { sanitizeJobId } from "./run-history.mjs";
 import { createJobNetwork, egressEnv, networkNameFor, removeJobNetwork } from "./egress.mjs";
 import { buildSandboxRunArgs, launchSandbox, listRunningSandboxes, parsePublish, resolveSandbox, sandboxContainerName } from "./sandbox.mjs";
 import { listSandboxes, pinSandbox } from "./sandbox-store.mjs";
 
+/**
+ * Would this deployment's venues put a job's retained directory out of this command's reach? Exported ONLY
+ * so it can be driven.
+ *
+ * The refusal it answers is unreachable today: the table holds one backend and it is local, so
+ * `PI_BACKENDS` cannot name a remote venue. An unreachable rule with no test is a rule a mutation pass
+ * deletes in silence -- which is precisely what happened to the trigger-side remote refusal one slice ago,
+ * so this one is a pure predicate over an explicit config from the start.
+ *
+ * TRUE when any blessed venue is remote, not merely when the DEFAULT is: a job that ran there has a
+ * retained directory this host never had, and `pi-dispatch sandbox <jobId>` takes a job id rather than a
+ * venue, so it cannot know which one it is being asked about until the directory is already missing.
+ */
+export function sandboxUnreachableFrom(config) {
+	const names = config?.backends ?? [config?.defaultBackend ?? DEFAULT_BACKEND];
+	return names.some((n) => backendFor(n)?.remote !== false);
+}
+
 /**
  * `pi-dispatch sandbox` -- re-open a finished run's sandbox as an interactive shell
  * (REQ-RESURRECTABLE-SANDBOX).
@@ -78,6 +97,23 @@ export async function runSandbox(argv = [], { env = process.env, deps = {} } = {
 		return fail(err, error.message);
 	}
 
+	// #227. THE SANDBOX IS LOCAL-ONLY, and this refusal is what keeps that true rather than accidental.
+	//
+	// `buildSandboxRunArgs` is a SECOND container producer, outside the `runContainer` seam and hard-wired to
+	// this host's docker CLI. It reopens a retained job directory: `manifest.workspace` is a path on THIS
+	// machine, which for a job that ran in another venue either does not exist or exists and reproduces a
+	// run from the wrong host, silently. `INT-SANDBOX-CONTRACT` is already "a SIBLING ... never an
+	// amendment" of the container contract, and this is the clause that says which sibling.
+	//
+	// Refused BEFORE `resolveSandbox` reads anything, so a deployment that blesses a remote venue is told
+	// what is wrong rather than handed a confusing missing-directory message.
+	if (sandboxUnreachableFrom(config)) {
+		return fail(
+			err,
+			`pi-dispatch sandbox opens a shell on THIS host's docker daemon against the job's retained directory, so it cannot reach a job that ran in a remote venue (PI_BACKENDS: ${config.backends.join(", ")}). Run it on the host that ran the job.`,
+		);
+	}
+
 	const resolved = resolveSandbox({
 		jobId,
 		sandboxDir: config.sandboxDir,

package/src/schedules.mjs

@@ -137,7 +137,7 @@ function normalizeCronSchedule({ on, run }, path, existsSync, fleet) {
 	// key. A command trigger carries no flow/task at all (the validator enforces the XOR), so those two
 	// keys hold undefined here and drop at JSON serialization -- the command schedule's data is exactly
 	// kind/folder/command plus the shared fields.
-	const data = { kind: "local", folder: run.folder, flow: run.flow, task: run.task, ...(run.command !== undefined && { command: run.command }), provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages: run.packages, image: run.image, ...(run.skillsDir !== undefined && { skillsDir: run.skillsDir }), ...(run.secrets !== undefined && { secrets: run.secrets }), ...(run.secretsProfile !== undefined && { secretsProfile: run.secretsProfile }), resume: run.resume, trigger: { id: on.id, pattern: on.pattern } };
+	const data = { kind: "local", folder: run.folder, flow: run.flow, task: run.task, ...(run.command !== undefined && { command: run.command }), provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages: run.packages, image: run.image, ...(run.backend !== undefined && { backend: run.backend }), ...(run.skillsDir !== undefined && { skillsDir: run.skillsDir }), ...(run.secrets !== undefined && { secrets: run.secrets }), ...(run.secretsProfile !== undefined && { secretsProfile: run.secretsProfile }), resume: run.resume, trigger: { id: on.id, pattern: on.pattern } };
 	// Retention only; the deterministic repeat:<id>:<millis> jobId supplies dedup, so no jobId here, and
 	// scheduler jobs are not retried (DES-CRON-VIA-BULLMQ-SCHEDULER) so no attempts/backoff.
 	const opts = { removeOnComplete: { age: 24 * 3600 }, removeOnFail: { age: 7 * 24 * 3600 } };