@edgehero/pi-dispatch 1.9.0 → 1.10.0

package/.env.example

@@ -130,6 +130,15 @@ PI_JOB_IMAGE=pi-job:latest          # the DEFAULT job image. Any trigger may nam
 
 PI_SCHEDULER_STALL_MAX=2            # tear down a scheduler after N consecutive stalls (money backstop)
 
+# --- Container backend: where a job's container is built, and what that place guarantees (docs/backends.md) ---
+# PI_BACKENDS=                       # which backends this deployment blesses, comma separated (default: local, the docker daemon on this host).
+#                                    # A trigger's run.backend selects among these; one that names none runs on the first, so the list must include local. An unknown name refuses to boot.
+# PI_BACKEND_FLOOR=                  # the minimum every backend above must declare, as property=word pairs: e.g. egress=enforced,nonRoot=asserted
+#                                    # Words are enforced (this worker builds it and a test reads it back), asserted (something outside the worker provides it, unverifiable from here) or absent.
+#                                    # A floor naming a switched-off control refuses to boot: asking for egress=enforced while PI_EGRESS=0 is a bound you would believe in and not have.
+#                                    # `pi-dispatch doctor` prints enforced quietly, asserted as a warning naming who asserts it, and absent as a failure.
+#                                    # Env-only, deliberately: a bound that can be widened from the surface it bounds is not a bound.
+
 # --- Egress policy: what a job container may reach on the network (docs/egress.md) ---
 # ON by default. Every job runs on its own --internal Docker network with no route anywhere except an
 # allowlist proxy, and a job whose policy cannot serve it is refused BEFORE it spends a budget slot.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@edgehero/pi-dispatch",
-  "version": "1.9.0",
+  "version": "1.10.0",
   "type": "module",
   "description": "Self-hosted job harness for the pi coding agent: a BullMQ worker that drains the queue, mints scoped forge tokens, and runs one container per job — plus the pi-dispatch CLI (init, up, doctor, service).",
   "keywords": [
@@ -52,7 +52,12 @@
     "./connection": "./src/connection.mjs",
     "./job-id": "./src/job-id.mjs",
     "./forges": "./src/forges.mjs",
-    "./triggers": "./src/triggers.mjs",
+    "./backend-local": "./src/backend-local.mjs",
+		"./backends": "./src/backends.mjs",
+		"./backend-registry": "./src/backend-registry.mjs",
+		"./container-spec": "./src/container-spec.mjs",
+		"./backend-conformance": "./src/backend-conformance.mjs",
+		"./triggers": "./src/triggers.mjs",
     "./triggers-file": "./src/triggers-file.mjs",
     "./packages": "./src/packages.mjs",
     "./pause-windows": "./src/pause-windows.mjs",

package/src/backend-conformance.mjs

@@ -0,0 +1,262 @@
+/**
+ * THE CONFORMANCE SUITE a backend runs against itself (issue #227).
+ *
+ * Every earlier slice of this issue ends with the same sentence: the table's words are a CONTRACT and not a
+ * FINDING, because nothing verifies a declaration against behaviour. `backends.mjs` says so in its own
+ * header, `backend-local.mjs` says its completeness check "proves ARITY AND NOTHING ELSE", and
+ * `container-spec.mjs` says the transfer abstraction is an adapter-facing contract awaiting a consumer.
+ * This module is that consumer. It is what turns `enforced` from a word into something that can be wrong.
+ *
+ * IT SHIPS IN `src/`, NOT IN `test/`, and that is the whole point. A suite that lived beside this repo's own
+ * tests could only ever check this repo's own backend. An adapter is written elsewhere, by someone who will
+ * never run `npm test` here, and the question they need answered is "does MY backend hold the contract" --
+ * so the checks have to be importable. `docs/backends.md` is the page that tells them to import it.
+ *
+ * WHAT IT CAN AND CANNOT DO, stated at its true size. It verifies the SHAPE of a bundle, the INTERNAL
+ * CONSISTENCY of its declaration, and the behaviour of whatever the caller's PROBES produce. It cannot
+ * start a real container, reach a real daemon, or prove a kernel enforced anything.
+ *
+ * THE PROBES ARE THE ADAPTER'S OWN CODE, and that is a real limit rather than a detail. How you make a
+ * container exit 2, or make an enumeration fail, is the runtime's business and cannot be written
+ * generically -- so `checkExitCodes` and `checkAbortFlag` verify what your probe REPORTS, and a probe that
+ * fabricates its answer instead of routing through your `runContainer` will pass while proving nothing.
+ * The suite cannot detect that, which is why it is written here rather than left for a reader to discover.
+ * `docs/backends.md` says the same thing where an adapter author will actually read it.
+ *
+ * `OQ-012`'s line applies to the harness itself: a check that proves intent is not a check that proves
+ * conformance, and the difference has to be stated rather than blurred.
+ */
+
+import { BACKENDS, PROPERTY_NAMES, isDeclaration } from "./backends.mjs";
+import { containerSpec, copyDowngrades } from "./container-spec.mjs";
+
+/** A finding. `ok: false` is a contract violation; `unverifiable` is a property this harness cannot reach. */
+const pass = (check, detail) => ({ check, ok: true, detail });
+const fail = (check, detail) => ({ check, ok: false, detail });
+const abstain = (check, detail) => ({ check, ok: true, unverifiable: true, detail });
+
+/**
+ * EXIT-CODE FIDELITY. The runner's integer must reach the processor unmodified.
+ *
+ * `INT-RUNNER-EXIT-CODE-PROTOCOL` rests on this and so does every retry decision: 0 is completed, 1 is
+ * infra (retry), 2 is policy (no retry). A backend that clamped, remapped or swallowed a code would turn a
+ * policy refusal into a paid retry loop, or a real failure into a clean success. The zero case matters most
+ * -- a backend that returns a falsy code as `null` reads as "unknown exit" downstream.
+ */
+async function checkExitCodes(backend, { probe }) {
+	const out = [];
+	for (const code of [0, 1, 2, 3, 137, 255]) {
+		const got = await probe(backend, { exitCode: code });
+		if (got?.code !== code) {
+			out.push(fail("exitCodes", `a runner exit of ${code} arrived as ${JSON.stringify(got?.code)}; every retry decision reads this integer`));
+			return out;
+		}
+	}
+	out.push(pass("exitCodes", "0, 1, 2, 3, 137 and 255 all arrive unmodified"));
+	return out;
+}
+
+/**
+ * THE ABORT FLAG must be distinguishable from the exit code.
+ *
+ * `INT-RUNNER-EXIT-CODE-PROTOCOL`: a worker SIGKILL and a kernel OOM BOTH surface as 137, so the code alone
+ * cannot say which happened. The processor classifies an abort as POLICY (no retry) and an OOM as INFRA
+ * (retry), so a backend that reports only the integer makes a hung job retry forever and an OOM look like a
+ * deliberate stop. This is the single most transferable clause in the contract and the easiest to drop.
+ */
+async function checkAbortFlag(backend, { probe }) {
+	const stopped = await probe(backend, { exitCode: 137, aborted: true });
+	const oomed = await probe(backend, { exitCode: 137, aborted: false });
+	if (stopped?.aborted !== true) return [fail("abortable", "a container stopped by the worker did not report aborted: true; 137 alone cannot be told from a kernel OOM")];
+	if (oomed?.aborted === true) return [fail("abortable", "a container that exited 137 on its own reported aborted: true; an OOM would be classified as a deliberate stop and never retried")];
+	return [pass("abortable", "the abort flag is reported independently of the exit code")];
+}
+
+/**
+ * THE REAPER'S TRI-STATE. `{reaped: true}` means "this host has ESTABLISHED it holds no job containers".
+ *
+ * `makeScopeClaimSweeper` gates a money decision on it: it may only delete a scope claim naming this host
+ * once the host has proven it holds nothing. A backend whose reap returns true after a FAILED enumeration
+ * frees slots for containers that may still be running and lets another host start more alongside them --
+ * a spend overrun rather than a tidy-up. Returning `[]` on a failed listing is the specific shape that gets
+ * this wrong, because an empty list and an unanswerable question look identical.
+ */
+async function checkReaperTriState(backend, { withBrokenEnumeration }) {
+	if (typeof backend?.reap !== "function") return []; // the shape check already said so; do not say it twice
+	if (typeof withBrokenEnumeration !== "function") {
+		return [abstain("reap", "no `withBrokenEnumeration` probe supplied, so a failed enumeration could not be simulated")];
+	}
+	const broken = await withBrokenEnumeration(backend);
+	if (broken?.reaped !== false) {
+		return [fail("reap", `a reap whose enumeration failed returned ${JSON.stringify(broken)}; it must be {reaped:false}, or a scope sweep frees slots for containers that may still be running`)];
+	}
+	const clean = await backend.reap();
+	if (clean?.reaped !== true) return [fail("reap", `a successful reap returned ${JSON.stringify(clean)}; it must be {reaped:true} or no host can ever sweep its own stale claims`)];
+	return [pass("reap", "the tri-state distinguishes a failed enumeration from an empty one")];
+}
+
+/**
+ * THE BUNDLE'S SHAPE, and the name it answers to.
+ *
+ * Cheap, and it catches the mistake an adapter author actually makes: a bundle whose `name` does not match
+ * the table entry it declares against. The registry keys on `name` and the table keys on the same string,
+ * so a mismatch silently gives the adapter a different backend's declaration.
+ */
+function checkShape(backend) {
+	const out = [];
+	// `containerName` is here because the REGISTRY calls it -- `registry.containerName(job)` builds the
+	// name the abort then stops. A bundle without it built at boot and threw at the first pickup, which is
+	// exactly the property the registry claims to have.
+	for (const fn of ["runContainer", "imagePreflight", "egressPreflight", "stopContainer", "reap", "containerName"]) {
+		if (typeof backend?.[fn] !== "function") out.push(fail("shape", `the bundle has no ${fn}()`));
+	}
+	if (!Array.isArray(backend?.neverStartedExits)) {
+		out.push(fail("shape", "neverStartedExits must be an array -- [] if this runtime normalises to container-never-started itself"));
+	} else {
+		// The set must not CLAIM a code the protocol already means. 137 is the OOM/SIGKILL code, so claiming
+		// it turns every kernel OOM into a refunded "never started" -- the container ran, spent its slot, and
+		// the refund hands the slot back. 0/1/2 are the runner's own completed/infra/policy codes, so claiming
+		// one of those converts a real outcome into an infra retry that can never resolve. Both are money.
+		const reserved = backend.neverStartedExits.filter((c) => [0, 1, 2, 137].includes(c));
+		if (reserved.length > 0) {
+			out.push(fail("shape", `neverStartedExits claims ${reserved.join(", ")}, which INT-RUNNER-EXIT-CODE-PROTOCOL already means (0 completed, 1 infra, 2 policy, 137 OOM/SIGKILL); claiming one refunds a slot the container actually spent`));
+		}
+	}
+	const entry = BACKENDS[backend?.name];
+	if (!entry) out.push(fail("shape", `the bundle's name ${JSON.stringify(backend?.name)} has no entry in the backend table, so nothing declares what it guarantees`));
+	else {
+		// BY VALUE, not by reference. An adapter written outside this repo cannot hold the table's own frozen
+		// object, and refusing a faithful copy made `checkDeclaration` below unreachable -- every bundle that
+		// passed shape held the table's own words, so the declaration gate could only ever restate a shape
+		// failure. What matters is that the words AGREE, since disagreement is what lets an adapter say one
+		// thing while doctor prints another.
+		const drift = PROPERTY_NAMES.filter((p) => backend.declares?.[p] !== entry.declares[p]);
+		if (drift.length > 0) out.push(fail("shape", `the bundle declares ${drift.join(", ")} differently from the table, so doctor would print one answer and the adapter believe another`));
+	}
+	return out.length > 0 ? out : [pass("shape", "six functions, a declared exit set, and a name the table knows")];
+}
+
+/**
+ * THE DECLARATION ITSELF, against the closed list and the three words.
+ *
+ * A backend that omits a property is not admitted for it (`backends.mjs`'s rule), and a word outside the
+ * vocabulary ranks with `absent`, so a typo would silently downgrade rather than fail.
+ */
+function checkDeclaration(backend) {
+	const declares = backend?.declares ?? {};
+	const out = [];
+	for (const property of PROPERTY_NAMES) {
+		if (!isDeclaration(declares[property])) out.push(fail("declaration", `${property} is declared ${JSON.stringify(declares[property])}, which is not one of enforced/asserted/absent`));
+	}
+	for (const property of Object.keys(declares)) {
+		if (!PROPERTY_NAMES.includes(property)) out.push(fail("declaration", `${property} is not a property of the closed list, so nothing reads it`));
+	}
+	// An `asserted` word with no source is not actionable: it says the worker is not providing the property
+	// without saying who is, so an operator cannot go and check.
+	for (const property of PROPERTY_NAMES) {
+		if (declares[property] === "asserted" && !BACKENDS[backend?.name]?.asserts?.[property]) {
+			out.push(fail("declaration", `${property} is asserted but nothing names WHO asserts it; "not us" without "them" leaves an operator nothing to check`));
+		}
+	}
+	return out.length > 0 ? out : [pass("declaration", "every property carries one of the three words, and every asserted one names its source")];
+}
+
+/**
+ * THE TRANSFER CONTRACT, and the check that gives it teeth.
+ *
+ * This is the consumer `container-spec.mjs` was written for. A backend that COPIES rather than bind-mounts
+ * loses the kernel's enforcement of `/job`'s read-only mount -- `DES-JOB-FILES-VIA-VOLUME-SUBPATH` says
+ * `docker cp` "cannot give /job a kernel-enforced read-only mount, which INT-CONTAINER-JOB-INPUTS depends
+ * on" -- so a backend declaring `binds: false` and `readOnlyJobInputs: enforced` is making exactly the
+ * claim `CONST-EGRESS-POLICY-IN-THE-ARGV` calls worse than no claim at all.
+ */
+function checkTransfers(backend) {
+	const declares = backend?.declares ?? {};
+	// `binds` is the adapter's own statement about how it moves files. Absent means "bind-mounts", which is
+	// what the local backend does and what every reader assumed before this field existed.
+	// FAIL-CLOSED on an unstated `binds`. An earlier draft treated "not false" as "binds", so a copying
+	// adapter that never set the field got a positive assertion about a property nothing had examined --
+	// the believed-in control, produced by the harness written to prevent it. A backend that does not say
+	// how it moves files has not earned the word, and this abstains rather than failing because not saying
+	// is a documentation gap rather than a false claim.
+	if (backend?.binds === undefined) {
+		return [abstain("readOnlyJobInputs", "this backend does not declare `binds`, so whether /job's read-only is the kernel's or a convention could not be determined; set binds: true (bind-mounts) or false (copies)")];
+	}
+	if (backend.binds === true) {
+		return [pass("readOnlyJobInputs", "this backend bind-mounts, so /job's read-only is the kernel's")];
+	}
+	const spec = containerSpec({ image: "i", name: "n", jobDir: "/j", workspace: "/w" });
+	const lost = copyDowngrades(spec).map((d) => d.container);
+	if (lost.length > 0 && declares.readOnlyJobInputs === "enforced") {
+		return [fail("readOnlyJobInputs", `this backend copies, so ${lost.join(", ")} is read-only by convention rather than by the kernel -- declaring it enforced is a control an operator would believe in and not have`)];
+	}
+	return [pass("readOnlyJobInputs", `this backend copies and declares ${JSON.stringify(declares.readOnlyJobInputs)}, which matches what a copy can hold`)];
+}
+
+/**
+ * Run every check against one backend bundle.
+ *
+ * `probe` is the one thing an adapter must supply: `(backend, { exitCode, aborted }) => result`, arranging
+ * for the backend's own `runContainer` to produce a container that exits that way. It cannot be written
+ * generically -- how you make a container exit 2 is the runtime's business -- and it is the reason this is a
+ * function taking probes rather than a fixed suite.
+ *
+ * Returns `{ ok, findings }`. `ok` is false if ANY check failed; a check that abstained does not fail the
+ * run but is reported, because a property nobody could verify is not a property that was verified.
+ */
+export async function runBackendConformance(backend, probes = {}) {
+	try {
+		return await conformance(backend, probes);
+	} catch (error) {
+		// A backend whose own function THREW is a conformance failure, not a harness crash. An adapter
+		// author running this for the first time is the person most likely to hit it, and they need the
+		// message rather than a stack trace from someone else's code.
+		return { ok: false, findings: [fail("harness", `a backend function threw while being driven: ${error?.message ?? error}`)] };
+	}
+}
+
+async function conformance(backend, probes) {
+	const findings = [
+		...checkShape(backend),
+		...checkDeclaration(backend),
+		...checkTransfers(backend),
+	];
+	// A bundle that is missing a function cannot be DRIVEN, and a harness that threw here would be useless
+	// in exactly the case it exists for: an adapter author's first run, against a bundle they have not
+	// finished. The shape findings above already name what is missing.
+	const drivable = ["runContainer", "stopContainer", "reap"].every((fn) => typeof backend?.[fn] === "function");
+	if (typeof probes.probe === "function" && drivable) {
+		findings.push(...(await checkExitCodes(backend, probes)));
+		findings.push(...(await checkAbortFlag(backend, probes)));
+	} else if (!drivable) {
+		findings.push(abstain("exitCodes", "the bundle is incomplete, so nothing could be driven to an exit"));
+		findings.push(abstain("abortable", "the bundle is incomplete, so an abort could not be told from an OOM"));
+	} else {
+		findings.push(abstain("exitCodes", "no `probe` supplied, so no container was driven to an exit"));
+		findings.push(abstain("abortable", "no `probe` supplied, so an abort could not be told from an OOM"));
+	}
+	findings.push(...(await checkReaperTriState(backend, probes)));
+	return { ok: findings.every((f) => f.ok), findings };
+}
+
+/**
+ * The properties this harness DOES NOT verify, and why -- printed alongside the findings so a green run is
+ * never mistaken for a conformant backend.
+ *
+ * Every one of these needs a live container on the target runtime, which is exactly what this repo's own
+ * offline CI cannot have. Naming them is the difference between a suite that is honest about its reach and
+ * one that lets a green tick stand for something it never checked. `verify-image.sh` is the shape the
+ * missing half would take, and it "runs ON THE HOST THAT HOLDS THE IMAGE, which is the only place it can".
+ */
+export const UNVERIFIED_BY_THIS_HARNESS = Object.freeze({
+	isolation: "needs a live container: read the capability set and no-new-privileges back from inside it",
+	ephemeral: "needs two runs of the same job id and a check that no container survived the first",
+	mountSet: "needs a live container: enumerate its mounts and assert nothing beyond the declared set",
+	egress: "needs a live container and a blocked destination",
+	jobToJobIsolation: "needs two live containers and an attempted connection between them",
+	imagePinning: "needs a run against an image absent from the target runtime",
+	nonRoot: "needs `id -u` inside a live container (`verify-image.sh` is the shape)",
+	secretsCustody: "needs a canary secret and an audit of every log, record and forge comment the run produced",
+	credentialTransit: "needs to observe what actually crossed the network to the runtime",
+	localFolders: "needs a bind-mounted host folder and a write read back outside the container",
+});

package/src/backend-local.mjs

@@ -0,0 +1,222 @@
+/**
+ * THE `local` BACKEND: the Docker daemon on this worker's own host (issue #227).
+ *
+ * `backends.mjs` says WHAT each backend guarantees; this module is the first thing that has to be true.
+ * It bundles the functions that actually build and run a container into one value, so that "a backend" is
+ * a thing the worker holds rather than a shape spread across four `deps` keys nobody names together.
+ *
+ * WHY THE TABLE DOES NOT HOLD A `make()`. The obvious design is one entry per backend carrying its own
+ * factory, and it cannot work here: `backends.mjs` imports NOTHING on purpose, because `doctor`, the config
+ * loader and the receiver all have to read a declaration without pulling the Docker implementation into
+ * their graph. A `make()` in the table is an import edge from the leaf to every adapter, which is the leaf
+ * property gone. So the table declares and this module implements, and the two are joined by NAME -- the
+ * same split `forges.mjs` uses against the forge hosts.
+ *
+ * FIVE FUNCTIONS, and the last two arrived late on purpose. `stopContainer` was a one-line literal inside
+ * `index.mjs`'s `createWorker` and unreachable from `startWorker`; `reap` lived in `start.mjs` and returns a
+ * TRI-STATE that `makeScopeClaimSweeper` gates a money decision on. Neither could move without its reasoning
+ * moving too, so an earlier slice declared `abortable` in the table, named both as deferred, and REFUSED a
+ * bundle that tried to supply them -- because an adapter author who passes `stopContainer` and has it
+ * silently dropped believes a runaway job can be stopped through their backend when the abort path still
+ * calls docker directly. That refusal is now gone because the seam is real.
+ */
+
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { BACKENDS, DEFAULT_BACKEND, DOCKER_NEVER_STARTED_EXITS } from "./backends.mjs";
+
+const execDocker = promisify(execFile);
+
+/**
+ * `pi-job-` -- the container-name namespace, and a LOAD-BEARING string rather than a prefix chosen for
+ * readability. TWO sweeps match it as a SUBSTRING, both at boot in `start.mjs`: the container reaper's
+ * `docker ps` filter and the network reaper's `docker network ls` filter. The sandbox tooling is the
+ * counterpart rather than a third sweep -- it names itself `pi-sandbox-` precisely to stay OUTSIDE this
+ * namespace, so a worker restart cannot tear down the shell an operator is sitting in, and it reaps its own
+ * by job id rather than by name.
+ *
+ * Exported and imported rather than re-typed at each site for the reason `CONTAINER_GLOBAL_PI_DIR` is: the
+ * namespace and the filters that sweep it are ONE fact, and two literals in two modules is how a rename
+ * lands in the producer and not in the reaper, leaving every crashed worker's containers behind forever with
+ * both test suites green.
+ */
+export const JOB_NAME_PREFIX = "pi-job-";
+
+/**
+ * `pi-job-<jobId>`. The name a running job answers to, for `docker stop` on the 30-minute timeout, for the
+ * per-job egress network derived from it, and for the reaper's filter.
+ *
+ * Not sanitised here: BullMQ ids are already `[A-Za-z0-9._-]`, and the one place a job id comes from
+ * anywhere else (a sandbox) goes through `sanitizeJobId` under its own prefix.
+ */
+export function jobContainerName(jobId) {
+	return `${JOB_NAME_PREFIX}${jobId}`;
+}
+
+/**
+ * Bundle the local backend's already-built functions into one checked value.
+ *
+ * Takes them BUILT rather than building them from config, because each is constructed in `start.mjs` from a
+ * different slice of the deployment and behind its own injectable factory that the wiring tests drive. This
+ * function's job is not to own that construction; it is to be the one place that says these functions
+ * together are a backend, and to REFUSE a bundle that is missing one.
+ *
+ * The refusal is worth having and worth not overselling. It catches a bundle assembled with a key MISSING or
+ * not callable, which is a wiring mistake that would otherwise surface as an unhelpful `undefined is not a
+ * function` deep inside a paid job. It proves ARITY AND NOTHING ELSE: `makeEgressPreflight({ armed: false })`
+ * returns a function that answers `{ ok: true }` and spawns nothing, so a bundle can pass this check with a
+ * gate that does no gating. `backend-conformance.mjs` does not close that either: it never invokes
+ * `imagePreflight` or `egressPreflight`, so a stubbed gate remains exactly what neither check can see. What
+ * the harness does verify is listed in its own header, and what it cannot is listed beside it.
+ */
+/** The functions a bundle carries today. Deferred members are refused BY NAME below, never ignored. */
+export const BACKEND_FUNCTIONS = ["runContainer", "imagePreflight", "egressPreflight", "stopContainer", "reap"];
+
+/**
+ * Members `makeLocalBackend` SETS rather than takes. An adapter written elsewhere supplies them itself; the
+ * local bundle knows its own, so passing them here is refused as an unknown member like any other.
+ */
+export const BACKEND_PROVIDED = ["name", "declares", "namePrefix", "containerName", "neverStartedExits", "binds"];
+
+/**
+ * Non-function members every bundle must also carry. Separate from the list above because the completeness
+ * check tests callability, and these are values -- but they are just as required: `neverStartedExits` gates
+ * a budget REFUND, so a bundle that omitted it would keep the slot and let BullMQ retry, burning a second
+ * one per never-started job. That is the exact bug the explicit `case 125/126/127` was added to fix,
+ * reachable again by an adapter simply not setting a property.
+ */
+export const BACKEND_VALUES = ["neverStartedExits"];
+
+export function makeLocalBackend(parts = {}) {
+	const { runContainer, imagePreflight, egressPreflight, stopContainer, reap } = parts ?? {};
+	const missing = Object.entries({ runContainer, imagePreflight, egressPreflight, stopContainer, reap })
+		.filter(([, fn]) => typeof fn !== "function")
+		.map(([k]) => k);
+	if (missing.length > 0) {
+		throw new Error(`backend "${DEFAULT_BACKEND}": cannot build a backend missing ${missing.join(", ")}`);
+	}
+
+	// An unknown key is REFUSED rather than dropped, and the two this slice defers are named as deferred.
+	// An adapter author who supplies `stopContainer` has read the issue and reasonably expects it to be
+	// wired; silently ignoring it would leave them believing a runaway job can be stopped through their
+	// backend while the abort path still goes straight to the local docker CLI. That is the believed-in
+	// control again, arriving through a dropped argument.
+	for (const key of Object.keys(parts ?? {})) {
+		if (BACKEND_FUNCTIONS.includes(key) || BACKEND_VALUES.includes(key)) continue;
+		throw new Error(`backend "${DEFAULT_BACKEND}": unknown bundle member ${JSON.stringify(key)} (this factory takes ${BACKEND_FUNCTIONS.join(", ")} and sets ${BACKEND_PROVIDED.join(", ")} itself)`);
+	}
+
+	return {
+		name: DEFAULT_BACKEND,
+		// The declaration is READ from the table, never re-typed here. An adapter that stated its own
+		// guarantees inline could drift from what `doctor` prints and what the boot refusal checks, and an
+		// operator would then be told one thing by the thing that decides and another by the thing that ran.
+		// This is the SAME object the table holds, and that is safe only because the table is deeply FROZEN:
+		// an unfrozen alias would let any holder of a bundle rewrite what `doctor`, the boot refusal and the
+		// receiver are all told about this backend, process-wide and invisibly, while the source still read
+		// `enforced`. A defensive copy would hide such a mutation rather than prevent it.
+		declares: BACKENDS[DEFAULT_BACKEND].declares,
+		namePrefix: JOB_NAME_PREFIX,
+		containerName: jobContainerName,
+		runContainer,
+		imagePreflight,
+		egressPreflight,
+		stopContainer,
+		reap,
+		// The integers this runtime uses for "the runner never ran". The processor asks the BACKEND rather
+		// than assuming docker's triple, because those numbers collide with the runner's own exit channel.
+		neverStartedExits: LOCAL_NEVER_STARTED_EXITS,
+		// This runtime BIND-MOUNTS, so `/job`'s read-only is the kernel's. The conformance harness abstains
+		// rather than passing when a bundle does not say, because a copy downgrades that to a convention.
+		binds: true,
+	};
+}
+
+/**
+ * The exit codes that mean THE RUNNER NEVER RAN, as this runtime spells them.
+ *
+ * Docker's own convention, and defined in `backends.mjs` rather than here so the processor can default to
+ * it without importing this module. Re-exported under the local backend's name because that is what the
+ * bundle carries and what an adapter author reads.
+ */
+export const LOCAL_NEVER_STARTED_EXITS = DOCKER_NEVER_STARTED_EXITS;
+
+/**
+ * `docker stop` on the job's container name, fired by the abort (the 30-minute timeout or a shutdown).
+ *
+ * MOVED HERE from `index.mjs`'s `createWorker`, where it was a one-line literal inside the processor's
+ * construction and could not be reached from `startWorker` at all. That is why `abortable` was declared in
+ * the table two slices before this function existed: a second backend could have passed every other check
+ * with no way to stop a runaway container, and nothing in the table would have moved.
+ *
+ * `-t 5` is SIGTERM then SIGKILL after five seconds. The runner exits, `docker run` returns, and
+ * `runContainer`'s promise resolves -- so the abort's effect reaches the processor through the container's
+ * own exit rather than through this call's return value, which is why nothing awaits it.
+ *
+ * `INT-RUNNER-EXIT-CODE-PROTOCOL` is what makes this transferable: the discriminator is the abort FLAG the
+ * processor already holds, not the exit code, because a worker SIGKILL and a kernel OOM both surface as
+ * 137. An adapter implements "stop this job" however its runtime spells it and the classification is
+ * unchanged.
+ */
+export function makeStopContainer({ exec = execDocker } = {}) {
+	return async function stopContainer(name) {
+		return exec("docker", ["stop", "-t", "5", name]);
+	};
+}
+
+/**
+ * Boot-time reaper: clear stray `pi-job-*` containers a previous worker crash left behind.
+ *
+ * MOVED HERE from `start.mjs` (issue #227). It belongs to the backend because the containers it sweeps are
+ * that backend's, and a second backend's crashed containers are unreachable by this one's `docker ps`.
+ *
+ * THE TRI-STATE IS THE POINT and moved with it: `{ reaped: true }` means this host has ESTABLISHED that it
+ * holds no job containers, `{ reaped: false }` means it could not establish that. `makeScopeClaimSweeper`
+ * gates a money decision on the difference -- it may only delete a scope claim naming this host once the
+ * host has proven it holds nothing -- so returning `[]` or `true` on a failed enumeration would free slots
+ * for containers that may still be running and let another host start more alongside them. That is a spend
+ * overrun rather than a tidy-up, which is why the catch below returns false rather than swallowing.
+ */
+export function makeReaper({ log, exec = execDocker }) {
+	return async function reap() {
+		try {
+			const { stdout } = await exec("docker", ["ps", "--filter", `name=${JOB_NAME_PREFIX}`, "--format", "{{.Names}}"]);
+			const names = stdout
+				.split("\n")
+				.map((n) => n.trim())
+				.filter(Boolean);
+			for (const name of names) {
+				await exec("docker", ["rm", "-f", name]);
+				log("reaped_container", { name });
+			}
+			// REQ-EGRESS-ALLOWLIST: the per-job networks those containers were on. Swept AFTER the containers,
+			// because a network with a member still attached cannot be removed -- and swept by the SAME
+			// `pi-job-` filter, so the namespace rule that keeps an operator's live sandbox safe from the
+			// container reaper keeps their sandbox NETWORK safe too, with no second rule to remember.
+			//
+			// A crashed worker is the case this exists for: `runContainer`'s own finally removes the network
+			// on every ordinary path, so anything still here outlived a process that did not get to run it.
+			// A network still in use by something else fails to remove and is skipped, which is correct: this
+			// is a best-effort sweep and never a reason not to boot.
+			const { stdout: nets } = await exec("docker", ["network", "ls", "--filter", `name=${JOB_NAME_PREFIX}`, "--format", "{{.Name}}"]);
+			for (const net of nets.split("\n").map((n) => n.trim()).filter(Boolean)) {
+				try {
+					await exec("docker", ["network", "rm", net]);
+					log("reaped_network", { network: net });
+				} catch {} // still in use, or already gone -- either way not this boot's problem
+			}
+			// Whether the enumeration HAPPENED, which the scope-claim sweep depends on: it may only delete a
+			// claim naming this host once this host has actually established that it holds no containers.
+			return { reaped: true };
+		} catch (err) {
+			// The `docker ps` is inside this try, so this path CANNOT establish that this host holds no
+			// containers -- whether it failed before listing anything or after reaping some and then losing
+			// the daemon. Either way the claim "I hold nothing" is unproven, and sweeping on it would free
+			// slots for containers that may STILL BE RUNNING, letting another host start more alongside
+			// them: a money overrun rather than a tidy-up. Conservative in the only safe direction.
+			log("reaper_skipped", { reason: err?.message });
+			return { reaped: false };
+		}
+	};
+}
+