@edgehero/pi-dispatch 1.10.3 → 2.0.0

package/src/backends.mjs

@@ -2,9 +2,10 @@
  * THE BACKEND TABLE -- the one place that says where a job's container can run, and what each place is
  * ABLE to guarantee (issue #227).
  *
- * A backend is where the box is built. Today there is one, `local`, which is the Docker daemon on the
- * worker's own host and is what every deployment has always used. The table exists so a second one can be
- * added without reading the worker's source, and so that adding one cannot quietly weaken a control.
+ * A backend is where the box is built. `local` is the Docker daemon on the worker's own host and is what
+ * every deployment has always used; since issue #354 `podman` is the worker account's own rootless Podman.
+ * The table exists so another can be added without reading the worker's source, and so that adding one
+ * cannot quietly weaken a control.
  *
  * This module imports NOTHING, deliberately, for `forges.mjs`'s reason: it is a leaf, and `doctor`, the
  * config loader and (later) the receiver all need to read a declaration without pulling the Docker
@@ -62,10 +63,12 @@
  * and the same is true one level out. The value of the table is not that a vendor is verified; it is that a
  * MISMATCH becomes a refusal instead of a silent downgrade. `backend-conformance.mjs` verifies THREE of the
  * thirteen -- exit-code fidelity, the abort flag's independence from the code, and the read-only downgrade a
- * copying runtime takes -- plus the shape of a bundle and the internal consistency of its declaration. The
- * other ten need a live container on the target runtime, and the harness names each one and what it would
- * take rather than passing them in silence. So a backend can still declare all of this and do most of it:
- * these words are a contract with three of them checked, which is more than none and less than proof.
+ * copying runtime takes -- plus the shape of a bundle and the internal consistency of its declaration. EIGHT more
+ * are read back off a live container (issues #278 and #344): for `local` by `doctor --live` (`live-probes.mjs`), for
+ * any other runtime through the harness's `readBack` probe, which the adapter supplies. The remaining TWO the harness
+ * names with what each would take rather than passing them in silence. So a backend can still declare all of this
+ * and do most of it: these words are a contract with three of them checked and eight readable, which is more than
+ * none and less than proof -- the read-back is of a fixture with the job image, not of every job.
  *
  * THE PROPERTIES ARE NOT A RENDERING OF `ISOLATION_FLAGS`, and must not become one. That array is the
  * literal, value-free, unconditional set the local argv splices in, and two tests assert every member of it
@@ -78,8 +81,10 @@
 /**
  * The exit codes a DOCKER-shaped runtime uses for "the runner never ran": 125 is `docker run` itself
  * failing, 126 an entrypoint that is not executable, 127 an entrypoint that was not found. In all three the
- * daemon never handed control to the runner, so nothing was spent -- which is what `container-never-started`
- * means, and why they refund the budget slot instead of keeping it.
+ * daemon normally never handed control to the runner, so nothing was spent -- which is what `container-never-started`
+ * means, and why they refund the budget slot instead of keeping it. NOT always (issue #345, measured): a CLI that lost
+ * the daemon's API mid-run exits 125 while its container runs on, which `run-container.mjs` finds through the cidfile
+ * and reports as `detached`, unrefunded.
  *
  * HERE, in the leaf, rather than in `backend-local.mjs`, so `processor.mjs` can default to them without
  * importing the local adapter and dragging `node:child_process` and the docker CLI machinery into its
@@ -263,14 +268,301 @@ export function declarationOf(name, property) {
 		// Only meaningful for an asserted word, and null otherwise rather than absent, so a consumer that
 		// prints it unconditionally renders nothing rather than "undefined".
 		assertedBy: word === ASSERTED ? (entry.asserts?.[property] ?? null) : null,
+		// The observation this word holds only while (#278), or null. A consumer that prints the word without it
+		// would print `enforced` for a daemon that is elsewhere.
+		observedBy: entry.observedBy?.[property] ?? null,
 	};
 }
 
+/**
+ * The word a backend EARNS for a property given what is observed on this host (issue #278). The declared word
+ * when the property is not observation-gated or its observation is `true`; otherwise at most `asserted`, since
+ * the guarantee then rests on whoever arranged for the observation to be false. `undefined` observations get no
+ * credit, this file's standing polarity. `undefined` for an unknown backend or property.
+ */
+export function effectiveWord(name, property, observations = {}) {
+	const entry = backendFor(name);
+	if (!entry || !isProperty(property)) return undefined;
+	const word = entry.declares[property] ?? ABSENT;
+	const observation = entry.observedBy?.[property];
+	if (!observation || observations?.[observation] === true) return word;
+	return (RANK[word] ?? 0) > RANK[ASSERTED] ? ASSERTED : word;
+}
+
+/**
+ * Which floored properties a backend declares strongly enough but does not EARN given the observations, as
+ * `[{ backend, property, have, want, observedBy }]` -- the floor misses the observation alone causes. The
+ * capability misses stay `floorShortfall`'s, checked at config load; this is the half only a live read can
+ * answer, so it is checked where the read happens (boot, and before each job's spend).
+ */
+export function unobservedFloor(backends, floor, observations = {}) {
+	const out = [];
+	for (const backend of backends ?? []) {
+		const entry = backendFor(backend);
+		if (!entry) continue;
+		for (const [property, want] of Object.entries(floor ?? {})) {
+			if (!isProperty(property) || !isDeclaration(want) || want === ABSENT) continue;
+			const declared = entry.declares[property] ?? ABSENT;
+			const have = effectiveWord(backend, property, observations);
+			if (meets(declared, want) && !meets(have, want)) out.push({ backend, property, have, want, observedBy: entry.observedBy[property] });
+		}
+	}
+	return out;
+}
+
+/**
+ * The refusals `unobservedFloor` implies, as operator-facing messages; empty when the floor holds. Outside
+ * `backendRefusals` because `loadConfig` is synchronous and cannot observe anything; the caller that made the
+ * observation passes it, with `evidence` -- `{ [observation]: "what was seen" }` -- for the message.
+ */
+export function observationRefusals({ backends = [], backendFloor = {}, observations = {}, evidence = {}, only = null } = {}) {
+	// `only` narrows the check to the named observations, for a caller that has read just those so far (boot reads the
+	// endpoint before the daemon, and must not refuse on an observation it has not asked yet).
+	const misses = unobservedFloor(backends, backendFloor, observations).filter((m) => !only || only.includes(m.observedBy));
+	if (misses.length === 0) return [];
+	const lines = misses.map((m) => `  ${m.backend}: ${m.property}=${m.want} holds only while ${OBSERVATIONS[m.observedBy] ?? m.observedBy}; ${evidence[m.observedBy] ?? "that was not observed"}`);
+	// One remedy per observation that missed, in the closed list's order, so a bounds miss is never told to repoint the CLI.
+	const remedies = Object.keys(OBSERVATION_FIX).filter((o) => misses.some((m) => m.observedBy === o)).map((o) => OBSERVATION_FIX[o]);
+	return [`PI_BACKEND_FLOOR asks for something this host is not observed to provide:\n${lines.join("\n")}\n${remedies.join("\n")}`];
+}
+
+/**
+ * Whether the floor misses `observationRefusals` would report rest ONLY on observations that were not answered (`null`
+ * or `undefined`, as opposed to an answered `false`): a daemon still starting, a read that timed out. Such a refusal is
+ * transient, and the caller retries rather than refusing for good. `false` when nothing misses.
+ */
+export function observationRefusalIsTransient({ backends = [], backendFloor = {}, observations = {} } = {}) {
+	const missed = unobservedFloor(backends, backendFloor, observations).map((m) => m.observedBy);
+	return missed.length > 0 && missed.every((o) => typeof observations?.[o] !== "boolean");
+}
+
 /** Is `name` one of the closed list? `Object.hasOwn`, so `"toString"` is not a property. */
 export function isProperty(name) {
 	return Object.hasOwn(PROPERTIES, name);
 }
 
+/** The observation that the docker CLI on this host resolves an endpoint on this host (issue #278). */
+export const DOCKER_ENDPOINT_LOCAL = "dockerEndpointLocal";
+
+/**
+ * The observation that the daemon reports it applies a container's pid and memory bounds (issue #345), from the same
+ * `docker info` read the job user is decided from. `true` only for `PidsLimit` and `MemoryLimit` both true on a daemon
+ * that is neither rootless nor Podman: Podman's Docker API hard-codes `PidsLimit` and derives `MemoryLimit` from the root
+ * cgroup, so on Podman the booleans say nothing about a container (source; measured true on a rootless host applying
+ * neither). CPU is never read: rootful Podman reports `CpuCfsQuota: false` while applying `--cpus` (measured).
+ */
+export const DAEMON_APPLIES_BOUNDS = "daemonAppliesBounds";
+
+/**
+ * The observation that the container runtime mounts nothing of its own into a job container (issue #345). Docker adds
+ * none; rootful Podman mounts `/run/secrets` from its default `mounts.conf` into every container, invisible in
+ * `.Mounts` (measured: host subscription files readable by the job user). On Podman it holds only with the documented
+ * empty `/etc/containers/mounts.conf` override and no `volumes`/`mounts` key in any containers.conf this host reads
+ * (`runtime-observations.mjs`).
+ */
+export const RUNTIME_ADDS_NO_MOUNTS = "runtimeAddsNoMounts";
+
+/**
+ * The observation that a ROOTLESS Podman applies a job container's pid, memory and cpu bounds (issue #354; the facts it
+ * rests on corrected by issue #453's measurements). From the `podman info` read the venue's job user is decided from
+ * (rootless, cgroup v2, the cgroup manager Podman uses) and this host's files: the account's systemd user manager
+ * running with `pids`, `memory` and `cpu` delegated to it (its `user@<uid>.service` cgroup), and Podman able to put a
+ * container under it (the `systemd` cgroup manager, or the worker itself inside that manager). Measured on Podman
+ * 5.8.1, Fedora 44: without a reachable user manager a container lands in the caller's own cgroup and its bounds read
+ * back `max`, exit 0, and `--cgroups=disabled` does the same; so the word rests on those facts being OBSERVED, and a
+ * `cgroups` key in a containers.conf this user's Podman reads withholds it. `pi-dispatch doctor --live` reads
+ * `pids.max` and `memory.max` back off a container.
+ */
+export const PODMAN_BOUNDS_DELEGATED = "podmanBoundsDelegated";
+
+/**
+ * The observation that this worker account's rootless Podman adds no mounts of its own to a job container (issue #354):
+ * the mounts.conf that wins (the user's `~/.config/containers/mounts.conf` when it exists, which OVERRIDES
+ * `/etc/containers/mounts.conf`, measured) is empty, no containers.conf this user's Podman reads sets `volumes`,
+ * `mounts`, `devices` or `hooks_dir`, no OCI hook is installed and FIPS mode is off (`runtime-observations.mjs`,
+ * `observePodman`).
+ */
+export const PODMAN_ADDS_NO_MOUNTS = "podmanAddsNoMounts";
+
+/**
+ * The observation that the podman CLI this worker spawns runs its containers in-process on this host rather than through
+ * a remote service (issue #354): `podman info` reports `host.serviceIsRemote: false`. `CONTAINER_HOST`, `--remote` or a
+ * `containers.conf` service destination turn it `true` (measured), and a remote service is where the provider key and
+ * the per-job forge token would travel as `-e NAME=VALUE`.
+ */
+export const PODMAN_SERVICE_LOCAL = "podmanServiceLocal";
+
+/**
+ * The podman venue's refusal of an account containers.conf that widens every job (issue #428), and the run record's
+ * `reason` for a job it refuses. Here, in the leaf, rather than in backend-podman.mjs beside the check, so the processor
+ * can name it without importing the venue's module (which pulls in the run path and pi-ai).
+ */
+export const PODMAN_CONF_WIDENS_JOB = "podman-conf-widens-job";
+
+/**
+ * The containers.conf keys that refusal refuses on presence (issues #428, #450 and #448), in the order every text names
+ * them. The ONE list for the podman venue: backend-podman.mjs builds `WIDENING_KEY` and its refusal texts from it, and
+ * `podman-doc.test.mjs` requires every list of these keys in the specs, the docs and the source to be exactly this one,
+ * since three copies drifted when #450 added two. The rule for a key issue #448 measured: REFUSED when the job saw its
+ * value cross a boundary the worker's argv sets (what the job may reach, run, read or be limited by), or when it could
+ * not be measured; documented as INERT when the argv's own pins overrode it; documented as HARMLESS
+ * (`PODMAN_HARMLESS_KEYS`) when it reached the job but moved nothing that is a boundary. Measured with the podman venue's
+ * own argv on rootless Podman 5.8.1 (Fedora 44) and 4.9.3 (Ubuntu 24.04), each key alone in the account's own
+ * containers.conf, on `--network=private` and on an `--internal` network, one parenthesis each:
+ * `default_sysctls` (a sysctl set in the job, the vendor's own block excepted), `default_ulimits` (a ulimit set),
+ * `seccomp_profile` (the job ran under the named profile), `init_path` (the named binary as its PID 1), `dns_servers`
+ * (its nameserver, on `--network=private`), `dns_options` (its resolver options), `dns_searches` (its search list),
+ * `base_hosts_file` (its /etc/hosts), `oom_score_adj` (its OOM score), `privileged` (a full capability bounding set, no
+ * seccomp filter, the host's devices and an unconfined SELinux label, though `--cap-drop=ALL` still empties its
+ * effective set), and from gate round 1 of PR #473: `label` (false ran it as `spc_t` on Fedora), `cgroup_conf` (its
+ * pids.max became max past `--pids-limit`), `host_containers_internal_ip` (the address it reaches as
+ * host.containers.internal), `runtimes` (the `[engine.runtimes]` table: a wrapper runtime ran for every job),
+ * `conmon_path` (a wrapper conmon ran for every job), `cgroups` (disabled left its pids and memory bounds unapplied) and
+ * `umask` (its umask became the one named, so what it writes to the host is as open as that says).
+ */
+export const PODMAN_WIDENING_KEYS = Object.freeze([
+	"pasta_options",
+	"network_cmd_options",
+	"annotations",
+	"env",
+	"helper_binaries_dir",
+	"network_cmd_path",
+	"default_sysctls",
+	"default_ulimits",
+	"seccomp_profile",
+	"init_path",
+	"dns_servers",
+	"dns_options",
+	"dns_searches",
+	"base_hosts_file",
+	"oom_score_adj",
+	"privileged",
+	"label",
+	"cgroup_conf",
+	"host_containers_internal_ip",
+	"runtimes",
+	"conmon_path",
+	"cgroups",
+	"umask",
+]);
+
+/**
+ * The keys of `PODMAN_WIDENING_KEYS` that shape this account's shared rootless network helper (pasta or slirp4netns, the
+ * program behind it, and `env`, whose `[engine]` form can point Podman at another containers.conf that does): only these
+ * leave a running network carrying the setting after the key is gone, so only their remedy resets that network (issue
+ * #450). Every other refused key is applied per container, and the next job after its removal runs (gate round 1 of PR
+ * #473, which found the reset asked for every key).
+ */
+export const PODMAN_NETWORK_HELPER_KEYS = Object.freeze(["pasta_options", "network_cmd_options", "env", "helper_binaries_dir", "network_cmd_path"]);
+
+/**
+ * The keys MEASURED INERT for a podman venue job (issue #448), on rootless Podman 5.8.1 and 4.9.3 with the venue's own
+ * argv, one parenthesis each: `userns` (the argv's `--userns=keep-id`), `pidns` (its `--pid=private`), `ipcns` (its
+ * `--ipc=private`), `utsns` (its `--uts=private`), `cgroupns` (its `--cgroupns=private`), `netns` (its `--network`),
+ * `apparmor_profile` (rootless Podman applies no AppArmor profile: the job's label was `crun (unconfined)` with and
+ * without it on Ubuntu 24.04), `default_capabilities` (`--cap-drop=ALL`), `no_new_privileges` (`no-new-privileges`),
+ * `init` (`--init`), `pids_limit` (`--pids-limit`), `shm_size` (`--shm-size`), `env_host` (`--env-host=false`) and
+ * `http_proxy` (`--http-proxy=false`: a proxy in the account's environment did not reach the job). Documented, not
+ * refused.
+ */
+export const PODMAN_ROOTLESS_INERT_KEYS = Object.freeze(["userns", "pidns", "ipcns", "utsns", "cgroupns", "netns", "apparmor_profile", "default_capabilities", "no_new_privileges", "init", "pids_limit", "shm_size", "env_host", "http_proxy"]);
+
+/**
+ * The containers.conf keys the `local` venue refuses where rootful Podman's Docker API service on this host runs its jobs
+ * (issue #448), whatever the value, under `PODMAN_WIDENING_KEYS`' rule, each MEASURED with the job argv the worker
+ * builds, on the default network and on an `--internal` one, on Fedora 44 with rootful Podman 5.8.1 and on Ubuntu 24.04
+ * with 4.9.3 (`apparmor_profile` on Ubuntu alone, which runs AppArmor; `label` reached the job on Fedora alone, which runs
+ * SELinux). What each did, key by key, one parenthesis each so no two keys are read as a list:
+ *   `annotations` (the job got the service's own supplementary groups), `env` (a variable in every job),
+ *   `helper_binaries_dir` (the netavark and aardvark-dns Podman ran as root for the job's network were the named
+ *   directory's), `default_sysctls` (a sysctl set in the job), `default_ulimits` (a ulimit set in the job), `userns`
+ *   (with `auto` and no subordinate range the container could not be created), `pidns` (a host PID namespace, refused
+ *   at create against the argv's `--init`), `ipcns` (a host IPC namespace, refused at create against `--shm-size`),
+ *   `utsns` (the host's UTS namespace and hostname), `cgroupns` (the host's cgroup namespace), `netns` (the host's
+ *   network namespace, on the default network), `seccomp_profile` (the job ran under the named profile),
+ *   `apparmor_profile` (`unconfined` replaced the job's `containers-default` profile), `init_path` (the job's PID 1 was
+ *   the named binary), `dns_servers` (its resolv.conf nameserver, default network), `dns_options` (its resolv.conf
+ *   options), `dns_searches` (its resolv.conf search list), `base_hosts_file` (its /etc/hosts), `label` (false ran it as
+ *   `spc_t`), `cgroup_conf` (its pids.max became max), `host_containers_internal_ip` (the address of
+ *   host.containers.internal), `runtimes` (a wrapper runtime ran for every job), `conmon_path` (a wrapper conmon ran for
+ *   every job) and `cgroups` (disabled left its pids and memory bounds unapplied).
+ * The one value that is not refused is the vendor's own `default_sysctls = ["net.ipv4.ping_group_range=0 0"]`,
+ * uncommented in the stock containers.conf of Fedora 44 and Ubuntu 24.04 (`STOCK_CONF_BLOCKS` in
+ * runtime-observations.mjs). The rest are `PODMAN_ROOTFUL_INERT_KEYS` and `PODMAN_HARMLESS_KEYS`. The first three are in
+ * `PODMAN_WIDENING_KEYS`' order, and `podman-doc.test.mjs` holds every three-key list of these keys to exactly one of the
+ * derived lists.
+ */
+export const PODMAN_ROOTFUL_WIDENING_KEYS = Object.freeze([
+	"annotations",
+	"env",
+	"helper_binaries_dir",
+	"default_sysctls",
+	"default_ulimits",
+	"userns",
+	"pidns",
+	"ipcns",
+	"utsns",
+	"cgroupns",
+	"netns",
+	"seccomp_profile",
+	"apparmor_profile",
+	"init_path",
+	"dns_servers",
+	"dns_options",
+	"dns_searches",
+	"base_hosts_file",
+	"label",
+	"cgroup_conf",
+	"host_containers_internal_ip",
+	"runtimes",
+	"conmon_path",
+	"cgroups",
+]);
+
+/**
+ * The keys MEASURED INERT for a rootful local job (issue #448), on both hosts, one parenthesis each: `pasta_options`
+ * (rootful Podman runs no pasta for these networks; netavark sets them up), `network_cmd_options` (nor slirp4netns),
+ * `network_cmd_path` (no slirp4netns ran), `default_capabilities` (the argv's `--cap-drop=ALL`), `no_new_privileges`
+ * (the argv's `no-new-privileges`), `init` (the argv's `--init`), `oom_score_adj` (the compat API sends its own, on
+ * Fedora and on Ubuntu), `pids_limit` (the argv's `--pids-limit`), `shm_size` (the argv's `--shm-size`), `privileged`
+ * (the compat API sends its own, on both), `env_host` (the service's environment did not reach the job), `umask` (the
+ * job's umask stayed 0022) and `http_proxy` (a proxy in the service's environment reached no job, with the key absent,
+ * true or false): nothing the job saw changed. Documented, not refused.
+ */
+export const PODMAN_ROOTFUL_INERT_KEYS = Object.freeze(["pasta_options", "network_cmd_options", "network_cmd_path", "default_capabilities", "no_new_privileges", "init", "oom_score_adj", "pids_limit", "shm_size", "privileged", "env_host", "umask", "http_proxy"]);
+
+/**
+ * The keys MEASURED REACHING a job, on both venues and both hosts, that move nothing the worker's argv sets as a boundary
+ * (issue #448, gate round 1 of PR #473), so they are documented and not refused: `tz` (the job's clock read the named
+ * zone) and `no_hosts` (Podman wrote no /etc/hosts into the job, so it has less, not more). Refusing them would refuse
+ * a harmless preference, and a refusal that fires on harmless settings stops being read.
+ */
+export const PODMAN_HARMLESS_KEYS = Object.freeze(["tz", "no_hosts"]);
+
+/**
+ * The closed list of observations a backend's `observedBy` may name, each with what it means. Closed for the
+ * reason `PROPERTIES` is: a typo in a table entry must not become an observation nobody makes, which would
+ * degrade a word forever with nothing saying why.
+ */
+export const OBSERVATIONS = Object.freeze({
+	[DOCKER_ENDPOINT_LOCAL]: "the docker endpoint this host's docker CLI resolves is on this host",
+	[DAEMON_APPLIES_BOUNDS]: "the daemon reports that it applies a container's pid and memory bounds, and is neither rootless nor Podman",
+	[RUNTIME_ADDS_NO_MOUNTS]: "the container runtime adds no mounts of its own to a job container",
+	[PODMAN_BOUNDS_DELEGATED]: "this worker's rootless Podman runs on cgroup v2 with a systemd user manager running for its account, the pids, memory and cpu controllers delegated to that manager, and Podman putting containers under it, and no containers.conf it reads sets `cgroups`",
+	[PODMAN_ADDS_NO_MOUNTS]: "this worker's rootless Podman adds no mounts of its own to a job container",
+	[PODMAN_SERVICE_LOCAL]: "the podman CLI this worker spawns runs containers on this host, not through a remote service",
+});
+
+/** What to do about each observation that a floor needed and did not get, keyed like `OBSERVATIONS` and pinned to it. */
+export const OBSERVATION_FIX = Object.freeze({
+	[DOCKER_ENDPOINT_LOCAL]: "Point the docker CLI back at this host (DOCKER_HOST, DOCKER_CONTEXT or `docker context use`), or lower that entry to `asserted` if the redirect is deliberate.",
+	[DAEMON_APPLIES_BOUNDS]: "Run jobs on a rootful Docker Engine that reports PidsLimit and MemoryLimit, or lower that entry to `asserted`: on Podman and rootless daemons `pi-dispatch doctor --live` reads pids.max and memory.max off a real container instead.",
+	[RUNTIME_ADDS_NO_MOUNTS]: "On Podman, create an empty /etc/containers/mounts.conf and remove any `volumes` or `mounts` key from containers.conf (and write it in plain ASCII with no escaped key or multi-line string, which the check refuses rather than guesses at), then restart podman.service if it is running, since it keeps the containers.conf it started with, or lower that entry to `asserted`.",
+	[PODMAN_BOUNDS_DELEGATED]: "Turn on linger for the worker account (`loginctl enable-linger <account>`, which keeps its systemd user manager running with no one logged in) and run the worker as a systemd user service (`pi-dispatch service install`), or give Podman the account's user bus so it uses the systemd cgroup manager; delegate the cpu, memory and pids controllers to that manager on cgroup v2 (a systemd `Delegate=` drop-in for user@.service), remove any `cgroups` key from its containers.conf (written in plain ASCII with no escaped key or multi-line string, which the check refuses rather than guesses at), or lower that entry to `asserted`: `pi-dispatch doctor --live` reads pids.max and memory.max off a real container.",
+	[PODMAN_ADDS_NO_MOUNTS]: "As the worker account, create an empty ~/.config/containers/mounts.conf (or, when it has none, an empty /etc/containers/mounts.conf), remove any `volumes`, `mounts`, `devices` or `hooks_dir` key from every containers.conf it reads (and write those in plain ASCII with no escaped key or multi-line string, which the check refuses rather than guesses at) and any OCI hook, or lower that entry to `asserted`.",
+	[PODMAN_SERVICE_LOCAL]: "Unset CONTAINER_HOST and any containers.conf service destination for the worker account so `podman info` reports serviceIsRemote false, or lower that entry to `asserted`.",
+});
+
 const BACKENDS_TABLE = {
 	local: {
 		/**
@@ -285,15 +577,20 @@ const BACKENDS_TABLE = {
 			// against the imported array by two tests. `dockerArgsFromSpec` additionally refuses a
 			// `dockerExtra` carrying a flag that would supersede one of them -- membership in an argv is not
 			// effectiveness of that argv, and without that guard those two assertions would pass on an argv
-			// with no boundary left. It is a deny-list, so it NARROWS that gap rather than closing it; what
-			// closes it today is that the only production caller passes fixed literals.
+			// with no boundary left. The deny-list only NARROWED that gap; since issue #341 an allow-list of what
+			// the callers pass (`DOCKER_EXTRA_ALLOWED`) closes it, and every caller still passes fixed literals.
+			//
+			// AND ONLY WHILE OBSERVED (issue #345): the pid and memory bounds in those flags are the daemon's to apply, and a
+			// rootless daemon without cgroup delegation accepts `--pids-limit` and applies nothing (measured). So the word is
+			// earned by `daemonAppliesBounds` below, and degrades to `asserted` where the daemon is not observed applying them.
 			isolation: ENFORCED,
 			// `--rm` leads ISOLATION_FLAGS and the container name carries the job id, so no container is
 			// reachable to reuse even in principle.
 			ephemeral: ENFORCED,
 			// The mount list is built by `containerSpec` from a fixed set of named host paths and nothing
 			// else. There is no pass-through, and the shared session store under PI_SESSIONS_DIR is never
-			// among them -- only this job's own copy.
+			// among them -- only this job's own copy. AND ONLY WHILE OBSERVED (issue #345): a runtime can mount things of
+			// its own that no argv names (rootful Podman's `/run/secrets`), so the word is earned by `runtimeAddsNoMounts`.
 			mountSet: ENFORCED,
 			// CAPABILITY, gated by PI_EGRESS. When armed: `--network=pi-job-<id>-net`, `--internal`, created
 			// before the spawn and removed in a finally. When PI_EGRESS=0 the flag is absent and the job is
@@ -324,15 +621,23 @@ const BACKENDS_TABLE = {
 			// and `buildRecord` is an explicit literal with no spread. Where the values GO is
 			// `credentialTransit`'s question, not this one's.
 			secretsCustody: ENFORCED,
-			// ASSERTED, and this is the second honest one. The intent is that the daemon is on this host, so
-			// nothing leaves it -- but every spawn is `docker` with the worker's own environment inherited,
-			// and `DOCKER_HOST=tcp://...` or a `docker context` redirects that connection to another machine
-			// with the provider key and the per-job forge token riding along as `-e NAME=VALUE`. `DOCKER_HOST`
-			// appears NOWHERE in this repository: no code sets it, no check refuses it, no test reads it back.
-			// By this file's own definition of `enforced` -- "this worker CAN build it, in its own code, and a
-			// test in this repo reads it back" -- there is no such code, so the word would be exactly the
-			// overclaim `nonRoot` avoids one property up. A boot check on DOCKER_HOST would earn `enforced`.
-			credentialTransit: ASSERTED,
+			// ENFORCED, AND ONLY WHILE OBSERVED (issue #278). Every spawn is `docker` with the worker's own
+			// environment inherited, and `DOCKER_HOST`, `DOCKER_CONTEXT` or a context selected in the CLI's config
+			// can send that connection to another machine with the provider key and the per-job forge token riding
+			// along as `-e NAME=VALUE`. This was `asserted` while nothing in the repository looked. Now the worker
+			// asks the CLI which endpoint it resolves (`backend-local.mjs`, `makeDockerEndpointResolver`) at boot
+			// and before every job's spend, and the word is what that answer earns: `observedBy` below names the
+			// observation, and `effectiveWord` degrades this to `asserted` -- asserted by the operator who pointed
+			// the CLI elsewhere -- whenever the endpoint is not observed on this host. A floor asking for
+			// `enforced` then refuses; a floor asking for `asserted` holds, because pointing the worker at a daemon
+			// is the operator's call. That split is `unarmedFloor`'s lesson: a floor is not met by capability alone.
+			//
+			// WHAT THE OBSERVATION CANNOT SEE, stated rather than claimed away: it judges the endpoint's FORM, so a
+			// unix socket or a loopback port that is really a tunnel (`ssh -L`, socat) to another machine reads as
+			// local, as does a `localhost` this host's resolver maps elsewhere; and a redirect that lands between the per-job read and that job's `docker run` spawn is not
+			// caught for that job. That window is not small: the read sits before the spend, so it spans the secret
+			// resolution, the token mint, the clone and the reservation.
+			credentialTransit: ENFORCED,
 			// The whole reason `DES-WORKER-ON-HOST` reversed the containerised worker.
 			localFolders: ENFORCED,
 		},
@@ -346,12 +651,101 @@ const BACKENDS_TABLE = {
 		 * rather than aspirational. For a vendor adapter this is where "the vendor's documentation" goes.
 		 */
 		asserts: {
-			nonRoot: "the job image's USER directive (this repo's builds `USER pi`; an operator-built image may not)",
-			credentialTransit: "the docker endpoint DOCKER_HOST resolves to, which is this host unless something redirects it",
+			// Issue #341: on a daemon that enforces bind-mount ownership the argv supplies the uid instead, validated
+			// non-zero by the builder; the word stays asserted because on Docker Desktop, and for a uid-1001 worker, the
+			// image still provides it.
+			nonRoot: "the job image's USER directive (this repo's builds `USER pi`; an operator-built image may not), or, on a daemon that enforces bind-mount ownership with a worker uid other than 1001, the worker's own non-zero uid passed as `--user`",
+		},
+		/**
+		 * Which declared words hold only while something about THIS HOST is observed (issue #278), as
+		 * `{ property: observation }`. A per-BACKEND map rather than a field on `PROPERTIES` like `armedBy`:
+		 * `armedBy` names a deployment switch that means the same thing for every venue, while an observation of
+		 * this host's docker CLI means nothing for a remote venue, whose `credentialTransit` a vendor might assert.
+		 */
+		observedBy: {
+			isolation: DAEMON_APPLIES_BOUNDS,
+			mountSet: RUNTIME_ADDS_NO_MOUNTS,
+			credentialTransit: DOCKER_ENDPOINT_LOCAL,
+		},
+	},
+	podman: {
+		/**
+		 * The worker account's own ROOTLESS Podman, driven through the real `podman` CLI (issue #354,
+		 * `DES-PODMAN-NATIVE-ROOTLESS-BACKEND`). Not `local` pointed at Podman: that route goes through Podman's Docker
+		 * API and keeps `local`'s words. This one runs `podman run` itself, always as the worker's own `<euid>:<egid>`
+		 * under `--userns=keep-id`, and REFUSES what it was not measured on: a rootful Podman (keep-id there is not
+		 * refused, and adds supplementary group 0, measured), a remote service, a root worker, and any platform but
+		 * Linux (`backend-podman.mjs`, `decidePodmanJobUser`). Every word below was measured on rootless Podman 5.8.1,
+		 * Fedora 44, 2026-09-25.
+		 */
+		describe: "this worker account's own rootless Podman, through the podman CLI",
+		// The containers run on this host, as this account. A remote service is refused rather than declared remote, so
+		// the venue is never one; `credentialTransit`'s observation is what notices a CLI that tries.
+		remote: false,
+		declares: {
+			// The same `ISOLATION_FLAGS` through the same builder (`buildPodmanRunArgs` shares `argsFromSpec`), read back on
+			// a live rootless container: CapEff 0, NoNewPrivs 1, seccomp 2, pids.max 512, memory.max 4 GiB. AND ONLY WHILE
+			// OBSERVED: the bounds are cgroup v2's, and a user without the controllers delegated accepts the flags and
+			// applies nothing, so the word is earned by `podmanBoundsDelegated`.
+			isolation: ENFORCED,
+			// `--rm` leads ISOLATION_FLAGS and the name carries the job id; measured, `--rm` removes the container (and its
+			// cidfile) when it exits.
+			ephemeral: ENFORCED,
+			// The same `containerSpec` mount list. AND ONLY WHILE OBSERVED: rootless Podman mounts what the winning
+			// mounts.conf lists (`/run/secrets` on Fedora and RHEL by default), invisible in `.Mounts`, so the word is earned
+			// by `podmanAddsNoMounts`. Nothing is relabelled but the job's own per-job directories (`:Z`).
+			mountSet: ENFORCED,
+			// CAPABILITY, gated by PI_EGRESS, as on `local`: a per-job `--internal` network and the proxy attached to it.
+			// Measured rootless: an `--internal` network reaches NOTHING on the host (its IP, host.containers.internal,
+			// 10.0.2.2 and the gateway all fail), and a proxy started on a NAMED bridge network under the same rootless
+			// Podman can be `network connect`ed to it (a pasta or slirp4netns one cannot: exit 125).
+			egress: ENFORCED,
+			// Same switch: the job and the proxy are the only endpoints on an `--internal` network. Measured, a peer on
+			// another job's network is unreachable (ENOTFOUND / ENETUNREACH).
+			jobToJobIsolation: ENFORCED,
+			// `--pull=never` in ISOLATION_FLAGS; measured, an absent image exits 125 "image not known" with no lookup.
+			imagePinning: ENFORCED,
+			// `spawn` on the CLI, and a payload's exit N reaches `close` as N (measured). The one residual, named in the
+			// design entry and not normalised: with `--init`, an entrypoint catatonit cannot exec exits 1, which only its
+			// stderr tells from a payload's own exit 1, so it retries as infra.
+			exitCodes: ENFORCED,
+			// `podman stop -t 5` on the name, and the abort FLAG is the discriminator, as on `local`.
+			abortable: ENFORCED,
+			// `-v <host>:/job:ro`: the kernel's, and the bundle says it binds.
+			readOnlyJobInputs: ENFORCED,
+			// ENFORCED here where `local` can only assert it, because the ARGV supplies the uid on every job: `--user
+			// <euid>:<egid>` beside `--userns=keep-id`, which the builder refuses to emit without a user and refuses for a
+			// uid or gid of 0 (`assertJobUser`). A root worker and a rootful Podman are refused before any spawn, so the uid
+			// inside is the worker's own unprivileged one on the host (measured rootless: runs as 1234, no root group).
+			nonRoot: ENFORCED,
+			// The same resolver, record and comment code as `local`; where the values go is `credentialTransit`'s.
+			secretsCustody: ENFORCED,
+			// ENFORCED, AND ONLY WHILE OBSERVED, for `local`'s reason with Podman's knob: `CONTAINER_HOST`, `--remote` or a
+			// service destination in containers.conf send the run, and its `-e` values, to a service elsewhere. The worker
+			// asks `podman info` whether the service is remote, and the word is earned by `podmanServiceLocal`.
+			credentialTransit: ENFORCED,
+			// A bind mount of the operator's folder, as on `local`, never relabelled (on an SELinux host it needs the
+			// `semanage fcontext` doctor names), and writable because the job runs as the account that owns it.
+			localFolders: ENFORCED,
+		},
+		// Nothing asserted: every word above is this worker's own argv or an observation of this host.
+		asserts: {},
+		// Podman's OWN observations, never `local`'s: those read `docker info` and the docker CLI's endpoint, which say
+		// nothing about the podman CLI's rootless store.
+		observedBy: {
+			isolation: PODMAN_BOUNDS_DELEGATED,
+			mountSet: PODMAN_ADDS_NO_MOUNTS,
+			credentialTransit: PODMAN_SERVICE_LOCAL,
 		},
 	},
 };
 
+/**
+ * The podman venue's name (issue #354): the table key, the bundle's `name` and what `PI_BACKENDS` spells, one constant so
+ * the adapter, the boot wiring and doctor cannot disagree about it.
+ */
+export const PODMAN_BACKEND = "podman";
+
 /**
  * FROZEN, deeply. `makeLocalBackend` hands `BACKENDS[name].declares` out on the bundle it returns, so
  * without this a consumer holding a backend could assign one word and silently rewrite what every later
@@ -360,15 +754,47 @@ const BACKENDS_TABLE = {
  */
 export const BACKENDS = Object.freeze(
 	Object.fromEntries(
-		Object.entries(BACKENDS_TABLE).map(([name, entry]) => [name, Object.freeze({ ...entry, declares: Object.freeze({ ...entry.declares }) })]),
+		// EVERY nested map, not just `declares`: `asserts` is what `doctor` prints beside a word and `observedBy` is
+		// what decides whether the word holds, and an unfrozen one is the shared mutable declaration this freeze
+		// exists to prevent (#278 found `asserts` had been left mutable).
+		Object.entries(BACKENDS_TABLE).map(([name, entry]) => [
+			name,
+			Object.freeze({ ...entry, declares: Object.freeze({ ...entry.declares }), asserts: Object.freeze({ ...(entry.asserts ?? {}) }), observedBy: Object.freeze({ ...(entry.observedBy ?? {}) }) }),
+		]),
 	),
 );
 
 export const BACKEND_NAMES = Object.freeze(Object.keys(BACKENDS));
 
-/** The default, and the name a deployment that has never heard of this table is running. */
+/**
+ * The default, and the name a deployment that has never heard of this table is running: what `PI_BACKENDS`
+ * means when it is unset. NOT "a venue every deployment holds": since issue #354 a deployment may bless a list
+ * without `local`, and its default is then that list's first entry (`config.defaultBackend`). Code that means
+ * the local adapter by name, rather than the unset default, compares against this same word on purpose and
+ * says so where it does.
+ */
 export const DEFAULT_BACKEND = "local";
 
+/**
+ * The venue an artifact that NAMES NO VENUE was produced in (issue #277): a session transcript with no stamp
+ * beside it, or a sandbox manifest with no `backend` key. Both are written with a venue from #277 on, so one
+ * without was written before venues were recorded, when `local` was the only entry this table had -- a fact
+ * about the past rather than a setting.
+ *
+ * DELIBERATELY NOT THE DEPLOYMENT DEFAULT, though today the two spell the same word. The default is
+ * `PI_BACKENDS[0]`, which a deployment can change; reading an unstamped transcript as "whatever the default
+ * is now" would hand a local transcript to a remote venue the day `PI_BACKENDS` lists that venue first. The
+ * past does not move when the configuration does.
+ *
+ * STILL `local` ON A HOST THAT NO LONGER BLESSES IT (issue #354), and that is the rule rather than an oversight.
+ * Such a host's default is another venue, so an unstamped transcript there matches no venue it runs and the key
+ * cold-starts once as `venue-changed`, then carries the new venue's stamp. Resolving absence to the default
+ * instead would resume a Docker-written transcript in a container another runtime built, on the strength of a
+ * stamp that was never written: the one thing the stamp exists to refuse. A sandbox manifest with no key is
+ * likewise a `local` run, reopened only through the local adapter's own CLI.
+ */
+export const UNATTRIBUTED_BACKEND = "local";
+
 /**
  * One backend's entry, or `undefined`. The caller decides how loudly an unknown name fails.
  *
@@ -416,7 +842,8 @@ export function shortfall(name, want = {}) {
 
 /**
  * `PI_BACKENDS` -- which backends this deployment blesses, comma separated. Unset means `[local]`, which is
- * what every deployment that has never heard of this table is already running.
+ * what every deployment that has never heard of this table is already running. Set, any non-empty list of
+ * known names is a deployment, `local` or not; its first entry is the default.
  *
  * ENV-ONLY, never the settings overlay and never the deployment pointer, on `config.mjs`'s rule for
  * `PI_SECRET_RESOLVER_ROOTS`: "a bound that can be widened from the surface it bounds is not a bound". The
@@ -430,26 +857,51 @@ export function shortfall(name, want = {}) {
  * Throws a plain Error; `config.mjs` re-tags it as a config error, which is `egressArmed`'s arrangement and
  * for its reason: this module imports nothing, so it cannot reach for that tagger itself.
  */
-export function parseBackendList(raw) {
+export function parseBackendList(raw, { known = BACKEND_NAMES } = {}) {
 	const names = (raw ?? "")
 		.split(",")
 		.map((s) => s.trim())
 		.filter((s) => s.length > 0);
 	if (names.length === 0) return [DEFAULT_BACKEND];
 	for (const name of names) {
-		if (!Object.hasOwn(BACKENDS, name)) {
-			throw new Error(`PI_BACKENDS names an unknown backend ${JSON.stringify(name)} (known: ${BACKEND_NAMES.join(", ")})`);
+		// `known` is a TEST seam: it was added while `local` was the table's one entry, when a list without it could not
+		// be written through the real table (since issue #354 `podman` can). Every production caller passes nothing and
+		// gets the table's own names.
+		if (!known.includes(name)) {
+			throw new Error(`PI_BACKENDS names an unknown backend ${JSON.stringify(name)} (known: ${known.join(", ")})`);
 		}
 	}
-	// Deduplicated, order preserved: the first entry is what a deployment means by "the default one".
-	const unique = [...new Set(names)];
-	// The DEFAULT backend must stay in the set. A trigger that names no venue is dispatched to
-	// `backends[0]`, and the boot registry refuses a default it does not hold -- so a set excluding it would
-	// describe a deployment whose unflagged triggers, which is nearly all of them, have nowhere to run.
-	if (!unique.includes(DEFAULT_BACKEND)) {
-		throw new Error(`PI_BACKENDS must include ${JSON.stringify(DEFAULT_BACKEND)}: a trigger that names no backend is dispatched there, so a set without it leaves every unflagged trigger nowhere to run`);
+	// Deduplicated, order preserved: the first entry is what a deployment means by "the default one", and it is
+	// where a trigger that names no venue is dispatched.
+	//
+	// `local` IS NOT REQUIRED (issue #354, the operator's decision: `PI_BACKENDS=podman` alone is a deployment). The
+	// guard that stood here refused any set without it, on the reasoning that an unflagged trigger is dispatched to
+	// the default and the default had to be `local`. It never had to: unflagged triggers go to `backends[0]`,
+	// whatever it names, and the boot registry refuses a default it does not hold, so a set whose first entry is
+	// built is runnable with or without `local` in it. What the guard actually did was force a host with no Docker
+	// to spawn `docker` at every boot for a venue it would never use.
+	return [...new Set(names)];
+}
+
+/**
+ * Which venues run jobs here (issue #354), from the same parse the worker boots with. An unparseable PI_BACKENDS reads as
+ * the unset default, `local` alone, so every docker line is exactly what it always was and only the backend section,
+ * which reports the parse, fails on it: guessing another venue from a list the worker refuses would be doctor inventing
+ * a deployment.
+ *
+ * Lives here rather than in doctor (where it started) since issue #430: `up` asks the same question to decide whether to
+ * drive docker, podman or both, and two copies of "which venues does this list bless" is two answers to one question.
+ * A caller that must REFUSE on an unparseable list (`service install`, which would otherwise install a docker-shaped
+ * unit for a deployment that meant podman) calls `parseBackendList` itself first.
+ */
+export function venuesOf(env) {
+	let blessed;
+	try {
+		blessed = parseBackendList(env?.PI_BACKENDS);
+	} catch {
+		blessed = [DEFAULT_BACKEND];
 	}
-	return unique;
+	return { localUsed: blessed.includes(DEFAULT_BACKEND), podmanUsed: blessed.includes(PODMAN_BACKEND), podmanDefault: blessed[0] === PODMAN_BACKEND };
 }
 
 /**