@edgehero/pi-dispatch 4.3.1 → 5.1.0

package/src/backend-podman.mjs

@@ -30,7 +30,7 @@
 
 import { readdirSync, readFileSync, statSync } from "node:fs";
 import { homedir } from "node:os";
-import { JOB_NAME_PREFIX, execDockerBounded, jobContainerName, makeReaper, makeStopContainer } from "./backend-local.mjs";
+import { JOB_NAME_PREFIX, execDockerBounded, isRuntimeName, jobContainerName, makeContainerGone, makeJobContainerLister, makePauseVerbs, makeReaper, makeStopContainer } from "./backend-local.mjs";
 import { BACKENDS, PODMAN_ADDS_NO_MOUNTS, PODMAN_BACKEND, PODMAN_CONF_WIDENS_JOB, PODMAN_NETWORK_HELPER_KEYS, PODMAN_WIDENING_KEYS, PODMAN_BOUNDS_DELEGATED, PODMAN_SERVICE_LOCAL, observationRefusalIsTransient, observationRefusals, unobservedFloor } from "./backends.mjs";
 import { CONTAINER_HOME, SHIPPED_IMAGE_UID } from "./container-spec.mjs";
 import { buildPodmanRunArgs } from "./docker-run.mjs";
@@ -981,9 +981,15 @@ export const PODMAN_BACKEND_OPTIONS = Object.freeze([
 	"authFromPi",
 	"forgeHosts",
 	"onOutput",
+	"openLiveTranscript",
+	// issue #646: the inbox relay's opener, as start.mjs hands local's
+	"openInboxRelay",
 	// the floor the per-job observation preflight judges, and the boot-built reaper
 	"backendFloor",
 	"reap",
+	// issue #681: the host budget's leftover listing and gone-check, as start.mjs hands them (its listing seam)
+	"listJobContainers",
+	"containerGone",
 	// the facts, the identity and the files the observations and the job user come from
 	"readInfo",
 	"platform",
@@ -1002,6 +1008,33 @@ export const PODMAN_BACKEND_OPTIONS = Object.freeze([
 	"makeStopContainer",
 ]);
 
+/**
+ * Whether this venue can FREEZE a job (issue #645, the `pausable` row), from a `podman info` read: `{ pausable, reason }`,
+ * with `pausable` `true`, `false`, or `null` when the read did not answer. Rootless Podman cannot pause on cgroup v1
+ * (`podman pause` refuses it there: the v1 freezer is a hierarchy the user is never delegated), and every podman venue
+ * is rootless because a rootful one is refused, so the cgroup version decides it. The version comes from the SAME read
+ * the venue admits jobs on and `podmanBoundsDelegated` judges, so the freeze and the bounds cannot disagree about which
+ * version this host runs. A version `podman info` did not give is not v1: the CLI is then left to answer for itself,
+ * which it does with a non-zero exit the verb returns as `{ ok: false }`. The bundle's verbs judge the KEPT read only, so
+ * an unanswered or not-yet-taken read (`pausable: null`) also leaves the CLI to answer.
+ */
+export function podmanPausable(read) {
+	if (read?.answered !== true || !read.info) return { pausable: null, reason: read?.reason ?? "unanswered" };
+	if (read.info.cgroupVersion === "v1") return { pausable: false, reason: "cgroup-v1-rootless" };
+	return { pausable: true, reason: null };
+}
+
+/** An execFile-shaped runner (resolves on exit 0, rejects with `.code` otherwise) as `execDockerBounded`'s `{ code, error }`. */
+function settledRun(execFn) {
+	return (cmd, args) =>
+		Promise.resolve()
+			.then(() => execFn(cmd, args))
+			.then(
+				() => ({ code: 0, error: null }),
+				(err) => ({ code: typeof err?.code === "number" ? err.code : null, error: err ?? { code: null } }),
+			);
+}
+
 /** The keeper read's bound: one `podman inspect`, on a pre-spend gate, like the proxy's own read beside it. */
 export const NETNS_KEEPER_READ_TIMEOUT_MS = 10_000;
 
@@ -1061,7 +1094,7 @@ export function keeperPreflight(proxyPreflight, { armed, proxy, info, spawnFn =
 
 /**
  * The podman bundle: `{ name, declares, neverStartedExits, containerName, namePrefix, binds, runContainer,
- * imagePreflight, egressPreflight, stopContainer, reap, jobUserPreflight, observationPreflight }`.
+ * imagePreflight, egressPreflight, stopContainer, pause, unpause, reap, jobUserPreflight, observationPreflight }`.
  *
  * BUILT here from PR #425's factories with `bin: "podman"` (and `buildPodmanRunArgs` for the run), rather than taken
  * built as `makeLocalBackend` does, because the pairing IS the adapter: a run under one binary and a stop, a preflight or
@@ -1087,8 +1120,12 @@ export function makePodmanBackend(opts = {}) {
 		authFromPi = false,
 		forgeHosts = {},
 		onOutput,
+		openLiveTranscript = null,
+		openInboxRelay = null,
 		backendFloor = {},
 		reap,
+		listJobContainers,
+		containerGone,
 		readInfo = makePodmanInfoReader(),
 		platform = process.platform,
 		euid = process.geteuid?.(),
@@ -1105,9 +1142,25 @@ export function makePodmanBackend(opts = {}) {
 		makeStopContainer: makeStopContainerFn = makeStopContainer,
 	} = opts ?? {};
 	if (reap !== undefined && typeof reap !== "function") throw new Error(`backend "${PODMAN_BACKEND}": reap must be a function (makePodmanReaper)`);
+	for (const [key, fn] of Object.entries({ listJobContainers, containerGone })) {
+		if (fn !== undefined && typeof fn !== "function") throw new Error(`backend "${PODMAN_BACKEND}": ${key} must be a function when it is passed`);
+	}
 	const spawnSeam = spawnFn ? { spawnFn } : {};
 	const execSeam = exec ? { exec } : spawnFn ? { exec: execViaSpawn(spawnFn) } : {};
 	const info = cachedPodmanInfo(readInfo, { log });
+	// The freezer verbs (issue #645) under the venue's own binary, through the same exec seam as the stop, so a test's fake
+	// child sees every podman spawn. In production the bounded runner; `makePauseVerbs` bounds an injected one too.
+	const verbs = makePauseVerbs({ bin: "podman", ...(execSeam.exec ? { run: settledRun(execSeam.exec) } : {}) });
+	// The name first, then cgroup v1, both BEFORE any spawn. The version comes from the KEPT answered `podman info`
+	// (`peek`), never a fresh read: a read here would be a second spawn with a bound of its own beside the verb's 15 s,
+	// and every job is admitted on an answered read (`observationPreflight`), so a running job always has one kept.
+	// With none kept (no job admitted yet) the verb runs and `podman pause`'s own refusal comes back as `{ ok: false }`.
+	const gated = (fn) => async (name) => {
+		if (!isRuntimeName(name)) return { ok: false, reason: "invalid-name" };
+		const can = podmanPausable(info.peek?.() ?? null);
+		if (can.pausable === false) return { ok: false, reason: can.reason };
+		return fn(name);
+	};
 
 	const runContainer = makeRunContainerFn({
 		image,
@@ -1116,6 +1169,8 @@ export function makePodmanBackend(opts = {}) {
 		egressProxy,
 		...(openJobLog ? { openJobLog } : {}),
 		...(onOutput ? { onOutput } : {}),
+		...(openLiveTranscript ? { openLiveTranscript } : {}),
+		...(openInboxRelay ? { openInboxRelay } : {}),
 		globalPiDir,
 		allowGlobalExtensions,
 		packagePaths,
@@ -1192,7 +1247,13 @@ export function makePodmanBackend(opts = {}) {
 		imagePreflight: makeImagePreflightFn({ image, bin: "podman", ...spawnSeam }),
 		egressPreflight: keeperPreflight(makeEgressPreflightFn({ proxy: egressProxy, armed: egress, bin: "podman", ...spawnSeam }), { armed: egress, proxy: egressProxy, info, spawnFn }),
 		stopContainer: makeStopContainerFn({ bin: "podman", ...execSeam }),
+		pause: gated(verbs.pause),
+		unpause: gated(verbs.unpause),
 		reap: reap ?? makePodmanReaper({ log, ...execSeam }),
+		// Issue #681: this store's leftover job containers and whether one is gone, asked of podman through the same exec
+		// seam as the stop and the reaper. The wiring used to pick the CLI from the venue's name instead.
+		listJobContainers: listJobContainers ?? makeJobContainerLister({ bin: "podman", ...execSeam }),
+		containerGone: containerGone ?? makeContainerGone({ binOf: () => "podman", ...execSeam }),
 		jobUserPreflight,
 		observationPreflight,
 	};

package/src/backend-registry.mjs

@@ -66,8 +66,19 @@ export const BACKEND_NOT_REGISTERED = "BACKEND_NOT_REGISTERED";
 export const OPTIONAL_PREFLIGHT_DEFAULTS = Object.freeze({
 	observationPreflight: () => ({ ok: true }),
 	jobUserPreflight: () => ({ user: null, home: null }),
+	// Issue #704 part 3: a project secret bundle judged against the venue (`(job, { endpoints, vars, shadowed })`). A
+	// venue that carries none admits every bundle, which is what a venue whose container holds its env as values does:
+	// the variables join the env, and `endpoints` means nothing there. Only a venue whose own layer holds credentials
+	// (both OpenShell venues, which carry one) has a narrower answer.
+	bundleGate: () => ({ ok: true }),
 });
 
+/** The freezer verbs a bundle MAY carry (issue #645, the `pausable` row), always as a pair. */
+export const PAUSE_VERBS = Object.freeze(["pause", "unpause"]);
+
+/** Whether a (validated) bundle carries both freezer verbs. */
+const isPausable = (bundle) => typeof bundle?.pause === "function" && typeof bundle?.unpause === "function";
+
 /** The venue's own member when it carries one, else the absent answer, as a promise either way. */
 async function optionalPreflight(bundle, fn, ...args) {
 	return typeof bundle[fn] === "function" ? bundle[fn](...args) : OPTIONAL_PREFLIGHT_DEFAULTS[fn]();
@@ -131,6 +142,9 @@ export function resolveBackendName(data, defaultName) {
  * @param bundles  the blessed backend bundles (each from a `make<Name>Backend`), keyed by their own `name`
  * @param defaultName  the venue a job that names none runs in -- `PI_BACKENDS[0]`
  */
+/** The functions every registered bundle must carry, checked at boot. */
+export const REQUIRED_FUNCTIONS = Object.freeze(["runContainer", "imagePreflight", "egressPreflight", "stopContainer", "reap", "containerName", "listJobContainers", "containerGone"]);
+
 export function makeBackendRegistry({ bundles = [], defaultName, blessed = null, reaps = null } = {}) {
 	const byName = new Map();
 	for (const bundle of bundles) {
@@ -144,7 +158,10 @@ export function makeBackendRegistry({ bundles = [], defaultName, blessed = null,
 		// `containerName` is in this list because THIS MODULE calls it: `registry.containerName(job)` builds
 		// the name the abort then stops. Omitting it built fine and threw at the first pickup, which is the
 		// property this constructor claims to have and did not.
-		for (const fn of ["runContainer", "imagePreflight", "egressPreflight", "stopContainer", "reap", "containerName"]) {
+		// `listJobContainers` and `containerGone` (issue #681) because the host budget asks every venue both: its leftover
+		// containers at boot, and whether an orphan's container is gone before its hold is freed. Without them the wiring
+		// guessed the runtime from the venue's name, and every venue but podman was asked through docker.
+		for (const fn of REQUIRED_FUNCTIONS) {
 			if (typeof bundle[fn] !== "function") throw new Error(`backend registry: ${JSON.stringify(bundle.name)} has no ${fn}()`);
 		}
 		if (!Array.isArray(bundle.neverStartedExits) || !bundle.neverStartedExits.every(Number.isInteger)) {
@@ -159,6 +176,13 @@ export function makeBackendRegistry({ bundles = [], defaultName, blessed = null,
 		for (const fn of Object.keys(OPTIONAL_PREFLIGHT_DEFAULTS)) {
 			if (bundle[fn] !== undefined && typeof bundle[fn] !== "function") throw new Error(`backend registry: ${JSON.stringify(bundle.name)} has a ${fn} that is not a function`);
 		}
+		// The freezer verbs (issue #645) are optional too, but only as a PAIR: a venue that could pause and not unpause
+		// would freeze a paid job for good at its first live checkpoint. Refused at boot, like a typo'd preflight.
+		const verbs = PAUSE_VERBS.filter((fn) => bundle[fn] !== undefined);
+		for (const fn of verbs) {
+			if (typeof bundle[fn] !== "function") throw new Error(`backend registry: ${JSON.stringify(bundle.name)} has a ${fn} that is not a function`);
+		}
+		if (verbs.length === 1) throw new Error(`backend registry: ${JSON.stringify(bundle.name)} carries ${verbs[0]} without its pair (pause and unpause come together, or neither)`);
 		byName.set(bundle.name, bundle);
 	}
 	if (byName.size === 0) throw new Error("backend registry: at least one backend must be registered");
@@ -215,15 +239,38 @@ export function makeBackendRegistry({ bundles = [], defaultName, blessed = null,
 		 * the abort path had to change: the container's NAME is not enough to find the runtime that holds
 		 * it once there is more than one, and `index.mjs` had only the name.
 		 */
-		runContainer: (args) => backendFor(args?.job).runContainer(args),
+		// The venue's freezer verbs ride on the call bag (issue #645), from the SAME bundle that runs the container, so a
+		// live checkpoint can never freeze a job through another venue's CLI. A venue without them gets none, and its
+		// container gets no live checkpoints; the call bag is then exactly the caller's.
+		// Issue #646: a steer job is also told whether its venue BINDS mounts (the bundle's own `binds`), which runContainer
+		// arms steering on: a copying venue would upload the inbox once and never see a later message. Only for a steer job,
+		// so every other call bag stays exactly the caller's (plus the verbs above).
+		runContainer: (args) => {
+			const bundle = backendFor(args?.job);
+			const withVerbs = isPausable(bundle) ? { ...args, pause: bundle.pause, unpause: bundle.unpause } : args;
+			return bundle.runContainer(args?.job?.steer === true ? { ...withVerbs, binds: bundle.binds === true } : withVerbs);
+		},
 		imagePreflight: (job) => backendFor(job).imagePreflight(job),
 		egressPreflight: (job) => backendFor(job).egressPreflight(job),
-		// The two OPTIONAL per-venue preflights (issue #354), dispatched through the same resolution as every
-		// required one, so an unregistered name still throws BACKEND_NOT_REGISTERED rather than being waved
+		// The OPTIONAL per-venue preflights (issue #354; `bundleGate`, #704 part 3), dispatched through the same
+		// resolution as every required one, so an unregistered name still throws BACKEND_NOT_REGISTERED rather than being waved
 		// through by a default. A venue that carries none gets the answer the processor's own defaults give.
 		observationPreflight: (job) => optionalPreflight(backendFor(job), "observationPreflight", job),
 		jobUserPreflight: (job, opts) => optionalPreflight(backendFor(job), "jobUserPreflight", job, opts),
+		bundleGate: (job, opts) => optionalPreflight(backendFor(job), "bundleGate", job, opts),
 		stopContainer: (name, job) => backendFor(job).stopContainer(name, job),
+		// Issue #645: whether this job's venue can freeze it, and the verbs themselves, dispatched like every per-job
+		// function. A venue without verbs answers `{ ok: false, reason: "not-pausable" }`, never a throw: the caller is a
+		// best-effort checkpoint beside a running job.
+		pausable: (job) => isPausable(backendFor(job)),
+		pause: async (name, job) => {
+			const bundle = backendFor(job);
+			return isPausable(bundle) ? bundle.pause(name) : { ok: false, reason: "not-pausable" };
+		},
+		unpause: async (name, job) => {
+			const bundle = backendFor(job);
+			return isPausable(bundle) ? bundle.unpause(name) : { ok: false, reason: "not-pausable" };
+		},
 		// ON THE SURFACE, not reached for through `backendFor` by a call site. Both are per-job backend
 		// FACTS rather than functions, and an earlier draft left them off: the wiring rebuilt
 		// `neverStartedExits` at the call site and `index.mjs` imported `jobContainerName` from the local
@@ -231,6 +278,16 @@ export function makeBackendRegistry({ bundles = [], defaultName, blessed = null,
 		// module's header says is structurally impossible -- and the container NAME is the argument the
 		// abort's `stopContainer` receives, so building it locally while resolving the venue per job was the
 		// same defect in the one call the slice exists to make dispatchable.
+		// Issue #681: the host budget's gone-check, asked of the venue that holds the container. `venue` is the job's data or
+		// `{ backend }`, as the budget keeps it. A venue no bundle answers for is `null` (unknown), never a throw: the
+		// caller is the budget's sweep, and `null` keeps the hold, which is the safe answer for room nobody can account for.
+		containerGone: async (name, venue) => {
+			const bundle = byName.get(resolveBackendName(venue ?? {}, defaultName));
+			return bundle ? bundle.containerGone(name) : null;
+		},
+		// Each registered venue's leftover-container lister, keyed by name, for the budget's boot seed.
+		// Called ON the bundle, as `containerGone` is, so an adapter whose lister is a method reading `this` still works.
+		listers: () => Object.fromEntries([...byName].map(([name, bundle]) => [name, () => bundle.listJobContainers()])),
 		neverStartedExits: (job) => backendFor(job).neverStartedExits,
 		containerName: (job) => backendFor(job).containerName(job?.id),
 		// The reaper map, validated above against the registered set, so the boot sweep can be handed