@edgehero/pi-dispatch 4.3.1 → 5.1.0

package/src/backend-local.mjs

@@ -21,6 +21,7 @@
  * calls docker directly. That refusal is now gone because the seam is real.
  */
 
+import { JOB_NAME_PREFIX, jobContainerName } from "./container-names.mjs";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 import { scrubCredentials } from "./redact.mjs";
@@ -51,7 +52,9 @@ export function reaperExec({ execFileFn = execFile, timeoutMs = REAPER_STEP_TIME
 	// 1 MiB, the facts readers' own); every other step keeps the reaper's 30 s.
 	return (bin, args, opts = {}) =>
 		new Promise((resolve, reject) => {
-			execFileFn(bin, [...args], { timeout: opts.timeoutMs ?? timeoutMs, killSignal: "SIGKILL", maxBuffer: opts.maxBuffer ?? 1024 * 1024 }, (err, stdout, stderr) => {
+			// `env` (issue #684): a step that hands the CLI a credential through its own environment, never its argv
+			// (the openshell venue's `provider create --credential NAME`); every other step inherits the worker's.
+			execFileFn(bin, [...args], { timeout: opts.timeoutMs ?? timeoutMs, killSignal: "SIGKILL", maxBuffer: opts.maxBuffer ?? 1024 * 1024, ...(opts.env ? { env: opts.env } : {}) }, (err, stdout, stderr) => {
 				if (err) reject(Object.assign(err, { stdout: String(stdout ?? ""), stderr: String(stderr ?? "") }));
 				else resolve({ stdout: String(stdout ?? ""), stderr: String(stderr ?? "") });
 			});
@@ -59,22 +62,8 @@ export function reaperExec({ execFileFn = execFile, timeoutMs = REAPER_STEP_TIME
 }
 const execReaperBounded = reaperExec();
 
-/**
- * `pi-job-` -- the container-name namespace, and a LOAD-BEARING string rather than a prefix chosen for
- * readability. TWO sweeps match it as a SUBSTRING, both at boot in `start.mjs`: the container reaper's
- * `docker ps` filter and the network reaper's `docker network ls` filter. They share the NAME rule
- * (`isJobNamespace`) and deliberately not the STATE question -- see `reapNetwork` for why one lists running
- * containers and the other asks about members in any state. The sandbox tooling is the
- * counterpart rather than a third sweep -- it names itself `pi-sandbox-` precisely to stay OUTSIDE this
- * namespace, so a worker restart cannot tear down the shell an operator is sitting in, and it reaps its own
- * by job id rather than by name.
- *
- * Exported and imported rather than re-typed at each site for the reason `CONTAINER_GLOBAL_PI_DIR` is: the
- * namespace and the filters that sweep it are ONE fact, and two literals in two modules is how a rename
- * lands in the producer and not in the reaper, leaving every crashed worker's containers behind forever with
- * both test suites green.
- */
-export const JOB_NAME_PREFIX = "pi-job-";
+// Naming is a leaf so operator guidance uses the runtime's exact rule without loading its implementation.
+export { JOB_NAME_PREFIX, jobContainerName } from "./container-names.mjs";
 
 /**
  * The container states `docker network inspect` lists in `.Containers`, and therefore the ones the daemon
@@ -87,23 +76,7 @@ export const JOB_NAME_PREFIX = "pi-job-";
  */
 export { ENDPOINT_LISTED_STATES };
 
-/**
- * `pi-job-<jobId>`. The name a running job answers to, for `docker stop` on the 30-minute timeout, for the
- * per-job egress network derived from it, and for the reaper's filter.
- *
- * Sanitised to the runtimes' own name rule, `[a-zA-Z0-9][a-zA-Z0-9_.-]*` (docker's `RestrictedNameChars`, and
- * Podman's, measured), every other character becoming `_`, as `sanitizeJobId` does for file names. It said here that
- * BullMQ ids are already that shape, and a cron job's is not: the job scheduler mints `repeat:<schedulerId>:<millis>`,
- * so `pi-job-repeat:...` was refused at create (exit 125, measured on Podman 5.8.1) and so was its `-net` network, and
- * every cron job ended `container-never-started` before it ever ran (issue #435). The prefix supplies the first
- * character. Deterministic, because every path that must find the container again (the timeout's stop, the cancel,
- * the network, the log sink) asks this function for the name rather than deriving it. Not injective: `repeat:a:1`
- * and a literal `repeat_a_1` share a name. BullMQ ids are unique per queue and no producer here mints an id with `_`
- * where a scheduler id has `:`, so it is a residual rather than a collision anything reaches.
- */
-export function jobContainerName(jobId) {
-	return `${JOB_NAME_PREFIX}${String(jobId).replace(/[^A-Za-z0-9._-]/g, "_")}`;
-}
+
 
 /**
  * Bundle the local backend's already-built functions into one checked value.
@@ -139,6 +112,19 @@ export const BACKEND_PROVIDED = ["name", "declares", "namePrefix", "containerNam
  */
 export const BACKEND_VALUES = ["neverStartedExits"];
 
+/**
+ * Functions a bundle carries that `makeLocalBackend` builds ITSELF unless the caller passes its own (issue #645): the
+ * freezer verbs. Optional to pass rather than required like `BACKEND_FUNCTIONS`, because every caller of this factory
+ * (start.mjs and the tests) predates them and the local adapter knows how to spell them for docker; a test seam is
+ * the only reason to pass one. Still a NAMED list, so a misspelt `pasue` is refused as an unknown member like any other
+ * key rather than silently replaced by the default.
+ *
+ * Issue #681: the host budget's two venue questions joined them, for the same reason. `listJobContainers` lists this
+ * venue's leftover job containers for the boot seed and `containerGone` settles an orphan's hold; both are docker's
+ * here, and a bundle carries its own so the budget never asks one runtime about another's container.
+ */
+export const BACKEND_OPTIONAL_FUNCTIONS = ["pause", "unpause", "listJobContainers", "containerGone"];
+
 export function makeLocalBackend(parts = {}) {
 	const { runContainer, imagePreflight, egressPreflight, stopContainer, reap } = parts ?? {};
 	const missing = Object.entries({ runContainer, imagePreflight, egressPreflight, stopContainer, reap })
@@ -155,9 +141,14 @@ export function makeLocalBackend(parts = {}) {
 	// control again, arriving through a dropped argument.
 	for (const key of Object.keys(parts ?? {})) {
 		if (BACKEND_FUNCTIONS.includes(key) || BACKEND_VALUES.includes(key)) continue;
+		if (BACKEND_OPTIONAL_FUNCTIONS.includes(key)) {
+			if (typeof parts[key] !== "function") throw new Error(`backend "${DEFAULT_BACKEND}": ${key} must be a function when it is passed`);
+			continue;
+		}
 		throw new Error(`backend "${DEFAULT_BACKEND}": unknown bundle member ${JSON.stringify(key)} (this factory takes ${BACKEND_FUNCTIONS.join(", ")} and sets ${BACKEND_PROVIDED.join(", ")} itself)`);
 	}
 
+	const verbs = makePauseVerbs({ bin: "docker" });
 	return {
 		name: DEFAULT_BACKEND,
 		// The declaration is READ from the table, never re-typed here. An adapter that stated its own
@@ -175,6 +166,14 @@ export function makeLocalBackend(parts = {}) {
 		egressPreflight,
 		stopContainer,
 		reap,
+		// The freezer verbs (issue #645), `pausable` in the table. Never throw; `{ ok: false, reason }` is the caller's cue to
+		// carry on without a frozen read.
+		pause: parts.pause ?? verbs.pause,
+		unpause: parts.unpause ?? verbs.unpause,
+		// The host budget's two questions about THIS venue's containers (issue #681), asked of docker because docker holds
+		// them. Before, the wiring derived the CLI from the job's venue name and sent every venue but podman to docker.
+		listJobContainers: parts.listJobContainers ?? makeJobContainerLister({ bin: "docker" }),
+		containerGone: parts.containerGone ?? makeContainerGone({ binOf: () => "docker" }),
 		// The integers this runtime uses for "the runner never ran". The processor asks the BACKEND rather
 		// than assuming docker's triple, because those numbers collide with the runner's own exit channel.
 		neverStartedExits: LOCAL_NEVER_STARTED_EXITS,
@@ -218,6 +217,76 @@ export function makeStopContainer({ exec = execDocker, bin = "docker" } = {}) {
 	};
 }
 
+/**
+ * The bound on one `pause` or `unpause` (issue #645). 15 s, not `execDockerBounded`'s 5 s default: a freeze waits for
+ * every task in the container's cgroup to reach the frozen state, and a container mid-`fork` storm or in
+ * uninterruptible I/O takes visibly longer than a CLI round trip, while a caller holding a job's checkpoint cannot wait
+ * on a wedged daemon forever. Past it the verb answers `{ ok: false, reason: "timeout" }` and the child is SIGKILLed.
+ */
+export const PAUSE_TIMEOUT_MS = 15_000;
+
+/**
+ * A runtime's container name, which is also what keeps the argument from reading as a FLAG: `docker pause --help`
+ * exits 0 and freezes nothing, so a name beginning with `-` would come back as a successful pause of no container.
+ * `jobContainerName` only mints names of this shape (the runtimes' own rule), so refusing anything else costs no job.
+ */
+const RUNTIME_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
+
+/** Whether `name` is a runtime container name the freezer verbs accept. Exported so a venue's own gate checks it FIRST. */
+export function isRuntimeName(name) {
+	return typeof name === "string" && RUNTIME_NAME.test(name);
+}
+
+/**
+ * `{ pause(name), unpause(name) }` for the venue's CLI (`docker pause` / `podman pause` and their `unpause`), each
+ * resolving `{ ok: true }` or `{ ok: false, reason }` and NEVER rejecting (issue #645, the `pausable` row).
+ *
+ * WHAT IT IS FOR. Pausing uses the cgroup freezer: every process in the container stops where it stands, nothing is
+ * killed, and the bind-mounted workspace holds still, so the host can read it as one consistent state while nothing in
+ * the container runs. That is the whole of it; a frozen container is granted nothing and its mounts do not change.
+ *
+ * NEVER THROWS, because its caller is a best-effort read beside a running job: a freeze that did not take means the
+ * read is taken live or skipped, never that the job fails. A failure is a fixed token (`timeout`, `exit-<n>`,
+ * `<bin>-not-found`, `spawn-failed`, `invalid-name`) and never the CLI's stderr, which can repeat a `DOCKER_HOST` with
+ * credentials in it (issue #339's reason for `execDockerBounded` not reading it by default).
+ *
+ * BOUNDED TWICE ON PURPOSE. `execDockerBounded` (the default `run`) settles on its own timer and kills the child, which
+ * is the bound in production; the race below is the same bound for an injected `run`, so "never hangs" is a property of
+ * the verb rather than of whichever seam a caller passed. `docker stop` is not bounded here, which is why this uses
+ * `execDockerBounded` and not `makeStopContainer`'s `exec`: the abort path does not await its stop and this caller does.
+ */
+export function makePauseVerbs({ bin = "docker", timeoutMs = PAUSE_TIMEOUT_MS, run = (cmd, args, opts) => execDockerBounded(args, { bin: cmd, timeoutMs: opts.timeoutMs }) } = {}) {
+	const verb = (word) =>
+		async function pauseVerb(name) {
+			if (!isRuntimeName(name)) return { ok: false, reason: "invalid-name" };
+			let timer = null;
+			const expired = new Promise((resolve) => {
+				timer = setTimeout(() => resolve({ code: null, error: { timedOut: true } }), timeoutMs);
+			});
+			let result;
+			try {
+				result = await Promise.race([Promise.resolve().then(() => run(bin, [word, name], { timeoutMs })), expired]);
+			} catch (err) {
+				result = { code: null, error: err };
+			} finally {
+				clearTimeout(timer);
+			}
+			if (!result?.error && result?.code === 0) return { ok: true };
+			return { ok: false, reason: pauseFailure(bin, result) };
+		};
+	return { pause: verb("pause"), unpause: verb("unpause") };
+}
+
+/** The fixed token for a verb that did not take; values Node supplies only, never CLI text. */
+function pauseFailure(bin, result) {
+	const error = result?.error ?? null;
+	if (error?.timedOut || error?.killed) return "timeout";
+	if (error?.code === "ENOENT") return `${bin}-not-found`;
+	if (typeof result?.code === "number") return `exit-${result.code}`;
+	if (typeof error?.code === "number") return `exit-${error.code}`;
+	return "spawn-failed";
+}
+
 /**
  * What the CLI resolved, as a phrase that is never EMPTY. Four call sites interpolate an endpoint into a
  * sentence of the form "resolves <this>, which is not shown to be on this host", and a context can carry no