@edgehero/pi-dispatch 3.1.0 → 4.0.0

package/src/run-container.mjs

@@ -2,7 +2,8 @@ import { spawn } from "node:child_process";
 import { randomBytes } from "node:crypto";
 import { readFileSync, rmSync } from "node:fs";
 import { DOCKER_NEVER_STARTED_EXITS } from "./backends.mjs";
-import { CONTAINER_HOME } from "./container-spec.mjs";
+import { CONTAINER_HOME, memoryBytesOfArgs } from "./container-spec.mjs";
+import { cpuCeilingCenti, cpuRangeRefusal, DEFAULT_JOB_SIZE } from "./job-size.mjs";
 import { buildDockerRunArgs, CONTAINER_SESSION_FILE, insideDir } from "./docker-run.mjs";
 import { createJobNetwork, networkNameFor, removeJobNetwork } from "./egress.mjs";
 import { buildContainerEnv } from "./env-allowlist.mjs";
@@ -11,7 +12,7 @@ import { InfraRetry } from "./processor.mjs";
 
 /**
  * The real `runContainer` the processor injects. Launches one job container and returns
- * `{ code, aborted, turns, tokens, session, usage, context, exitReason }`, where `aborted` records whether the WORKER initiated the stop (docker stop on
+ * `{ code, aborted, turns, tokens, session, usage, context, exitReason, resources?, exitOomKilled?, memoryLimit? }`, where `aborted` records whether the WORKER initiated the stop (docker stop on
  * the 30-min timeout or graceful shutdown), which the processor classifies as POLICY (no retry) per
  * INT-RUNNER-EXIT-CODE-PROTOCOL. The numeric `code` alone cannot say this: a worker SIGKILL and a
  * kernel OOM both surface as 137, so the abort FLAG -- not the code -- is the discriminator.
@@ -64,6 +65,10 @@ export function makeRunContainer({
 	log = () => {},
 	// Issue #545: the per-job exit-line key, 32 random bytes as hex. A seam so a test can name the key it verifies with.
 	mintExitKey = () => randomBytes(32).toString("hex"),
+	// Issue #596: called when the runtime refused this job's `--cpus` as above its CPU count (`cpu_ceiling_stale`), so
+	// the wiring drops the cached runtime facts the ceiling came from and the next pickup reads them again. The local
+	// venue's is the job-user resolver's `invalidate`; a no-op elsewhere (Podman accepts any `--cpus`, measured).
+	onCpuCeilingStale = () => {},
 }) {
 	// async so a synchronous throw (e.g. buildContainerEnv on an unconfigured provider) surfaces as
 	// a rejection, uniformly awaitable by the processor and by tests.
@@ -75,7 +80,15 @@ export function makeRunContainer({
 	// `exitAuth` (issue #545) is the processor's, off the image preflight: true when the job image declares `exitAuth`, so
 	// its runner reads a key from stdin and signs its exit line with it. Defaults off, so a caller that predates it, and
 	// every image that does not declare it, runs exactly as before and is read exactly as before.
-	return async function runContainer({ job, token, prepared, secrets = {}, name, signal, user = null, home = null, relabel = false, modelEndpoints = null, exitAuth = false }) {
+	// `size` (issue #596) is the job's `{ memMiB, cpuCenti, source }`, resolved at pickup from the limits snapshot and
+	// handed here as an argument, never through `job.data`; absent, the built-in 4g and 2. `hostCpus` is the runtime's own
+	// CPU count from the job user's facts read, which sets the `--cpus` ceiling; null leaves `--cpus` off (fails open, and
+	// says so in `cpu_ceiling_unknown`). `cpuBudgetCenti` (phase 2) is the host's CPU budget at pickup, which replaces that
+	// ceiling while it is in force (`cpuCeilingCenti`).
+	// `unenforced` (issue #596) is the size flags this runtime said it drops (`unenforcedSizeFlags`: `--memory-swap` where
+	// Docker reports SwapLimit false, `--cpu-shares` where it reports CPUShares false), logged per job as
+	// `size_bound_unenforced` so a run past its size's bound is named where it happens. The flags stay on the argv.
+	return async function runContainer({ job, token, prepared, secrets = {}, name, signal, user = null, home = null, relabel = false, modelEndpoints = null, exitAuth = false, size = DEFAULT_JOB_SIZE, hostCpus = null, cpuBudgetCenti = null, unenforced = [] }) {
 		if (signal?.aborted) return { code: 137, aborted: true, turns: null, tokens: null, session: null, usage: null, context: null, exitReason: null }; // killed before it could start
 		// Minted per attempt, held in this closure and the sink's, and handed to the container on its stdin only: never in
 		// the argv (a host `ps` shows it), never in the env (the container's /proc/1/environ shows it), never logged.
@@ -175,6 +188,9 @@ export function makeRunContainer({
 			sessionDir: prepared.session?.hostDir,
 			globalPiDir, // undefined/null -> docker-run's guard skips the /opt/pi-global mount
 			name,
+			size, // issue #596: memory, swap equal to it, the CPU weight and /dev/shm, through containerSpec's one function
+			hostCpus, // issue #596: the `--cpus` ceiling every job shares; null leaves the flag off
+			cpuBudgetCenti, // issue #596, phase 2: the host's CPU budget, the ceiling while it is in force
 			network, // REQ-EGRESS-ALLOWLIST: null when no policy is armed, and the flag is then absent
 			user, // issue #341: the worker's own "<uid>:<gid>" on a daemon that enforces bind-mount ownership, else null
 			cidFile, // issue #345: where the CLI writes this attempt's container ID, read below when the run exits "never started"
@@ -190,6 +206,13 @@ export function makeRunContainer({
 			...(exitKey !== null ? { extraFlags: ["-i"] } : {}),
 		});
 
+		// Issue #596: the memory bound this container actually got, in bytes, read off the argv the runtime is handed (both
+		// builders spell it `--memory=`, from the job's size), never re-derived from the environment or the size. The
+		// processor confirms an OOM only when the run's peak reached 90% of it. null when the argv names none.
+		const memoryLimit = memoryBytesOfArgs(args);
+		if (!args.some((a) => typeof a === "string" && a.startsWith("--cpus="))) log("cpu_ceiling_unknown", { container: name });
+		if (Array.isArray(unenforced) && unenforced.length > 0) log("size_bound_unenforced", { container: name, flags: unenforced.filter((f) => typeof f === "string" && /^--[a-z-]{1,32}$/.test(f)) });
+
 		// REQ-EGRESS-ALLOWLIST. This job's own --internal network, created here rather than at boot because
 		// it holds exactly two endpoints -- this container and the proxy -- and that is what makes job-to-job
 		// traffic structurally impossible rather than merely discouraged. A shared network could not do it:
@@ -205,6 +228,10 @@ export function makeRunContainer({
 		// Host-side per-job log sink, teed off `onOutput`. `name` is `pi-job-<jobId>`; the sink
 		// sanitizes internally. No container mount, no env var -- the sink lives on this side only.
 		const sink = openJobLog(name, { exitKey });
+		// Issue #596: the head of the CLI's stderr, kept only to recognise the runtime's refusal of a stale `--cpus`
+		// (`cpuRangeRefusal`). Bounded, and never logged: only the count it names leaves this function.
+		let stderrHead = "";
+		const STDERR_HEAD_MAX = 4096;
 
 		const run = new Promise((resolve, reject) => {
 			const child = spawnFn(bin, args, { stdio: [exitKey !== null ? "pipe" : "ignore", "pipe", "pipe"] });
@@ -223,7 +250,10 @@ export function makeRunContainer({
 				} catch {}
 			};
 			child.stdout?.on("data", tee);
-			child.stderr?.on("data", tee);
+			child.stderr?.on("data", (chunk) => {
+				if (stderrHead.length < STDERR_HEAD_MAX) stderrHead = (stderrHead + String(chunk)).slice(0, STDERR_HEAD_MAX);
+				tee(chunk);
+			});
 			// docker not found / daemon down -- a transient infra fault, so tag it retryable
 			// (CONST-RETRY-INFRA-ONLY). `reason` also cues the processor to release the budget slot,
 			// since a container that never started spent nothing.
@@ -243,19 +273,23 @@ export function makeRunContainer({
 				// RUNNER_POLICY_REASONS set and to a line that itself said code 2. The processor still decides
 				// the retry class from `code` alone; this only picks the label inside exit 2.
 				let exitReason = null;
-				// Issue #501 (PR #542's review, round 3): the LAST exit line's own `code`, which the dollar settlement
-				// compares with the container's real exit code before it trusts that line's cost.
+				// Issue #501 (PR #542's review, round 3): the decisive exit line's own `code` (decisiveExitLine), which the
+				// dollar settlement compares with the container's real exit code before it trusts that line's cost.
 				let exitLineCode = null;
 				// Issue #507: the cost guard's rule on a `cost-cap` line, already filtered by parseExitWhy to COST_CAP_WHYS.
 				let exitWhy = null;
 				// Issue #545: null (no key issued), "verified" or "unverified" (a key, and no exit line carried it).
 				let exitAuthResult = null;
+				// Issue #596: what the container used, off its exit line (parseExitResources), or null.
+				let resources = null;
+				// Issue #596: the image's supervisor reported the runner killed for memory (parseExitOomKilled).
+				let exitOomKilled = false;
 				try {
 					// `context = null` is a DEFAULT rather than a plain destructure: an injected sink that
 					// predates the field returns no such key, and `undefined` would then reach the record's
 					// shape where every other absence is spelled `null`.
 					// `exitReason` defaults the same way, for the same reason.
-					({ turns, tokens, session, usage, context = null, exitReason = null, exitLineCode = null, exitWhy = null, exitAuth: exitAuthResult = null } = await sink.close());
+					({ turns, tokens, session, usage, context = null, exitReason = null, exitLineCode = null, exitWhy = null, exitAuth: exitAuthResult = null, resources = null, exitOomKilled = false } = await sink.close());
 				} catch {
 					turns = null;
 					tokens = null;
@@ -265,6 +299,8 @@ export function makeRunContainer({
 					exitReason = null;
 					exitLineCode = null;
 					exitWhy = null;
+					resources = null;
+					exitOomKilled = false;
 					// The sink could not say, and a key was issued: nothing it returned was verified.
 					exitAuthResult = exitKey !== null ? "unverified" : null;
 				}
@@ -279,12 +315,17 @@ export function makeRunContainer({
 					exitReason = null;
 					exitLineCode = null;
 					exitWhy = null;
+					resources = null;
+					exitOomKilled = false;
 				}
 				// Spread only when a key was issued, so a run without one resolves the very object it always did.
 				const auth = exitKey !== null ? { exitAuth: exitAuthResult } : {};
 				// `exitWhy` only when the line named one, so every other run resolves the object it always did.
 				const why = exitWhy !== null ? { exitWhy } : {};
-				resolve(aborted ? { code: code ?? 137, aborted: true, turns, tokens, session, usage, context, exitReason, exitLineCode, ...why, ...auth } : { code: code ?? 1, aborted: false, turns, tokens, session, usage, context, exitReason, exitLineCode, ...why, ...auth });
+				// `resources` (issue #596) only when the exit line carried a block, so every other run resolves the object it always did.
+				// `memoryLimit` only beside the OOM report, the one place it is read, for the same reason.
+				const used = { ...(resources !== null && resources !== undefined ? { resources } : {}), ...(exitOomKilled === true ? { exitOomKilled: true, memoryLimit } : {}) };
+				resolve(aborted ? { code: code ?? 137, aborted: true, turns, tokens, session, usage, context, exitReason, exitLineCode, ...why, ...used, ...auth } : { code: code ?? 1, aborted: false, turns, tokens, session, usage, context, exitReason, exitLineCode, ...why, ...used, ...auth });
 			});
 		});
 
@@ -293,6 +334,19 @@ export function makeRunContainer({
 		// that answer. What a failure leaves behind is a memberless network, which the boot reaper sweeps.
 		try {
 			const result = await run;
+			// Issue #596: the runtime refused the ceiling as above its CPU count. The count it was computed from is stale
+			// (Docker Desktop's VM given fewer CPUs since the facts were read), and every later job would fail the same
+			// way until a restart, so the facts are dropped and the next pickup reads them again. This attempt stays a
+			// never-started one (refunded and retried as infrastructure), and the line names both counts.
+			const runtimeCpus = !result.aborted && (neverStartedExits ?? []).includes(result.code) && Number.isSafeInteger(hostCpus) ? cpuRangeRefusal(stderrHead) : null;
+			if (runtimeCpus !== null) {
+				log("cpu_ceiling_stale", { container: name, cpus: (cpuCeilingCenti(hostCpus, cpuBudgetCenti) ?? 0) / 100, hostCpus, runtimeCpus });
+				try {
+					onCpuCeilingStale();
+				} catch {
+					// a wiring fault must not rewrite this attempt's answer; the facts age out on their own (`maxAgeMs`)
+				}
+			}
 			// Issue #345: an exit that says "never started" is checked against the cidfile BEFORE the network goes, so a
 			// container found running is stopped while its network still exists. Only when the worker did not abort it.
 			if (!result.aborted && (neverStartedExits ?? []).includes(result.code) && (await stopDetached({ spawnFn, cidFile, fs, bin, ...detachedCheck }))) {