@edgehero/pi-dispatch 3.0.0 → 4.0.0

package/src/processor.mjs

@@ -8,7 +8,7 @@ import { PROJECT_CAP_REASON } from "./scoped-limits.mjs";
 import { DEFAULT_SECRETS_PROFILE, secretsArmed } from "./secrets.mjs";
 import { RESERVED_ENV_NAMES } from "./triggers.mjs";
 import { EXIT_COMPLETED, EXIT_INFRA, EXIT_POLICY } from "./exit-code.mjs";
-import { COST_CAP_WHYS, RUNNER_POLICY_REASONS } from "./run-history.mjs";
+import { COST_CAP_WHYS, EXIT_OOM_KILLED, RUNNER_POLICY_REASONS } from "./run-history.mjs";
 import { DEFAULT_EGRESS_PROXY } from "./egress.mjs";
 import { CAPABILITY_GATES, EXIT_AUTH_CAPABILITY } from "./image-preflight.mjs";
 import { modelListProblem, modelOnList, splitModelEntry } from "./model-ref.mjs";
@@ -83,6 +83,29 @@ export const OBSERVATION_COMMENT_UNNAMED = "the venue this job runs on did not c
  * into the queue's retry behaviour -- that is INT-RUNNER-EXIT-CODE-PROTOCOL.
  */
 
+/** The exit code of a container whose main process was SIGKILLed: a worker's stop or the kernel's OOM killer. */
+const EXIT_SIGKILL = 137;
+
+/**
+ * Issue #596: whether a run's memory peak reached the container's own limit closely enough for its OOM kill to be the
+ * job's own: `memPeak` (bytes, off the decisive exit line) at 90% of `limit` (bytes, the `--memory=` the worker passed)
+ * or more. Both must be known; anything else is false, so the 137 stays infrastructure and retries.
+ *
+ * Why the check exists: `memory.events` `oom_kill` counts a kill by ANY OOM killer, the HOST's included, and the runner
+ * tree's `oom_score_adj` of 1000 makes a job the host's first victim when the machine itself runs short. A kill of
+ * that kind says nothing about the job's size, and a retry may well pass. A cgroup OOM happens only at the limit, so its
+ * peak sits there: measured in the issue #596 lab, `memory.peak` read exactly the bound (64m, 80m) or just under it on
+ * every venue. 90% and not 100% is the margin for that "just under"; nothing in between is read as a measurement.
+ *
+ * What it cannot see (the residual): a host OOM that strikes a job which is ALREADY at 90% of its own limit, page
+ * cache included (a job that read large files counts that cache), reads as the job's own OOM. And the peak is a
+ * high-water mark over the whole run, so a job that touched 90% early and was killed by the host later reads the same.
+ */
+export function peakReachedLimit(memPeak, limit) {
+	if (!Number.isSafeInteger(memPeak) || !Number.isSafeInteger(limit) || memPeak < 0 || limit <= 0) return false;
+	return memPeak >= Math.ceil((limit * 9) / 10);
+}
+
 // The post-spend terminal comments (issue #288). Every FREE refusal above the container already comments;
 // these are the paths where money was spent and the run still ended without the agent's own status step,
 // which used to tell the issue nothing (REQ-JOB-STATUS-COMMENTS' acceptance -- "exactly one completion or
@@ -106,6 +129,9 @@ export const TERMINAL_COMMENTS = {
 	"model-not-allowed": "Stopped: the run tried to call an AI model this trigger does not allow, or to change an AI request in a way it does not allow, so the call was not made. Partial work may exist. Not retried.",
 	"cost-cap-unenforceable": "Stopped: this run has a cost limit, and the job image could not enforce it before each AI call, so nothing was sent to the AI provider. The operator needs to update the job image. Not retried.",
 	"model-policy-unenforceable": "Stopped: this run is limited to certain AI models, and the job image could not enforce that before each AI call, so nothing was sent to the AI provider. The operator needs to update the job image. Not retried.",
+	// Issue #596. Never the size or the project: both are operator configuration, and the reader may be an issue author
+	// who can act on neither. The worker log and the run record carry them.
+	[EXIT_OOM_KILLED]: "Stopped: the job's container ran out of memory and was stopped. Partial work may exist. Not retried, because the same size would stop the same way. The operator can raise this job's memory size.",
 };
 
 // Issue #502: the `model-unknown` refusal's comment. Names no model: the reader may be an issue author.
@@ -238,7 +264,8 @@ export async function runJob(job, deps) {
 		// Issue #341: which uid this job's container runs as. `(job, { capabilities, observed }) =>` `{ user, home }`
 		// (`user` null = the image's own USER), `{ refused, cause }` or `{ unavailable, reason }`. The default runs
 		// every job as the image's user, exactly as before, so a wiring that omits it changes nothing. A non-refused answer
-		// may also carry `relabel: true` (issue #355), which reaches `runContainer` beside the user it was decided with.
+		// may also carry `relabel: true` (issue #355), which reaches `runContainer` beside the user it was decided with, and
+		// `hostCpus` (issue #596), the runtime's CPU count from the same facts read, which sets the `--cpus` ceiling.
 		jobUserPreflight = async () => ({ user: null, home: null }),
 		// (session, { piVersion, context }) => { promoted, reason, bytes }. Promotes this job's transcript back into
 		// the store, on a COMPLETED exit only. Never throws. The default is a no-op so a wiring that omits
@@ -300,8 +327,11 @@ export async function runJob(job, deps) {
 		mintToken,
 		isDefaultBranchProtected, // (job, token) => boolean; same reason -- the forge is the job's, not the process's
 		prepareWorkspace, // (job, token) => { workspaceDir, jobDir }  (clone+materialise+prompt)
-		// runContainer({ job, token, prepared, secrets, name, signal, user, home, modelEndpoints? }) => { code, aborted, abortReason, turns, tokens, session, usage, context, exitReason }.
+		// runContainer({ job, token, prepared, secrets, name, signal, user, home, modelEndpoints?, size?, hostCpus?, unenforced? }) => { code, aborted, abortReason, turns, tokens, session, usage, context, exitReason }.
+		// `size` (issue #596) is `jobSize` below; `hostCpus` the runtime's CPU count off the job-user gate's own facts read.
+		// `unenforced` (issue #596) is the size flags that same read says the runtime drops, passed only when there are some.
 		// `exitReason` (issue #437) is parseExitReason's closed-set label, read only inside the exit-2 branch.
+		// `resources` and `exitOomKilled` (issue #596) are the exit line's cgroup block and the supervisor's OOM report.
 		// `user`/`home` are the job-user gate's answer (issue #341), null for the image's own USER.
 		// `secrets` is the resolved map from the gate above: values, already fetched, host-side. It MUST honour
 		// `signal`: stop the container on abort, and reject/exit promptly if `signal.aborted` is already
@@ -356,6 +386,14 @@ export async function runJob(job, deps) {
 		// after prepare, before any reserve. Absent on a bare wiring, which checks nothing.
 		pickupProject = null,
 		folderProject = null,
+		// Issue #596: the job's size `{ memMiB, cpuCenti, source }`, resolved at pickup from the same limits snapshot as
+		// every other gate (index.mjs). Handed to `prepareWorkspace` (a retained run's manifest records it, so a sandbox
+		// reopens the run at its size) and to `runContainer`, never through `job.data`. null on a bare wiring: neither call
+		// then carries it, and the container gets the built-in 4g and 2.
+		jobSize = null,
+		// Issue #596, phase 2: the host's CPU budget at pickup in hundredths (`host-budget.mjs`), or null when it is off or
+		// unknown. It becomes every job's `--cpus` (capped at the runtime's count), so no job can use the reserve.
+		cpuBudgetCenti = null,
 		// Issue #503 part 7: the builtin catalog's model object for (provider, id), or null (model-catalog.mjs
 		// `builtinModel`). Read only by the zero-rated check; the default knows no builtin model, so an unwired
 		// processor judges overlay models alone and reserves for every other.
@@ -412,6 +450,8 @@ export async function runJob(job, deps) {
 	// the record says about it (INT-RUN-HISTORY-FILE-CONTRACT), null until the reservation step ran.
 	let dollarHold = null;
 	let dollars = null;
+	// Issue #596: what the container used (`resources` off its exit line), null until a container ran and reported it.
+	let resources = null;
 
 	try {
 		// The one-shot pre-spend check (issue #231), FIRST on the ladder: one file read, cheaper than
@@ -1067,7 +1107,7 @@ export async function runJob(job, deps) {
 		// `podmanStore` (issue #429) only where the venue's job user carried one: the podman store the container ran in.
 		// `portfolio` (issue #505) only for a job the gate above confirmed: prepare then writes /job/portfolio.json, after
 		// asking the live file once more. Absent otherwise, so every other job's prepare call is unchanged.
-		prepared = await prepareWorkspace(job, token, { piVersion, jobUser: { user: jobUser?.user ?? null, home: jobUser?.home ?? null }, ...(typeof jobUser?.store === "string" ? { podmanStore: jobUser.store } : {}), ...(portfolio ? { portfolio: true } : {}) }); // resolves SHA, clones, materialises .pi/, writes prompt
+		prepared = await prepareWorkspace(job, token, { piVersion, jobUser: { user: jobUser?.user ?? null, home: jobUser?.home ?? null }, ...(typeof jobUser?.store === "string" ? { podmanStore: jobUser.store } : {}), ...(portfolio ? { portfolio: true } : {}), ...(jobSize ? { size: jobSize } : {}) }); // resolves SHA, clones, materialises .pi/, writes prompt
 
 		// A determinate prepare refusal -- sha-gone (the default branch advanced past the resolved tip),
 		// or a `pi-*` materialiser cap breach (the repo's .pi/ is too large to place in /job, issue #60)
@@ -1270,12 +1310,29 @@ export async function runJob(job, deps) {
 		// then only a signed line is read (run-container.mjs, run-history.mjs `authenticExitLines`). An image that does not
 		// declare it is read as before, under the #542 trust rule below alone.
 		const exitAuth = (img.capabilities ?? []).includes(EXIT_AUTH_CAPABILITY);
-		const { code, aborted, abortReason, turns, tokens, session, usage, context, detached, exitReason, exitWhy = null, exitLineCode = null, exitAuth: exitAuthResult = null } = await runContainer({ job: containerJob, token, prepared, secrets, user: jobUser?.user ?? null, home: jobUser?.home ?? null, relabel: jobUser?.relabel === true, ...(modelEndpoints?.endpoints?.length > 0 ? { modelEndpoints } : {}), ...(exitAuth ? { exitAuth: true } : {}) });
+		const { code, aborted, abortReason, turns, tokens, session, usage, context, detached, exitReason, exitWhy = null, exitLineCode = null, exitAuth: exitAuthResult = null, exitOomKilled = false, memoryLimit = null, resources: ranResources = null } = await runContainer({ job: containerJob, token, prepared, secrets, user: jobUser?.user ?? null, home: jobUser?.home ?? null, relabel: jobUser?.relabel === true, ...(modelEndpoints?.endpoints?.length > 0 ? { modelEndpoints } : {}), ...(exitAuth ? { exitAuth: true } : {}), ...(jobSize ? { size: jobSize } : {}), ...(Number.isSafeInteger(jobUser?.hostCpus) ? { hostCpus: jobUser.hostCpus } : {}), ...(Number.isSafeInteger(cpuBudgetCenti) ? { cpuBudgetCenti } : {}), ...(Array.isArray(jobUser?.unenforced) && jobUser.unenforced.length > 0 ? { unenforced: jobUser.unenforced } : {}) });
 		containerRan = true;
+		// Issue #596: what the container used, off its exit line, rebuilt by the sink (null from a runContainer that predates
+		// the field). Every result and every throw below carries it, so a retried attempt's record says what it used too.
+		resources = ranResources ?? null;
+		// Issue #596: CONFIRMED killed for memory. The image's supervisor (image/runner/src/supervise.mjs) outlives the runner,
+		// whose process tree it gives the highest OOM score, and when the runner dies of SIGKILL with the cgroup's
+		// `oom_kill` above 0 it writes the signed line `code: 137, reason: "oom-killed"` (parseExitOomKilled). All three
+		// facts must agree: that line, verified under this run's key (an unsigned line is a tool's), and the container's
+		// own exit 137. Docker's `oom` event is not used: it fires also when only a child was killed and the job went on
+		// to exit 0, and Podman has no such event at all (both measured in the issue #596 lab). And a fourth: the line's
+		// `memPeak` at 90% of the `--memory` this container got (`memoryLimit`, from runContainer) or more, because
+		// `oom_kill` counts the HOST's OOM killer too (`peakReachedLimit`). A report below it stays infrastructure.
+		const oomReported = exitOomKilled === true && exitAuthResult === "verified" && code === EXIT_SIGKILL;
+		const oomKilled = oomReported && peakReachedLimit(resources?.memPeak ?? null, memoryLimit);
+		// Numbers only (bytes), never a path or a project: the operator's trace of a kill read as the host's. Never on a run
+		// the worker stopped itself (a timeout, a cancel, a shutdown): that 137 is the worker's own, the abort decides it
+		// below, and a line naming the host's OOM killer would send the operator after a kill that never happened.
+		if (oomReported && !oomKilled && !aborted) log("oom_report_below_limit", { jobId: job.id ?? null, memPeak: resources?.memPeak ?? null, memoryLimit });
 		// `exitAuth: "unverified"` is a run whose image signs its exit line and no signed line was found: the runner died
 		// before writing one, or a line was forged or taken off the pipe. Its tokens read as unknown and its dollars settle
 		// at the floor, the same as a container that wrote no exit line at all.
-		log("container_exit", { exitCode: code, aborted, ...(detached === true ? { detached: true } : {}), ...(exitAuthResult !== null ? { exitAuth: exitAuthResult } : {}) });
+		log("container_exit", { exitCode: code, aborted, ...(detached === true ? { detached: true } : {}), ...(exitAuthResult !== null ? { exitAuth: exitAuthResult } : {}), ...(oomKilled ? { oomKilled: true } : {}) });
 
 		// Record token spend post-run (the check-AFTER half of the lagging token cap). The container ran,
 		// so it spent real tokens on EVERY path that reaches here -- abort, completed, policy, AND the infra
@@ -1334,14 +1391,15 @@ export async function runJob(job, deps) {
 		// mid-job). It DID start, so this is never refunded as never-started: it keeps its slot and retries as infrastructure,
 		// BEFORE the exit-code switch, where the same code would read as a free never-started exit.
 		if (detached === true) {
-			throw new InfraRetry(`the container outlived its docker run, exit ${code}`, { reason: "container-detached", exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), dollars });
+			throw new InfraRetry(`the container outlived its docker run, exit ${code}`, { reason: "container-detached", exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), dollars, resources });
 		}
 
 		// A WORKER-initiated stop (30-min timeout via cancelJob, graceful-shutdown docker stop, or an
 		// operator's cancel, issue #287) kills the container -> exit 143/137. That is our decision, not an
 		// infra fault: it is POLICY and must NOT retry, or a wedged job re-runs into a second PR / double
 		// spend. Keyed on the abort FLAG, not the code -- an unbidden 137 (kernel OOM) carries
-		// `aborted: false`, falls to the switch, and stays infra-retryable.
+		// `aborted: false`, and falls to the OOM branch below when the runtime confirmed it, else to the switch, where it
+		// stays infra-retryable (issue #596).
 		// WHO aborted is an exact-match on `abortReason` (the wiring maps it off `signal.reason`), and the
 		// match is deliberately closed: "job-timeout-30m", "shutdown", undefined and any future garbage all
 		// classify as worker-abort, so a pin bump that changes what rides the signal can widen nothing.
@@ -1352,7 +1410,23 @@ export async function runJob(job, deps) {
 			// Awaited bare like every determinate refusal above: the adapter never throws by contract, and
 			// the one swallowed comment in this file (the catch's) justifies itself by its position.
 			await comment(job, TERMINAL_COMMENTS[reason]);
-			return { outcome: "policy", reason, exitCode: code, turns, tokens, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}) };
+			return { outcome: "policy", reason, exitCode: code, turns, tokens, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}) };
+		}
+
+		// Issue #596: a runner the kernel killed for memory, CONFIRMED (`oomKilled` above). After the abort branch, so a
+		// worker's own stop is never relabelled, and before the switch, where the same 137 is an unknown exit and retries.
+		// The same size would be killed the same way on every retry, so it is POLICY: returned, never retried, its slot
+		// kept and its dollars settled above like any other paid stop. An UNCONFIRMED 137 (an image without the supervisor,
+		// an unsigned line, a SIGKILL with no OOM kill in the cgroup, a peak short of the limit, which is how a kill by the
+		// host's OOM killer reads) falls through and retries, exactly as before.
+		//
+		// When only a CHILD was killed, the runner survives and ends on its own code, so this branch is not taken: the
+		// outcome is the runner's, and `resources.oomKills` in the record says a process was killed for memory.
+		if (oomKilled) {
+			// The project and the size go to the log and the record, never to the comment (the TERMINAL_COMMENTS rule).
+			log("oom_killed", { jobId: job.id ?? null });
+			await comment(job, TERMINAL_COMMENTS[EXIT_OOM_KILLED]);
+			return { outcome: "policy", reason: EXIT_OOM_KILLED, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...(resources ? { resources } : {}) };
 		}
 
 		switch (code) {
@@ -1394,6 +1468,7 @@ export async function runJob(job, deps) {
 					// Only when the collector returned a plan (a file, or a confirmed portfolio job with none, `plan-absent`): the
 					// record's `plan` is null otherwise, and every other result is unchanged.
 					...(plan ? { plan } : {}),
+					...(resources ? { resources } : {}),
 				};
 			}
 			case EXIT_POLICY: {
@@ -1420,7 +1495,7 @@ export async function runJob(job, deps) {
 				// `why`. parseExitWhy keeps only a member of the closed COST_CAP_WHYS off a `cost-cap` line that said code 2,
 				// and it rides only beside a `cost-cap` reason, so a forged line can at worst name the wrong rule of three.
 				const why = reason === "cost-cap" && COST_CAP_WHYS.includes(exitWhy) ? { why: exitWhy } : {};
-				return { outcome: "policy", reason, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...why };
+				return { outcome: "policy", reason, exitCode: code, turns, tokens, usage: usage ?? null, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), budgetReserved: true, ...(dollars ? { dollars } : {}), ...why, ...(resources ? { resources } : {}) };
 			}
 			case EXIT_INFRA:
 				// NO comment on any infra throw, here or in the catch: an InfraRetry may be retried and
@@ -1428,7 +1503,7 @@ export async function runJob(job, deps) {
 				// the whole infra class lives at the terminal seam -- start.mjs's failed listener, guarded on
 				// BullMQ's own finishedOn -- which also catches the stall-kill and wait-gate paths this
 				// function never sees (issue #288).
-				throw new InfraRetry(`infra failure, container exit ${code}`, { exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), dollars });
+				throw new InfraRetry(`infra failure, container exit ${code}`, { exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), dollars, resources });
 			default:
 				// THE RUNTIME NEVER HANDED CONTROL TO THE RUNNER, in whatever integers this venue spells that
 				// (issue #227). For docker it is 125 (`docker run` itself failed), 126 (the entrypoint exists
@@ -1444,9 +1519,9 @@ export async function runJob(job, deps) {
 				// and normalises to this outcome itself.
 				if (neverStarted) {
 					// No `dollars` here: the hold is still standing, and the catch refunds it whole with the job-count slots.
-					throw new InfraRetry(`the runtime could not start the container, exit ${code}`, { reason: "container-never-started", exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session) });
+					throw new InfraRetry(`the runtime could not start the container, exit ${code}`, { reason: "container-never-started", exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), resources });
 				}
-				throw new InfraRetry(`unknown container exit ${code}`, { exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), dollars });
+				throw new InfraRetry(`unknown container exit ${code}`, { exitCode: code, turns, tokens, usage, provider: job.provider ?? null, model: job.model ?? null, session: mergeSession(prepared, session), dollars, resources });
 		}
 	} catch (e) {
 		// A CONFIG-tagged throw is a determinate policy refusal wearing an exception, and issue #310 is the
@@ -1625,7 +1700,7 @@ function isNeverStartedRetry(e) {
 }
 
 export class InfraRetry extends Error {
-	constructor(message, { cause, reason, exitCode, turns, tokens, session, usage, provider, model, budgetReserved, dollars } = {}) {
+	constructor(message, { cause, reason, exitCode, turns, tokens, session, usage, provider, model, budgetReserved, dollars, resources } = {}) {
 		super(message, cause ? { cause } : undefined);
 		this.name = "InfraRetry";
 		this.piDispatchRetry = true;
@@ -1647,6 +1722,8 @@ export class InfraRetry extends Error {
 		// Issue #501: the dollar reservation's outcome (`dollarsRecord`), or null when no dollar window applied. Set by
 		// the processor on a throw after the reservation, so a retried attempt's record says what its window was charged.
 		this.dollars = dollars ?? null;
+		// Issue #596: what the container used, off its exit line, or null; set on a throw after a container ran.
+		this.resources = resources ?? null;
 	}
 }
 

package/src/reserved-env.mjs

@@ -64,11 +64,12 @@ export const CONTAINER_ENV_NAMES = new Set([
 
 /**
  * Issue #500: the names the RUNNER sets in its own environment inside the container, for its descendants: the child
- * ledger directory and the runner's pid (image/runner/src/usage-meter.mjs, openChildLedger). The worker never writes
+ * ledger directory and the runner's pid (image/runner/src/usage-meter.mjs, openChildLedger), and the hash of the price
+ * table the runner hands its children (publishPriceTable, issue #587's review). The worker never writes
  * them, so they are not in the set above (which a test pins to what buildContainerEnv emits). They are reserved all the
  * same: a trigger binding one through `run.secrets`, or a host value forwarded through `PI_FORWARD_ENV`, would arrive in
  * the runner's environment before the runner sets its own, and a value the runner then failed to replace would point
  * every pi child's ledger at a directory the agent chose. Refused at load in both lists, and deleted by
  * buildContainerEnv after both loops as the backstop.
  */
-export const RUNNER_ENV_NAMES = new Set(["PI_DISPATCH_CHILD_LEDGER", "PI_DISPATCH_RUNNER_PID"]);
+export const RUNNER_ENV_NAMES = new Set(["PI_DISPATCH_CHILD_LEDGER", "PI_DISPATCH_PRICE_TABLE", "PI_DISPATCH_RUNNER_PID"]);

package/src/run-container.mjs

@@ -2,7 +2,8 @@ import { spawn } from "node:child_process";
 import { randomBytes } from "node:crypto";
 import { readFileSync, rmSync } from "node:fs";
 import { DOCKER_NEVER_STARTED_EXITS } from "./backends.mjs";
-import { CONTAINER_HOME } from "./container-spec.mjs";
+import { CONTAINER_HOME, memoryBytesOfArgs } from "./container-spec.mjs";
+import { cpuCeilingCenti, cpuRangeRefusal, DEFAULT_JOB_SIZE } from "./job-size.mjs";
 import { buildDockerRunArgs, CONTAINER_SESSION_FILE, insideDir } from "./docker-run.mjs";
 import { createJobNetwork, networkNameFor, removeJobNetwork } from "./egress.mjs";
 import { buildContainerEnv } from "./env-allowlist.mjs";
@@ -11,7 +12,7 @@ import { InfraRetry } from "./processor.mjs";
 
 /**
  * The real `runContainer` the processor injects. Launches one job container and returns
- * `{ code, aborted, turns, tokens, session, usage, context, exitReason }`, where `aborted` records whether the WORKER initiated the stop (docker stop on
+ * `{ code, aborted, turns, tokens, session, usage, context, exitReason, resources?, exitOomKilled?, memoryLimit? }`, where `aborted` records whether the WORKER initiated the stop (docker stop on
  * the 30-min timeout or graceful shutdown), which the processor classifies as POLICY (no retry) per
  * INT-RUNNER-EXIT-CODE-PROTOCOL. The numeric `code` alone cannot say this: a worker SIGKILL and a
  * kernel OOM both surface as 137, so the abort FLAG -- not the code -- is the discriminator.
@@ -64,6 +65,10 @@ export function makeRunContainer({
 	log = () => {},
 	// Issue #545: the per-job exit-line key, 32 random bytes as hex. A seam so a test can name the key it verifies with.
 	mintExitKey = () => randomBytes(32).toString("hex"),
+	// Issue #596: called when the runtime refused this job's `--cpus` as above its CPU count (`cpu_ceiling_stale`), so
+	// the wiring drops the cached runtime facts the ceiling came from and the next pickup reads them again. The local
+	// venue's is the job-user resolver's `invalidate`; a no-op elsewhere (Podman accepts any `--cpus`, measured).
+	onCpuCeilingStale = () => {},
 }) {
 	// async so a synchronous throw (e.g. buildContainerEnv on an unconfigured provider) surfaces as
 	// a rejection, uniformly awaitable by the processor and by tests.
@@ -75,7 +80,15 @@ export function makeRunContainer({
 	// `exitAuth` (issue #545) is the processor's, off the image preflight: true when the job image declares `exitAuth`, so
 	// its runner reads a key from stdin and signs its exit line with it. Defaults off, so a caller that predates it, and
 	// every image that does not declare it, runs exactly as before and is read exactly as before.
-	return async function runContainer({ job, token, prepared, secrets = {}, name, signal, user = null, home = null, relabel = false, modelEndpoints = null, exitAuth = false }) {
+	// `size` (issue #596) is the job's `{ memMiB, cpuCenti, source }`, resolved at pickup from the limits snapshot and
+	// handed here as an argument, never through `job.data`; absent, the built-in 4g and 2. `hostCpus` is the runtime's own
+	// CPU count from the job user's facts read, which sets the `--cpus` ceiling; null leaves `--cpus` off (fails open, and
+	// says so in `cpu_ceiling_unknown`). `cpuBudgetCenti` (phase 2) is the host's CPU budget at pickup, which replaces that
+	// ceiling while it is in force (`cpuCeilingCenti`).
+	// `unenforced` (issue #596) is the size flags this runtime said it drops (`unenforcedSizeFlags`: `--memory-swap` where
+	// Docker reports SwapLimit false, `--cpu-shares` where it reports CPUShares false), logged per job as
+	// `size_bound_unenforced` so a run past its size's bound is named where it happens. The flags stay on the argv.
+	return async function runContainer({ job, token, prepared, secrets = {}, name, signal, user = null, home = null, relabel = false, modelEndpoints = null, exitAuth = false, size = DEFAULT_JOB_SIZE, hostCpus = null, cpuBudgetCenti = null, unenforced = [] }) {
 		if (signal?.aborted) return { code: 137, aborted: true, turns: null, tokens: null, session: null, usage: null, context: null, exitReason: null }; // killed before it could start
 		// Minted per attempt, held in this closure and the sink's, and handed to the container on its stdin only: never in
 		// the argv (a host `ps` shows it), never in the env (the container's /proc/1/environ shows it), never logged.
@@ -175,6 +188,9 @@ export function makeRunContainer({
 			sessionDir: prepared.session?.hostDir,
 			globalPiDir, // undefined/null -> docker-run's guard skips the /opt/pi-global mount
 			name,
+			size, // issue #596: memory, swap equal to it, the CPU weight and /dev/shm, through containerSpec's one function
+			hostCpus, // issue #596: the `--cpus` ceiling every job shares; null leaves the flag off
+			cpuBudgetCenti, // issue #596, phase 2: the host's CPU budget, the ceiling while it is in force
 			network, // REQ-EGRESS-ALLOWLIST: null when no policy is armed, and the flag is then absent
 			user, // issue #341: the worker's own "<uid>:<gid>" on a daemon that enforces bind-mount ownership, else null
 			cidFile, // issue #345: where the CLI writes this attempt's container ID, read below when the run exits "never started"
@@ -190,6 +206,13 @@ export function makeRunContainer({
 			...(exitKey !== null ? { extraFlags: ["-i"] } : {}),
 		});
 
+		// Issue #596: the memory bound this container actually got, in bytes, read off the argv the runtime is handed (both
+		// builders spell it `--memory=`, from the job's size), never re-derived from the environment or the size. The
+		// processor confirms an OOM only when the run's peak reached 90% of it. null when the argv names none.
+		const memoryLimit = memoryBytesOfArgs(args);
+		if (!args.some((a) => typeof a === "string" && a.startsWith("--cpus="))) log("cpu_ceiling_unknown", { container: name });
+		if (Array.isArray(unenforced) && unenforced.length > 0) log("size_bound_unenforced", { container: name, flags: unenforced.filter((f) => typeof f === "string" && /^--[a-z-]{1,32}$/.test(f)) });
+
 		// REQ-EGRESS-ALLOWLIST. This job's own --internal network, created here rather than at boot because
 		// it holds exactly two endpoints -- this container and the proxy -- and that is what makes job-to-job
 		// traffic structurally impossible rather than merely discouraged. A shared network could not do it:
@@ -205,6 +228,10 @@ export function makeRunContainer({
 		// Host-side per-job log sink, teed off `onOutput`. `name` is `pi-job-<jobId>`; the sink
 		// sanitizes internally. No container mount, no env var -- the sink lives on this side only.
 		const sink = openJobLog(name, { exitKey });
+		// Issue #596: the head of the CLI's stderr, kept only to recognise the runtime's refusal of a stale `--cpus`
+		// (`cpuRangeRefusal`). Bounded, and never logged: only the count it names leaves this function.
+		let stderrHead = "";
+		const STDERR_HEAD_MAX = 4096;
 
 		const run = new Promise((resolve, reject) => {
 			const child = spawnFn(bin, args, { stdio: [exitKey !== null ? "pipe" : "ignore", "pipe", "pipe"] });
@@ -223,7 +250,10 @@ export function makeRunContainer({
 				} catch {}
 			};
 			child.stdout?.on("data", tee);
-			child.stderr?.on("data", tee);
+			child.stderr?.on("data", (chunk) => {
+				if (stderrHead.length < STDERR_HEAD_MAX) stderrHead = (stderrHead + String(chunk)).slice(0, STDERR_HEAD_MAX);
+				tee(chunk);
+			});
 			// docker not found / daemon down -- a transient infra fault, so tag it retryable
 			// (CONST-RETRY-INFRA-ONLY). `reason` also cues the processor to release the budget slot,
 			// since a container that never started spent nothing.
@@ -243,19 +273,23 @@ export function makeRunContainer({
 				// RUNNER_POLICY_REASONS set and to a line that itself said code 2. The processor still decides
 				// the retry class from `code` alone; this only picks the label inside exit 2.
 				let exitReason = null;
-				// Issue #501 (PR #542's review, round 3): the LAST exit line's own `code`, which the dollar settlement
-				// compares with the container's real exit code before it trusts that line's cost.
+				// Issue #501 (PR #542's review, round 3): the decisive exit line's own `code` (decisiveExitLine), which the
+				// dollar settlement compares with the container's real exit code before it trusts that line's cost.
 				let exitLineCode = null;
 				// Issue #507: the cost guard's rule on a `cost-cap` line, already filtered by parseExitWhy to COST_CAP_WHYS.
 				let exitWhy = null;
 				// Issue #545: null (no key issued), "verified" or "unverified" (a key, and no exit line carried it).
 				let exitAuthResult = null;
+				// Issue #596: what the container used, off its exit line (parseExitResources), or null.
+				let resources = null;
+				// Issue #596: the image's supervisor reported the runner killed for memory (parseExitOomKilled).
+				let exitOomKilled = false;
 				try {
 					// `context = null` is a DEFAULT rather than a plain destructure: an injected sink that
 					// predates the field returns no such key, and `undefined` would then reach the record's
 					// shape where every other absence is spelled `null`.
 					// `exitReason` defaults the same way, for the same reason.
-					({ turns, tokens, session, usage, context = null, exitReason = null, exitLineCode = null, exitWhy = null, exitAuth: exitAuthResult = null } = await sink.close());
+					({ turns, tokens, session, usage, context = null, exitReason = null, exitLineCode = null, exitWhy = null, exitAuth: exitAuthResult = null, resources = null, exitOomKilled = false } = await sink.close());
 				} catch {
 					turns = null;
 					tokens = null;
@@ -265,6 +299,8 @@ export function makeRunContainer({
 					exitReason = null;
 					exitLineCode = null;
 					exitWhy = null;
+					resources = null;
+					exitOomKilled = false;
 					// The sink could not say, and a key was issued: nothing it returned was verified.
 					exitAuthResult = exitKey !== null ? "unverified" : null;
 				}
@@ -279,12 +315,17 @@ export function makeRunContainer({
 					exitReason = null;
 					exitLineCode = null;
 					exitWhy = null;
+					resources = null;
+					exitOomKilled = false;
 				}
 				// Spread only when a key was issued, so a run without one resolves the very object it always did.
 				const auth = exitKey !== null ? { exitAuth: exitAuthResult } : {};
 				// `exitWhy` only when the line named one, so every other run resolves the object it always did.
 				const why = exitWhy !== null ? { exitWhy } : {};
-				resolve(aborted ? { code: code ?? 137, aborted: true, turns, tokens, session, usage, context, exitReason, exitLineCode, ...why, ...auth } : { code: code ?? 1, aborted: false, turns, tokens, session, usage, context, exitReason, exitLineCode, ...why, ...auth });
+				// `resources` (issue #596) only when the exit line carried a block, so every other run resolves the object it always did.
+				// `memoryLimit` only beside the OOM report, the one place it is read, for the same reason.
+				const used = { ...(resources !== null && resources !== undefined ? { resources } : {}), ...(exitOomKilled === true ? { exitOomKilled: true, memoryLimit } : {}) };
+				resolve(aborted ? { code: code ?? 137, aborted: true, turns, tokens, session, usage, context, exitReason, exitLineCode, ...why, ...used, ...auth } : { code: code ?? 1, aborted: false, turns, tokens, session, usage, context, exitReason, exitLineCode, ...why, ...used, ...auth });
 			});
 		});
 
@@ -293,6 +334,19 @@ export function makeRunContainer({
 		// that answer. What a failure leaves behind is a memberless network, which the boot reaper sweeps.
 		try {
 			const result = await run;
+			// Issue #596: the runtime refused the ceiling as above its CPU count. The count it was computed from is stale
+			// (Docker Desktop's VM given fewer CPUs since the facts were read), and every later job would fail the same
+			// way until a restart, so the facts are dropped and the next pickup reads them again. This attempt stays a
+			// never-started one (refunded and retried as infrastructure), and the line names both counts.
+			const runtimeCpus = !result.aborted && (neverStartedExits ?? []).includes(result.code) && Number.isSafeInteger(hostCpus) ? cpuRangeRefusal(stderrHead) : null;
+			if (runtimeCpus !== null) {
+				log("cpu_ceiling_stale", { container: name, cpus: (cpuCeilingCenti(hostCpus, cpuBudgetCenti) ?? 0) / 100, hostCpus, runtimeCpus });
+				try {
+					onCpuCeilingStale();
+				} catch {
+					// a wiring fault must not rewrite this attempt's answer; the facts age out on their own (`maxAgeMs`)
+				}
+			}
 			// Issue #345: an exit that says "never started" is checked against the cidfile BEFORE the network goes, so a
 			// container found running is stopped while its network still exists. Only when the worker did not abort it.
 			if (!result.aborted && (neverStartedExits ?? []).includes(result.code) && (await stopDetached({ spawnFn, cidFile, fs, bin, ...detachedCheck }))) {