@edgehero/pi-dispatch 3.0.0 → 4.0.0

package/src/live-probes.mjs

@@ -28,7 +28,9 @@
  */
 
 import { READ_BACK_BY_A_LIVE_PROBE } from "./backend-conformance.mjs";
-import { containerSpec } from "./container-spec.mjs";
+import { containerSpec, memoryBytes } from "./container-spec.mjs";
+import { CGROUP_PARENT, DEFAULT_JOB_SIZE } from "./job-size.mjs";
+import { cpuMaxLine, parseCpuMaxRead } from "./cpu-reserve.mjs";
 import { ISOLATION_FLAGS, buildDockerRunArgs } from "./docker-run.mjs";
 import { DEFAULT_EGRESS_PROXY, EGRESS_PROXY_PORT, createJobNetworkWith, networkEndpoints, networkNameFor, removeNetworkOrSay } from "./egress.mjs";
 import { detachBlockedSentence, makeDetachGate } from "./netns-keeper.mjs";
@@ -112,8 +114,14 @@ export function liveFixture(root) {
 // `relabel` (issue #355) is a job's too: where a job's own mounts carry `:Z`, so do the probe's, because a probe mounted
 // the way no job is would read back a container no job gets. The fixture is doctor's own directory, so its workspace
 // is relabelled like a forge job's clone (`workspaceOwned`); its global overlay directory never is, exactly as a job's.
-function probeOptions({ image, name, fixture, user = null, relabel = false }) {
-	return { image, name, env: {}, network: "none", user, relabel: relabel === true, workspaceOwned: true, ...fixture };
+//
+// `size` and `hostCpus` (issue #596) are a job's too: the deployment's default size and the `--cpus` ceiling of this
+// runtime's own CPU count, so the probe is bounded exactly as a job without a project size is, and reads that back.
+//
+// `cgroupParent` (issue #596, phase 2) is a job's too: the jobs' parent cgroup, or null where this venue runs jobs without
+// it (`cgroupParentFor`), so the probe sits where a job sits and `cgroupParentVerdict` reads that back.
+function probeOptions({ image, name, fixture, user = null, relabel = false, size = DEFAULT_JOB_SIZE, hostCpus = null, cgroupParent = CGROUP_PARENT }) {
+	return { image, name, env: {}, network: "none", user, relabel: relabel === true, workspaceOwned: true, size, hostCpus, cgroupParent, ...fixture };
 }
 
 // `buildArgs` (issue #354) is the RUNTIME's job builder, never a second one: each probe below is built by whichever
@@ -121,32 +129,32 @@ function probeOptions({ image, name, fixture, user = null, relabel = false }) {
 // The default is docker's, so every argv here is byte-for-byte what it was.
 
 /** The probe container's argv: the job builder's, detached, with `sleep <derived seconds>` as its whole program. */
-export function liveProbeRunArgs({ image, name, fixture, sleepSeconds = liveSleepSeconds(), user = null, relabel = false, buildArgs = buildDockerRunArgs }) {
-	return [...buildArgs({ ...probeOptions({ image, name, fixture, user, relabel }), extraFlags: ["-d", "--entrypoint", "sleep"] }), String(sleepSeconds)];
+export function liveProbeRunArgs({ image, name, fixture, sleepSeconds = liveSleepSeconds(), user = null, relabel = false, buildArgs = buildDockerRunArgs, size = DEFAULT_JOB_SIZE, hostCpus = null, cgroupParent = CGROUP_PARENT }) {
+	return [...buildArgs({ ...probeOptions({ image, name, fixture, user, relabel, size, hostCpus, cgroupParent }), extraFlags: ["-d", "--entrypoint", "sleep"] }), String(sleepSeconds)];
 }
 
 /**
  * The pinning probe's argv: the job builder's, detached, against an image this host does not have. Detached so a
  * container that WAS created prints the ID it is removed by; with `--pull=never` in the builder none should be.
  */
-export function pinningProbeRunArgs({ name, nonce, fixture, user = null, relabel = false, buildArgs = buildDockerRunArgs }) {
-	return buildArgs({ ...probeOptions({ image: absentImageRef(nonce), name, fixture, user, relabel }), extraFlags: ["-d"] });
+export function pinningProbeRunArgs({ name, nonce, fixture, user = null, relabel = false, buildArgs = buildDockerRunArgs, size = DEFAULT_JOB_SIZE, hostCpus = null, cgroupParent = CGROUP_PARENT }) {
+	return buildArgs({ ...probeOptions({ image: absentImageRef(nonce), name, fixture, user, relabel, size, hostCpus, cgroupParent }), extraFlags: ["-d"] });
 }
 
 /**
  * One ephemeral run's argv (issue #344): the job builder's, detached, running EPHEMERAL_SCRIPT with the nonce and the
  * run's number. The same NAME both times, because "a job id run twice" is the question.
  */
-export function ephemeralRunArgs({ image, name, fixture, nonce, run, user = null, relabel = false, buildArgs = buildDockerRunArgs }) {
-	return [...buildArgs({ ...probeOptions({ image, name, fixture, user, relabel }), extraFlags: ["-d", "--entrypoint", "sh"] }), "-c", EPHEMERAL_SCRIPT, "sh", nonce, String(run)];
+export function ephemeralRunArgs({ image, name, fixture, nonce, run, user = null, relabel = false, buildArgs = buildDockerRunArgs, size = DEFAULT_JOB_SIZE, hostCpus = null, cgroupParent = CGROUP_PARENT }) {
+	return [...buildArgs({ ...probeOptions({ image, name, fixture, user, relabel, size, hostCpus, cgroupParent }), extraFlags: ["-d", "--entrypoint", "sh"] }), "-c", EPHEMERAL_SCRIPT, "sh", nonce, String(run)];
 }
 
 /**
  * One peer's argv (issue #344): the job builder's, detached, on its OWN job network, running PEER_SCRIPT, which answers
  * every connection with the nonce for `seconds`. Built with `network` set exactly as a job with egress armed is.
  */
-export function peerRunArgs({ image, name, fixture, network, nonce, seconds = liveSleepSeconds(), user = null, relabel = false, buildArgs = buildDockerRunArgs }) {
-	return [...buildArgs({ ...probeOptions({ image, name, fixture, user, relabel }), network, extraFlags: ["-d", "--entrypoint", "node"] }), "--eval", PEER_SCRIPT, nonce, String(PEER_PORT), String(seconds)];
+export function peerRunArgs({ image, name, fixture, network, nonce, seconds = liveSleepSeconds(), user = null, relabel = false, buildArgs = buildDockerRunArgs, size = DEFAULT_JOB_SIZE, hostCpus = null, cgroupParent = CGROUP_PARENT }) {
+	return [...buildArgs({ ...probeOptions({ image, name, fixture, user, relabel, size, hostCpus, cgroupParent }), network, extraFlags: ["-d", "--entrypoint", "node"] }), "--eval", PEER_SCRIPT, nonce, String(PEER_PORT), String(seconds)];
 }
 
 /**
@@ -156,6 +164,9 @@ export function peerRunArgs({ image, name, fixture, network, nonce, seconds = li
 export const STATUS_SCRIPT = [
 	"cat /proc/1/status",
 	'if [ -f /sys/fs/cgroup/cgroup.controllers ]; then echo "cgroup:v2"; echo "pids.max:$(cat /sys/fs/cgroup/pids.max 2>/dev/null)"; echo "memory.max:$(cat /sys/fs/cgroup/memory.max 2>/dev/null)";',
+	// Issue #596: the swap bound (0 when --memory-swap equals --memory), the CPU weight --cpu-shares became, and the
+	// --cpus ceiling. cgroup v2 only: v1 spells all three differently, and every venue measured is v2.
+	'echo "memory.swap.max:$(cat /sys/fs/cgroup/memory.swap.max 2>/dev/null)"; echo "cpu.weight:$(cat /sys/fs/cgroup/cpu.weight 2>/dev/null)"; echo "cpu.max:$(cat /sys/fs/cgroup/cpu.max 2>/dev/null)";',
 	'else echo "cgroup:v1"; echo "pids.max:$(cat /sys/fs/cgroup/pids/pids.max 2>/dev/null)"; echo "memory.max:$(cat /sys/fs/cgroup/memory/memory.limit_in_bytes 2>/dev/null)"; fi',
 ].join("\n");
 
@@ -246,6 +257,9 @@ export function parseStatus(output) {
 		cgroup: field("cgroup"),
 		pidsMax: field("pids.max"),
 		memoryMax: field("memory.max"),
+		swapMax: field("memory.swap.max"),
+		cpuWeight: field("cpu.weight"),
+		cpuMax: field("cpu.max"),
 	};
 }
 
@@ -257,9 +271,34 @@ export function expectedPidsLimit(flags = ISOLATION_FLAGS) {
 
 /** The memory bound in bytes, from the spec's own default (`4g`), never a literal. */
 export function expectedMemoryBytes(memory = containerSpec({ image: "i", name: "n", workspace: "/w" }).memory) {
-	const m = /^(\d+)([kmg]?)$/i.exec(String(memory));
-	if (!m) return null;
-	return Number(m[1]) * { "": 1, k: 1024, m: 1024 ** 2, g: 1024 ** 3 }[m[2].toLowerCase()];
+	return memoryBytes(memory);
+}
+
+/**
+ * What a container of this size reads back (issue #596), from the SAME spec builder the probe's argv came from:
+ * `{ pidsLimit, memoryBytes, cpuShares, cpuMax }`, `cpuMax` being the `cpu.max` line `--cpus` writes (`<quota> 100000`,
+ * or `max 100000` with no ceiling).
+ */
+export function expectedBounds(size = DEFAULT_JOB_SIZE, hostCpus = null, cpuBudgetCenti = null, cgroupParent = CGROUP_PARENT) {
+	const spec = containerSpec({ image: "i", name: "n", workspace: "/w", size, hostCpus, cpuBudgetCenti, cgroupParent });
+	// ROUNDED (issue #596, gate round 3 of phase 1): under a CPU budget the ceiling may be fractional (`3.3`), and
+	// `3.3 * 100000` is 329999.99999999994 in floating point, while the runtime writes the integer quota 330000.
+	return { pidsLimit: expectedPidsLimit(), memoryBytes: memoryBytes(spec.memory), cpuShares: spec.cpuShares, cpuMax: spec.cpus === null ? "max 100000" : `${Math.round(Number(spec.cpus) * 100000)} 100000` };
+}
+
+/**
+ * The `cpu.weight` a `--cpu-shares` value becomes, under the two mappings measured (issue #596): runc 1.1.13 and crun
+ * 1.14 map linearly (`1 + ((shares - 2) * 9999) / 262142`, integer division: 1024 to 39, 2048 to 79), runc 1.5.1 and
+ * crun 1.27 map so 1024 is 100 (`10^((l^2 + 125l) / 612 - 7/34)` rounded up, l = log2(shares): 2048 to 174). Both
+ * reproduce every value of the lab's sweep. A weight that is neither is a runtime nobody measured, said as such.
+ */
+export function cpuWeightsFor(shares) {
+	if (!Number.isSafeInteger(shares) || shares < 2) return [];
+	if (shares >= 262144) return [10000];
+	const linear = 1 + Math.floor(((shares - 2) * 9999) / 262142);
+	const l = Math.log2(shares);
+	const curved = Math.ceil(10 ** ((l * l + 125 * l) / 612 - 7 / 34));
+	return [...new Set([linear, curved])];
 }
 
 const verdict = (property, ok, detail, extra = {}) => ({ property, ok, detail, ...extra });
@@ -273,7 +312,7 @@ const notReadBack = (property, why) => verdict(property, false, `not read back:
  * FAILURE (the bound was not applied, as on rootless docker without delegation); a bound that could not be read at
  * all is "not read back", never a pass.
  */
-export function isolationVerdict(status, { pidsLimit = expectedPidsLimit(), memoryBytes = expectedMemoryBytes() } = {}) {
+export function isolationVerdict(status, { pidsLimit = expectedPidsLimit(), memoryBytes = expectedMemoryBytes(), cpuShares = null, cpuMax = null } = {}) {
 	if (!status || status.capBnd === null || status.noNewPrivs === null) return notReadBack("isolation", "PID 1's status did not show CapBnd and NoNewPrivs");
 	const failures = [];
 	if (!/^0+$/.test(status.capBnd)) failures.push(`CapBnd ${status.capBnd} (the capability bounding set is not empty)`);
@@ -287,10 +326,30 @@ export function isolationVerdict(status, { pidsLimit = expectedPidsLimit(), memo
 		if (Number(got) !== want) failures.push(`${name} ${got}, expected ${want}`);
 		return null;
 	};
-	const unread = [bound("pids.max", status.pidsMax, pidsLimit), bound("memory.max", status.memoryMax, memoryBytes)].filter(Boolean);
+	const unread = [bound("pids.max", status.pidsMax, pidsLimit), bound("memory.max", status.memoryMax, memoryBytes)];
+	// Issue #596, read only by a caller that passes the size's bounds (`expectedBounds`). The swap bound is part of the
+	// memory bound: `--memory-swap` equal to `--memory` makes it 0, and anything else lets a job swap past its size.
+	// `cpu.max` is the host ceiling (`--cpus`), which keeps the host's reserve, so a different one is a failure too.
+	// `cpu.weight` is the job's share, whose number depends on the runtime's version: one neither measured mapping
+	// gives is said, not failed, because the ORDER between jobs is what it carries and an unknown mapping may keep it.
+	let cpuNote = "";
+	if (cpuMax !== null) {
+		const swap = status.swapMax;
+		if (swap === null || swap === undefined || swap === "") unread.push("memory.swap.max not readable");
+		else if (swap !== "0") failures.push(`memory.swap.max ${swap}, expected 0 (a job may swap beyond its memory)`);
+		const max = status.cpuMax;
+		if (max === null || max === undefined || max === "") unread.push("cpu.max not readable");
+		else if (max !== cpuMax) failures.push(`cpu.max ${max}, expected ${cpuMax}`);
+		const weight = status.cpuWeight;
+		const known = cpuWeightsFor(cpuShares);
+		if (weight === null || weight === undefined || weight === "") unread.push("cpu.weight not readable");
+		else if (!known.includes(Number(weight))) unread.push(`cpu.weight ${weight} is neither mapping measured for --cpu-shares=${cpuShares} (${known.join(" or ")})`);
+		else cpuNote = `, memory.swap.max 0, cpu.max ${cpuMax}, cpu.weight ${weight} (--cpu-shares=${cpuShares})`;
+	}
+	const missing = unread.filter(Boolean);
 	if (failures.length > 0) return verdict("isolation", false, failures.join("; "));
-	if (unread.length > 0) return notReadBack("isolation", `${unread.join(" and ")} (cgroup ${status.cgroup ?? "unknown"}); CapBnd 0 and NoNewPrivs 1 did hold`);
-	return verdict("isolation", true, `CapBnd 0, NoNewPrivs 1, pids.max ${pidsLimit}, memory.max ${memoryBytes}`);
+	if (missing.length > 0) return notReadBack("isolation", `${missing.join(" and ")} (cgroup ${status.cgroup ?? "unknown"}); CapBnd 0 and NoNewPrivs 1 did hold`);
+	return verdict("isolation", true, `CapBnd 0, NoNewPrivs 1, pids.max ${pidsLimit}, memory.max ${memoryBytes}${cpuNote}`);
 }
 
 /** NON-ROOT: all four Uid fields (real, effective, saved, filesystem) nonzero, on the process the job would be. */
@@ -612,6 +671,60 @@ export function peerTargetsOf(inspectOutput, { network, name }) {
 	return { targets: [...[...hosts].map((h) => `${h}:${PEER_PORT}`), ...addresses], addresses };
 }
 
+/**
+ * WHERE THE RUNTIME PUT A JOB-BUILT CONTAINER (issue #596, phase 2, INT-LIVE-PROBE-CONTRACT): `{ ok, warn?, detail,
+ * placed, quota }`, from `<bin> inspect --format={{.HostConfig.CgroupParent}}|{{.State.Pid}}` and this host's own files.
+ *
+ * Three readings, each as far as this host can see. The runtime's RECORD of the parent must be the one the argv named
+ * (`expected`; none where the venue runs jobs without it). The process's own cgroup (`/proc/<pid>/cgroup`, cgroup v2's
+ * `0::<path>`) must lie under `/<parent>/`, which proves the placement rather than the flag: readable for a local Linux
+ * daemon and rootless Podman, not for a daemon in a VM (Docker Desktop), where the record alone is said as such. The
+ * container cannot see any of this itself (a private cgroup namespace shows it `0::/`, measured). Then the parent's
+ * `cpu.max`, where readable, against the CPU budget (`cpuBudgetCenti`: an integer, `Infinity` for off, null not
+ * checked): a quota that is not the budget is a WARNING, "no host CPU reserve across jobs", never a failure of the
+ * placement, because on a systemd host only the operator can set it.
+ */
+export function cgroupParentVerdict({ inspected, expected = CGROUP_PARENT, cpuBudgetCenti = null, readFile = () => {
+	throw new Error("no reader");
+}, bin = "docker" }) {
+	const out = (ok, detail, extra = {}) => ({ ok, detail, placed: null, quota: undefined, ...extra });
+	if (inspected?.code !== 0) return out(false, `not read back: ${bin} inspect did not answer`, { warn: true });
+	const [recorded = "", pidText = ""] = String(inspected.stdout ?? "").trim().split("|");
+	if (expected === null) {
+		if (recorded.includes(CGROUP_PARENT)) return out(false, `the runtime recorded the parent ${recorded} for a venue that runs jobs without one`);
+		return out(true, `this venue runs jobs without the ${CGROUP_PARENT} parent (Podman's cgroup manager is not systemd), so their CPU weight is capped at 1024: the egress proxy and Valkey get a fair share of the CPU, not a reserve`, { warn: true });
+	}
+	// The bare name, or a path ending in it (a runtime may record where the slice resolved); the process's own cgroup
+	// below is the proof either way.
+	if (recorded.replace(/^\//, "") !== expected && !recorded.endsWith(`/${expected}`)) return out(false, `the runtime recorded the parent ${JSON.stringify(recorded)}, not ${expected}, so this container is outside the jobs' CPU reserve`);
+	let cgroupText = null;
+	if (/^[1-9][0-9]{0,9}$/.test(pidText)) {
+		try {
+			cgroupText = readFile(`/proc/${pidText}/cgroup`);
+		} catch {
+			cgroupText = null;
+		}
+	}
+	const path = cgroupText === null ? null : (/^0::(\/\S*)$/m.exec(String(cgroupText))?.[1] ?? null);
+	if (path === null) return out(true, `the runtime recorded the parent ${expected}; the container's own cgroup is not readable from this host (its process runs in the runtime's VM or another namespace), so the placement and the parent's quota were not read here`, { warn: true });
+	const at = path.indexOf(`/${expected}/`);
+	if (at === -1) return out(false, `the runtime recorded the parent ${expected}, but the container's cgroup is ${path}, not under it`);
+	const parentDir = `/sys/fs/cgroup${path.slice(0, at + expected.length + 1)}`;
+	let quota;
+	try {
+		quota = parseCpuMaxRead(readFile(`${parentDir}/cpu.max`));
+	} catch {
+		quota = null;
+	}
+	const placed = path.slice(0, at + expected.length + 1);
+	if (quota === null) return out(true, `the container's cgroup is under ${placed}; the parent's cpu.max is not readable here`, { warn: true, placed });
+	const shown = quota.cpuCenti === null ? "no quota" : `a quota of ${quota.cpuCenti / 100} CPUs (${cpuMaxLine(quota.cpuCenti)})`;
+	if (cpuBudgetCenti === null || cpuBudgetCenti === undefined) return out(true, `the container's cgroup is under ${placed}, which has ${shown}`, { placed, quota: quota.cpuCenti });
+	const want = cpuBudgetCenti === Infinity ? null : cpuBudgetCenti;
+	if (quota.cpuCenti === want) return out(true, `the container's cgroup is under ${placed}, which has ${want === null ? "no quota, as the CPU budget is off" : `${shown}, the host's CPU budget, so all jobs together keep the reserve`}`, { placed, quota: quota.cpuCenti });
+	return out(true, `the container's cgroup is under ${placed}, which has ${shown}, not ${want === null ? "none (the CPU budget is off)" : `the CPU budget of ${want / 100}`}: no host CPU reserve across jobs`, { warn: true, placed, quota: quota.cpuCenti });
+}
+
 /**
  * The whole sequence. Returns `{ ran, reason, verdicts, notes, swept, ranAs }`: `ran` false with a `reason` when nothing
  * was read back; the verdicts in READ_BACK_BY_A_LIVE_PROBE's order otherwise; `notes` for a teardown that failed, on
@@ -660,6 +773,14 @@ export async function runLiveProbes({
 	// Issue #452, gate round 3: the runtime as this pass already read it (`{ podman, rootless, version } | null`), for the
 	// detach gate, so doctor asks the daemon once per run. Absent, the gate reads it itself.
 	readRuntime = null,
+	// Issue #596: the size every probe container is built at (a job's default) and the runtime's CPU count for the
+	// `--cpus` ceiling; the isolation verdict reads both back.
+	size = DEFAULT_JOB_SIZE,
+	hostCpus = null,
+	// Issue #596, phase 2: the parent cgroup a job on this venue runs under (null: none, `cgroupParentFor`) and the CPU
+	// budget its quota should be (an integer in hundredths, `Infinity` for off, null or absent: not checked).
+	cgroupParent = CGROUP_PARENT,
+	cpuBudgetCenti = null,
 }) {
 	const notRun = (reason) => ({ ran: false, reason, verdicts: [], notes: [], swept: [] });
 	const notLocal = notRun(`this shell's ${bin} CLI is not observed to point at this host, so bind paths and .Mounts would describe another machine; nothing was run`);
@@ -764,7 +885,7 @@ export async function runLiveProbes({
 		}
 
 		// --- the reading container: mounts, status, writes ---
-		const reading = await start("probe container", names.probe, liveProbeRunArgs({ image, name: names.probe, fixture, sleepSeconds: liveSleepSeconds(stepTimeoutMs), user, relabel, buildArgs }));
+		const reading = await start("probe container", names.probe, liveProbeRunArgs({ image, name: names.probe, fixture, sleepSeconds: liveSleepSeconds(stepTimeoutMs), user, relabel, buildArgs, size, hostCpus, cgroupParent }));
 		const probeId = reading.entry.id;
 		if (reading.result?.code !== 0 || probeId === null) {
 			// The runtime's own words, when it printed any (issue #453, gate round 1): "did not start" alone left the operator
@@ -773,7 +894,7 @@ export async function runLiveProbes({
 			return { ran: false, reason: `the probe container did not start${said ? ` (${bin} said: ${said})` : ""}, so nothing was read back`, verdicts: [], notes, swept };
 		}
 
-		const expected = containerSpec(probeOptions({ image, name: names.probe, fixture, user, relabel })).mounts;
+		const expected = containerSpec(probeOptions({ image, name: names.probe, fixture, user, relabel, size, hostCpus, cgroupParent })).mounts;
 		const inspected = await step(["inspect", "--format={{json .Mounts}}", probeId]);
 		// Issue #345: the mount table as the container itself sees it, by a constant `cat`, for what `.Mounts` does not list.
 		const mountinfo = await step(["exec", probeId, "cat", "/proc/self/mountinfo"]);
@@ -781,8 +902,12 @@ export async function runLiveProbes({
 
 		const statusRun = await step(["exec", probeId, "sh", "-c", STATUS_SCRIPT]);
 		const status = statusRun?.code === 0 ? parseStatus(statusRun.stdout) : null;
-		const isolation = status ? isolationVerdict(status) : notReadBack("isolation", "the status probe did not run in the container");
+		const isolation = status ? isolationVerdict(status, expectedBounds(size, hostCpus, null, cgroupParent)) : notReadBack("isolation", "the status probe did not run in the container");
 		const nonRoot = status ? nonRootVerdict(status) : notReadBack("nonRoot", "the status probe did not run in the container");
+		// Issue #596, phase 2: where the runtime put the container (its record, and the process's own cgroup where this host
+		// can read it) and, where readable, the parent's quota. Its own line, not one of the declared properties.
+		const placed = await step(["inspect", "--format={{.HostConfig.CgroupParent}}|{{.State.Pid}}", probeId]);
+		const parentRead = cgroupParentVerdict({ inspected: placed, expected: cgroupParent, cpuBudgetCenti, readFile: (path) => fs.readFileSync(path, "utf8"), bin });
 
 		const written = await step(["exec", probeId, "sh", "-c", WRITE_SCRIPT, "sh", nonce]);
 		const readBack = (dir) => {
@@ -807,7 +932,7 @@ export async function runLiveProbes({
 		await release(reading.entry);
 
 		// --- the pinning container: an image this host does not have ---
-		const pinning = await start("pinning container", names.pin, pinningProbeRunArgs({ name: names.pin, nonce, fixture, user, relabel, buildArgs }));
+		const pinning = await start("pinning container", names.pin, pinningProbeRunArgs({ name: names.pin, nonce, fixture, user, relabel, buildArgs, size, hostCpus, cgroupParent }));
 		const after = await step(["image", "inspect", absentImageRef(nonce)]);
 		const stillAbsent = after?.code === 0 ? false : typeof after?.code === "number" ? true : null;
 		const imagePinning = imagePinningVerdict({ code: pinning.result?.code, output: `${pinning.result?.stdout ?? ""}${pinning.result?.stderr ?? ""}`, stillAbsent, bin });
@@ -815,7 +940,7 @@ export async function runLiveProbes({
 
 		// --- the ephemeral pair (issue #344): one name, two runs, each waited on until it is gone ---
 		const runEphemeral = async (n) => {
-			const { result, entry } = await start(`ephemeral container (run ${n})`, names.ephemeral, ephemeralRunArgs({ image, name: names.ephemeral, fixture, nonce, run: n, user, relabel, buildArgs }));
+			const { result, entry } = await start(`ephemeral container (run ${n})`, names.ephemeral, ephemeralRunArgs({ image, name: names.ephemeral, fixture, nonce, run: n, user, relabel, buildArgs, size, hostCpus, cgroupParent }));
 			const started = result?.code === 0 && entry.id !== null;
 			// A HELD NAME is the daemon refusing the create for the name, in its own words (measured: Docker "Conflict. ...
 			// is already in use", Podman "that name is already in use"). Not a listed container: after the first run was
@@ -864,7 +989,7 @@ export async function runLiveProbes({
 						} catch {
 							break;
 						}
-						const { result, entry } = await start(`${key} container`, names[key], peerRunArgs({ image, name: names[key], fixture: peerFixture, network: networkOf[key], nonce, seconds: liveSleepSeconds(stepTimeoutMs), user, relabel, buildArgs }));
+						const { result, entry } = await start(`${key} container`, names[key], peerRunArgs({ image, name: names[key], fixture: peerFixture, network: networkOf[key], nonce, seconds: liveSleepSeconds(stepTimeoutMs), user, relabel, buildArgs, size, hostCpus, cgroupParent }));
 						peers.push(entry);
 						if (result?.code !== 0 || entry.id === null) break;
 						ids[key] = entry.id;
@@ -901,7 +1026,7 @@ export async function runLiveProbes({
 		const byProperty = { isolation, ephemeral, mountSet, egress: egressVerdict(egress ?? {}), jobToJobIsolation, imagePinning, nonRoot, localFolders };
 		// The uid PID 1 actually ran as, for doctor's job-user line: the decision it was given, read back.
 		const ranAs = Array.isArray(status?.uids) && /^\d+$/.test(status.uids[1] ?? "") ? Number(status.uids[1]) : null;
-		return { ran: true, verdicts: READ_BACK_BY_A_LIVE_PROBE.map((p) => byProperty[p]), notes, swept, ranAs };
+		return { ran: true, verdicts: READ_BACK_BY_A_LIVE_PROBE.map((p) => byProperty[p]), notes, swept, ranAs, cgroupParent: parentRead };
 	} finally {
 		for (const entry of owned) await release(entry);
 		for (const entry of networks) await dropNetwork(entry);

package/src/model-catalog.mjs

@@ -91,7 +91,7 @@ function overlayDeclares(overlay, provider, id) {
 
 /**
  * Would pi COMPOSE this overlay provider (PR #536's review)? A file that passes pi's schema can still lose a provider
- * at the next step: pi 0.99.1's `applyModelsJson` and `modelFromJson` (`pi-coding-agent/dist/core/provider-composer.js`)
+ * at the next step: the pinned pi's `applyModelsJson` and `modelFromJson` (`pi-coding-agent/dist/core/provider-composer.js`)
  * throw for `oauth` with no `baseUrl`, and per model no resolvable `api` or
  * `baseUrl`, or a `contextWindow` or `maxTokens` at or below zero. pi then keeps the provider's BUILTIN models
  * (`ModelRuntime.composeProvider` falls back to the base) and drops every model the overlay added, so such a model

package/src/model-endpoints.mjs

@@ -518,6 +518,28 @@ export function unreportedUsageModels(models, { builtinModel = () => null, built
 /** The exact `apiKey` a keyless provider's models.json entry carries: pi resolves `$NAME` from the job's environment. */
 export const KEYLESS_API_KEY = `$${KEYLESS_ENV_NAME}`;
 
+/**
+ * Provider ids pi renamed, old to new (issue #587: pi 1.0.3 renamed `azure-openai-responses` to `azure`, and its catalog
+ * file with it; the api id `azure-openai-responses` is unchanged). Pinned against the catalog by
+ * worker/test/env-allowlist.test.mjs: every old id is gone from pi, every new id is there. A rename is not a refusal of
+ * its own: the old id is already refused before spend as a provider pi does not have, and this only lets that refusal
+ * and doctor say which id was meant.
+ */
+export const RENAMED_PROVIDERS = Object.freeze({ "azure-openai-responses": "azure" });
+
+/**
+ * The "did you mean" for a provider pi renamed, or "" (issue #587). Only for an old id pi no longer has, whose new id pi
+ * does have, and that the overlay's models.json (`models`, parsed, or null when there is none) does not declare: a
+ * provider the file declares under the old id is the operator's own custom provider, and the hint would be wrong there.
+ * `overlayUnread`: the overlay exists but could not be read, so whether it declares the id is not known: no guess.
+ */
+export function providerRenameHint(provider, { models = null, piProviders = [], overlayUnread = false } = {}) {
+	if (typeof provider !== "string" || !Object.hasOwn(RENAMED_PROVIDERS, provider)) return "";
+	const renamed = RENAMED_PROVIDERS[provider];
+	if (overlayUnread || piProviders.includes(provider) || !piProviders.includes(renamed) || providerOf(models, provider) !== null) return "";
+	return ` pi renamed the provider "${provider}" to "${renamed}" in pi 1.0.3: did you mean "${renamed}"?`;
+}
+
 /**
  * The second way into the credential gate, said once for every refusal and doctor line that names it (issue #503).
  */

package/src/models-json.mjs

@@ -1,5 +1,5 @@
 /**
- * The overlay `models.json`, read the way pi reads it (issue #502, PR #536's review): pi 0.99.1's
+ * The overlay `models.json`, read the way pi reads it (issue #502, PR #536's review): pi's
  * `ModelConfig.load` (`pi-coding-agent/dist/core/model-config.js`) strips a leading BOM, strips `//` comments and
  * trailing commas outside strings, parses, and then checks the whole document against its TypeBox schema. ANY
  * schema error drops the WHOLE file: pi then knows none of its providers or models.
@@ -121,7 +121,8 @@ const T = {
 const opt = (check) => [check, true];
 const req = (check) => [check, false];
 
-// The schema, transcribed from model-config.js at the 0.99.1 pin, in its order.
+// The schema, transcribed from model-config.js at the pin (1.0.3; worker/test/models-json.test.mjs holds the file's
+// content hash), in its order.
 const PercentileCutoffs = T.obj({ p50: opt(T.num()), p75: opt(T.num()), p90: opt(T.num()), p99: opt(T.num()) });
 const numOrStr = T.union(T.num(), T.str());
 const OpenRouterRouting = T.obj({
@@ -142,6 +143,9 @@ const OpenRouterRouting = T.obj({
 const VercelGatewayRouting = T.obj({ only: opt(T.arr(T.str())), order: opt(T.arr(T.str())) });
 const ThinkingValue = T.union(T.str(), T.nul());
 const ThinkingLevelMap = T.obj({ off: opt(ThinkingValue), minimal: opt(ThinkingValue), low: opt(ThinkingValue), medium: opt(ThinkingValue), high: opt(ThinkingValue), xhigh: opt(ThinkingValue), max: opt(ThinkingValue) });
+// pi 1.0.2 (issue #587): a model's sampling parameters, and the same per thinking level, merged per call over them.
+const SamplingParams = T.rec(T.unknown());
+const SamplingParamsByThinkingLevel = T.obj({ off: opt(SamplingParams), minimal: opt(SamplingParams), low: opt(SamplingParams), medium: opt(SamplingParams), high: opt(SamplingParams), xhigh: opt(SamplingParams), max: opt(SamplingParams) });
 const KwargScalar = T.union(T.str(), T.num(), T.bool(), T.nul());
 const KwargVariable = T.obj({ $var: req(T.union(T.lit("thinking.enabled"), T.lit("thinking.effort"))), omitWhenOff: opt(T.bool()) });
 const Kwarg = T.union(KwargScalar, KwargVariable);
@@ -214,7 +218,8 @@ const ModelDefinition = T.obj({
 	promptCache: opt(ModelPromptCache),
 	contextWindow: opt(T.num()),
 	maxTokens: opt(T.num()),
-	samplingParams: opt(T.rec(T.unknown())),
+	samplingParams: opt(SamplingParams),
+	samplingParamsByThinkingLevel: opt(SamplingParamsByThinkingLevel),
 	headers: opt(headers),
 	compat: opt(ProviderCompat),
 });
@@ -228,7 +233,8 @@ const ModelOverride = T.obj({
 	promptCache: opt(ModelPromptCache),
 	contextWindow: opt(T.num()),
 	maxTokens: opt(T.num()),
-	samplingParams: opt(T.rec(T.unknown())),
+	samplingParams: opt(SamplingParams),
+	samplingParamsByThinkingLevel: opt(SamplingParamsByThinkingLevel),
 	headers: opt(headers),
 	compat: opt(ProviderCompat),
 });

package/src/output-cap.mjs

@@ -13,7 +13,7 @@
 
 import { composedCost, endpointsForModel, isZeroCost, modelEntryOf } from "./model-endpoints.mjs";
 
-/** The hosts pi's catalog serves openai-completions on: the runner's `COMPLETIONS_CATALOG_HOSTS`, copied. */
+/** The hosts pi's catalog serves openai-completions on: the runner's `COMPLETIONS_CATALOG_HOSTS`, generated the same way. */
 export const COMPLETIONS_CATALOG_HOSTS = Object.freeze([
 	"api.ant-ling.com",
 	"api.cerebras.ai",
@@ -56,7 +56,7 @@ export function completionsOwnServer(model) {
 }
 
 /**
- * The parts of a model the rule reads, composed as pi 0.99.1's provider-composer.js composes them from the overlay
+ * The parts of a model the rule reads, composed as the pinned pi's provider-composer.js composes them from the overlay
  * `models.json` (`models`, parsed, or null) and the builtin catalog (`builtinModel(provider, id)`, injected):
  *   - a model the overlay DEFINES: its own `api` and `baseUrl`, else its provider's, else those of pi's DEFAULTS
  *     model (`findModelDefaults`, mirrored in `definedDefaults`); its compat field from its `modelOverrides` entry,
@@ -94,7 +94,7 @@ export function outputCapView({ models, provider, modelId, builtinModel = () =>
 const str = (v) => (typeof v === "string" ? v : undefined);
 
 /**
- * The `{ api, baseUrl }` pi 0.99.1 composes for the overlay-defined model `modelId` (provider-composer.js
+ * The `{ api, baseUrl }` the pinned pi composes for the overlay-defined model `modelId` (provider-composer.js
  * `applyModelsJson`): the provider's chat models start as the catalog's (each on the provider's `baseUrl` when it sets
  * one), and each definition in the file's order is composed and then replaces the model of its id or joins the list.
  * A definition's `api` is its own, else the provider's, else its DEFAULTS model's, and its `baseUrl` likewise, where the

package/src/prepare.mjs

@@ -14,6 +14,7 @@ import { buildForgejoPrompt } from "./forgejo-prompt.mjs";
 import { buildAzurePrompt } from "./azure-prompt.mjs";
 import { prepareLocalWorkspace } from "./prepare-local.mjs";
 import { copySkillTree } from "./copy-tree.mjs";
+import { recordedJobSize } from "./job-size.mjs";
 
 /**
  * The subdirectory of the per-job dir a trigger's injected skills are copied into, so they reach the
@@ -82,7 +83,7 @@ export function makePrepareWorkspace({
 	removeDir = (dir) => rmSync(dir, { recursive: true, force: true }),
 }) {
 	ensureDir(jobsDir);
-	return async function prepareWorkspace(job, token, { queueJobId, piVersion = null, jobUser = null, podmanStore = null, portfolio = false } = {}) {
+	return async function prepareWorkspace(job, token, { queueJobId, piVersion = null, jobUser = null, podmanStore = null, portfolio = false, size = null } = {}) {
 		ensureDir(jobsDir);
 		const jobDir = mkdtempSync(join(jobsDir, "job-"));
 		// Issue #524: a THROW out of anything below leaves `jobDir` to nobody. The processor tears down only what
@@ -92,7 +93,7 @@ export function makePrepareWorkspace({
 		// covers the thrown ones, in one place for every kind, rather than in each preparer that can throw.
 		// A retry makes a fresh directory, so nothing is lost by removing this one.
 		try {
-			return await prepareInto(job, token, jobDir, { queueJobId, piVersion, jobUser, podmanStore, portfolio });
+			return await prepareInto(job, token, jobDir, { queueJobId, piVersion, jobUser, podmanStore, portfolio, size });
 		} catch (error) {
 			// GUARDED: a removal that fails (a busy mount, a permission flipped mid-job) must never replace the error
 			// that is the job's actual outcome. The directory is then left, which is what happened before this catch.
@@ -103,7 +104,7 @@ export function makePrepareWorkspace({
 		}
 	};
 
-	async function prepareInto(job, token, jobDir, { queueJobId, piVersion, jobUser, podmanStore, portfolio }) {
+	async function prepareInto(job, token, jobDir, { queueJobId, piVersion, jobUser, podmanStore, portfolio, size }) {
 		// The trigger's injected skills (REQ-PER-TRIGGER-SKILLS, issue #60), COPIED here rather than
 		// mounted, and copied ONCE for every job kind because this is where local and forge converge.
 		//
@@ -137,6 +138,9 @@ export function makePrepareWorkspace({
 			...(jobUser ? { jobUser: { user: jobUser.user ?? null, home: jobUser.home ?? null } } : {}),
 			// Issue #429: the podman store the run's container lived in, for a podman run only.
 			...(typeof podmanStore === "string" && podmanStore !== "" ? { podmanStore } : {}),
+			// Issue #596: the size the run's container had, so a re-opened sandbox gets the same memory and CPU weight.
+			// Rebuilt (`recordedJobSize`), and only when the processor resolved one; a direct call stamps nothing.
+			...(recordedJobSize(size) ? { size: recordedJobSize(size) } : {}),
 		};
 		if (job.kind === "local") {
 			// Harness text above, operator DATA below: the fixed pointer line names /job/event.json so a