@edgehero/pi-dispatch 3.0.0 → 4.0.0

package/src/job-size.mjs

@@ -0,0 +1,286 @@
+/**
+ * How big one job's container is (issue #596, phase 1): its memory and its CPU weight, per project.
+ *
+ * A LEAF, importing nothing, for the reason `container-spec.mjs` is one: that module builds every container's argv
+ * from a size, and `packages.mjs` relies on it importing nothing heavy. So the project row is matched here by its
+ * scope string (`project:<id>`) rather than through `scoped-limits.mjs`; a test holds the two spellings equal.
+ *
+ * A size is held as TWO INTEGERS, never as the strings an operator writes: `memMiB` (mebibytes) and `cpuCenti`
+ * (hundredths of a CPU). Integers so a size compares, sums and records exactly (the host budget, `host-budget.mjs`, adds them
+ * up), and so the one spelling the runtime is handed is derived from them (`formatMemory`), never passed through.
+ *
+ * WHAT A SIZE BECOMES (`containerSizing`, read by `containerSpec`):
+ *   - `--memory=<size>` and `--memory-swap=<size>`, equal, so a job gets NO swap beyond its memory. Docker and Podman
+ *     give a container swap equal to its memory by default; with the two equal `memory.swap.max` reads 0 on every
+ *     venue measured, rootless Podman included (lab, issue #596). A job that needs more memory needs a bigger size,
+ *     not a slower one.
+ *   - `--cpu-shares=round(cpus x 1024)`: a WEIGHT, not a cap. Under contention a larger size gets more CPU than a
+ *     smaller one, in order; an idle host lets any job use the idle cores. This is the fair share decision 2 of the
+ *     issue asked for. EXACTLY proportional only on a runtime with the linear mapping (below): measured, sizes 3:1
+ *     split CPU about 3:1 on runc 1.1.13 and crun 1.14, and about 2.4:1 on runc 1.5.1 and crun 1.27, whose curved
+ *     mapping compresses the weights. The order holds on both, which is what a share promises here.
+ *   - `--shm-size=min(1g, memory/2)`: Chromium needs a large `/dev/shm` (the reason `1g` was there), and `/dev/shm`
+ *     is charged to the container's memory, so it may not be most of a small size.
+ *   - `--cpus=<host ceiling>`: the host's CPU budget, else its CPU count minus a reserve (`cpuCeilingCenti`), the same for every job. It
+ *     bounds ONE job: no single job can use more than the ceiling. It does not keep a core free across jobs, because
+ *     each container's quota is its own and they do not sum (measured: two busy jobs on a 4-core cpuset with
+ *     `--cpus=3` used 4.06 cores between them). The reserve across every job is the quota of the one parent cgroup
+ *     they all run under (`CGROUP_PARENT`, `cpu-reserve.mjs`). It is not the job's size either, which would be the hard cap decision 2
+ *     rejected; the host budget bounds the jobs' SIZES together, not their use.
+ *     Absent when the runtime did not say how many CPUs it has (see `hostCpuCeiling`).
+ *
+ * THE cpu.weight A SHARE BECOMES DEPENDS ON THE OCI RUNTIME'S VERSION (measured, issue #596): runc 1.1.13 and crun
+ * 1.14 map shares linearly (1024 to 39, 2048 to 79), runc 1.5.1 and crun 1.27 map 1024 to 100 and 2048 to 174, and a
+ * container started without `--cpu-shares` gets 100 on all four. Both mappings keep the ORDER of shares, and every job
+ * container carries a share, so the order between jobs, which is what a fair share is about, holds everywhere. What an
+ * old runtime changes is a job's weight against a container started WITHOUT a share (the egress proxy, a Valkey): a
+ * default job's 79 is below their 100. That is the right direction (the proxy serves every job), and no such container
+ * competes with jobs for long, so the flag is passed as is rather than scaled per runtime version.
+ *
+ * THE OTHER DIRECTION is closed by the parent cgroup, not by a weight scale (issue #596, phase 2). At `cpus` 256 the
+ * share is 262144, which a current runtime maps to `cpu.weight` 10000 against the 100 of the egress proxy, Valkey and the
+ * host's own services; with no parent such jobs left a proxy-like container 0.01 of a core (measured). Every job runs
+ * under `CGROUP_PARENT`, where a job's weight competes only with its sibling jobs and the parent competes as ONE group of
+ * weight 100, so the shares are kept as they are and still order the jobs (on Fedora's kernel 6.19 the ratio is
+ * compressed under the parent's throttling, the order holds). Only a job run WITHOUT the parent has its shares capped,
+ * at `PARENTLESS_SHARES_MAX`.
+ */
+
+/** The smallest memory a job may be given: 512 MiB. The runner, pi and a shell need about 200 MB before any tool runs. */
+export const JOB_MEMORY_FLOOR_MIB = 512;
+/** The largest: 1024 GiB. A bound against a typo (`40000g`), and it keeps every byte count a safe integer. */
+export const JOB_MEMORY_CEILING_MIB = 1024 * 1024;
+/** The smallest CPU size: 0.25, as hundredths. Its share, 256, is well inside the runtime's range. */
+export const JOB_CPUS_FLOOR_CENTI = 25;
+/** The largest CPU size: 256, as hundredths, because round(256 x 1024) is 262144, the top of `--cpu-shares`. */
+export const JOB_CPUS_CEILING_CENTI = 256 * 100;
+/** The valid range of `--cpu-shares` on cgroup v2 (runc and crun clamp to it; 2 is the kernel's cgroup v1 minimum). */
+export const CPU_SHARES_MIN = 2;
+export const CPU_SHARES_MAX = 262144;
+/**
+ * The one parent cgroup every job container is started under (issue #596, phase 2): a slice name with NO dash, because
+ * a dash nests (`pd-jobs.slice` lands in `/pd.slice/pd-jobs.slice/`, measured on docker and podman). Its quota is the
+ * host's CPU budget; `cpu-reserve.mjs` says who sets it on which venue and why it is always on the argv.
+ */
+export const CGROUP_PARENT = "pidispatch.slice";
+
+/**
+ * The `--cpu-shares` cap for a job that runs WITHOUT the parent (`cpu-reserve.mjs` `cgroupParentFor`): 1024 is weight
+ * 100 on current runtimes (39 on old ones), the egress proxy's and Valkey's default, so no job outweighs them. A FAIR
+ * share, not a reserve: six busy jobs at 1024 left a proxy-like container 0.57 of a core, one seventh (measured).
+ */
+export const PARENTLESS_SHARES_MAX = 1024;
+/** `/dev/shm`'s ceiling, the `--shm-size=1g` every job had before sizes. */
+export const SHM_CEILING_MIB = 1024;
+
+/** The size every job had before issue #596, and the one a deployment that sets nothing still gets. */
+export const DEFAULT_JOB_MEMORY = "4g";
+export const DEFAULT_JOB_CPUS = "2";
+
+/** Where a resolved size came from: the job's project row, the deployment's `PI_JOB_*` settings, or the built-in default. */
+export const JOB_SIZE_SOURCES = Object.freeze(["project", "env", "default"]);
+
+/** The scope prefix of a project row, as `scoped-limits.mjs` spells it (`PROJECT_SCOPE_PREFIX`; a test holds them equal). */
+const PROJECT_ROW_PREFIX = "project:";
+
+/**
+ * A memory size, written as an operator writes it (`512m`, `1536m`, `4g`), in MiB. Throws an Error naming the rule for
+ * anything else. Lower-case `m` or `g` only, no zero and no leading zero, no fraction, no bytes or kilobytes: one
+ * spelling per size, so a row reads the same in the file, the panel and the record. Strings only: a bare number would
+ * have to mean bytes to agree with the runtime, and nobody writes a job's memory in bytes.
+ */
+export function parseMemory(value) {
+	if (typeof value !== "string" || !/^[1-9]\d*[mg]$/.test(value)) {
+		throw new Error(`a memory size is a whole number of megabytes or gigabytes such as "512m", "1536m" or "4g" (got ${JSON.stringify(value)})`);
+	}
+	const digits = value.slice(0, -1);
+	// Compared as digits first, so a value of any length is refused without ever becoming an unsafe number.
+	const ceilingDigits = String(JOB_MEMORY_CEILING_MIB).length;
+	const mib = digits.length > ceilingDigits + 1 ? Infinity : Number(digits) * (value.endsWith("g") ? 1024 : 1);
+	if (mib < JOB_MEMORY_FLOOR_MIB) throw new Error(`a memory size must be at least ${JOB_MEMORY_FLOOR_MIB}m (got ${JSON.stringify(value)})`);
+	if (mib > JOB_MEMORY_CEILING_MIB) throw new Error(`a memory size must be at most ${JOB_MEMORY_CEILING_MIB / 1024}g (got ${JSON.stringify(value)})`);
+	return mib;
+}
+
+/**
+ * A CPU size (`0.5`, `2`, `"1.25"`), in hundredths. A number or a decimal string: above 0, at most two decimals, at
+ * least 0.25 and at most 256. Throws an Error naming the rule for anything else. A number is judged by its own decimal
+ * spelling (`String(n)`), so `1e-7`, `NaN`, `Infinity` and `0.125` are refused rather than rounded into a size.
+ */
+export function parseCpus(value) {
+	const text = typeof value === "number" ? String(value) : value;
+	if (typeof text !== "string" || !/^(?:0|[1-9]\d{0,5})(?:\.\d{1,2})?$/.test(text)) {
+		throw new Error(`a CPU size is a number above 0 with at most two decimals, such as 0.5, 2 or 1.25 (got ${JSON.stringify(value)})`);
+	}
+	const [whole, fraction = ""] = text.split(".");
+	const centi = Number(whole) * 100 + Number(fraction.padEnd(2, "0"));
+	if (centi < JOB_CPUS_FLOOR_CENTI) throw new Error(`a CPU size must be at least ${JOB_CPUS_FLOOR_CENTI / 100} (got ${JSON.stringify(value)})`);
+	if (centi > JOB_CPUS_CEILING_CENTI) throw new Error(`a CPU size must be at most ${JOB_CPUS_CEILING_CENTI / 100} (got ${JSON.stringify(value)})`);
+	return centi;
+}
+
+/**
+ * The one spelling of a memory size: `<n>g` when it is whole gigabytes, else `<n>m`. Every `--memory`, `--memory-swap`
+ * and stored row is written by this function, and `container-spec.mjs`'s `memoryBytes` reads every value it can write
+ * (a test walks the whole accepted range), so the OOM rule's 90%-of-the-limit comparison works at every size.
+ */
+export function formatMemory(memMiB) {
+	return memMiB % 1024 === 0 ? `${memMiB / 1024}g` : `${memMiB}m`;
+}
+
+/** The one spelling of a CPU size: `2`, `0.5`, `1.25`. Built from the integer, so no float ever prints as `0.30000000000000004`. */
+export function formatCpus(cpuCenti) {
+	const whole = Math.floor(cpuCenti / 100);
+	const fraction = String(cpuCenti % 100).padStart(2, "0").replace(/0+$/, "");
+	return fraction === "" ? String(whole) : `${whole}.${fraction}`;
+}
+
+/** `--cpu-shares` for a size: round(cpus x 1024), held inside the runtime's range. */
+export function cpuSharesOf(cpuCenti) {
+	return Math.min(CPU_SHARES_MAX, Math.max(CPU_SHARES_MIN, Math.round((cpuCenti * 1024) / 100)));
+}
+
+/** `/dev/shm` for a size, in MiB: half the memory, at most 1g. */
+export function shmMiBOf(memMiB) {
+	return Math.min(SHM_CEILING_MIB, Math.floor(memMiB / 2));
+}
+
+/**
+ * The CPUs any ONE job may use at most, `--cpus`: the host's count minus the reserve (one CPU when the host has four
+ * or more, else none), or null when the count is unknown. Per container: several busy jobs together can still use
+ * every core, the reserved one included, unless the parent cgroup they share holds its quota (`cpu-reserve.mjs`).
+ *
+ * `hostCpus` is the RUNTIME's own count (`docker info`'s `NCPU`, `podman info`'s `host.cpus`), never the worker's
+ * `os.availableParallelism()`: on Docker Desktop the daemon runs in a VM with its own count, and Docker refuses a
+ * `--cpus` above the CPUs it has. Under a CPU budget the ceiling is the budget instead (`cpuCeilingCenti`), which
+ * carries an operator's reserve and a rootless service's own `cpu.max`; this is the ceiling without one.
+ *
+ * NULL FAILS OPEN, and says so: the argv then carries no `--cpus`, so a job may use every core, reserve included, as a
+ * container with no bound does. The shares still apply. The worker logs `cpu_ceiling_unknown` when it happens; it
+ * needs a daemon whose `info` did not answer while the job still ran, which only a desktop platform's job user allows.
+ */
+export function hostCpuCeiling(hostCpus) {
+	if (!Number.isSafeInteger(hostCpus) || hostCpus < 1) return null;
+	return hostCpus - (hostCpus >= 4 ? 1 : 0);
+}
+
+/**
+ * The `--cpus` every job gets, in hundredths (issue #596, phase 2): the host's CPU BUDGET when one is in force (an
+ * integer `cpuBudgetCenti`, `host-budget.mjs`), capped at the runtime's own count where that is known (Docker refuses a
+ * `--cpus` above it), else the phase 1 ceiling (`hostCpuCeiling`: the count minus the default reserve), else null. With
+ * the budget off or unknown the phase 1 ceiling stands, so a deployment that sets nothing gets the same flag as before
+ * whenever `auto` reads the same count. It bounds ONE job; the budget's ledger is what bounds them together.
+ */
+export function cpuCeilingCenti(hostCpus, cpuBudgetCenti = null) {
+	const runtime = Number.isSafeInteger(hostCpus) && hostCpus >= 1 ? hostCpus * 100 : null;
+	if (Number.isSafeInteger(cpuBudgetCenti) && cpuBudgetCenti > 0) return runtime === null ? cpuBudgetCenti : Math.min(cpuBudgetCenti, runtime);
+	const ceiling = hostCpuCeiling(hostCpus);
+	return ceiling === null ? null : ceiling * 100;
+}
+
+/**
+ * The highest CPU count a runtime's own refusal of a `--cpus` names, or null when `text` holds no such refusal. Docker
+ * refuses a `--cpus` above the CPUs its daemon has, at create, exit 125: `Range of CPUs is from 0.01 to 4.00, as there
+ * are only 4 CPUs available` (measured, Docker 27.4.0). That is what a ceiling computed from a stale count meets after
+ * Docker Desktop's VM was given fewer CPUs, so the worker reads its facts again (`cpu_ceiling_stale`). Podman 4.9.3 and
+ * 5.8.1 accept a `--cpus` above the host's count (measured, rootless and rootful), so no Podman text matches: a stale
+ * count there is a ceiling that bounds less, never a job that cannot start. Only the leading digits are read.
+ */
+export function cpuRangeRefusal(text) {
+	const match = /range of cpus is from [0-9.]+ to ([0-9]{1,5})(?:\.[0-9]+)?/i.exec(String(text ?? ""));
+	return match ? Number(match[1]) : null;
+}
+
+/**
+ * The size flags a runtime said it will not enforce, from its parsed `docker info` facts (`parseDaemonFacts`): an
+ * empty array, or `--memory-swap` where Docker reports `SwapLimit` false and `--cpu-shares` where it reports
+ * `CPUShares` false. Docker then drops the flag with a client warning only and runs the container anyway (no swap
+ * accounting, no cpu controller), so a job would run past the bound its size promised and nothing would say so. The
+ * worker logs `size_bound_unenforced` per job and doctor warns. A fact that is null (Podman, or a key absent) is no
+ * evidence and adds nothing.
+ */
+export function unenforcedSizeFlags(facts) {
+	const flags = [];
+	if (facts?.swapLimit === false) flags.push("--memory-swap");
+	if (facts?.cpuShares === false) flags.push("--cpu-shares");
+	return flags;
+}
+
+/**
+ * What a size becomes on a container's argv, as `containerSpec` fields: `{ memory, memorySwap, cpus, cpuShares,
+ * shmSize }`. `memory` and `memorySwap` are the SAME string by construction; `cpus` is the host ceiling
+ * (`cpuCeilingCenti`, which may be fractional, `3.5`, under a CPU budget) or null.
+ */
+export function containerSizing(size, hostCpus = null, cpuBudgetCenti = null) {
+	assertJobSize(size);
+	const memory = formatMemory(size.memMiB);
+	const ceiling = cpuCeilingCenti(hostCpus, cpuBudgetCenti);
+	return { memory, memorySwap: memory, cpus: ceiling === null ? null : formatCpus(ceiling), cpuShares: cpuSharesOf(size.cpuCenti), shmSize: formatMemory(shmMiBOf(size.memMiB)) };
+}
+
+/** Throws unless `size` is `{ memMiB, cpuCenti }` within the floors and ceilings above. */
+export function assertJobSize(size) {
+	const mem = size?.memMiB;
+	const cpu = size?.cpuCenti;
+	if (!Number.isSafeInteger(mem) || mem < JOB_MEMORY_FLOOR_MIB || mem > JOB_MEMORY_CEILING_MIB || !Number.isSafeInteger(cpu) || cpu < JOB_CPUS_FLOOR_CENTI || cpu > JOB_CPUS_CEILING_CENTI) {
+		throw new Error(`container spec: refusing a job size that is not { memMiB, cpuCenti } within ${JOB_MEMORY_FLOOR_MIB}..${JOB_MEMORY_CEILING_MIB} MiB and ${JOB_CPUS_FLOOR_CENTI}..${JOB_CPUS_CEILING_CENTI} hundredths: ${JSON.stringify(size ?? null)}`);
+	}
+}
+
+/** The built-in size, `{ memMiB: 4096, cpuCenti: 200, source: "default" }`, derived from the two defaults above. */
+export const DEFAULT_JOB_SIZE = Object.freeze({ memMiB: parseMemory(DEFAULT_JOB_MEMORY), cpuCenti: parseCpus(DEFAULT_JOB_CPUS), source: "default" });
+
+/**
+ * The deployment's default size from `PI_JOB_MEMORY` and `PI_JOB_CPUS` (unset or empty is the built-in value), as `{
+ * memMiB, cpuCenti, memSet, cpuSet }`. Throws an Error naming the key for a value either parser refuses; the worker's
+ * config runs this at boot, so a bad value stops the worker there (exit 2) rather than at a job.
+ */
+export function jobSizeDefaults(env = {}) {
+	const read = (key, parse, fallback) => {
+		const raw = env?.[key];
+		if (raw === undefined || raw === null || raw === "") return { value: parse(fallback), set: false };
+		try {
+			return { value: parse(raw), set: true };
+		} catch (error) {
+			throw new Error(`${key}: ${error.message}`);
+		}
+	};
+	const mem = read("PI_JOB_MEMORY", parseMemory, DEFAULT_JOB_MEMORY);
+	const cpu = read("PI_JOB_CPUS", parseCpus, DEFAULT_JOB_CPUS);
+	return { memMiB: mem.value, cpuCenti: cpu.value, memSet: mem.set, cpuSet: cpu.set };
+}
+
+/**
+ * The size a job runs at, `{ memMiB, cpuCenti, source }`, resolved ONCE at pickup from the same limits snapshot every
+ * other gate reads (issue #596). `project` is the pickup project's id (or null), `limits` the parsed scoped-limits
+ * rows, `env` the deployment's settings (`PI_JOB_MEMORY`, `PI_JOB_CPUS`).
+ *
+ * Field by field: the project row's `memory` and `cpus` win, then the deployment's settings, then the built-in 4g and
+ * 2. `source` names the most specific place that supplied ANY field: `project` when the row set memory or CPUs (a row
+ * that sets only one takes the other from the deployment), `env` when a setting did and the row none, else `default`.
+ *
+ * Pure. Never throws on a parsed limits list (its sizes were validated at load); throws on a refused `env` value, which
+ * a booted worker cannot have.
+ */
+export function resolveJobSize({ project = null, limits = [], env = {} } = {}) {
+	const defaults = jobSizeDefaults(env);
+	const row = typeof project === "string" && Array.isArray(limits) ? (limits.find((l) => l?.scope === `${PROJECT_ROW_PREFIX}${project}`) ?? null) : null;
+	const rowMem = typeof row?.memory === "string" ? parseMemory(row.memory) : null;
+	const rowCpu = row?.cpus !== null && row?.cpus !== undefined ? parseCpus(row.cpus) : null;
+	const source = rowMem !== null || rowCpu !== null ? "project" : defaults.memSet || defaults.cpuSet ? "env" : "default";
+	return { memMiB: rowMem ?? defaults.memMiB, cpuCenti: rowCpu ?? defaults.cpuCenti, source };
+}
+
+/**
+ * A size as a record carries it (INT-RUN-HISTORY-FILE-CONTRACT): `{ memMiB, cpuCenti, source }` REBUILT as an explicit
+ * literal, or null for anything that is not one. Integers and a fixed word, so the record stays PII-free.
+ */
+export function recordedJobSize(size) {
+	if (size === null || typeof size !== "object" || Array.isArray(size)) return null;
+	try {
+		assertJobSize(size);
+	} catch {
+		return null;
+	}
+	return JOB_SIZE_SOURCES.includes(size.source) ? { memMiB: size.memMiB, cpuCenti: size.cpuCenti, source: size.source } : null;
+}

package/src/job-user.mjs

@@ -207,9 +207,36 @@ export function jobUserRefusal(causeOrDecision) {
  * NOT cached: `unknown`, `unmappable` `runtime-unreadable`, and any decision made without an answered read (an `image`
  * decided on macOS while the daemon was still starting). Each describes an answer rather than a daemon, and a cached one
  * would retry or refuse every later job on that endpoint, or leave the observations unread, until the worker restarted.
- * A cached decision is otherwise kept until the endpoint state changes: a daemon reconfigured behind an unchanged
- * endpoint (rootful to rootless on one socket path) is read again only after a restart, a residual the design entry names.
+ * A cached decision is otherwise kept until the endpoint state changes or for `maxAgeMs` (issue #596), whichever comes
+ * first, and `resolveJobUser.invalidate()` drops it at once. The age exists for the facts that move behind an unchanged
+ * endpoint: Docker Desktop's VM resized to another CPU count changes every job's `--cpus` ceiling, and a daemon
+ * reconfigured on one socket path (rootful to rootless) changes the job user. Ten minutes bounds how long either goes
+ * unseen; a lowered CPU count is caught sooner, because Docker refuses the stale `--cpus` and the job path invalidates
+ * (`cpu_ceiling_stale`, `run-container.mjs`).
+ *
+ * STALE WHILE ERROR (issue #596, gate round 2). A read past the age is a fresh read, and one that does not answer (a
+ * timeout, a daemon restarting, an unreadable reply) or throws keeps serving the last answered value for the same key
+ * until a read answers, logging `job_user_facts_stale` with the failed read's reason once per run of failures. Deciding
+ * such a read as a first read would make that pickup unavailable (an infrastructure retry) where, before the age
+ * existed, the cached answer was simply reused; the age is there to SEE a change, never to manufacture an outage.
+ * `invalidate()` is different on purpose: it is called because the job path just proved the cached CPU count wrong
+ * (Docker refused its `--cpus`, exit 125 "Range of CPUs"), so it drops the value entirely. Serving that count again
+ * would refuse every job the same way, so after an invalidate the next pickup reads fresh and, if that read fails, is
+ * unavailable exactly as a first read on a new endpoint is.
+ */
+export const JOB_USER_FACTS_MAX_AGE_MS = 10 * 60_000;
+
+/**
+ * How long a kept answer may be served while every re-read fails (issue #596, phase 1 gate round 3, carried to phase 2).
+ * Stale-while-error exists so one slow read is not an outage; it must not let a runtime that stopped answering a day ago
+ * keep deciding jobs from what it said then. Past this the kept answer is no longer served and a pickup is decided as a
+ * first read is (unavailable, retried as infrastructure, said per job), which is loud where it belongs. Chosen over
+ * repeating the stale line every max-age plus a doctor field: one comparison in each cache, no new surface, and the
+ * existing per-job unavailability is the honest signal for a runtime that has not answered in a day. Shared by the
+ * podman venue's info cache (`cachedPodmanInfo`).
  */
+export const STALE_FACTS_CEILING_MS = 24 * 60 * 60_000;
+
 export function makeJobUserResolver({
 	readFacts,
 	platform = process.platform,
@@ -217,18 +244,46 @@ export function makeJobUserResolver({
 	euid = process.geteuid?.(),
 	egid = process.getegid?.(),
 	stat = statSync,
+	now = Date.now,
+	maxAgeMs = JOB_USER_FACTS_MAX_AGE_MS,
+	log = () => {},
 } = {}) {
 	let cached = null;
+	// True while a run of failed re-reads is being answered from `cached`, so the line is written once per run.
+	let staleSaid = false;
 	const inFlight = new Map();
+	resolveJobUser.invalidate = () => {
+		cached = null;
+		staleSaid = false;
+	};
 	return resolveJobUser;
 	async function resolveJobUser({ endpoint, key }) {
-		if (cached && cached.key === key) return cached.value;
+		if (cached && cached.key === key && now() - cached.at < maxAgeMs) return cached.value;
 		if (inFlight.has(key)) return inFlight.get(key);
 		const work = (async () => {
 			// Asked on EVERY platform and endpoint, although a VM-backed platform or an endpoint on another machine decides
 			// `image` before any daemon fact is read: the same read is where the runtime observations come from (issue
 			// #345), and a Docker Desktop host skipped here would never be credited with the bounds its daemon applies.
-			const daemon = await readFacts();
+			let daemon;
+			let threw = null;
+			try {
+				daemon = await readFacts();
+			} catch (error) {
+				threw = error;
+				daemon = { answered: false, reason: "read-threw", transient: true };
+			}
+			// The value an expired entry may still serve when this read failed (see STALE WHILE ERROR above). Taken AFTER
+			// the read, so an `invalidate()` that landed while it was out is honoured by this very pickup.
+			// Never past `STALE_FACTS_CEILING_MS`: an answer that old is not served, whatever the read says now.
+			const stale = cached && cached.key === key && now() - cached.at < STALE_FACTS_CEILING_MS ? cached : null;
+			if (threw && !stale) throw threw;
+			if (stale && daemon?.answered !== true) {
+				if (!staleSaid) {
+					staleSaid = true;
+					log("job_user_facts_stale", { reason: daemon?.reason ?? "no-daemon-facts", ageMs: now() - stale.at });
+				}
+				return stale.value;
+			}
 			// A local docker endpoint's display form IS its unix path (credentials never ride a unix URL); with no endpoint,
 			// Podman's own shape names the service socket.
 			const socketPath = endpoint?.local === true && typeof endpoint.endpoint === "string" && endpoint.endpoint.startsWith("unix://")
@@ -237,9 +292,15 @@ export function makeJobUserResolver({
 			const socket = socketFacts(socketPath, { stat });
 			const decision = decideJobUser({ platform, release, euid, egid, endpoint, daemon, socket });
 			const value = { decision, facts: daemon?.answered ? daemon.facts : null, daemon, socket };
+			// An ANSWERED read ends a run of failures even when it decides nothing cacheable (an `unknown`, a
+			// `runtime-unreadable`), so the next failure after it is said again (gate round 3 of phase 1).
+			if (daemon?.answered === true) staleSaid = false;
 			// Cached only for an ANSWERED read: an `image` decided on a VM-backed platform while its daemon was still starting
 			// must not pin "not read" for the runtime observations until the endpoint changes.
-			if (decision.mode !== "unknown" && decision.cause !== "runtime-unreadable" && daemon?.answered === true) cached = { key, value };
+			if (decision.mode !== "unknown" && decision.cause !== "runtime-unreadable" && daemon?.answered === true) {
+				cached = { key, value, at: now() };
+				staleSaid = false;
+			}
 			return value;
 		})();
 		inFlight.set(key, work);