@edgehero/pi-dispatch 5.1.0 → 5.2.1

package/src/service-env.mjs

@@ -237,7 +237,8 @@ export function readDeploymentEnv({ dir, fs, uid = process.geteuid?.(), names =
 /**
  * The `.env` of the deployment in `dir`, read the way doctor reads it (a regular file only, as bytes, one read) and
  * resolved for `keys` (`resolveServiceEnv`). For a caller that has no file of its own in hand (the admin panel).
- * Returns the resolution plus `path`, `unreadable` (a reason, when something is at the path and could not be read) and
+ * Returns the resolution plus `path`, `absent` (true when nothing is at the path: issue #845's console tells a file that
+ * went away from one that never was), `unreadable` (a reason, when something is at the path and could not be read) and
  * `untrusted` (`envFileTrust`'s reason); in either case nothing was taken from it. `shellRunsLine` is the line a sourcing shell does
  * not read as an assignment, whatever keys were asked for: it can set any of them without spelling one (`. ./more.env`, `eval`), so
  * `hazardSkipped`, which needs the key spelled, says nothing of it. Only for that loader: systemd ignores such a line and the cmd wrapper
@@ -245,11 +246,11 @@ export function readDeploymentEnv({ dir, fs, uid = process.geteuid?.(), names =
  */
 export function deploymentServiceEnv({ env, dir, keys, platform = process.platform, fs, uid = process.geteuid?.() }) {
 	// Issue #471 (gate rounds 1 and 2): one descriptor, and nothing taken from a file another account can change.
-	const { path, bytes, unreadable, untrusted } = readDeploymentEnv({ dir, fs, uid });
+	const { path, bytes, absent, unreadable, untrusted } = readDeploymentEnv({ dir, fs, uid });
 	const raw = untrusted ? null : bytes;
 	const file = raw === null ? null : serviceEnvFileOf(raw, path, serviceEnvLoader(platform));
 	const shellRunsLine = file?.loader === "shell" && file.hazard !== null && file.hazard.shape !== "shell-crlf-comment" ? file.hazard.line : null;
-	return { ...resolveServiceEnv({ env, file, keys }), path, unreadable, untrusted, hazard: file?.hazard ?? null, shellRunsLine };
+	return { ...resolveServiceEnv({ env, file, keys }), path, absent, unreadable, untrusted, hazard: file?.hazard ?? null, shellRunsLine };
 }
 
 /**

package/src/size-suggest.mjs

@@ -39,6 +39,12 @@
  *   4. otherwise it fits.
  *   A FACT (no call) rides along when runs at the limit were also under real memory pressure (`memFullUsec` above 1%
  *   of the wall time): information, never an instruction.
+ *   An `oom-killed` raise also says what CPU count the killed runs' tools counted at least (issue #865, `oomCeiling`,
+ *   worded as `memoryCeiling`): on a venue whose `--cpus` is the host ceiling (`CEILING_VENUES`), the fewest whole CPUs
+ *   a run's ceiling gives `os.availableParallelism()` (`seenCpusOf`), from its record's `capacity`, when that is more
+ *   than its size in whole CPUs (`PI_JOB_SIZE_CPUS`). A tool that starts a worker per CPU it sees starts one per such CPU inside the size's memory,
+ *   so a smaller pool may be a cheaper fix than a larger size. Information beside the raise, which is the same either
+ *   way.
  *   A size is rounded UP to a step: 256m up to 2g, 512m up to 8g, then 1g.
  * CPUs: fewer than `SUGGEST_MIN_SAMPLES` runs is not enough runs; the p95 cores used (CPU time over wall time) below
  * 0.4x the current cpus lowers to 1.25x that p95, never below 1.25x the window's largest cores used, nor 0.25;
@@ -59,7 +65,7 @@
  * land on the side the rule says, on every host.
  */
 
-import { JOB_CPUS_CEILING_CENTI, JOB_CPUS_FLOOR_CENTI, JOB_MEMORY_CEILING_MIB, JOB_MEMORY_FLOOR_MIB, formatCpus, formatMemory, recordedJobSize } from "./job-size.mjs";
+import { HOST_CPUS_MAX, JOB_CPUS_CEILING_CENTI, JOB_CPUS_FLOOR_CENTI, JOB_MEMORY_CEILING_MIB, JOB_MEMORY_FLOOR_MIB, cpuCeilingCenti, formatCpus, formatMemory, recordedJobSize, wholeCpusOf } from "./job-size.mjs";
 
 /** How far back a suggestion reads: 30 days. */
 export const SUGGEST_WINDOW_DAYS = 30;
@@ -93,6 +99,18 @@ export const MEMORY_HELD = Object.freeze(["cap", "largest", "no-cap"]);
 export const MEMORY_CAP_MISSING = Object.freeze(["unread", "none-holds", "off"]);
 /** The facts a suggestion may carry with no call: memory `pressure`, CPU `ceiling`. */
 export const SIZE_FACTS = Object.freeze(["pressure", "ceiling"]);
+/**
+ * Issue #865: the venues whose job containers get the host's CPU ceiling as `--cpus`, the same for every job
+ * (job-size.mjs `containerSizing`, on the docker and podman builders alike), so a job's tools count at least the ceiling
+ * (`seenCpusOf`) and never its size. OpenShell's venues are not here: their `--cpu` is the job's size itself
+ * (backend-openshell.mjs `sizeArgs`), so on cgroup v2 a tool there reads the size rounded down from one CPU up, and
+ * under one CPU (0.25 to 0.99) the gateway's whole runtime, whose CPU count the record does not hold (its
+ * `capacity.cpus` is this host's runtime's, and the gateway may be another machine). That last case is a residual,
+ * named in `DES-JOB-SIZE-IN-THE-ENV`, not a clause said on a guess. Restated, since this module imports nothing heavy, and held
+ * to backends.mjs by a test: every venue is in this list or in `OPENSHELL_BACKENDS`, so a new venue fails it until
+ * someone says which it is.
+ */
+export const CEILING_VENUES = Object.freeze(["local", "podman"]);
 
 const MIB = 1024 * 1024;
 const DAY_MS = 24 * 60 * 60 * 1000;
@@ -117,9 +135,60 @@ function rank(sorted, pct) {
 
 const isCount = (v) => Number.isSafeInteger(v) && v >= 0;
 
+/**
+ * Issue #865: the whole CPUs a run's tools read through `os.availableParallelism()`, as its record gives it, or null. The
+ * `--cpus` ceiling comes from the record's `capacity` (`cpus`, the runtime's CPU count, and `cpuCenti`, the host's CPU
+ * budget, each judged again here) through the worker's own `cpuCeilingCenti`, on a venue in `CEILING_VENUES` only, and
+ * becomes a count by `seenCpusOf`. Null for any other venue, a record with no readable `capacity` (every record before
+ * issue #599, and every refusal before a slot), a capacity that names neither, and a ceiling under one CPU on a record
+ * whose runtime count is not known.
+ * The record holds what the host offered at admission, not the argv itself, so this is the ceiling the worker computes
+ * from those facts. Where the budget's read of the runtime gave no CPU count, the record holds the worker's own count
+ * instead (index.mjs `capacityNow`), which on Docker Desktop is the Mac's and not the VM's, so the value is then an
+ * estimate of what the tools read. Accepted, rather than a record field for the argv's own `--cpus`: the clause is
+ * information beside a raise that does not depend on it, and that read failing while a job still ran is rare.
+ */
+function seenOf(record) {
+	if (!CEILING_VENUES.includes(record.backend)) return null;
+	const c = record.capacity;
+	if (c === null || typeof c !== "object" || Array.isArray(c)) return null;
+	const cpus = Number.isSafeInteger(c.cpus) && c.cpus >= 1 && c.cpus <= HOST_CPUS_MAX ? c.cpus : null;
+	const budget = Number.isSafeInteger(c.cpuCenti) && c.cpuCenti >= 1 && c.cpuCenti <= HOST_CPUS_MAX * 100 ? c.cpuCenti : null;
+	const ceilingCenti = cpuCeilingCenti(cpus, budget);
+	return ceilingCenti === null ? null : seenCpusOf(ceilingCenti, cpus);
+}
+
+/**
+ * Issue #865: the libuv the rule in `seenCpusOf` was read at, the one the job image's Node bundles (the Node
+ * image/Dockerfile notes its base resolves to). The next minor release, 1.52.0, changed it
+ * (`uv__get_cgroupv2_constrained_cpu` answers 1 for a quota under one CPU, where this release answers 0 and the caller
+ * skips it), so the image job compares the built image's `process.versions.uv` and `process.versions.node` with this
+ * and with that note (.github/scripts/libuv-pin-check.mjs, in "the job image holds its contract") and fails on any
+ * other, saying to read the rule again at the new libuv before moving this.
+ */
+export const SEEN_CPUS_LIBUV = "1.51.0";
+
+/**
+ * Issue #865: the fewest whole CPUs a tool counts through `os.availableParallelism()` in a container with a `--cpus`
+ * ceiling in hundredths, or null, where `runtimeCpus` is the runtime's own count (the container's affinity mask holds all
+ * of it, so `nproc` reads it). A LOWER BOUND, since what libuv reads depends on the host's cgroup version, which the
+ * record does not hold. Read at libuv `SEEN_CPUS_LIBUV`: on cgroup v2, `uv__get_constrained_cpu` (src/unix/linux.c)
+ * sets the quota to `cpu.max`'s limit over its period in INTEGER division, and `uv_available_parallelism`
+ * (src/unix/core.c) takes it only `if (quota > 0 && quota < rc)`, where `rc` is the affinity count: a ceiling of 2.5
+ * reads as 2 (ROUNDED DOWN, where `PI_JOB_SIZE_CPUS` rounds the size up), and one under one CPU gives a quota of 0,
+ * which is skipped, so the tool reads the WHOLE runtime. On cgroup v1, `uv__cgroup1_find_cpu_controller` turns
+ * `N:cpu,cpuacct:/path` into a `/sys/fs/cgroup/cpuacct:/path` that does not exist, so the read fails and the tool reads
+ * the whole runtime at any ceiling. So a tool counts at least the ceiling rounded down, and under one CPU the whole
+ * runtime: null when the record does not say how many that is.
+ */
+function seenCpusOf(ceilingCenti, runtimeCpus) {
+	if (ceilingCenti >= 100) return Math.floor(ceilingCenti / 100);
+	return Number.isSafeInteger(runtimeCpus) && runtimeCpus >= 1 ? runtimeCpus : null;
+}
+
 /**
  * One record as a suggestion reads it, or null when it is not evidence: `{ at, size, memPeak, oom, wallUsec, cpuUsec,
- * throttledUsec, memFullUsec }`. `memPeak` (bytes, clamped to the record's own limit) is null when absent or not a
+ * throttledUsec, memFullUsec, seenCpus }`. `memPeak` (bytes, clamped to the record's own limit) is null when absent or not a
  * count; `wallUsec` is null unless the span is readable; `cpuUsec` is null unless it can be read with a wall time (CPU
  * time clamped to the CPU ceiling over the wall), and `throttledUsec` likewise on its own (clamped to the wall): cgroup v2
  * writes `throttled_usec` in `cpu.stat` only where the cpu controller is enabled, so a run without it is still a CPU
@@ -145,7 +214,7 @@ function evidenceOf(record, project, nowMs) {
 	// unclamped: it is only ever compared (exactly, in BigInt) with 1% of the wall, and a stall past the wall is above it
 	const memFullUsec = wallUsec !== null && isCount(r.memFullUsec) ? r.memFullUsec : null;
 	if (memPeak === null && cpu.cpuUsec === null) return null;
-	return { at, size, memPeak, oom: record.reason === "oom-killed", wallUsec, ...cpu, memFullUsec };
+	return { at, size, memPeak, oom: record.reason === "oom-killed", wallUsec, ...cpu, memFullUsec, seenCpus: seenOf(record) };
 }
 
 /** a x b > c x d, exactly. */
@@ -164,6 +233,25 @@ function heldToCap(wanted, current, cap) {
 	return { suggested: wanted, held: null };
 }
 
+/**
+ * Issue #865: what CPU count the killed runs' tools counted at least, against their size, or null: over the
+ * `oom-killed` runs that decide a raise (the same ones `ooms` counts), those where the fewest CPUs their tools could
+ * count (`seenOf`) is more than their size rounded up, which is what `PI_JOB_SIZE_CPUS` tells them. At or below it nothing can be said:
+ * a pool sized from the variable may be the pool the ceiling gave. `{ runs, seenCpus, seenMaxCpus, sizeCenti,
+ * sizeVaries }`: how many such runs, the fewest and the most CPUs their tools counted at least (equal when every run
+ * gives the same bound), and the smallest CPU size among them and whether their sizes differ. Either can vary alone
+ * (one host's ceiling under projects of several sizes, or one size across hosts), and the words say a range for that
+ * one only.
+ */
+function oomCeilingOf(relevant) {
+	const over = relevant.filter((e) => e.oom && e.seenCpus !== null && e.seenCpus > wholeCpusOf(e.size.cpuCenti));
+	if (over.length === 0) return null;
+	const seen = over.map((e) => e.seenCpus);
+	const sizes = over.map((e) => e.size.cpuCenti);
+	const sizeCenti = Math.min(...sizes);
+	return { runs: over.length, seenCpus: Math.min(...seen), seenMaxCpus: Math.max(...seen), sizeCenti, sizeVaries: sizes.some((n) => n !== sizeCenti) };
+}
+
 function suggestMemory(runs, current, cap) {
 	const measured = runs.filter((e) => e.memPeak !== null);
 	const relevant = measured.filter((e) => e.size.memMiB >= current);
@@ -176,7 +264,7 @@ function suggestMemory(runs, current, cap) {
 	// at the limit (90% of it or more: peak x 10 >= limit x 9) AND stalled for memory more than 1% of the wall.
 	const pressured = relevant.filter((e) => !productAbove(e.size.memMiB * MIB, 9, e.memPeak, 10) && e.memFullUsec !== null && productAbove(e.memFullUsec, 100, e.wallUsec, 1)).length;
 	// `smaller`: the measured runs below the current size, which a chart draws and this verdict does not count.
-	const evidence = { samples, smaller: measured.length - samples, p95MiB: p95 === null ? null : Math.ceil(p95 / MIB), maxPeakMiB: maxPeak === null ? null : Math.ceil(maxPeak / MIB), ooms, largestOomMiB: largestOom, pressured };
+	const evidence = { samples, smaller: measured.length - samples, p95MiB: p95 === null ? null : Math.ceil(p95 / MIB), maxPeakMiB: maxPeak === null ? null : Math.ceil(maxPeak / MIB), ooms, largestOomMiB: largestOom, pressured, oomCeiling: oomCeilingOf(relevant) };
 	const fact = pressured > 0 ? "pressure" : null;
 	const out = (reason, suggested, more = {}) => ({ current, suggested: suggested === null || suggested === current ? null : suggested, reason, evidence, fact, held: null, wanted: null, ...more });
 	if (ooms > 0) {
@@ -419,12 +507,33 @@ export function notEnoughRunsText(dim, sizeText) {
 	return `not enough runs at ${sizeText(dim.current)} (${samples}${smaller})`;
 }
 
+/**
+ * Issue #865: the CPU count the killed runs' tools counted at least, in words, or "" when the evidence carries none
+ * (`oomCeilingOf`): `size per-CPU worker pools from PI_JOB_SIZE_CPUS: the killed run's tools counted at least 15 CPUs
+ * at a size of 2 CPUs`. "At least", since the count is a lower bound (`seenCpusOf`). A word of its own
+ * (`memoryCeiling`), not part of the OOM words: every surface prints it AFTER the suggestion, a held raise and a refusal
+ * (doctor ahead of its two facts; the console's card as the last of the suggestion's notes, before the card's advisory
+ * note, with the call's block below the notes; the panel and the confirm after their facts), so where a surface cuts a
+ * line (doctor publishes a label at most `DOCTOR_LABEL_MAX` long) the facts go first, then this, never the call, a held
+ * raise or a refusal. Short for the same cap, and the variable first, so a cut keeps the pointer as long as it can. A
+ * pointer, never a call: the raise is the same either way.
+ */
+function oomCeilingWords(me) {
+	const o = me?.oomCeiling;
+	if (o === null || typeof o !== "object" || !Number.isSafeInteger(o.runs) || o.runs < 1 || !Number.isSafeInteger(o.seenCpus)) return "";
+	const ooms = Number.isSafeInteger(me.ooms) && me.ooms >= o.runs ? me.ooms : o.runs;
+	const whose = ooms === 1 ? "the killed run's tools" : o.runs === ooms ? "the killed runs' tools" : `the tools of ${o.runs} of the ${ooms} killed runs`;
+	const seen = `${o.seenCpus}${Number.isSafeInteger(o.seenMaxCpus) && o.seenMaxCpus > o.seenCpus ? ` to ${o.seenMaxCpus}` : ""} CPUs`;
+	const size = o.sizeVaries === true ? `sizes from ${cpusText(o.sizeCenti)}` : `a size of ${cpusText(o.sizeCenti)}`;
+	return `size per-CPU worker pools from PI_JOB_SIZE_CPUS: ${whose} counted at least ${seen} at ${size}`;
+}
+
 /**
  * The evidence of a suggestion in words, for doctor, the panel and the insights page alike: `{ memory, cpu, memoryHeld,
- * memoryFact, cpuFact }`, each a short clause (empty when it does not apply), such as `2 runs ended oom-killed (the
- * largest size killed 4g)` or `p95 peak 2560m, largest 3g, over 24 runs`. Ids and numbers only. A fact is worded as
- * information and a held raise as what the host offers: none of them is an instruction, and none advises growing a
- * host's budget.
+ * memoryFact, cpuFact, memoryCeiling }`, each a short clause (empty when it does not apply), such as `2 runs ended
+ * oom-killed (the largest size killed 4g)` or `p95 peak 2560m, largest 3g, over 24 runs`. Ids and numbers only. A fact
+ * is worded as information and a held raise as what the host offers, and `memoryCeiling` (issue #865) points to a
+ * variable: none of them is a call, and none advises growing a host's budget.
  */
 export function suggestionEvidence(suggestion) {
 	const m = suggestion?.memory ?? {};
@@ -457,5 +566,6 @@ export function suggestionEvidence(suggestion) {
 		memoryHeld: heldWords[m.held] ?? "",
 		memoryFact: m.fact === "pressure" ? `${me.pressured} of ${plural(me.samples, "run")} reached the memory limit while stalled for memory more than 1% of their time` : "",
 		cpuFact: c.fact === "ceiling" ? `the median run was held back ${ce.throttledPct}% of its time by this host's CPU ceiling (its CPU budget, shared by every job), which a job's cpus do not change` : "",
+		memoryCeiling: m.reason === "oom-killed" ? oomCeilingWords(me) : "",
 	};
 }

package/src/touch-outbox.mjs

@@ -54,6 +54,11 @@ export async function trackTouchAction({ store, outbox = null, event, act, succe
 	const writer = outbox ?? { begin: (e) => store.begin(e), applied: (intent) => store.write(intent.event), abort: (intent) => store.abort(intent.event) };
 	const intent = await writer.begin(event);
 	let result;
+	// Aborted on any throw, also one whose answer was lost (`valkeyMayStillRun`, issue #867): a cancel's removal that then
+	// lands is a stop not counted, so the stops count can be one short (the rate is unaffected, since stops never enter
+	// its denominator). REJECTED: keeping the intent pending as unknown. Nothing resolves it (a rerun finds no job, the CLI
+	// has no outbox, `repair()` only begins it again), so the store reads `touch-actions-unresolved` with its coverage reset
+	// to now, the Zero touch page reads unavailable, and every later tracked action renews its 30-day TTL.
 	try { result = await act(); } catch (error) { await writer.abort(intent); throw error; }
 	if (!succeeded(result)) { await writer.abort(intent); return result; }
 	const tracked = await writer.applied(intent);

package/src/valkey-endpoint.mjs

@@ -58,7 +58,7 @@ export function defaultValkeyContext() {
 }
 
 /**
- * Where a client stands (issue #464, gate round 3): `{ envPath, shared, rootRefused, error, password }`.
+ * Where a client stands (issue #464, gate round 3): `{ envPath, shared, rootRefused, error, absent, password, url }`.
  *
  * The owner decision itself does not depend on this (gate round 3's simpler rule): on Linux EVERY client refuses a
  * listener held by a uid that is neither this account's euid, nor one of its subordinate uids, nor root, from any cwd.
@@ -86,10 +86,12 @@ export function valkeyClientContext({ env = process.env, cwd = process.cwd(), pl
 	const envPath = join(cwd, ".env");
 	let content = null;
 	let error = null;
+	let absent = false;
 	try {
 		content = readEnv(envPath);
 	} catch (err) {
-		if (err?.code !== "ENOENT") error = `${envPath} could not be read (${err?.code ?? err?.message})`;
+		if (err?.code === "ENOENT") absent = true;
+		else error = `${envPath} could not be read (${err?.code ?? err?.message})`;
 	}
 	const fileKeys = {};
 	if (content !== null) {
@@ -104,7 +106,7 @@ export function valkeyClientContext({ env = process.env, cwd = process.cwd(), pl
 		if (keys.error) error ??= keys.error;
 		else for (const k of [VALKEY_SHARED_KEY, VALKEY_PASSWORD_KEY, "VALKEY_URL"]) if (typeof keys.keys[k] === "string") fileKeys[k] = keys.keys[k];
 	}
-	return valkeyContextFromKeys({ env, envPath, fileKeys, error, platform, rootRefused });
+	return valkeyContextFromKeys({ env, envPath, fileKeys, error, absent, platform, rootRefused });
 }
 
 /**
@@ -138,9 +140,10 @@ export const VALKEY_CONTEXT_KEYS = Object.freeze(["VALKEY_URL", VALKEY_PASSWORD_
  * A context from keys a caller has already read out of the deployment's `.env` (`fileKeys`, the `VALKEY_CONTEXT_KEYS`
  * it supplied) and this process's environment. `valkeyClientContext` reads the file itself; the admin panel passes what
  * issue #471's resolver read, so the panel has one `.env` reader. `error` is a reason the file could not be read, as
- * `valkeyClientContext`'s.
+ * `valkeyClientContext`'s. `absent` (issue #845) is true when nothing is at `envPath`: no file, which is not a file that
+ * sets nothing, so a client that reads its password again at each connect keeps the one it has (`freshPassword`).
  */
-export function valkeyContextFromKeys({ env = process.env, envPath = null, fileKeys = {}, error = null, platform = process.platform, rootRefused } = {}) {
+export function valkeyContextFromKeys({ env = process.env, envPath = null, fileKeys = {}, error = null, absent = false, platform = process.platform, rootRefused } = {}) {
 	const fileVenue = typeof fileKeys.PI_BACKENDS === "string" ? { PI_BACKENDS: fileKeys.PI_BACKENDS } : {};
 	const shared = valkeySharedOn(fileKeys[VALKEY_SHARED_KEY]);
 	const filePassword = fileKeys[VALKEY_PASSWORD_KEY] || null;
@@ -156,7 +159,7 @@ export function valkeyContextFromKeys({ env = process.env, envPath = null, fileK
 	const engine = typeof env.PI_OPENSHELL_ENGINE === "string" ? env.PI_OPENSHELL_ENGINE : fileKeys.PI_OPENSHELL_ENGINE;
 	const derived = platform === "linux" && ((venues.podmanUsed && !venues.localUsed) || openshellSetupOf(venues, { PI_OPENSHELL_ENGINE: engine }).valkeyEngine === "podman");
 	const envUrl = typeof env.VALKEY_URL === "string" ? env.VALKEY_URL : null;
-	return { envPath, shared, rootRefused: typeof rootRefused === "boolean" ? rootRefused : derived, error, password: { environment: envPassword, file: filePassword }, url: { environment: envUrl, file: fileUrl } };
+	return { envPath, shared, rootRefused: typeof rootRefused === "boolean" ? rootRefused : derived, error, absent: absent === true, password: { environment: envPassword, file: filePassword }, url: { environment: envUrl, file: fileUrl } };
 }
 
 /**