@edgehero/pi-dispatch 3.1.0 → 4.0.0

package/src/start.mjs

@@ -25,9 +25,12 @@ import { builtinModel, checkModelsKnown } from "./model-catalog.mjs";
 import { capabilityTokens, serializeCaps } from "./capabilities.mjs";
 import { cronFingerprint } from "./fingerprint.mjs";
 import { makeHostRegistry } from "./host-registry.mjs";
+import { budgetField, readUserServiceLimits } from "./host-budget.mjs";
+import { makeCpuReserve, reservePlan } from "./cpu-reserve.mjs";
 import { makeImagePreflight } from "./image-preflight.mjs";
 import { createWorker, JOB_TIMEOUT_MS, STALLED_FAILED_REASON } from "./index.mjs";
 import { BOOT_REFUSING_JOB_USER_CAUSES, DAEMON_FACTS_TIMEOUT_MS, jobUserRefusal, makeDaemonFactsReader, makeJobUserResolver, relabelsPrivateMounts, resolveImageUser } from "./job-user.mjs";
+import { unenforcedSizeFlags } from "./job-size.mjs";
 import { makeCollectChain } from "./outbox.mjs";
 import { makeCollectPlan } from "./outbox-plan.mjs";
 import { makePortfolioSnapshot } from "./portfolio-snapshot.mjs";
@@ -41,7 +44,7 @@ import { scrubCredentials } from "./redact.mjs";
 import { makeCheckOnceSpent, makeCheckPortfolioFlag, makeCheckWaitSkew, makeDisarmOnce } from "./triggers-file.mjs";
 import { WATCH_DEBOUNCE_MS, changedWhileArming, makeWatchCloser, readBeforeArming } from "./watch-closer.mjs";
 import { loadPauseWindows, pauseUntilMs } from "./pause-windows.mjs";
-import { checkProjectRows, danglingProjectRows, dollarRowsWithoutCap, loadScopedLimits, scopeClaimRows } from "./scoped-limits.mjs";
+import { checkProjectRows, danglingProjectRows, dollarRowsWithoutCap, loadScopedLimits, SCOPED_LIMITS_VERSION, scopeClaimRows } from "./scoped-limits.mjs";
 import { escapeControls, loadProjects, projectOf, projectsFingerprint } from "./projects.mjs";
 import { envelopeDigest, envelopeInsideJobPaths, loadEnvelopeChecked } from "./envelope.mjs";
 import { NO_ENVELOPE_FINGERPRINT, makeAllocationAudit, makeAllocationLogReaper, makeAllocationState } from "./allocation.mjs";
@@ -50,7 +53,7 @@ import { makeOnFailure } from "./on-failure.mjs";
 import { makeWaitChecker } from "./wait-check.mjs";
 import { makeWaitState } from "./wait-state.mjs";
 import { hostQueueName, makeQueue } from "./queue.mjs";
-import { endpointShown, makeDockerEndpointResolver, makeLocalBackend, makeReaper, makeStopContainer, quotedShown } from "./backend-local.mjs";
+import { endpointShown, execDockerBounded, makeContainerGone, makeDockerEndpointResolver, makeJobContainerLister, makeLocalBackend, makeReaper, makeStopContainer, quotedShown } from "./backend-local.mjs";
 import { NETNS_KEEPER_MIN_AGE_MS, NETNS_KEEPER_YOUNG_MARGIN_MS, runtimeFromFacts } from "./netns-keeper.mjs";
 import { makeBackendRegistry, reapAll, resolveBackendName } from "./backend-registry.mjs";
 import { DEFAULT_BACKEND, DOCKER_ENDPOINT_LOCAL, PODMAN_ADDS_NO_MOUNTS, PODMAN_BACKEND, PODMAN_BOUNDS_DELEGATED, PODMAN_SERVICE_LOCAL, backendFor, isPerMachineHost, observationRefusalIsTransient, observationRefusals, unobservedFloor } from "./backends.mjs";
@@ -60,7 +63,7 @@ import { PODMAN_RESTART_HOLD_EXPIRED, makePodmanServiceReader, onceFs, makeRootf
 import { makeRunContainer } from "./run-container.mjs";
 import { resolveProviderCredential } from "./env-allowlist.mjs";
 import { makeSecretsResolver } from "./secrets.mjs";
-import { buildRecord, makeFindPreviousRun, makeLogReaper, makeLogSink, makeReadRecord, makeRecordWriter, makeSettledRecord, RUNNER_POLICY_REASONS, sanitizeJobId } from "./run-history.mjs";
+import { buildRecord, EXIT_OOM_KILLED, makeFindPreviousRun, makeLogReaper, makeLogSink, makeReadRecord, makeRecordWriter, makeSettledRecord, RUNNER_POLICY_REASONS, sanitizeJobId } from "./run-history.mjs";
 import { makeRunMirror, readMirroredRecord } from "./run-mirror.mjs";
 import { readOverlay, resolveSettings } from "./runtime-settings.mjs";
 import { usdFingerprint } from "./dollar-fingerprint.mjs";
@@ -575,6 +578,7 @@ export async function startWorker(
 		makeAuth = makeGitHubAuth,
 		makeHost = makeGitHubHost,
 		createWorkerFn = createWorker,
+		makeCpuReserve: makeCpuReserveFn = makeCpuReserve,
 		makeReaper: makeReaperFn = makeReaper,
 		makeBackendRegistry: makeBackendRegistryFn = makeBackendRegistry,
 		// Additional backend bundles, in registration order after `local` (which is built only while blessed,
@@ -606,6 +610,13 @@ export async function startWorker(
 		// Issue #341: the one `docker info` the job-user decision reads, and the process facts it reads beside it.
 		// Seams for the same reason as the endpoint: a wiring test decides what the daemon and the process say.
 		readDaemonFacts: readDaemonFactsFn = makeDaemonFactsReader(),
+		// Issue #596, phase 2: a cgroup file's text (a rootless account's `memory.max` and `cpu.max`), for the host budget.
+		readCgroupFile: readCgroupFileFn = (path) => readFileSync(path, "utf8"),
+		// Issue #596, phase 2: whether an orphaned job container is gone, per venue's CLI.
+		containerGone: containerGoneFn = null,
+		// Issue #596, gate round 1 of phase 2: the job containers a venue's CLI still lists after the boot reaper,
+		// with their size labels, `(bin) => async () => [{ name, memMiB, cpuCenti }]`, throwing when the CLI does not answer.
+		listJobContainers: listJobContainersFn = (bin) => makeJobContainerLister({ bin }),
 		// `home` (issue #354) is the account whose rootless Podman runs the podman venue's jobs: its own mounts.conf and
 		// containers.conf are read from there. Absent in a test's identity, it falls to the observation's own default.
 		jobUserIdentity = { platform: process.platform, release: osRelease(), euid: process.geteuid?.(), egid: process.getegid?.(), home: homedir() },
@@ -822,6 +833,9 @@ export async function startWorker(
 	// The per-job read (below, `observationPreflight`) logs only when the answer CHANGES from the last one, so a
 	// deliberate, standing redirect writes one line at boot rather than one per job.
 	let endpointSeen = bootEndpoint ? dockerEndpointState(bootEndpoint) : null;
+	// Issue #596, phase 2: the endpoint the last job-user read asked about, so the host budget's tick reads the SAME cached
+	// facts a job was decided from (and re-reads them when they age), never a second daemon of its own choosing.
+	let budgetEndpoint = bootEndpoint ? { endpoint: bootEndpoint, key: endpointSeen } : null;
 
 	// Issue #341: WHO job containers run as on this daemon (`DES-JOB-USER-INFERRED-READ-BACK-ON-REQUEST`). Decided
 	// from facts, never a probe container, cached per endpoint state. Bounded like the boot image read, because a
@@ -829,7 +843,7 @@ export async function startWorker(
 	// default venue: rootless, userns-remap, a root worker and Docker Desktop on Linux cannot run any local job here.
 	// An unknown answer (a daemon still starting) boots, so a unit with RestartPreventExitStatus=2 is never stranded
 	// by one; so does `runtime-unreadable`, which a later job re-reads.
-	const resolveJobUser = makeJobUserResolver({ readFacts: readDaemonFactsFn, ...jobUserIdentity });
+	const resolveJobUser = makeJobUserResolver({ readFacts: readDaemonFactsFn, ...jobUserIdentity, now, log });
 	// Issue #345: a floor that needs a DAEMON observation waits the facts read's own bound (plus a margin), not the image
 	// read's 5 s: a busy host's `docker info` is the slow read, and a floor this boot met from the table before would
 	// otherwise exit 1 on every restart of a healthy daemon.
@@ -897,7 +911,42 @@ export async function startWorker(
 	// Wrapped once in `cachedPodmanInfo` and the SAME wrapper is handed to the bundle below, so an answer read here is the
 	// one the first job is decided from rather than a second spawn. Bounded twice: the reader's own timeout (docker info's
 	// 15 s, reused) and this fuse two seconds past it, because a spawn that never settles has no timeout to fire.
-	const podmanInfo = podmanBlessed ? cachedPodmanInfo(readPodmanInfoFn) : null;
+	const podmanInfo = podmanBlessed ? cachedPodmanInfo(readPodmanInfoFn, { now, log }) : null;
+	// Issue #596, phase 2: what the host budget's `auto` is computed from, read on its tick (off every job path) from the
+	// two cached readers above: each blessed venue's memory and CPU count, the SMALLER where both answered (two venues on
+	// one host share it; a desktop VM is the smaller), and on rootless Podman the user service's own `memory.max` and
+	// `cpu.max` beside them. A venue that did not answer adds nothing, so the budget is unknown only when none did.
+	// The same reads also say how each venue manages cgroups (`reserveVenues`), from which the CPU reserve keeps the jobs'
+	// parent cgroup's quota at the budget (`cpu-reserve.mjs`, on every refresh, off every job path).
+	const readHostFacts = async () => {
+		const views = [];
+		const reserveVenues = [];
+		let user = {};
+		if (localBlessed && budgetEndpoint) {
+			const read = await resolveJobUser(budgetEndpoint).catch(() => null);
+			if (read?.daemon?.answered === true) {
+				views.push(read.daemon.facts);
+				reserveVenues.push({ venue: DEFAULT_BACKEND, facts: read.daemon.facts, endpointLocal: budgetEndpoint.endpoint?.local === true });
+			}
+		}
+		if (podmanInfo) {
+			const read = await podmanInfo().catch(() => null);
+			if (read?.answered === true) {
+				views.push(read.info);
+				reserveVenues.push({ venue: PODMAN_BACKEND, facts: read.info, endpointLocal: read.info?.serviceIsRemote === false });
+				if (read.info?.rootless === true) user = readUserServiceLimits({ uid: jobUserIdentity.euid, readFile: readCgroupFileFn });
+			}
+		}
+		const least = (key) => {
+			const known = views.map((v) => v?.[key]).filter((v) => Number.isSafeInteger(v));
+			return known.length > 0 ? Math.min(...known) : null;
+		};
+		return { memTotalMiB: least("memTotalMiB"), hostCpus: least("hostCpus"), ...user, reserveVenues };
+	};
+	// Issue #596, phase 2: the aggregate CPU reserve. One per worker; the helper container (Docker's cgroupfs driver) runs
+	// the job image this worker already pins, with `--pull=never`, never an image fetched for it.
+	const cpuReserve = makeCpuReserveFn({ run: (bin, args, { timeoutMs }) => execDockerBounded(args, { bin, timeoutMs }), image: config.jobImage, now, log });
+	const syncCpuReserve = (budget, facts) => cpuReserve.sync({ cpuCenti: budget.cpuCenti, plans: (facts?.reserveVenues ?? []).map((v) => reservePlan({ ...v, platform: jobUserIdentity.platform })) });
 	const bootPodmanRead = podmanInfo
 		? await settleWithin(
 				Promise.resolve()
@@ -1319,7 +1368,7 @@ export async function startWorker(
 	// would be bytes nothing reads. That is also what keeps a single-host deployment byte-identical, since
 	// no job then issues a single extra Valkey command.
 	const runMirror = config.workerNameDeclared ? makeRunMirrorFn({ redis, retentionDays: config.logRetentionDays, log }) : null;
-	const recordRun = ({ job, result, error, startedAt, endedAt, project }) => {
+	const recordRun = ({ job, result, error, startedAt, endedAt, project, size = null }) => {
 		// The project (issue #499) was resolved at the pickup gate and rides here as `project` (an id or null), so a live
 		// edit of projects.json mid-run cannot make the record disagree with what the job was counted against. A record
 		// path that ends BEFORE the pickup gate (the wait gate's refusals) passes none, and resolves from the live ref
@@ -1329,7 +1378,7 @@ export async function startWorker(
 		// four `recordRun` call sites byte-unchanged and `buildRecord` a pure function of its arguments.
 		// The default venue rides the same way and for the same reason (#277): it is the value the registry
 		// below is built with, so the record resolves a job's venue exactly as dispatch does.
-		const record = buildRecord({ job, result, error, startedAt, endedAt, host: config.workerName, defaultBackend: config.defaultBackend, project: projectId });
+		const record = buildRecord({ job, result, error, startedAt, endedAt, host: config.workerName, defaultBackend: config.defaultBackend, project: projectId, size });
 		writeRecord(record);
 		// STRICTLY AFTER the file, and deliberately not awaited. After, because a crash between the two must
 		// leave a record with no fleet row rather than a fleet row with no record -- the mirror is a VIEW,
@@ -1564,6 +1613,11 @@ export async function startWorker(
 	// Resolved once: `Intl` is not free, and this value cannot change without a restart.
 	const hostTz = Intl.DateTimeFormat().resolvedOptions().timeZone ?? "";
 	const registry = makeHostRegistryFn({ redis, name: config.workerName, log });
+	// Issue #596, phase 2: one integer of the host budget's snapshot for the beat, or "" before the worker exists.
+	const snapshotField = (key) => {
+		const snap = worker?.hostBudget?.snapshot?.();
+		return Number.isSafeInteger(snap?.[key]) ? String(snap[key]) : "";
+	};
 	// NOT awaited, and that is load-bearing rather than an optimisation. `makeRedisClient` sets
 	// `maxRetriesPerRequest: null` -- required for BullMQ's blocking connections -- which means a command
 	// issued against an unreachable server QUEUES FOREVER instead of rejecting. Awaiting the first beat
@@ -1617,6 +1671,29 @@ export async function startWorker(
 		// Issue #504 part B: the digest of this host's live envelope (`envelopeDigest`, 16 hex, never a value), so doctor can
 		// name a host whose envelope differs; such a host refuses governed jobs as `envelope-mismatch`. `none` without one.
 		fpEnvelope: () => envelope.digest ?? NO_ENVELOPE_FINGERPRINT,
+		// Issue #596: the highest scoped-limits version this build reads, so doctor can name a worker that would keep its
+		// last good file (so no size and no later edit to the file applies on it) once the file is version 3. An integer.
+		limitsVersion: SCOPED_LIMITS_VERSION,
+		// Issue #596, phase 2: this host's budget and what its jobs hold against it, integers (MiB and hundredths of a CPU),
+		// `off` for a budget switched off and "" while unknown, so doctor and a forge job's never-fits check can read which
+		// host a size fits on. Thunks over the worker's one budget, so every beat says what is held now.
+		budgetMemMiB: () => budgetField(worker?.hostBudget?.current().memMiB ?? null),
+		budgetCpuCenti: () => budgetField(worker?.hostBudget?.current().cpuCenti ?? null),
+		usedMemMiB: () => snapshotField("usedMemMiB"),
+		usedCpuCenti: () => snapshotField("usedCpuCenti"),
+		heldMemMiB: () => snapshotField("heldMemMiB"),
+		heldCpuCenti: () => snapshotField("heldCpuCenti"),
+		budgetRunning: () => snapshotField("running"),
+		budgetHolds: () => snapshotField("holds"),
+		budgetOrphans: () => snapshotField("orphans"),
+		// whether the boot listing of the job containers left from before this worker started has been read
+		// (`listed`); until a venue's is, the worker admits no job on it (`unlisted:<venue>[,<venue>]`, gate round 2 of
+		// phase 2), and doctor names the venues. "" before the worker exists.
+		budgetSeed: () => {
+			const snap = worker?.hostBudget?.snapshot?.();
+			if (typeof snap?.seeded !== "boolean") return "";
+			return snap.seeded ? "listed" : `unlisted:${(snap.unseeded ?? []).join(",")}`;
+		},
 	});
 
 
@@ -1653,6 +1730,7 @@ export async function startWorker(
 			if (endpoint.local === null && endpoint.transient) return { unavailable: true, reason: endpoint.reason };
 			return { refused: true, message: endpointRefusal, observations: [DOCKER_ENDPOINT_LOCAL] };
 		}
+		budgetEndpoint = { endpoint, key: state };
 		const jobUser = await resolveJobUser({ endpoint, key: state });
 		const unit = await readRootfulService({ endpoint, daemon: jobUser.daemon, readService: readPodmanServiceFn });
 		// One read of each host path for this job's two checks (`onceFs`, gate round 3 of PR #473), fresh per job.
@@ -1702,6 +1780,7 @@ export async function startWorker(
 		const venue = resolveBackendName(job, config.defaultBackend);
 		if (venue !== DEFAULT_BACKEND) return { user: null, home: null };
 		const endpoint = observed?.endpoint ?? (await resolveDockerEndpointFn());
+		if (!observed?.jobUser) budgetEndpoint = { endpoint, key: dockerEndpointState(endpoint) };
 		const admittedOn = observed?.jobUser ?? (await resolveJobUser({ endpoint, key: dockerEndpointState(endpoint) }));
 		const { decision, socket, facts } = admittedOn;
 		// Issue #452, gate round 5: the runtime THIS job is admitted on, recorded per job for its teardown's detach gate. The
@@ -1718,7 +1797,12 @@ export async function startWorker(
 		// undecidable daemon runs nothing), and only when true: every host this does not apply to keeps the answer
 		// shape, and so the argv, it had before.
 		if (chosen.refused || chosen.unavailable) return chosen;
-		return relabelsPrivateMounts(facts, endpoint, jobUserIdentity.platform) ? { ...chosen, relabel: true } : chosen;
+		// Issue #596: the daemon's CPU count from that same read, for the job's `--cpus` ceiling. Absent when it did not say.
+		// Beside it, the size flags the same daemon said it drops (SwapLimit or CPUShares false), which the job logs.
+		const unenforced = unenforcedSizeFlags(facts);
+		const counted = Number.isSafeInteger(facts?.hostCpus) ? { ...chosen, hostCpus: facts.hostCpus } : chosen;
+		const sized = unenforced.length > 0 ? { ...counted, unenforced } : counted;
+		return relabelsPrivateMounts(facts, endpoint, jobUserIdentity.platform) ? { ...sized, relabel: true } : sized;
 	};
 
 	// #227: WHERE this job's container runs. The three functions that decide whether a container may start and
@@ -1781,6 +1865,9 @@ export async function startWorker(
 						// reads. A refused teardown is logged with its token.
 						teardownRuntime: (job) => takeAdmittedRuntime(job),
 						log,
+						// Issue #596: Docker refused a job's `--cpus` as above its CPU count, so the count the resolver cached is
+						// stale (a resized Docker Desktop VM); the next pickup reads the daemon again.
+						onCpuCeilingStale: () => resolveJobUser.invalidate?.(),
 					}),
 				}),
 				observationPreflight: localObservationPreflight,
@@ -1897,7 +1984,8 @@ export async function startWorker(
 	// each already comments -- a delivery storm against a spent cap must not page anyone), and
 	// `operator-cancel`, because the operator initiated it and a push telling them what they just did is
 	// noise with a pager attached.
-	const HOOK_POLICY_REASONS = new Set(["worker-abort", "runner-policy", ...RUNNER_POLICY_REASONS]);
+	// `oom-killed` (issue #596) is a paid terminal the operator alone can fix (a job's memory size), so it pages too.
+	const HOOK_POLICY_REASONS = new Set(["worker-abort", "runner-policy", EXIT_OOM_KILLED, ...RUNNER_POLICY_REASONS]);
 	// One predicate for the completed listener and the lost-lock path below, so a record replays exactly the page its
 	// result would have sent.
 	const pagesAsPolicy = (result) => Boolean(onFailure) && result?.outcome === "policy" && HOOK_POLICY_REASONS.has(result.reason) && result.budgetReserved !== false;
@@ -1920,6 +2008,39 @@ export async function startWorker(
 
 	const worker = createWorkerFn({
 		connection: valkeyConn(),
+		// Issue #596, phase 2 (DES-HOST-BUDGET): the host budget's inputs. `createWorker` builds the ONE budget from them and
+		// shares it between both queues' processors. The settings and the default size were refused at boot if bad.
+		hostBudget: {
+			settings: config.hostBudget,
+			jobDefault: { memMiB: config.jobSize.memMiB, cpuCenti: config.jobSize.cpuCenti },
+			readFacts: readHostFacts,
+			onRefresh: syncCpuReserve,
+			containerGone: containerGoneFn ?? makeContainerGone({ binOf: (venue) => (resolveBackendName(venue ?? {}, config.defaultBackend) === PODMAN_BACKEND ? "podman" : "docker") }),
+			// AFTER the boot reaper (above), every blessed venue's remaining job containers, seeded into the ledger
+			// as orphans from their size labels. PER VENUE (gate round 2 of phase 2): a venue that cannot be listed
+			// stops only its own jobs until a tick reads it, because a container nobody counted is an overcommit, while the
+			// other venue's jobs run. A venue that holds no container by construction counts as none rather than as unread:
+			// its binary is absent (ENOENT, nothing of it can run), or its boot job-user decision is `unmappable` for a
+			// cause that refuses a boot (the venue's boot-refusing set: every job on it is refused before a container, so this
+			// worker starts none there). A cause decided from one unreadable answer is re-decided per job and can clear, so
+			// it leaves the venue unread.
+			survivors: Object.fromEntries(
+				[...(localBlessed ? [[DEFAULT_BACKEND, "docker", bootDecision, BOOT_REFUSING_JOB_USER_CAUSES]] : []), ...(podmanBlessed ? [[PODMAN_BACKEND, "podman", bootPodmanDecision, PODMAN_BOOT_REFUSING_CAUSES]] : [])].map(([backend, bin, decision, stable]) => [
+					backend,
+					async () => {
+						try {
+							return (await listJobContainersFn(bin)()).map((c) => ({ ...c, venue: { backend } }));
+						} catch (error) {
+							if (error?.code === "ENOENT" || (decision?.mode === "unmappable" && stable.has(decision.cause))) return [];
+							throw error;
+						}
+					},
+				]),
+			),
+			defaultVenue: config.defaultBackend,
+			now,
+			log,
+		},
 		// #227. The abort path's stop, resolved per job rather than hard-wired to docker. A container NAME is
 		// not enough to find the runtime holding it once there is more than one venue.
 		stopContainer: backends.stopContainer,
@@ -1947,6 +2068,9 @@ export async function startWorker(
 		scopedLimits: () => scopedLimits.current,
 		// Issue #499: the projects snapshot, read by the pickup gate once, beside the limits snapshot above.
 		projects: () => projects.current,
+		// Issue #596: the deployment's default job size, the two settings `loadConfig` already refused at boot if bad. ENV
+		// ONLY, never the settings overlay, so a size cannot change under a running worker.
+		jobSizeEnv: { PI_JOB_MEMORY: env.PI_JOB_MEMORY, PI_JOB_CPUS: env.PI_JOB_CPUS },
 		// Issue #504 part B: the live envelope and its digest, and the reconcile the pickup runs before it narrows a job's
 		// dollar ledgers by the applied split. Without an envelope `current()` is null, and `fleetGoverned` asks whether an
 		// applied split exists: if it does, this host's jobs refuse as envelope-mismatch rather than run ungoverned.

package/src/triggers.mjs

@@ -1309,7 +1309,8 @@ function validateMaxCostUsd(run, at, path) {
  * trigger, while naming what to take away cannot widen on a bump.
  *
  * MEMBERSHIP IS VALIDATED against `EXCLUDABLE_TOOL_NAMES` because pi ignores unknown names in
- * `excludeTools` silently (verified at the pin: the set is only ever consulted by a filter), which puts
+ * `excludeTools` silently (verified at the pin: pi matches entries by exact name or `*` pattern, so an
+ * unknown exact name matches nothing, silently), which puts
  * a misspelled exclusion in `run.backend`'s destructive-absence class -- the job runs WITH the tool
  * while the file reads as though it was off. The near-miss sweep covers the KEY for the same reason.
  * No charset check: membership subsumes it, and no known name carries the comma the container env