@edgehero/pi-dispatch 4.0.1 → 4.1.0

package/src/cli.mjs

@@ -52,6 +52,9 @@ const USAGE = `pi-dispatch — run pi coding-agent flows on your own folders
   pi-dispatch pause        stop taking new jobs (durable; survives worker restart)
   pi-dispatch resume       resume taking jobs
   pi-dispatch status       show paused state + job counts
+  pi-dispatch capacity [--since 24h|7d|30d] [--host <name>] [--json] [--valkey-url <url>]
+                           how busy each host was: busy and idle time, slots in use, memory and CPU
+                           promised and used, waits and projects, read from the run records (jobs only)
   pi-dispatch cancel <jobId>  stop one job: a queued or held job is removed (the line says whether it had
                            made attempts; cancel records nothing), a running one is aborted on whichever
                            host owns it (its record says operator-cancel)
@@ -235,6 +238,12 @@ export async function main(argv = process.argv.slice(2), env = process.env, { wr
 		return code;
 	}
 
+	if (cmd === "capacity") {
+		// Read-only, and on the kill switch's footing: the Valkey URL and the logs directory, never loadConfig (issue #599).
+		const { runCapacity } = await import("./capacity-cli.mjs");
+		return runCapacity(argv.slice(1), { env, write, valkeyRefusal, deploymentEnv: cliDeploymentEnv });
+	}
+
 	if (cmd === "cancel") {
 		// The kill switch's doctrine (issue #287): VALKEY_URL only, never loadConfig, so one misbehaving
 		// job can be stopped even when everything else about the deployment is misconfigured. On a shell/.env

package/src/config.mjs

@@ -19,6 +19,9 @@ import { jobSizeDefaults } from "./job-size.mjs";
 import { hostBudgetSettings } from "./host-budget.mjs";
 import { CONTAINER_ENV_NAMES, KEYLESS_ENV_NAME, RUNNER_ENV_NAMES } from "./reserved-env.mjs";
 import { modelListProblem } from "./model-ref.mjs";
+import { WORKER_NAME_RE } from "./worker-name.mjs";
+
+export { WORKER_NAME_RE };
 import { DOLLAR_ENV_NAMES, DOLLAR_WINDOW_KEYS, checkDollarInvariant, optionalUsdMicros } from "./money.mjs";
 
 /**
@@ -1146,26 +1149,7 @@ export function underOsTempDir(candidate, env = process.env, { realpath = realpa
 	return false;
 }
 
-/**
- * What a worker may call itself (issue #57). The CHARACTER CLASS is `sanitizeJobId`'s
- * (`[A-Za-z0-9._-]`), reused rather than invented so this project has one name-safe alphabet -- but that
- * function is a REPLACER, not a validator, so the three rules around the class are NEW and are claimed
- * as new here rather than borrowed:
- *
- *   - a leading alphanumeric, which is what refuses `..` and a leading `-` that reads as a flag;
- *   - a 64-character ceiling, because the name is a Valkey key segment and a log field on every line;
- *   - no `.json`/`.log` tail, which is not decoration. The class contains the dot, so `prod.json` is
- *     otherwise a legal name -- and a later slice writes a per-host marker file into `PI_LOGS_DIR`,
- *     where `<something>.json` is parsed as a run record by the admin and DELETED by the log reaper.
- *     A name is refused here rather than escaped there, because the escape would have to be remembered
- *     at every site that ever composes a filename from this value.
- *
- * The class is `:`-free, `,`-free and `#`-free, which is what lets the name be a Valkey key segment
- * UNHASHED. That is the point of validating instead of hashing (`scopeKeyPrefix` does the opposite for
- * a folder path, which was never chosen for key-safety and cannot be refused): the whole value of a host
- * registry is that `HGETALL host:h:mac-mini-1` is readable by a human.
- */
-export const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
+// WORKER_NAME_RE lives in worker-name.mjs (a leaf, for the capacity report) and is re-exported above; its reasons are there.
 
 /** True when the name would collide with the run-history filename namespace. See WORKER_NAME_RE. */
 const RESERVED_NAME_TAIL = /\.(json|log)$/i;

package/src/connection.mjs

@@ -189,8 +189,10 @@ export function parseConnection(url, { failFast = false, servername = null, cont
  * `parseConnection`'s options, so it connects through `JudgedConnector` like every other client. `lazyConnect` and
  * `failFast` as ioredis and `parseConnection` read them.
  */
-export function makeRedisClient(url, { servername = null, context = null, failFast = false, lazyConnect = false, judge = null, withoutPassword = false } = {}) {
-	return new Redis({ ...parseConnection(url, { failFast, servername, context, judge, withoutPassword }), ...(lazyConnect ? { lazyConnect: true } : {}) });
+export function makeRedisClient(url, { servername = null, context = null, failFast = false, lazyConnect = false, judge = null, withoutPassword = false, disconnectTimeoutMs = null } = {}) {
+	// `disconnectTimeoutMs` (issue #599): how long ioredis's `disconnect()` waits for a socket to close before it destroys
+	// it (its default is 2 s, on a ref'd timer). A one-shot read that must not hold its process passes 0.
+	return new Redis({ ...parseConnection(url, { failFast, servername, context, judge, withoutPassword }), ...(lazyConnect ? { lazyConnect: true } : {}), ...(Number.isSafeInteger(disconnectTimeoutMs) && disconnectTimeoutMs >= 0 ? { disconnectTimeout: disconnectTimeoutMs } : {}) });
 }
 
 /**

package/src/doctor.mjs

@@ -99,6 +99,8 @@ import { CONTAINER_HOME, SHIPPED_IMAGE_UID, SIZE_LABEL_CPU, SIZE_LABEL_MEM } fro
 import { DEFAULT_JOB_SIZE, cpuCeilingCenti, formatCpus, formatMemory, jobSizeDefaults, resolveJobSize } from "./job-size.mjs";
 import { SUGGEST_WINDOW_DAYS, cpusText, hostCap, refusalWords, sizeRefusal, suggestSize, suggestionCall, suggestionEvidence } from "./size-suggest.mjs";
 import { SIZING_RECORD_MAX_BYTES, readSizingRecords } from "./size-records.mjs";
+import { CAPACITY_WINDOWS, computeCapacity } from "./capacity.mjs";
+import { JOBS_ONLY, durationText, milliText, notSharedWhy, percentText, shareText } from "./capacity-cli.mjs";
 import { CGROUP_PARENT, cgroupParentFor, operatorQuotaCommand, readQuota, reservePlan, userQuotaCommand } from "./cpu-reserve.mjs";
 import { HOST_BUDGET_KEYS, computeHostBudget, hostBudgetSettings, largestFit, neverFits, projectBudgetRow, publishedBudget, readUserServiceLimits } from "./host-budget.mjs";
 import { makeImagePreflight, normalizeImageId } from "./image-preflight.mjs";
@@ -220,6 +222,9 @@ export async function runDoctor(shellVars = process.env, deps = {}) {
 		dollarKeysExist,
 		// Issue #504 part B: the applied split's envelope digest (`alloc:plan`), read once. Undefined means the default.
 		readAppliedSplit,
+		// Issue #599, phase 2: the capacity line's read of the run history (the run mirror and the logs directory).
+		// Undefined means the default, which reads the deployment's real Valkey and files.
+		readCapacity,
 		// --live (issue #278, INT-LIVE-PROBE-CONTRACT): read the backend declarations back off short-lived real containers.
 		// STRICTLY `=== true`, so only the CLI's own flag arms it: a truthy string from a caller that forwarded an
 		// option bag runs nothing. The fs, PID-liveness and nonce are seams so the sequence is driven without Docker.
@@ -383,7 +388,7 @@ export async function runDoctor(shellVars = process.env, deps = {}) {
 						return { ...(await valkeyAuthState(url, { context, withoutPassword })), passwordSet: Boolean(sent.password), from: sent.from };
 					}
 				: null;
-	const seams = { cwd, out, spawn, probeValkey, valkeyAuth: valkeyAuthSeam, readHosts, ...(modelCatalog ? { modelCatalog } : {}), ...(piModelLoader ? { piModelLoader } : {}), ...(dollarKeysExist ? { dollarKeysExist } : {}), ...(readAppliedSplit ? { readAppliedSplit } : {}), fileExists, nodeVersion, mkdir, chmod, rm, agentDir, platform, home, providerOracle, facts, jobUserIdentity, stat, passwd, readUnit, readEnvFile: readEnvFileShared, observationFs, jobsDirFs, jobsDirUid, valkeyOwner: valkeyOwnerSeam, isAlive, pid, runTimeouts, live: live === true, wallClock, ...(readRunRecords ? { readRunRecords } : {}), venueChecks, userName, proxyFilesExist, proxyFileIsDirectory, ...(includeNeeds ? { includeNeeds } : {}), ...(declaredEndpoints ? { declaredEndpoints } : {}), ...(readOverlayFile ? { readOverlayFile } : {}), ...(lstatOverlayFile ? { lstatOverlayFile } : {}), ...(hostAddresses ? { hostAddresses } : {}), ...(readProxyConf ? { readProxyConf } : {}), ...(readPackagedConf ? { readPackagedProxyConf: readPackagedConf } : {}), ...(readPodmanService ? { readPodmanService } : {}), serviceEnvFile: envValues === null ? null : serviceEnvFileOf(envValues, envPath, serviceEnvLoader(platform)) };
+	const seams = { cwd, out, spawn, probeValkey, valkeyAuth: valkeyAuthSeam, readHosts, ...(modelCatalog ? { modelCatalog } : {}), ...(piModelLoader ? { piModelLoader } : {}), ...(dollarKeysExist ? { dollarKeysExist } : {}), ...(readAppliedSplit ? { readAppliedSplit } : {}), ...(readCapacity ? { readCapacity } : {}), fileExists, nodeVersion, mkdir, chmod, rm, agentDir, platform, home, providerOracle, facts, jobUserIdentity, stat, passwd, readUnit, readEnvFile: readEnvFileShared, observationFs, jobsDirFs, jobsDirUid, valkeyOwner: valkeyOwnerSeam, isAlive, pid, runTimeouts, live: live === true, wallClock, ...(readRunRecords ? { readRunRecords } : {}), venueChecks, userName, proxyFilesExist, proxyFileIsDirectory, ...(includeNeeds ? { includeNeeds } : {}), ...(declaredEndpoints ? { declaredEndpoints } : {}), ...(readOverlayFile ? { readOverlayFile } : {}), ...(lstatOverlayFile ? { lstatOverlayFile } : {}), ...(hostAddresses ? { hostAddresses } : {}), ...(readProxyConf ? { readProxyConf } : {}), ...(readPackagedConf ? { readPackagedProxyConf: readPackagedConf } : {}), ...(readPodmanService ? { readPodmanService } : {}), serviceEnvFile: envValues === null ? null : serviceEnvFileOf(envValues, envPath, serviceEnvLoader(platform)) };
 	// Issue #471: every other service key, resolved ONCE for the whole run (the fix pass's re-collect and `--live` judge the
 	// same resolution). THE RULE (PR #474's round cap, after three rounds of trust patches): no program doctor starts is
 	// handed anything from `.env`. Every child gets this shell's own environment, the one it had before #471; a `.env`
@@ -595,7 +600,7 @@ export const ENV_FILE_READABLE_KEYS = Object.freeze(["PI_PAUSE_WINDOWS_FILE", "P
 export const GITHUB_SERVICE_KEYS = Object.freeze(["GITHUB_AUTH_SOURCE", "GITHUB_APP_ID", "GITHUB_APP_INSTALLATION_ID", "GITHUB_APP_PRIVATE_KEY_PATH", "GITHUB_APP_PRIVATE_KEY"]);
 /** Issue #471: the worker's settings doctor judges, which it read from this shell alone while the service read them from
  *  `.env`. TEMP is TMPDIR's twin in the worker's temp root; PI_CODING_AGENT_DIR is where the worker reads auth.json. */
-export const WORKER_SERVICE_KEYS = Object.freeze(["PI_JOB_IMAGE", "PI_JOB_MEMORY", "PI_JOB_CPUS", "PI_CONCURRENCY", ...Object.values(HOST_BUDGET_KEYS), "PI_TRIGGERS_FILE", "PI_LOGS_DIR", "PI_SETTINGS_FILE", "PI_SESSIONS_DIR", "PI_SESSIONS_TTL_DAYS", "PI_SESSION_MAX_AGE_DAYS", "PI_SESSION_MAX_CONTEXT_PCT", "PI_SESSION_MAX_RESUME_CHAIN", "PI_GLOBAL_PI_DIR", "PI_GLOBAL_ALLOW_EXTENSIONS", "PI_FORWARD_ENV", "PI_AUTH_FROM_PI", "PI_CODING_AGENT_DIR", "PI_BACKEND_FLOOR", "PI_SECRET_PROFILES", "PI_SECRET_RESOLVER_ROOTS", "PI_WAIT_PROFILES", "PI_WAIT_AFTER_MAX_MS", "PI_SANDBOX_RETENTION_HOURS", "PI_ALLOWED_MODELS", "PI_DISPATCH_RUN_ROOTS", "GITHUB_PAT_VAR", "TEMP", ...Object.values(DOLLAR_ENV_NAMES)]);
+export const WORKER_SERVICE_KEYS = Object.freeze(["PI_JOB_IMAGE", "PI_JOB_MEMORY", "PI_JOB_CPUS", "PI_CONCURRENCY", ...Object.values(HOST_BUDGET_KEYS), "PI_TRIGGERS_FILE", "PI_LOGS_DIR", "PI_LOG_RETENTION_DAYS", "PI_SETTINGS_FILE", "PI_SESSIONS_DIR", "PI_SESSIONS_TTL_DAYS", "PI_SESSION_MAX_AGE_DAYS", "PI_SESSION_MAX_CONTEXT_PCT", "PI_SESSION_MAX_RESUME_CHAIN", "PI_GLOBAL_PI_DIR", "PI_GLOBAL_ALLOW_EXTENSIONS", "PI_FORWARD_ENV", "PI_AUTH_FROM_PI", "PI_CODING_AGENT_DIR", "PI_BACKEND_FLOOR", "PI_SECRET_PROFILES", "PI_SECRET_RESOLVER_ROOTS", "PI_WAIT_PROFILES", "PI_WAIT_AFTER_MAX_MS", "PI_SANDBOX_RETENTION_HOURS", "PI_ALLOWED_MODELS", "PI_DISPATCH_RUN_ROOTS", "GITHUB_PAT_VAR", "TEMP", ...Object.values(DOLLAR_ENV_NAMES)]);
 /** Issue #471: the receiver's keys doctor judges its boot by (the receiver's unit reads the same `.env`). */
 export const RECEIVER_SERVICE_KEYS = Object.freeze(["WEBHOOK_SECRET", "RECEIVER_PORT", "GITLAB_TOKEN", "GITLAB_URL", "GITLAB_WEBHOOK_MODE", "GITLAB_WEBHOOK_SECRET", "FORGEJO_URL", "FORGEJO_TOKEN", "FORGEJO_WEBHOOK_SECRET", "AZURE_ORG_URL", "AZURE_TOKEN", "AZURE_WEBHOOK_MODE", "AZURE_WEBHOOK_SECRET", "AZURE_WEBHOOK_HEADER"]);
 /**
@@ -2594,6 +2599,11 @@ export async function collectChecks(shellVars, seams) {
 		// Said, rather than silently absent: "no peers" and "could not ask" are different facts.
 		checks.push({ ok: true, label: `Fleet: could not read the host registry (${printable(fleet.unreachable)})` });
 	}
+	// Issue #599, phase 2: how busy each host was over the last 7 days, one fact line per host, never a warning (nothing
+	// decides on it). On every deployment, a single host too: its own files are read when the run mirror is not. Read from
+	// the same Valkey the fleet was, with the registry rows just read (their running jobs count up to now), every Valkey
+	// call bounded and the whole read under `CAPACITY_DOCTOR_TIMEOUT_MS`, so a slow mirror costs this line, not the run.
+	checks.push(...(await doctorCapacity({ seams, env, home, url: valkeyUsable ? valkeyTalkUrl : null, hosts: fleet.hosts ?? [], localHost: workerNameOf(declaredWorkerName) })));
 	// Issue #504 part B: the APPLIED split names the envelope it was made for, and every host
 	// whose envelope differs refuses its governed jobs. Read whenever this command may talk to the Valkey (above), from
 	// the same Valkey the fleet was read from; with no split there, or no answer, nothing is said.
@@ -4722,21 +4732,15 @@ export function appliedSplitChecks(applied, mine, myName, peers) {
  */
 export async function defaultReadAppliedSplit(url) {
 	try {
-		const { makeRedisClient } = await import("./connection.mjs");
-		const client = makeRedisClient(url, { failFast: true, lazyConnect: true });
-		client.on("error", () => {});
-		try {
-			await client.connect();
-			const text = await client.get(ALLOC_PLAN_KEY);
+		return await withDoctorClient(url, async (client) => {
+			const text = await boundedOp(client.get(ALLOC_PLAN_KEY), DOCTOR_VALKEY_OP_TIMEOUT_MS);
 			if (text === null || text === undefined) return null;
 			let digest = null;
 			try {
 				digest = JSON.parse(text)?.envelopeDigest;
 			} catch {}
 			return typeof digest === "string" && /^[0-9a-f]{16}$/.test(digest) ? { digest } : { undecodable: true };
-		} finally {
-			client.disconnect();
-		}
+		});
 	} catch {
 		return null;
 	}
@@ -7389,16 +7393,9 @@ function workerNameOf(declared) {
 
 async function defaultReadHosts(url) {
 	try {
-		const { makeRedisClient } = await import("./connection.mjs");
 		const { readLiveHosts } = await import("./host-registry.mjs");
-		const client = makeRedisClient(url, { failFast: true, lazyConnect: true });
-		client.on("error", () => {});
-		try {
-			await client.connect();
-			return await readLiveHosts(client);
-		} finally {
-			client.disconnect();
-		}
+		// readLiveHosts bounds each of its own commands.
+		return await withDoctorClient(url, (client) => readLiveHosts(client));
 	} catch (err) {
 		return { unreachable: err?.message ?? "registry unreadable" };
 	}
@@ -7447,35 +7444,60 @@ export async function dollarKeysExistWith(client, { now = () => new Date(), dead
 /** `dollarKeysExistWith` over a fail-fast client on `url`, always disconnected. */
 export async function defaultDollarKeysExist(url) {
 	try {
-		const { makeRedisClient } = await import("./connection.mjs");
-		const client = makeRedisClient(url, { failFast: true, lazyConnect: true });
-		client.on("error", () => {});
-		try {
-			await client.connect();
-			return await dollarKeysExistWith(client);
-		} finally {
-			client.disconnect();
-		}
+		// dollarKeysExistWith bounds its whole read.
+		return await withDoctorClient(url, (client) => dollarKeysExistWith(client));
 	} catch {
 		return false;
 	}
 }
 
 async function defaultProbeValkey(url) {
-	const { makeRedisClient } = await import("./connection.mjs");
-	const client = makeRedisClient(url, { failFast: true, lazyConnect: true });
-	client.on("error", () => {}); // swallow connect errors + retries; reachability is the ✓/✗, not a trace
 	try {
-		await client.connect();
-		await client.ping();
-		return true;
+		return await withDoctorClient(url, async (client) => (await boundedOp(client.ping(), DOCTOR_VALKEY_OP_TIMEOUT_MS), true));
 	} catch {
 		return false;
+	}
+}
+
+/** How long doctor waits for a Valkey connection to be ready (the TCP connect AND the ready check), and for one command. */
+export const DOCTOR_VALKEY_CONNECT_TIMEOUT_MS = 5_000;
+export const DOCTOR_VALKEY_OP_TIMEOUT_MS = 5_000;
+
+/**
+ * One fail-fast client for one of doctor's own Valkey reads, its connection bounded, always dropped. Measured: a
+ * server that accepts the connection and never answers holds ioredis in its ready check forever (`failFast` bounds
+ * only the TCP connect), so doctor never exited; and `disconnect()` arms a 2 s timer a closed socket never clears, so
+ * the client is built with a disconnect timeout of 0 and its stream destroyed. Throws when the connection is not ready
+ * in time; `fn`'s own commands are the caller's to bound (`boundedOp`).
+ */
+export async function withDoctorClient(url, fn, { connectTimeoutMs = DOCTOR_VALKEY_CONNECT_TIMEOUT_MS, makeClient } = {}) {
+	const make = makeClient ?? (await import("./connection.mjs")).makeRedisClient;
+	const client = make(url, { failFast: true, lazyConnect: true, disconnectTimeoutMs: 0 });
+	client.on?.("error", () => {}); // swallow connect errors + retries; reachability is the ✓/✗, not a trace
+	try {
+		await connectWithin(client, connectTimeoutMs);
+		return await fn(client);
 	} finally {
-		client.disconnect();
+		try {
+			client.disconnect?.();
+			client.stream?.destroy?.();
+		} catch {
+			// a release that failed has stopped mattering
+		}
 	}
 }
 
+/** A command's reply, or a rejection after `ms` (the client is dropped by `withDoctorClient` either way). */
+export function boundedOp(promise, ms) {
+	return new Promise((resolve, reject) => {
+		const t = setTimeout(() => reject(new Error("timeout")), ms);
+		Promise.resolve(promise).then(
+			(v) => (clearTimeout(t), resolve(v)),
+			(e) => (clearTimeout(t), reject(e)),
+		);
+	});
+}
+
 /**
  * The deployment's default job size as doctor reads it (issue #596): `PI_JOB_MEMORY` and `PI_JOB_CPUS` through the
  * worker's own rule (`jobSizeDefaults`), or the built-in 4g and 2 when they do not parse (`jobSizeChecks` reports that as
@@ -7756,6 +7778,162 @@ export function hostBudgetChecks(view, { concurrency = 3, limits = [], env = {},
 	return checks;
 }
 
+/** The most the capacity read may take in doctor, all of it: a full mirror reads in well under a second. */
+export const CAPACITY_DOCTOR_TIMEOUT_MS = 5_000;
+
+/**
+ * The capacity lines (issue #599, phase 2): the report over the last 7 days (`capacity.mjs`, the CLI's function), read
+ * by `seams.readCapacity` (default: `readCapacityRecords` through a fail-fast client, `prune`-free and writing nothing).
+ * Never throws and never warns: an unreadable history is one line saying so.
+ */
+export async function doctorCapacity({ seams, env, home, url, hosts, localHost }) {
+	try {
+		const window = CAPACITY_WINDOWS["7d"];
+		const nowMs = (typeof seams.wallClock === "function" ? seams.wallClock : Date.now)();
+		const retention = typeof env.PI_LOG_RETENTION_DAYS === "string" && /^\d{1,6}$/.test(env.PI_LOG_RETENTION_DAYS) ? Number(env.PI_LOG_RETENTION_DAYS) : 30;
+		// The read is told to stop when the bound passes (`signal`): giving up on it is not enough, since a client still
+		// waiting on its connection would keep this process alive after doctor has printed everything.
+		const stop = new AbortController();
+		const args = { url, logsDir: logsDirPath(env, home), sinceMs: nowMs - window.ms, nowMs, retentionDays: retention, localHost, signal: stop.signal };
+		let timer;
+		const read = await Promise.race([
+			Promise.resolve()
+				.then(() => (seams.readCapacity ?? defaultReadCapacity)(args))
+				.finally(() => clearTimeout(timer)),
+			new Promise((resolve) => {
+				timer = setTimeout(() => {
+					stop.abort();
+					resolve(null);
+				}, seams.capacityTimeoutMs ?? CAPACITY_DOCTOR_TIMEOUT_MS);
+			}),
+		]);
+		if (read === null) return [{ ok: true, label: "Capacity: the run history did not answer in time, so nothing is shown (pi-dispatch capacity waits up to 2 s per Valkey call)" }];
+		const report = computeCapacity({ records: read.records, live: hosts, windowStartMs: nowMs - window.ms, nowMs, bucketMs: window.bucketMs, coverage: read.coverage });
+		return capacityChecks(report, { since: "7d" });
+	} catch (err) {
+		if (err?.[FORBIDDEN_READ]) throw err;
+		return [{ ok: true, label: `Capacity: not read (${printable(err?.message ?? "error")})` }];
+	}
+}
+
+/**
+ * Set by the test helper (test/helpers/doctor.mjs): from then on the DEFAULT capacity read throws an error doctor does
+ * not swallow, so a test that reaches the developer's real Valkey and logs directory fails loudly instead of printing
+ * whatever that machine holds.
+ */
+let defaultCapacityReadForbidden = false;
+export function forbidDefaultCapacityRead() {
+	defaultCapacityReadForbidden = true;
+}
+const FORBIDDEN_READ = Symbol("forbidden-read");
+
+/** How long the default read waits for its Valkey connection to be ready (the TCP connect and the ready check). */
+export const CAPACITY_CONNECT_TIMEOUT_MS = 2_000;
+
+/**
+ * The default capacity read: a fail-fast client, its connection bounded (`connectWithin`: a server that accepts TCP and
+ * never answers holds ioredis in its ready check forever) and DISCONNECTED when `signal` aborts, so a read doctor gave
+ * up on cannot keep the process alive.
+ */
+async function defaultReadCapacity({ url, logsDir, sinceMs, nowMs, retentionDays, localHost, signal }) {
+	if (defaultCapacityReadForbidden) throw Object.assign(new Error("doctor reached its real capacity read in a test: pass readCapacity"), { [FORBIDDEN_READ]: true });
+	const { readCapacityRecords } = await import("./capacity-records.mjs");
+	let client = null;
+	// Measured on ioredis 5.11: `disconnect()` ends the socket and arms a timer (`disconnectTimeout`, 2 s by default)
+	// that destroys it unless the socket's `close` clears it first, and the client's own failed ready check disconnects
+	// again after the close, arming one that nothing clears: a server that never answered held the process 2 s past
+	// doctor's bound. So the client is built with a disconnect timeout of 0 (no timer outlives the socket), its stream
+	// is destroyed in the same tick (the socket is closed by the time doctor prints), and it is dropped once.
+	const drop = () => {
+		const c = client;
+		client = null;
+		try {
+			c?.disconnect?.();
+			c?.stream?.destroy?.();
+		} catch {
+			// a release that failed has stopped mattering
+		}
+	};
+	signal?.addEventListener?.("abort", drop, { once: true });
+	if (url && !signal?.aborted) {
+		try {
+			const { makeRedisClient } = await import("./connection.mjs");
+			client = makeRedisClient(url, { failFast: true, lazyConnect: true, disconnectTimeoutMs: 0 });
+			client.on("error", () => {});
+			await connectWithin(client, CAPACITY_CONNECT_TIMEOUT_MS, signal);
+		} catch {
+			drop();
+		}
+	}
+	try {
+		return await readCapacityRecords({ redis: client, logsDir, sinceMs, nowMs, retentionDays, localHost, noMirrorReason: url ? "the Valkey did not answer: only this host's files were read" : "no Valkey to read: only this host's files were read" });
+	} finally {
+		signal?.removeEventListener?.("abort", drop);
+		drop();
+	}
+}
+
+/**
+ * `client.connect()`, or a rejection after `ms` or as soon as `signal` aborts (the client is then the caller's to
+ * disconnect). The timer goes with an abort, so nothing of a read doctor gave up on holds the process.
+ */
+export function connectWithin(client, ms, signal) {
+	return new Promise((resolve, reject) => {
+		const stop = () => (clearTimeout(t), reject(new Error("connect stopped")));
+		const t = setTimeout(() => (signal?.removeEventListener?.("abort", stop), reject(new Error("connect timeout"))), ms);
+		signal?.addEventListener?.("abort", stop, { once: true });
+		Promise.resolve(client.connect()).then(
+			(v) => (clearTimeout(t), signal?.removeEventListener?.("abort", stop), resolve(v)),
+			(e) => (clearTimeout(t), signal?.removeEventListener?.("abort", stop), reject(e)),
+		);
+	});
+}
+
+/**
+ * One fact line per host of a capacity report (`INT-CAPACITY-REPORT`), `{ ok: true, label }` each, never a warning:
+ * `Host a: last 7d busy 63% (avg 2.1 of 4 slots, full 12%), promised 48% memory / 40% CPU, used 18% CPU of 8, wait p50
+ * 40s p95 6m, most busy: web`, each part only when it is known, then what the line cannot see: the running jobs not
+ * counted, and the coverage when the history is cut, not shared, or this host's files only. Every control character is
+ * removed (the report admits none; this is the printer not relying on it).
+ */
+export function capacityChecks(report, { since = "7d" } = {}) {
+	const out = [];
+	const localOnly = report?.coverage?.source === "local";
+	for (const h of Array.isArray(report?.hosts) ? report.hosts : []) {
+		const notes = [];
+		let line;
+		if (h.coveredMs === 0) {
+			line = `Host ${h.name}: last ${since} no history here`;
+			if (!h.shared) notes.push(notSharedWhy(h, report.coverage));
+		} else {
+			const c = h.capacity ?? {};
+			const slots = Number.isSafeInteger(c.slots) ? ` of ${c.slots} slots` : " at once";
+			const full = h.fullMs !== null ? `, full ${shareText(h.fullMs, h.coveredMs)}` : "";
+			const parts = [`busy ${shareText(h.busyMs, h.coveredMs)} (avg ${milliText(h.avgMilli ?? 0)}${slots}${full})`];
+			const promised = [h.promisedMemPerMille !== null ? `${percentText(h.promisedMemPerMille)} memory` : null, h.promisedCpuPerMille !== null ? `${percentText(h.promisedCpuPerMille)} CPU` : null].filter(Boolean);
+			if (promised.length > 0) parts.push(`promised ${promised.join(" / ")}`);
+			if (h.usedCpuPerMille !== null) parts.push(`used ${percentText(h.usedCpuPerMille)} CPU${Number.isSafeInteger(c.cpus) ? ` of ${c.cpus}` : ""}`);
+			if (h.waits?.n > 0) parts.push(`wait p50 ${durationText(h.waits.p50Ms)} p95 ${durationText(h.waits.p95Ms)}`);
+			if (h.projects?.length > 0) parts.push(`most busy: ${h.projects[0].project ?? "(no project)"}`);
+			line = `Host ${h.name}: last ${since} ${parts.join(", ")}`;
+			if (h.coverage.live > 0) notes.push(`${h.coverage.live} running now, counted to now`);
+			if (h.missingMs > 0) notes.push(`history from ${new Date(h.coverage.fromMs).toISOString().slice(0, 16).replace("T", " ")} UTC only${h.coverage.truncated ? " (the run mirror holds nothing older)" : ""}, earlier time counted as neither busy nor idle`);
+			// The reason with it: a run mirror that did not answer leaves a busy fleet reading nearly idle here.
+			// The reasons end in "only this host's files were read", which this clause already says.
+			const why = typeof report.coverage.reason === "string" ? report.coverage.reason.replace(/: only this host's files were read$/, "") : "";
+			if (localOnly) notes.push(`this host's files only${why ? ` (${why})` : ""}`);
+		}
+		if (h.coverage?.liveNotCounted > 0) notes.push(`${h.coverage.liveNotCounted} running now not counted`);
+		if (h.coverage?.liveUnreadable > 0) notes.push("its running jobs could not be read, not counted");
+		if (report.coverage?.liveRowMissing === h.name) notes.push("no live row read for it, so its running jobs are not known");
+		out.push({ ok: true, label: `${line}${notes.length > 0 ? `; ${notes.join("; ")}` : ""}`.replace(/[\u0000-\u001f\u007f-\u009f]/g, "") });
+	}
+	// What none of these lines can see, in the words every surface of the report uses (REQ-CAPACITY-INSIGHTS): a fact,
+	// never a warning, once after the hosts.
+	if (out.length > 0) out.push({ ok: true, label: JOBS_ONLY.replace(/\.$/, "") });
+	return out;
+}
+
 /**
  * The fleet's budgets (issue #596, phase 2), from the registry rows (this host's own included): one line per host that
  * publishes a budget (its budget, what its jobs hold, what its holds keep, and the largest project size that fits it),

package/src/host-budget.mjs

@@ -484,6 +484,9 @@ export function makeHostBudget({ settings, jobDefault = { memMiB: 4096, cpuCenti
 		return Number.isSafeInteger(limit) && limit >= 0 ? limit : null;
 	};
 
+	// The runtime's own CPU count from the last facts read (`docker info` NCPU, `podman info` host.cpus), or null: what a
+	// run record names as the host's CPUs (issue #599), never the worker's own count, which on Docker Desktop is the Mac's.
+	let hostCpus = null;
 	const refresh = async () => {
 		let facts = {};
 		try {
@@ -493,6 +496,7 @@ export function makeHostBudget({ settings, jobDefault = { memMiB: 4096, cpuCenti
 		}
 		const next = computeHostBudget(settings, facts, jobDefault);
 		budget = { memMiB: next.memMiB, cpuCenti: next.cpuCenti };
+		hostCpus = Number.isSafeInteger(facts.hostCpus) && facts.hostCpus >= 1 ? facts.hostCpus : null;
 		detail = next.detail;
 		for (const entry of ledger.values()) {
 			if (entry.guess) Object.assign(entry, pessimisticSize([], entry.guess, budget));
@@ -618,6 +622,8 @@ export function makeHostBudget({ settings, jobDefault = { memMiB: 4096, cpuCenti
 		refresh,
 		/** The budget in force: `{ memMiB, cpuCenti }`, each an integer, `Infinity` (off) or null (unknown). */
 		current: () => ({ ...budget }),
+		/** The runtime's CPU count from the last facts read, or null when no venue answered. */
+		hostCpus: () => hostCpus,
 		detail: () => detail,
 		/** null when `size` can start here some day, else `host` or `share` (`neverFits`). */
 		neverFits: (size, project, limits = scopedLimits()) => neverFits(size, budget, project ? rulesOf(limits).shareOf(project) : null),

package/src/host-registry.mjs

@@ -29,8 +29,14 @@
  * from the reader rather than the writer -- the panel is where an operator screenshots, and doctor
  * prints these rows. A value that must be carried but cannot satisfy the rule is HASHED before it gets
  * here (`scopeKeyPrefix`'s idiom), never abbreviated.
+ *
+ * `jobs` (issue #599, phase 2) is the one field that carries a list: JSON of the jobs this host runs now, ids and
+ * integers only, built and read back by `live-jobs.mjs`, whose header argues each field against this rule. The reader
+ * below parses it through that module's allowlist, so a row's `jobs` is an array (or null), never the raw string.
  */
 
+import { parseJobsMore, parseLiveJobs } from "./live-jobs.mjs";
+
 /** The index. A SET cannot expire its members, so the leak is handled by the reader, as `wait:held` does. */
 export const HOST_SET = "host:live";
 
@@ -270,7 +276,7 @@ export function makeHostRegistry({ redis, name, now = () => Date.now(), ttlMs =
  * one treats them alike: "there are no other hosts" and "I could not find out" differ, and a panel that
  * renders the second as the first tells an operator their fleet is gone when Valkey merely blinked.
  */
-export async function readLiveHosts(redis, { now = () => Date.now(), timeoutMs = REGISTRY_OP_TIMEOUT_MS } = {}) {
+export async function readLiveHosts(redis, { now = () => Date.now(), timeoutMs = REGISTRY_OP_TIMEOUT_MS, prune = true } = {}) {
 	let names;
 	try {
 		names = await bounded(redis.smembers(HOST_SET), timeoutMs);
@@ -285,12 +291,24 @@ export async function readLiveHosts(redis, { now = () => Date.now(), timeoutMs =
 		try {
 			const row = await bounded(redis.hgetall(hostKey(member)), timeoutMs);
 			if (!row || Object.keys(row).length === 0) {
-				await bounded(redis.srem(HOST_SET, member), timeoutMs).catch(() => {});
+				// `prune: false` for a reader that promises to write nothing (the capacity report, issue #599): the stale member
+				// is skipped the same, and left for the next pruning reader.
+				if (prune) await bounded(redis.srem(HOST_SET, member), timeoutMs).catch(() => {});
 				continue;
 			}
 			const beatAt = typeof row.beatAt === "string" && row.beatAt.trim() !== "" ? Number(row.beatAt) : NaN;
+			// The running jobs through the allowlist (issue #599, phase 2): an array, or null where the row publishes none (a
+			// worker from before the field) or none that parses. A listed entry that does not read is a running job not
+			// counted, so it joins `jobsMore` rather than vanishing.
+			const live = parseLiveJobs(row.jobs);
+			const more = parseJobsMore(row.jobsMore);
 			hosts.push({
 				...row,
+				jobs: live.jobs,
+				jobsMore: live.jobs === null ? more : (more ?? 0) + live.dropped,
+				// A row that HAS a `jobs` value that is not a list: how many it runs is unknown, which a reader must say
+				// rather than read as a worker from before the field.
+				jobsUnreadable: live.jobs === null && typeof row.jobs === "string" && row.jobs !== "",
 				name: row.name || member,
 				// Derived rather than stored, so the panel can say "stale 2m" about a row that still lives.
 				// A row whose clock is AHEAD of ours reads as 0 rather than negative: the difference is the