@edgehero/pi-dispatch 4.0.1 → 4.1.0

package/src/run-mirror.mjs

@@ -39,6 +39,158 @@ import { UNREADABLE_RECORD } from "./run-history.mjs";
 /** The index: sanitized jobId -> the run's end (or start) in millis. */
 export const RUNS_INDEX = "runs:index";
 
+/**
+ * The fleet's HISTORY HORIZON (issue #599): a one-member ZSET (`trim`) whose score is the newest instant before which a
+ * writer has trimmed runs out of the index. Every writer trims the SHARED index by its OWN retention (and the shared
+ * count cap), so a host with a short `PI_LOG_RETENTION_DAYS` removes every host's older runs; a reader cannot know
+ * that from its own settings, and without this key it read the gap as idle time. Raised only, never lowered (`ZADD
+ * GT`), so concurrent writers agree on the latest cut without a lock.
+ */
+export const RUNS_HORIZON = "runs:horizon";
+/** The horizon's one member. */
+export const RUNS_HORIZON_MEMBER = "trim";
+
+/**
+ * WHEN THIS MIRROR STARTED (issue #599): a hash, `at` the instant in millis from which the index holds every run that
+ * ended, and `member` a run of the index that vouches for it. Nothing above says it: the reader's other bounds (the
+ * depth, the horizon, the cap, an expired body) all live in keys that a flushed or replaced Valkey, or an index that
+ * expired, loses together with the runs, and the next write then recreates an index that claimed the whole window, so a
+ * peer read every lost run as idle.
+ *
+ * A start is TRUSTED only while its `member` is in the index. The start can outlive its index (a worker of 4.0.1 sets
+ * the index's expiry to its own retention while the start keeps the deepest window; a pruning reader can remove every
+ * member), and an older worker can then recreate the index without knowing this key: a member that is still there is
+ * the one proof that the index the start describes is the index that exists. Rejected: also trusting it while
+ * `runs:horizon` is at or past it, since the horizon outlives a lost index just the same.
+ *   - The write that finds no index starts it NOW (the write's time), vouched for by the run it adds.
+ *   - A write that finds an index whose start it cannot trust (none, or its member is gone) starts it at the index's
+ *     oldest run before this write, vouched for by that run: the most the index can say, so time before it reads as
+ *     missing. A trusted start is kept.
+ *   - Whatever removes the vouching run while the start is trusted (a trim, `PRUNE_SCRIPT`) hands it to the oldest run
+ *     that remains, `at` unchanged, and deletes the start with the last run. So no trim moves `at`, earlier or later.
+ * It expires with the index (one `PEXPIRE` each, in the same script). A reader that finds no trusted start starts the
+ * mirror at the index's oldest run.
+ */
+export const RUNS_SINCE = "runs:since";
+
+/**
+ * Lua shared by the three scripts (KEYS[1] the index, KEYS[3] the start): whether the start is a hash, whether its member
+ * is held, and handing a trusted start on once its member is removed. A start that is not a hash (a string from a
+ * development build, or a hand SET) is read as absent, and the writing scripts delete it first (`sinceClean`), so a
+ * stray value can never make HGET fail with WRONGTYPE and stop the mirror.
+ */
+const SINCE_LUA = `
+local function sinceIsHash()
+	local t = redis.call("TYPE", KEYS[3])
+	return (t.ok or t) == "hash"
+end
+local function sinceClean()
+	if redis.call("EXISTS", KEYS[3]) == 1 and not sinceIsHash() then redis.call("DEL", KEYS[3]) end
+end
+local function sinceHeld()
+	if not sinceIsHash() then return false end
+	local member = redis.call("HGET", KEYS[3], "member")
+	return member and redis.call("ZSCORE", KEYS[1], member) and true or false
+end
+local function sinceHandOn()
+	local first = redis.call("ZRANGE", KEYS[1], 0, 0)
+	if first[1] then redis.call("HSET", KEYS[3], "member", first[1]) else redis.call("DEL", KEYS[3]) end
+end
+`;
+
+/**
+ * The writer's index write and trim, in one script so the member, the start (`RUNS_SINCE`) and the horizon it raises
+ * are atomic (issue #599). KEYS: the index, the horizon and the start; ARGV: the age cutoff in millis, the count cap,
+ * the expiry all three keys get, the horizon's member, the run's score, its id, and the write's time in millis.
+ *
+ * - The start is checked and set as `RUNS_SINCE` says, from the index as it was before this write, and the member is
+ *   added (`ZADD`).
+ * - By age (scores below the cutoff, exclusive) and by count (all but the newest `cap`). A trim that REMOVED something
+ *   raises the horizon with `ZADD GT`: to the cutoff, or after a count trim to the oldest score that remains, which is
+ *   conservative (a removed run ended at or before it). Nothing removed, nothing written: a fleet whose writers trim
+ *   nothing keeps no horizon. A trim that removed the start's member hands the start on.
+ * - The keys expire after the DEEPEST window any reader asks for (`mirrorWindowMs(0)`), never the writer's own. The
+ *   index's members are trimmed by score anyway; a short-retention writer that set the shared index's TTL to its own
+ *   window made the whole index expire on a quiet day, every peer's history with it and no horizon to say so. The
+ *   expiry still rolls with traffic: a fleet that runs nothing for that long has nothing to show.
+ */
+export const TRIM_SCRIPT = `${SINCE_LUA}
+sinceClean()
+local oldest = redis.call("ZRANGE", KEYS[1], 0, 0, "WITHSCORES")
+if not oldest[2] then
+	redis.call("DEL", KEYS[3])
+	redis.call("HSET", KEYS[3], "at", ARGV[7], "member", ARGV[6])
+elseif not (redis.call("HGET", KEYS[3], "at") and sinceHeld()) then
+	redis.call("DEL", KEYS[3])
+	redis.call("HSET", KEYS[3], "at", string.format("%d", tonumber(oldest[2])), "member", oldest[1])
+end
+redis.call("ZADD", KEYS[1], ARGV[5], ARGV[6])
+redis.call("PEXPIRE", KEYS[3], ARGV[3])
+local removed = redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", "(" .. ARGV[1])
+local horizon = nil
+if removed > 0 then horizon = tonumber(ARGV[1]) end
+if redis.call("ZREMRANGEBYRANK", KEYS[1], 0, -tonumber(ARGV[2]) - 1) > 0 then
+	local kept = redis.call("ZRANGE", KEYS[1], 0, 0, "WITHSCORES")
+	if kept[2] then
+		local score = tonumber(kept[2])
+		if horizon == nil or score > horizon then horizon = score end
+	end
+end
+if horizon then
+	redis.call("ZADD", KEYS[2], "GT", string.format("%d", horizon), ARGV[4])
+	redis.call("PEXPIRE", KEYS[2], ARGV[3])
+	if not sinceHeld() then sinceHandOn() end
+end
+redis.call("PEXPIRE", KEYS[1], ARGV[3])
+return horizon and string.format("%d", horizon) or false
+`;
+
+/**
+ * A pruning reader's removal of members whose bodies are gone (`readMirroredRuns`), in one script so the cut is
+ * recorded with it (issue #599). The run a removed member stood for is no longer shown, so the time up to it is no
+ * longer whole: the horizon is raised (`ZADD GT`) to the highest score removed, as `TRIM_SCRIPT` raises it for a trim,
+ * and a trusted start whose member is removed is handed on. Without it, the reader's "expired body" bound was erased by
+ * the very read that saw it, and the time before it read as idle. KEYS: the index, the horizon and the start; ARGV: the
+ * horizon's expiry, its member, then the ids. Returns the highest score removed, or false.
+ */
+export const PRUNE_SCRIPT = `${SINCE_LUA}
+sinceClean()
+-- Handed on only when the start was trusted BEFORE this removal: a start whose run was already gone (an older worker
+-- recreated the index under it) must not be revalidated by handing it to a run of the new index.
+local held = sinceHeld()
+local top = nil
+for i = 3, #ARGV do
+	local score = redis.call("ZSCORE", KEYS[1], ARGV[i])
+	if score then
+		redis.call("ZREM", KEYS[1], ARGV[i])
+		score = tonumber(score)
+		if top == nil or score > top then top = score end
+	end
+end
+if top then
+	redis.call("ZADD", KEYS[2], "GT", string.format("%d", top), ARGV[2])
+	redis.call("PEXPIRE", KEYS[2], ARGV[1])
+	if held and not sinceHeld() then sinceHandOn() end
+end
+return top and string.format("%d", top) or false
+`;
+
+/**
+ * The capacity reader's one snapshot of the index (issue #599): its size, its oldest score, the horizon, the start's
+ * `at` and whether its member is held, and the members that ended after ARGV[2] (exclusive) with their scores, newest
+ * first. One script, so a flush or a write between these reads cannot mix two states into one report. KEYS: the index,
+ * the horizon and the start; ARGV: the horizon's member and the window's start.
+ */
+export const READ_SCRIPT = `${SINCE_LUA}
+local size = redis.call("ZCARD", KEYS[1])
+if size == 0 then return {0} end
+local oldest = redis.call("ZRANGE", KEYS[1], 0, 0, "WITHSCORES")
+local horizon = redis.call("ZSCORE", KEYS[2], ARGV[1])
+local at = sinceIsHash() and redis.call("HGET", KEYS[3], "at") or false
+local range = redis.call("ZREVRANGEBYSCORE", KEYS[1], "+inf", "(" .. ARGV[2], "WITHSCORES")
+return {size, oldest[2] or false, horizon, at, sinceHeld() and 1 or 0, range}
+`;
+
 /** One run's own bytes. */
 export const runRecordKey = (sanitizedJobId) => `runs:rec:${sanitizedJobId}`;
 
@@ -106,17 +258,13 @@ export function makeRunMirror({ redis, retentionDays, now = () => Date.now(), lo
 				const score = Number.isFinite(at) ? at : now();
 				const body = JSON.stringify(record);
 				await bounded(redis.set(runRecordKey(sanitizedJobId), body, "PX", windowMs), timeoutMs);
-				await bounded(redis.zadd(RUNS_INDEX, score, sanitizedJobId), timeoutMs);
-				// Trimmed by the WRITER, twice: by age, and by count. Two `ZREMRANGE`s against a run that
-				// took minutes is free, and it means no reader has to pay for a backlog it did not create.
-				await bounded(redis.zremrangebyscore(RUNS_INDEX, "-inf", `(${now() - windowMs}`), timeoutMs);
-				await bounded(redis.zremrangebyrank(RUNS_INDEX, 0, -indexMax - 1), timeoutMs);
-				// ROLLING expiry, deliberately unlike `budget.mjs`'s set-once rule and deliberately like
-				// `pi-dispatch:sched-stalls:<schedulerId>`. A budget window must not be pushed forward by traffic or a busy
-				// day never resets; an ACTIVITY index should roll with traffic, because that is what it
-				// describes. A fleet that stops running jobs loses its index one window later, which is
-				// correct: there is nothing left to show.
-				await bounded(redis.pexpire(RUNS_INDEX, windowMs), timeoutMs);
+				// Indexed, and trimmed by the WRITER, twice: by age, and by count, with the mirror's start (`RUNS_SINCE`) and
+				// the fleet's horizon written by the same script (`TRIM_SCRIPT`), so no crash or timeout can land between a
+				// member and the record of where the index starts, or between a cut and the record of it. A script, not a
+				// MULTI, because both depend on what the index held: the start on whether it existed, the horizon on what a
+				// trim REMOVED and what remains.
+				const writtenMs = now();
+				await bounded(redis.eval(TRIM_SCRIPT, 3, RUNS_INDEX, RUNS_HORIZON, RUNS_SINCE, writtenMs - windowMs, indexMax, mirrorWindowMs(0), RUNS_HORIZON_MEMBER, score, sanitizedJobId, writtenMs), timeoutMs);
 				warned = false;
 				return true;
 			} catch (err) {
@@ -180,7 +328,8 @@ export async function readMirroredRuns(redis, { limit = 50, sinceMs = 0, now = (
 	}
 	if (stale.length > 0) {
 		try {
-			await bounded(redis.zrem(RUNS_INDEX, ...stale), timeoutMs);
+			// Removed with the cut recorded (`PRUNE_SCRIPT`): the time up to a pruned run is no longer whole.
+			await bounded(redis.eval(PRUNE_SCRIPT, 3, RUNS_INDEX, RUNS_HORIZON, RUNS_SINCE, mirrorWindowMs(0), RUNS_HORIZON_MEMBER, ...stale), timeoutMs);
 		} catch {
 			// best-effort: a straggler in the index costs one skipped row next time, never a wrong one
 		}

package/src/size-suggest.mjs

@@ -42,7 +42,8 @@
  *   A size is rounded UP to a step: 256m up to 2g, 512m up to 8g, then 1g.
  * CPUs: fewer than `SUGGEST_MIN_SAMPLES` runs is not enough runs; the p95 cores used (CPU time over wall time) below
  * 0.4x the current cpus lowers to 1.25x that p95, never below 1.25x the window's largest cores used, nor 0.25;
- * otherwise it fits. A FACT (no call) rides along when the median run was throttled more than 25% of its wall time.
+ * otherwise it fits. A FACT (no call) rides along when the median run was throttled more than 25% of its wall time,
+ * read over the runs that report throttled time and only when at least `SUGGEST_MIN_SAMPLES` of them do.
  * The wall time is the record's pickup-to-end span, which includes the clone, so cores used read slightly LOW.
  *
  * THE CAP. A raise never goes past `cap` (this host's budget per dimension, or where that is off or unknown the host's
@@ -119,9 +120,10 @@ const isCount = (v) => Number.isSafeInteger(v) && v >= 0;
 /**
  * One record as a suggestion reads it, or null when it is not evidence: `{ at, size, memPeak, oom, wallUsec, cpuUsec,
  * throttledUsec, memFullUsec }`. `memPeak` (bytes, clamped to the record's own limit) is null when absent or not a
- * count; `wallUsec` is null unless the span is readable; `cpuUsec` and `throttledUsec` are null unless both can be read
- * with a wall time (CPU time clamped to the CPU ceiling over the wall, throttled time to the wall); `memFullUsec` is
- * null unless it can be read with a wall time.
+ * count; `wallUsec` is null unless the span is readable; `cpuUsec` is null unless it can be read with a wall time (CPU
+ * time clamped to the CPU ceiling over the wall), and `throttledUsec` likewise on its own (clamped to the wall): cgroup v2
+ * writes `throttled_usec` in `cpu.stat` only where the cpu controller is enabled, so a run without it is still a CPU
+ * sample, and only the throttle fact leaves it out. `memFullUsec` is null unless it can be read with a wall time.
  */
 function evidenceOf(record, project, nowMs) {
 	if (record === null || typeof record !== "object" || Array.isArray(record)) return null;
@@ -136,11 +138,10 @@ function evidenceOf(record, project, nowMs) {
 	const start = typeof record.startedAt === "string" ? Date.parse(record.startedAt) : NaN;
 	const wallMs = Number.isFinite(start) ? at - start : NaN;
 	const wallUsec = Number.isSafeInteger(wallMs) && wallMs > 0 && wallMs <= SUGGEST_MAX_WALL_MS ? wallMs * 1000 : null;
-	let cpu = { cpuUsec: null, throttledUsec: null };
-	if (wallUsec !== null && isCount(r.cpuUsec) && isCount(r.throttledUsec)) {
-		const cpuCap = (wallUsec * JOB_CPUS_CEILING_CENTI) / 100;
-		cpu = { cpuUsec: Math.min(r.cpuUsec, cpuCap), throttledUsec: Math.min(r.throttledUsec, wallUsec) };
-	}
+	const cpu = {
+		cpuUsec: wallUsec !== null && isCount(r.cpuUsec) ? Math.min(r.cpuUsec, (wallUsec * JOB_CPUS_CEILING_CENTI) / 100) : null,
+		throttledUsec: wallUsec !== null && isCount(r.cpuUsec) && isCount(r.throttledUsec) ? Math.min(r.throttledUsec, wallUsec) : null,
+	};
 	// unclamped: it is only ever compared (exactly, in BigInt) with 1% of the wall, and a stall past the wall is above it
 	const memFullUsec = wallUsec !== null && isCount(r.memFullUsec) ? r.memFullUsec : null;
 	if (memPeak === null && cpu.cpuUsec === null) return null;
@@ -174,7 +175,8 @@ function suggestMemory(runs, current, cap) {
 	const largestOom = oomSizes.length > 0 ? Math.max(...oomSizes) : null;
 	// at the limit (90% of it or more: peak x 10 >= limit x 9) AND stalled for memory more than 1% of the wall.
 	const pressured = relevant.filter((e) => !productAbove(e.size.memMiB * MIB, 9, e.memPeak, 10) && e.memFullUsec !== null && productAbove(e.memFullUsec, 100, e.wallUsec, 1)).length;
-	const evidence = { samples, p95MiB: p95 === null ? null : Math.ceil(p95 / MIB), maxPeakMiB: maxPeak === null ? null : Math.ceil(maxPeak / MIB), ooms, largestOomMiB: largestOom, pressured };
+	// `smaller`: the measured runs below the current size, which a chart draws and this verdict does not count.
+	const evidence = { samples, smaller: measured.length - samples, p95MiB: p95 === null ? null : Math.ceil(p95 / MIB), maxPeakMiB: maxPeak === null ? null : Math.ceil(maxPeak / MIB), ooms, largestOomMiB: largestOom, pressured };
 	const fact = pressured > 0 ? "pressure" : null;
 	const out = (reason, suggested, more = {}) => ({ current, suggested: suggested === null || suggested === current ? null : suggested, reason, evidence, fact, held: null, wanted: null, ...more });
 	if (ooms > 0) {
@@ -198,12 +200,17 @@ function suggestCpus(runs, current) {
 	const samples = relevant.length;
 	// ordered by the exact fraction, never a float: a/b < c/d  <=>  a x d < c x b.
 	const byFraction = (num) => (x, y) => (productAbove(num(y), x.wallUsec, num(x), y.wallUsec) ? -1 : productAbove(num(x), y.wallUsec, num(y), x.wallUsec) ? 1 : 0);
-	const throttle = samples > 0 ? rank([...relevant].sort(byFraction((e) => e.throttledUsec)), 50) : null;
+	// The throttle fact reads only the runs whose cgroup reported throttled time (see `evidenceOf`), and needs as many of
+	// them as a suggestion needs runs: one throttled run among twenty is not "the median run".
+	const throttled = relevant.filter((e) => e.throttledUsec !== null);
+	const throttle = throttled.length >= SUGGEST_MIN_SAMPLES ? rank([...throttled].sort(byFraction((e) => e.throttledUsec)), 50) : null;
 	const busy = samples > 0 ? rank([...relevant].sort(byFraction((e) => e.cpuUsec)), 95) : null;
 	const busiest = measured.length > 0 ? [...measured].sort(byFraction((e) => e.cpuUsec)).at(-1) : null;
 	const coresOf = (e) => (e === null ? null : Math.ceil((e.cpuUsec * 100) / e.wallUsec));
 	const evidence = {
 		samples,
+		smaller: measured.length - samples,
+		throttledSamples: throttled.length,
 		p95CoresCenti: coresOf(busy),
 		maxCoresCenti: coresOf(busiest),
 		throttledPct: throttle === null ? null : Math.round((throttle.throttledUsec * 100) / throttle.wallUsec),
@@ -211,7 +218,7 @@ function suggestCpus(runs, current) {
 	const out = (reason, suggested, fact = null) => ({ current, suggested: suggested === null || suggested === current ? null : suggested, reason, evidence, fact, held: null, wanted: null });
 	if (samples < SUGGEST_MIN_SAMPLES) return out("not-enough-runs", null);
 	// throttled more than 25% of the wall time (throttled x 4 > wall): a fact about the host's ceiling, never a call.
-	const fact = productAbove(throttle.throttledUsec, 4, throttle.wallUsec, 1) ? "ceiling" : null;
+	const fact = throttle !== null && productAbove(throttle.throttledUsec, 4, throttle.wallUsec, 1) ? "ceiling" : null;
 	// p95 cores below 0.4 x cpus: cpuUsec / wall < 0.4 x current / 100, that is cpuUsec x 1000 < 4 x current x wall.
 	if (productAbove(4 * current, busy.wallUsec, busy.cpuUsec, 1000)) {
 		const lower = Math.max(roundCpusUp(quarterMoreCenti(busy)), roundCpusUp(quarterMoreCenti(busiest)));
@@ -399,6 +406,19 @@ export const coresText = (centi) => `${formatCpus(centi)} core${centi === 100 ?
 /** CPUs in words: `1 CPU`, `0.5 CPUs`, `2 CPUs`. */
 export const cpusText = (centi) => `${formatCpus(centi)} CPU${centi === 100 ? "" : "s"}`;
 
+/**
+ * A "not enough runs" verdict in words for the panel's PROJECTS view and the dispatch_limit_edit preview (the insights
+ * page restates it; doctor says the same counts through `suggestionEvidence`): how many runs at the current size or
+ * larger it counted, and how many smaller ones it did not, which for memory the chart beside it draws:
+ * `not enough runs at 4g (0; 20 at smaller sizes)`. `sizeText` formats the dimension's size.
+ */
+export function notEnoughRunsText(dim, sizeText) {
+	const e = dim?.evidence ?? {};
+	const samples = Number.isSafeInteger(e.samples) ? e.samples : 0;
+	const smaller = Number.isSafeInteger(e.smaller) && e.smaller > 0 ? `; ${e.smaller} at smaller sizes` : "";
+	return `not enough runs at ${sizeText(dim.current)} (${samples}${smaller})`;
+}
+
 /**
  * The evidence of a suggestion in words, for doctor, the panel and the insights page alike: `{ memory, cpu, memoryHeld,
  * memoryFact, cpuFact }`, each a short clause (empty when it does not apply), such as `2 runs ended oom-killed (the
@@ -415,12 +435,12 @@ export function suggestionEvidence(suggestion) {
 	const cores = `p95 ${coresText(ce.p95CoresCenti ?? 0)} used, largest ${coresText(ce.maxCoresCenti ?? 0)}, over ${plural(ce.samples, "run")}`;
 	const memWords = {
 		"oom-killed": `${plural(me.ooms, "run")} ended oom-killed (the largest size killed ${formatMemory(me.largestOomMiB ?? 0)})`,
-		"not-enough-runs": `${me.samples} of the ${SUGGEST_MIN_SAMPLES} runs with measurements it needs`,
+		"not-enough-runs": `${me.samples} of the ${SUGGEST_MIN_SAMPLES} runs with measurements at ${formatMemory(m.current ?? 0)} or larger it needs${me.smaller > 0 ? `, ${me.smaller} more at smaller sizes` : ""}`,
 		oversized: peaks,
 		fits: peaks,
 	};
 	const cpuWords = {
-		"not-enough-runs": `${ce.samples} of the ${SUGGEST_MIN_SAMPLES} runs with measurements it needs`,
+		"not-enough-runs": `${ce.samples} of the ${SUGGEST_MIN_SAMPLES} runs with measurements at ${cpusText(c.current ?? 0)} or more it needs${ce.smaller > 0 ? `, ${ce.smaller} more at smaller sizes` : ""}`,
 		underused: cores,
 		fits: cores,
 	};

package/src/start.mjs

@@ -25,6 +25,7 @@ import { builtinModel, checkModelsKnown } from "./model-catalog.mjs";
 import { capabilityTokens, serializeCaps } from "./capabilities.mjs";
 import { cronFingerprint } from "./fingerprint.mjs";
 import { makeHostRegistry } from "./host-registry.mjs";
+import { liveJobsFields, liveJobsOf } from "./live-jobs.mjs";
 import { budgetField, readUserServiceLimits } from "./host-budget.mjs";
 import { makeCpuReserve, reservePlan } from "./cpu-reserve.mjs";
 import { makeImagePreflight } from "./image-preflight.mjs";
@@ -63,7 +64,7 @@ import { PODMAN_RESTART_HOLD_EXPIRED, makePodmanServiceReader, onceFs, makeRootf
 import { makeRunContainer } from "./run-container.mjs";
 import { resolveProviderCredential } from "./env-allowlist.mjs";
 import { makeSecretsResolver } from "./secrets.mjs";
-import { buildRecord, EXIT_OOM_KILLED, makeFindPreviousRun, makeLogReaper, makeLogSink, makeReadRecord, makeRecordWriter, makeSettledRecord, RUNNER_POLICY_REASONS, sanitizeJobId } from "./run-history.mjs";
+import { buildRecord, EXIT_OOM_KILLED, makeFindPreviousRun, makeLogReaper, makeLogSink, makeReadRecord, makeRecordWriter, makeSettledRecord, newerRecord, RUNNER_POLICY_REASONS, sanitizeJobId, UNREADABLE_RECORD } from "./run-history.mjs";
 import { makeRunMirror, readMirroredRecord } from "./run-mirror.mjs";
 import { readOverlay, resolveSettings } from "./runtime-settings.mjs";
 import { usdFingerprint } from "./dollar-fingerprint.mjs";
@@ -1368,7 +1369,7 @@ export async function startWorker(
 	// would be bytes nothing reads. That is also what keeps a single-host deployment byte-identical, since
 	// no job then issues a single extra Valkey command.
 	const runMirror = config.workerNameDeclared ? makeRunMirrorFn({ redis, retentionDays: config.logRetentionDays, log }) : null;
-	const recordRun = ({ job, result, error, startedAt, endedAt, project, size = null }) => {
+	const recordRun = ({ job, result, error, startedAt, endedAt, project, size = null, capacity = null, earlier = null }) => {
 		// The project (issue #499) was resolved at the pickup gate and rides here as `project` (an id or null), so a live
 		// edit of projects.json mid-run cannot make the record disagree with what the job was counted against. A record
 		// path that ends BEFORE the pickup gate (the wait gate's refusals) passes none, and resolves from the live ref
@@ -1378,7 +1379,7 @@ export async function startWorker(
 		// four `recordRun` call sites byte-unchanged and `buildRecord` a pure function of its arguments.
 		// The default venue rides the same way and for the same reason (#277): it is the value the registry
 		// below is built with, so the record resolves a job's venue exactly as dispatch does.
-		const record = buildRecord({ job, result, error, startedAt, endedAt, host: config.workerName, defaultBackend: config.defaultBackend, project: projectId, size });
+		const record = buildRecord({ job, result, error, startedAt, endedAt, host: config.workerName, defaultBackend: config.defaultBackend, project: projectId, size, capacity, earlier });
 		writeRecord(record);
 		// STRICTLY AFTER the file, and deliberately not awaited. After, because a crash between the two must
 		// leave a record with no fleet row rather than a fleet row with no record -- the mirror is a VIEW,
@@ -1399,10 +1400,23 @@ export async function startWorker(
 	// The record read back, for a job the queue lost the lock of after it finished (DES-TERMINAL-COMMENTS-AND-FAILURE-HOOK,
 	// CONST-RETRY-INFRA-ONLY): this host's own file first, then the fleet's copy where a mirror is armed, because the
 	// host that meets the stalled job need not be the one that ran it. A single host has no mirror and needs none.
+	const readRecord = makeReadRecord({ logsDir: config.logsDir });
 	const settledRecord = makeSettledRecord({
-		readRecord: makeReadRecord({ logsDir: config.logsDir }),
+		readRecord,
 		readMirrored: runMirror ? (jobId) => readMirroredRecordFn(redis, sanitizeJobId(jobId)) : null,
 	});
+	// The record a retry (or a pickup after a stall) is about to replace (issue #599), read BOTH ways: this host's file and
+	// the fleet's copy where a mirror is armed. Both, because each host keeps its own file: a job that ran on A, then B,
+	// then A again finds A's stale first attempt locally while B's second is in the mirror. `newerRecord` keeps the higher
+	// attempt (a tie: the later end). Its slot interval rides the new record as `earlier`. Never rejects: unreadable or
+	// unreachable is no record from that side.
+	const previousRecord = async (jobId) => {
+		const read = readRecord(jobId);
+		const local = read === UNREADABLE_RECORD ? null : read;
+		if (!runMirror) return local;
+		const mirrored = await readMirroredRecordFn(redis, sanitizeJobId(jobId)).catch(() => null);
+		return newerRecord(local, mirrored === UNREADABLE_RECORD ? null : mirrored);
+	};
 
 	// INT-CONFIG-OVERLAY-CONTRACT: the worker reads the runtime-settings overlay at EACH job start, so this
 	// closure -- not a value frozen at boot -- is what the processor calls per job. It resolves the fourteen
@@ -1618,6 +1632,18 @@ export async function startWorker(
 		const snap = worker?.hostBudget?.snapshot?.();
 		return Number.isSafeInteger(snap?.[key]) ? String(snap[key]) : "";
 	};
+	// Issue #599, phase 2: the jobs this host runs now and its budget's orphans, oldest first, as the row's two fields
+	// (`live-jobs.mjs`). Without a budget there is no ledger, so no orphan, and the list is the in-flight map alone.
+	const liveJobsNow = () => {
+		const read = (fn) => {
+			try {
+				return fn() ?? [];
+			} catch {
+				return [];
+			}
+		};
+		return liveJobsFields(liveJobsOf({ running: read(() => worker?.runningJobs?.list?.()), budgetEntries: read(() => worker?.hostBudget?.entries?.()) }));
+	};
 	// NOT awaited, and that is load-bearing rather than an optimisation. `makeRedisClient` sets
 	// `maxRetriesPerRequest: null` -- required for BullMQ's blocking connections -- which means a command
 	// issued against an unreachable server QUEUES FOREVER instead of rejecting. Awaiting the first beat
@@ -1684,6 +1710,14 @@ export async function startWorker(
 		heldMemMiB: () => snapshotField("heldMemMiB"),
 		heldCpuCenti: () => snapshotField("heldCpuCenti"),
 		budgetRunning: () => snapshotField("running"),
+		// Issue #599, phase 2: the jobs waiting for the budget ("" without one), and the jobs this host runs now: `jobs` is
+		// JSON of at most 32 `{ id, p, m, c, at, o? }` (job id, project id or null, size integers, admission millis, 1 for an
+		// orphan), oldest first, `jobsMore` how many are not listed. Ids and integers only (live-jobs.mjs says why that
+		// meets the content rule). Each thunk builds the list again; the beat reads every thunk with no await between, so
+		// the two agree. "" before the worker exists.
+		waiters: () => snapshotField("waiters"),
+		jobs: () => (worker ? liveJobsNow().jobs : ""),
+		jobsMore: () => (worker ? liveJobsNow().jobsMore : ""),
 		budgetHolds: () => snapshotField("holds"),
 		budgetOrphans: () => snapshotField("orphans"),
 		// whether the boot listing of the job containers left from before this worker started has been read
@@ -2059,6 +2093,7 @@ export async function startWorker(
 		redis,
 		recordRun,
 		settledRecord,
+		previousRecord,
 		extraClosers,
 		// REQ-SCOPED-PAUSE-WINDOWS: the processor defers a job whose folder/repo is inside an active window.
 		// Reads the live-reloaded ref, so an operator edit takes effect on the next job without a restart.

package/src/wait-for.mjs

@@ -40,6 +40,13 @@ function configError(message) {
  */
 export const WAIT_CONDITION_KEYS = ["after", "profile"];
 
+/**
+ * Every reason the wait gate refuses a job with (index.mjs `refuseWait`), each before the job holds a slot. A test
+ * holds this list equal to the processor's calls; the capacity report reads it to tell such a refusal from a run on
+ * a record from before `capacity` was recorded (issue #599).
+ */
+export const WAIT_REFUSAL_REASONS = Object.freeze(["wait-superseded", "wait-unreadable", "wait-profile-unknown", "wait-after-beyond-max", "wait-expired", "wait-refused", "wait-unanswerable"]);
+
 /**
  * The ceiling on conditions per trigger. Four, on `SECRETS_MAX`'s reasoning rather than `REPLICAS_MAX`':
  * this multiplies no spend, but every `profile` condition is a subprocess evaluated before the container,

package/src/worker-name.mjs

@@ -0,0 +1,25 @@
+/**
+ * The worker name rule (issue #57), in a module with NO imports, so a pure reader (the capacity report, issue #599) can
+ * hold a record's `host` to it without loading config.mjs and its fs and os. `config.mjs` re-exports the name.
+ */
+
+/**
+ * What a worker may call itself (issue #57). The CHARACTER CLASS is `sanitizeJobId`'s
+ * (`[A-Za-z0-9._-]`), reused rather than invented so this project has one name-safe alphabet -- but that
+ * function is a REPLACER, not a validator, so the three rules around the class are NEW and are claimed
+ * as new here rather than borrowed:
+ *
+ *   - a leading alphanumeric, which is what refuses `..` and a leading `-` that reads as a flag;
+ *   - a 64-character ceiling, because the name is a Valkey key segment and a log field on every line;
+ *   - no `.json`/`.log` tail, which is not decoration. The class contains the dot, so `prod.json` is
+ *     otherwise a legal name -- and a later slice writes a per-host marker file into `PI_LOGS_DIR`,
+ *     where `<something>.json` is parsed as a run record by the admin and DELETED by the log reaper.
+ *     A name is refused here rather than escaped there, because the escape would have to be remembered
+ *     at every site that ever composes a filename from this value.
+ *
+ * The class is `:`-free, `,`-free and `#`-free, which is what lets the name be a Valkey key segment
+ * UNHASHED. That is the point of validating instead of hashing (`scopeKeyPrefix` does the opposite for
+ * a folder path, which was never chosen for key-safety and cannot be refused): the whole value of a host
+ * registry is that `HGETALL host:h:mac-mini-1` is readable by a human.
+ */
+export const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;