@edgehero/pi-dispatch 4.0.1 → 4.1.0

package/src/index.mjs

@@ -1,3 +1,4 @@
+import { availableParallelism } from "node:os";
 import { DelayedError, UnrecoverableError, Worker } from "bullmq";
 import { assertJudgedConnection, onValkeyError } from "./connection.mjs";
 import { jobContainerName } from "./backend-local.mjs";
@@ -7,7 +8,7 @@ import { CANCEL_ACK_TTL_MS, cancelAckKey, cancelReqKey } from "./cancel-state.mj
 import { InfraRetry, NETNS_KEEPER_CRASH_LOOP, NETNS_KEEPER_NOT_HOLDING, TERMINAL_COMMENTS, runJob } from "./processor.mjs";
 import { NETNS_KEEPER_YOUNG_HOLD_MAX_MS, netnsKeeperCrashLoopSentence, netnsKeeperLoopAgainSentence } from "./netns-keeper.mjs";
 import { PODMAN_RESTART_HOLD_EXPIRED, PODMAN_RESTART_HOLD_MAX_MS, PODMAN_RESTART_HOLD_RECHECK_MS } from "./runtime-observations.mjs";
-import { targetFor } from "./run-history.mjs";
+import { earlierFrom, targetFor } from "./run-history.mjs";
 import { isPerMachineHost } from "./backends.mjs";
 import { hash16 } from "./fleet-lease.mjs";
 import { endpointsForModel } from "./model-endpoints.mjs";
@@ -190,7 +191,43 @@ export function effectiveJobOf(data, settings, allowedModels = null, log = () =>
 	};
 }
 
-export function makeProcessor({ cancelJob, stopContainer, containerName = (job) => jobContainerName(job.id), redis, getSettings, hostBudget = null, abortGraceMs = ABORT_GRACE_MS, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], projects = () => [], jobSizeEnv = {}, allocation = null, inFlight = makeInFlight(), hostBound = null, checkLease = null, scopeLease = null, endpointSlots = makeInFlight(), endpointLease = null, modelEndpoints = null, overlayModels = () => null, endpointSetFor = mainModelEndpoints, deps, recordRun = () => {}, settledRecord = null, timeoutMs = JOB_TIMEOUT_MS, cancelPollMs = 2_000, cancelStopBoundMs = CANCEL_STOP_BOUND_MS, hostName = "", multiHost = false, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, checkTimeoutMs = () => 10_000, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
+/**
+ * The jobs this host runs right now (issue #599, phase 2): one entry per PICKUP that holds a slot, `{ id, project, memMiB,
+ * cpuCenti, at }`, `at` its admission instant in millis (the record's `startedAt`). The registry beat publishes it
+ * (`jobs`, live-jobs.mjs), and the capacity report counts each as busy until its record exists.
+ *
+ * It works WITHOUT a host budget, which is why it is not the budget's ledger: the ledger exists only when a budget is
+ * set. Keyed by a per-pickup number, not the job id, so a second pickup of the same id (a stalled job handed back while
+ * the first still runs here) can never remove the first's entry. Process memory, `makeInFlight`'s reason: it counts
+ * this process's own pickups, and a restart that loses it has no pickup left to count.
+ *
+ * An entry is added where the job is admitted (`startedAt`, `capacity`) and removed only by `releaseAllHolds`, the one
+ * release every exit goes through, so it cannot outlive its pickup. An ORPHAN (the container's stop did not take) is
+ * removed like any other exit: the processor is done with it, and only the host budget keeps watching its container,
+ * so the beat flags orphans from the budget's ledger alone. Without a budget nothing watches such a container, and the
+ * list says only what this process still runs.
+ */
+export function makeRunningJobs() {
+	const entries = new Map();
+	let next = 0;
+	return {
+		/** Adds one pickup's entry; returns its key. Total: it never throws, so the admission block may call it. */
+		add(entry) {
+			next += 1;
+			entries.set(next, { ...entry });
+			return next;
+		},
+		/** Removes one pickup's entry; true only the first time. */
+		remove(key) {
+			return entries.delete(key);
+		},
+		/** Copies, never the live map. */
+		list: () => [...entries.values()].map((e) => ({ ...e })),
+		size: () => entries.size,
+	};
+}
+
+export function makeProcessor({ cancelJob, stopContainer, containerName = (job) => jobContainerName(job.id), redis, getSettings, hostBudget = null, abortGraceMs = ABORT_GRACE_MS, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], projects = () => [], jobSizeEnv = {}, allocation = null, inFlight = makeInFlight(), hostBound = null, checkLease = null, scopeLease = null, endpointSlots = makeInFlight(), endpointLease = null, modelEndpoints = null, overlayModels = () => null, endpointSetFor = mainModelEndpoints, deps, recordRun = () => {}, settledRecord = null, previousRecord = null, timeoutMs = JOB_TIMEOUT_MS, cancelPollMs = 2_000, cancelStopBoundMs = CANCEL_STOP_BOUND_MS, hostName = "", multiHost = false, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, checkTimeoutMs = () => 10_000, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random, cpus = availableParallelism, runningJobs = makeRunningJobs() }) {
 	// The lost-lock gate's rejection lines, said ONCE per job id, stall count and attempt. The gate runs before every
 	// deferral (pause window, wait, scope or endpoint busy), and BullMQ never resets a job's stall count, so a stalled
 	// scheduled job whose record is refused meets the gate again on every deferred pickup: a 30-minute scope-busy hold
@@ -233,6 +270,19 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			}
 		}
 
+		// THE EARLIER ATTEMPTS' SLOT TIME (issue #599): a retry, or a pickup after a stall, writes its record over the one
+		// before it (one file and one mirror key per job id), so that record is read NOW, before anything here can replace
+		// it, and its slot interval (with the ones it carried) rides every record this pickup writes (`earlier`). Only on
+		// such a pickup, so a first attempt reads nothing; bounded by the reader, and a fault reads as none.
+		let earlier = null;
+		if (typeof previousRecord === "function" && ((Number.isInteger(job.attemptsMade) && job.attemptsMade > 0) || Number(job.stalledCounter) > 0)) {
+			try {
+				earlier = earlierFrom(await previousRecord(job.id));
+			} catch {
+				earlier = null;
+			}
+		}
+
 		// Scoped pause windows (REQ-SCOPED-PAUSE-WINDOWS): if this job's folder/repo is inside an active pause
 		// window, DEFER it to the window end via BullMQ's delayed set -- the job keeps its identity/dedup and
 		// auto-resumes when re-picked. This is FIRST, before the kill timer, the settings read, and the budget
@@ -302,7 +352,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 					const result = { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, budgetReserved: false, hostBudget: { memMiB: budgetNow.memMiB, cpuCenti: budgetNow.cpuCenti, hostShare: share } };
 					// Above the wait gate, so through the recorder's own arguments rather than the bound one below it: the
 					// same pickup project and size, by hand once.
-					recordRun({ job, result, startedAt: at, endedAt: new Date().toISOString(), project, size });
+					recordRun({ job, result, startedAt: at, endedAt: new Date().toISOString(), project, size, earlier });
 					return result;
 				}
 				deps?.log?.("job_size_never_fits_here_deferred", { jobId: job.id, project, misfit, delayMs: NEVER_FITS_RECHECK_MS, ...sizeFields });
@@ -344,7 +394,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// resolver path or a vault topology -- `secret-profile-unknown` sets that rule.
 				if (sentence && deps?.comment) await Promise.resolve(deps.comment(job.data, sentence)).catch(() => {});
 				const result = { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, budgetReserved: false };
-				recordRun({ job, result, startedAt: at, endedAt: new Date().toISOString() });
+				recordRun({ job, result, startedAt: at, endedAt: new Date().toISOString(), earlier });
 				return result;
 			};
 
@@ -673,6 +723,8 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 		// guard) still frees every local slot before it rethrows; the fleet halves are release-if-mine and awaited where
 		// the caller can. Draining, so a second call releases nothing: the in-process map's release is not idempotent.
 		let budgetHeld = false;
+		// This pickup's entry in the running jobs (issue #599, phase 2), null until it is admitted below.
+		let runningKey = null;
 		let stopDidNotTake = false;
 		let name;
 		let venue;
@@ -692,6 +744,11 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			return Promise.all(taken.map((hold) => hold.fleet?.release?.()));
 		};
 		const releaseAllHolds = ({ orphan = false } = {}) => {
+			// The running-jobs entry first (issue #599, phase 2): this pickup no longer runs the job, orphan or not.
+			if (runningKey !== null) {
+				runningJobs.remove(runningKey);
+				runningKey = null;
+			}
 			const endpointsReleased = releaseEndpointHolds();
 			const scopesReleased = releaseScopeHolds();
 			if (hostHeld) {
@@ -718,11 +775,31 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			hostHeld = true;
 		}
 
+		// The host's capacity as the record carries it (issue #599). Read when the job is admitted, not here: the slot count,
+		// the budget and the runtime's CPU count are live, and the value a run is judged against is the one it started under.
+		const capacityNow = () => {
+			const read = (fn) => {
+				try {
+					return fn();
+				} catch {
+					return null;
+				}
+			};
+			const budget = hostBudget ? read(() => hostBudget.current()) : null;
+			// The CPUs are the RUNTIME's count where its facts were read (the budget's own read), the worker's only when
+			// they were not: on Docker Desktop the worker sees the Mac's cores while every job runs in the VM's.
+			const runtimeCpus = hostBudget ? read(() => hostBudget.hostCpus?.()) : null;
+			return { slots: read(concurrencyNow), memMiB: budget?.memMiB ?? null, cpuCenti: budget?.cpuCenti ?? null, cpus: Number.isSafeInteger(runtimeCpus) && runtimeCpus >= 1 ? runtimeCpus : read(cpus) };
+		};
 		// THE ONE RECORDER BELOW THE GATE, bound once, so the pickup project is a property of the path and not of each call
 		// site: every record from here on goes through it, and none can drop the field and fall back to the live ref in
 		// start.mjs, which would disagree with the pickup value exactly when projects.json was edited mid-run. A bolt in
 		// project-pickup.test.mjs refuses a bare `recordRun(` call below this line.
-		const recordAfterGate = (args) => recordRun({ ...args, project, size });
+		//
+		// `capacity` (issue #599) rides the same binding, and is null until the job is admitted below (where `startedAt` is
+		// set): every record of a refusal before that point says, by its null, that the job never held a slot.
+		let capacity = null;
+		const recordAfterGate = (args) => recordRun({ ...args, project, size, capacity, earlier });
 
 		// The MATCHED ROW's scope keys both the in-process slot and the fleet lease (issue #498), the same string
 		// `budgetCapsFor` hashes below and the boot sweeper hashes from the file: a qualified `github:acme/web` row holds
@@ -984,6 +1061,14 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			// exception, and it is named: the registry's refusal of an unheld venue is caught at the call, and
 			// anything else it throws propagates to the catch below, which releases what was acquired.
 			startedAt = new Date().toISOString();
+			// What this host offered as the job took its slot (issue #599, INT-RUN-HISTORY-FILE-CONTRACT): the live slot
+			// count, the host budget (null without one; `recordedCapacity` writes its `Infinity` as "off") and the CPUs the OS
+			// reports. Each read guarded: this block must not throw (above), and a fact that cannot be read is unknown, never
+			// a reason to fail a job that is about to run.
+			capacity = capacityNow();
+			// This host runs the job from here on (issue #599, phase 2): its entry, at the same instant as `startedAt`, given
+			// back by `releaseAllHolds` on every exit. `add` is total, so this block still cannot throw.
+			runningKey = runningJobs.add({ id: job.id, project, memMiB: size.memMiB, cpuCenti: size.cpuCenti, at: Date.parse(startedAt) });
 			// The producer of the name both boot reapers sweep by substring. Built from the shared prefix
 			// rather than typed here, so a rename cannot land in the producer and not in the sweeps (#227).
 			// From the VENUE that will build the container, not from the local adapter reached for directly:
@@ -1517,7 +1602,7 @@ export function keeperHoldState(job, at) {
 	return { at, since, startedMs };
 }
 
-export function createWorker({ connection, name, stopContainer, containerName, hostQueue = null, checkLease = null, scopeLease = null, checkTimeoutMs, concurrency, getSettings, redis, deps, recordRun, settledRecord = null, limiter, pauseUntil, scopedLimits, projects, jobSizeEnv = {}, allocation = null, inFlight = makeInFlight(), waitState, afterMaxMs, checkSlots = makeInFlight(), checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, hostSlots = makeInFlight(), endpointSlots = makeInFlight(), endpointLease = null, modelEndpoints = null, overlayModels, extraClosers = [], hostBudget: hostBudgetOptions = null }) {
+export function createWorker({ connection, name, stopContainer, containerName, hostQueue = null, checkLease = null, scopeLease = null, checkTimeoutMs, concurrency, getSettings, redis, deps, recordRun, settledRecord = null, previousRecord = null, limiter, pauseUntil, scopedLimits, projects, jobSizeEnv = {}, allocation = null, inFlight = makeInFlight(), waitState, afterMaxMs, checkSlots = makeInFlight(), checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, hostSlots = makeInFlight(), endpointSlots = makeInFlight(), endpointLease = null, modelEndpoints = null, overlayModels, extraClosers = [], hostBudget: hostBudgetOptions = null, runningJobs = makeRunningJobs() }) {
 	// One Worker per queue name (issue #57). A host-affine job -- one whose folder, secret resolver or wait
 	// check lives on THIS machine -- is enqueued to `pi-jobs@<name>` rather than filtered for at pickup,
 	// because BullMQ has no selective pop and the put-it-back alternative does not work: promotion out of
@@ -1626,6 +1711,10 @@ export function createWorker({ connection, name, stopContainer, containerName, h
 			// The run-record lookup a stalled job is checked against before it can run again (see the processor's
 			// first gate). `null` in a bare wiring, which keeps today's behaviour: the job runs.
 			settledRecord,
+			previousRecord,
+			// Issue #599, phase 2: SHARED across both workers for the host slot's reason: the registry row lists what this
+			// HOST runs, and two maps would each list half.
+			runningJobs,
 		});
 
 		// Issue #464: only a connection `parseConnection` built, which judges and pins the Valkey it dials.
@@ -1652,6 +1741,8 @@ export function createWorker({ connection, name, stopContainer, containerName, h
 	primary.hostWorker = workers[1] ?? null;
 	// Issue #596, phase 2: the host budget, for the registry beat's thunks (start.mjs) and doctor-facing snapshots.
 	primary.hostBudget = hostBudget;
+	// Issue #599, phase 2: the jobs this host runs now, for the registry beat's `jobs` thunk (start.mjs).
+	primary.runningJobs = runningJobs;
 
 	const stop = async () => {
 		// Abort active jobs (=> docker stop via onAbort), then close. Without the cancel,

package/src/job-size.mjs

@@ -54,6 +54,16 @@ export const JOB_MEMORY_CEILING_MIB = 1024 * 1024;
 export const JOB_CPUS_FLOOR_CENTI = 25;
 /** The largest CPU size: 256, as hundredths, because round(256 x 1024) is 262144, the top of `--cpu-shares`. */
 export const JOB_CPUS_CEILING_CENTI = 256 * 100;
+/**
+ * The most a host can have, as a run record's `capacity` may say it (issue #599, phase 4's review): a value past these is
+ * not a host but a fault or a forgery, so the writer records it as unknown and the capacity report counts a record that
+ * says it as unreadable, rather than judging a week against "1 of 1000000000 slots". The CPUs and the memory are the
+ * ceilings the runtime facts are read to (daemon-facts.mjs `cpuCount`, `memoryMiB`); the slots are as many of the
+ * smallest jobs (`JOB_CPUS_FLOOR_CENTI`) as those CPUs hold.
+ */
+export const HOST_CPUS_MAX = 4096;
+export const HOST_MEMORY_MAX_MIB = 64 * 1024 * 1024;
+export const HOST_SLOTS_MAX = (HOST_CPUS_MAX * 100) / JOB_CPUS_FLOOR_CENTI;
 /** The valid range of `--cpu-shares` on cgroup v2 (runc and crun clamp to it; 2 is the kernel's cgroup v1 minimum). */
 export const CPU_SHARES_MIN = 2;
 export const CPU_SHARES_MAX = 262144;
@@ -119,6 +129,12 @@ export function parseCpus(value) {
 	return centi;
 }
 
+/**
+ * The two never-fits refusals (index.mjs `SIZE_REFUSAL_COMMENTS`, issue #596), both decided before the job holds a slot.
+ * Here, beside the size they judge, so a leaf can read them; a test holds them equal to the processor's.
+ */
+export const SIZE_REFUSAL_REASONS = Object.freeze(["job-size-exceeds-host", "job-size-exceeds-share"]);
+
 /**
  * The one spelling of a memory size: `<n>g` when it is whole gigabytes, else `<n>m`. Every `--memory`, `--memory-swap`
  * and stored row is written by this function, and `container-spec.mjs`'s `memoryBytes` reads every value it can write

package/src/live-jobs.mjs

@@ -0,0 +1,135 @@
+/**
+ * The running jobs a host row publishes (issue #599, phase 2, INT-HOST-REGISTRY-CONTRACT `jobs`), written by the
+ * worker's beat and read back by every reader of the row: one module for both halves, so the writer can never publish
+ * a shape the reader drops, and pure (no clock, no filesystem, no Valkey), so the capacity report and the admin bundle
+ * can import it.
+ *
+ * WHAT IS LISTED. Every job this host's processors hold a slot for right now (the in-flight map, `makeRunningJobs` in
+ * index.mjs), and every ORPHAN of the host budget (a job whose container's stop did not take, or a container left
+ * from before the worker started), flagged `o: 1`. An orphan comes from the budget's ledger only, because only the
+ * budget keeps watching such a container until the runtime says it is gone; without a budget nothing does, and the
+ * job leaves the list when its processor ends. A job listed by both is listed once, as running.
+ *
+ * WHY THIS MEETS THE CONTENT RULE (names, integers and digests). Every field is an integer, the fixed 1, or an id:
+ *   - `id` is the job id, the one every run record already carries in `jobId` and `runs:rec:<id>` already puts in
+ *     Valkey. Each shape the project mints is charset-checked by construction: `gh-`, `gl-`, `fj-` and the other forge
+ *     prefixes with the forge's delivery id (and `-r<n>` for a replica), `repeat:<trigger id>:<millis>` and
+ *     `manual:<trigger id>:<millis>` (a trigger id is `[A-Za-z0-9._-]+`, the loader's rule), `local-<hex>`,
+ *     `chain-<hex>`, and a budget orphan's `container:<name>`. A delivery id is a forge's header value and is not
+ *     checked by the receiver, so the id is held to `LIVE_JOB_ID_RE` HERE, and one outside it (or longer than 128
+ *     characters) is published as its digest (`sha256:<16 hex>`), the rule's own idiom for a value that must be
+ *     carried and cannot satisfy it. No quote, backslash, slash or control character can reach the JSON, so the
+ *     writer's path-shape check never refuses the field.
+ *   - `p` is a project id (`isProjectId`) or null; `runs:rec:*` already carries project ids.
+ *   - `m`, `c` and `at` are integers (MiB, hundredths of a CPU, epoch millis), `o` is 1.
+ *
+ * ORDER AND SIZE. Oldest first (by `at`, then id), at most `LIVE_JOBS_MAX`: the jobs that have held a slot longest
+ * carry the most busy time and are the ones an operator looks for. The rest are counted in `jobsMore`, never dropped
+ * silently. A full list is about 7 KiB, once per beat.
+ */
+
+import { createHash } from "node:crypto";
+import { isProjectId } from "./project-id.mjs";
+
+/** The most jobs one row lists; the rest are counted (`jobsMore`). */
+export const LIVE_JOBS_MAX = 32;
+/** A job id as published: the charset every id shape the project mints is in, at most 128 characters. */
+export const LIVE_JOB_ID_RE = /^[A-Za-z0-9._:-]{1,128}$/;
+/** The most a `jobs` value may be before a reader declines to parse it: 32 entries at their longest fit well inside. */
+export const LIVE_JOBS_MAX_BYTES = 16 * 1024;
+
+const positiveInt = (v) => Number.isSafeInteger(v) && v > 0;
+
+/** A job id as it may be published: itself when inside `LIVE_JOB_ID_RE`, else its digest; null for no id. */
+export function publishedJobId(id) {
+	if (typeof id !== "string" || id === "") return null;
+	if (LIVE_JOB_ID_RE.test(id)) return id;
+	return `sha256:${createHash("sha256").update(id).digest("hex").slice(0, 16)}`;
+}
+
+/**
+ * The list a row publishes, before the cap: `[{ id, p, m, c, at, o? }]`, oldest first. `running` is the in-flight
+ * map's entries (`{ id, project, memMiB, cpuCenti, at }`), `budgetEntries` the host budget's ledger (`entries()`), of
+ * which only orphans are taken. An entry with no id or no instant is left out and counted in `skipped`.
+ */
+export function liveJobsOf({ running = [], budgetEntries = [] } = {}) {
+	const out = [];
+	const seen = new Set();
+	let skipped = 0;
+	const push = (e, orphan) => {
+		const id = publishedJobId(e?.id);
+		if (id === null || !Number.isSafeInteger(e?.at) || e.at <= 0) {
+			skipped++;
+			return;
+		}
+		if (seen.has(id)) return; // one job, two pickups (or running and orphaned): listed once
+		seen.add(id);
+		out.push({ id, p: isProjectId(e.project) ? e.project : null, ...(positiveInt(e.memMiB) ? { m: e.memMiB } : {}), ...(positiveInt(e.cpuCenti) ? { c: e.cpuCenti } : {}), at: e.at, ...(orphan ? { o: 1 } : {}) });
+	};
+	// Running first, so a job both running and in the ledger as an orphan (it cannot be: the gate defers it) reads as running.
+	for (const e of Array.isArray(running) ? running : []) push(e, false);
+	for (const e of Array.isArray(budgetEntries) ? budgetEntries : []) if (e?.orphan) push(e, true);
+	out.sort((a, b) => a.at - b.at || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+	return { jobs: out, skipped };
+}
+
+/**
+ * The two row fields: `jobs`, the JSON of at most `LIVE_JOBS_MAX` entries, and `jobsMore`, how many running jobs are
+ * not in it (past the cap, or with no id or instant), both strings as a row holds them.
+ */
+export function liveJobsFields(list) {
+	const { jobs, skipped } = list;
+	const kept = jobs.slice(0, LIVE_JOBS_MAX);
+	return { jobs: JSON.stringify(kept), jobsMore: String(jobs.length - kept.length + skipped) };
+}
+
+/**
+ * A row's `jobs` read back, through a per-field allowlist: `{ jobs, dropped }`, `jobs` an array of `{ id, p, m, c, at,
+ * o }` (`m`, `c` null when not published, `o` true for an orphan), or null when the field is absent, empty, too long
+ * or not a JSON list. An entry that is not an object, or whose id, project, size, instant or flag is not the shape
+ * above, is dropped and counted; past `LIVE_JOBS_MAX` entries the rest are counted too. A list this function already
+ * read (an array, its `o` a boolean) reads back the same, so a caller may hand either form. Never throws: a row is
+ * another host's text.
+ */
+export function parseLiveJobs(raw) {
+	if (Array.isArray(raw)) return parseList(raw);
+	if (typeof raw !== "string" || raw === "" || raw.length > LIVE_JOBS_MAX_BYTES) return { jobs: null, dropped: 0 };
+	let parsed;
+	try {
+		parsed = JSON.parse(raw);
+	} catch {
+		return { jobs: null, dropped: 0 };
+	}
+	return Array.isArray(parsed) ? parseList(parsed) : { jobs: null, dropped: 0 };
+}
+
+const optionalSize = (v) => (v === undefined || v === null ? null : positiveInt(v) ? v : undefined);
+
+function parseList(list) {
+	const jobs = [];
+	let dropped = 0;
+	for (const e of list) {
+		if (jobs.length >= LIVE_JOBS_MAX) {
+			dropped++;
+			continue;
+		}
+		const ok = e !== null && typeof e === "object" && !Array.isArray(e);
+		const m = ok ? optionalSize(e.m) : undefined;
+		const c = ok ? optionalSize(e.c) : undefined;
+		if (!ok || typeof e.id !== "string" || !LIVE_JOB_ID_RE.test(e.id) || !(e.p === undefined || e.p === null || isProjectId(e.p)) || m === undefined || c === undefined || !positiveInt(e.at) || !(e.o === undefined || e.o === 1 || typeof e.o === "boolean")) {
+			dropped++;
+			continue;
+		}
+		jobs.push({ id: e.id, p: e.p ?? null, m, c, at: e.at, o: e.o === 1 || e.o === true });
+	}
+	return { jobs, dropped };
+}
+
+/**
+ * A row's `jobsMore` read back: a non-negative integer, or null when absent or not one. A reader adds `dropped` from
+ * `parseLiveJobs` to it: a listed job it could not read is a running job it does not count.
+ */
+export function parseJobsMore(raw) {
+	if (Number.isSafeInteger(raw) && raw >= 0) return raw;
+	return typeof raw === "string" && /^\d{1,9}$/.test(raw) ? Number(raw) : null;
+}

package/src/prepare.mjs

@@ -15,6 +15,7 @@ import { buildAzurePrompt } from "./azure-prompt.mjs";
 import { prepareLocalWorkspace } from "./prepare-local.mjs";
 import { copySkillTree } from "./copy-tree.mjs";
 import { recordedJobSize } from "./job-size.mjs";
+import { scheduledForMillis } from "./repeat-slot.mjs";
 
 /**
  * The subdirectory of the per-job dir a trigger's injected skills are copied into, so they reach the
@@ -214,15 +215,6 @@ function localEventContext(job, queueJobId, findPreviousRun) {
 	return { source: "manual" };
 }
 
-/** Parse the millis out of a `repeat:<id>:<millis>` BullMQ scheduled jobId, or null. */
-function scheduledForMillis(queueJobId) {
-	if (typeof queueJobId !== "string" || !queueJobId.startsWith("repeat:")) return null;
-	const tail = queueJobId.slice(queueJobId.lastIndexOf(":") + 1);
-	if (tail === "") return null;
-	const millis = Number(tail);
-	return Number.isFinite(millis) ? millis : null;
-}
-
 /**
  * Attach the sandbox stamp to a successful prepare, and to nothing else.
  *

package/src/repeat-slot.mjs

@@ -0,0 +1,13 @@
+/**
+ * The scheduled-for instant of a BullMQ job scheduler's job, read from the deterministic `repeat:<id>:<millis>` id it
+ * mints (DES-CRON-VIA-BULLMQ-SCHEDULER). A leaf module, so the run record and the prepare step read one parser.
+ */
+
+/** Parse the millis out of a `repeat:<id>:<millis>` BullMQ scheduled jobId, or null. */
+export function scheduledForMillis(queueJobId) {
+	if (typeof queueJobId !== "string" || !queueJobId.startsWith("repeat:")) return null;
+	const tail = queueJobId.slice(queueJobId.lastIndexOf(":") + 1);
+	if (tail === "") return null;
+	const millis = Number(tail);
+	return Number.isFinite(millis) ? millis : null;
+}

package/src/run-earlier.mjs

@@ -0,0 +1,29 @@
+/**
+ * The rule for a run record's `earlier` (issue #599, INT-RUN-HISTORY-FILE-CONTRACT), in a module that imports only the
+ * worker name rule, so the record writer (run-history.mjs, which re-exports both names) and the capacity report, a pure
+ * reader, rebuild it with ONE validator.
+ */
+
+import { WORKER_NAME_RE } from "./worker-name.mjs";
+
+const recordInt = (v) => (Number.isSafeInteger(v) && v >= 0 ? v : null);
+
+/** The most earlier attempts a record carries: the newest are kept. */
+export const EARLIER_MAX = 4;
+
+/** One earlier attempt as a record carries it, or null: a worker name, two canonical ISO instants in order, two sizes. */
+function earlierEntry(e) {
+	if (e === null || typeof e !== "object" || Array.isArray(e)) return null;
+	const instant = (v) => (typeof v === "string" && Number.isFinite(Date.parse(v)) && new Date(Date.parse(v)).toISOString() === v ? v : null);
+	const startedAt = instant(e.startedAt);
+	const endedAt = instant(e.endedAt);
+	if (typeof e.host !== "string" || !WORKER_NAME_RE.test(e.host) || startedAt === null || endedAt === null || Date.parse(endedAt) < Date.parse(startedAt)) return null;
+	return { host: e.host, startedAt, endedAt, memMiB: recordInt(e.memMiB), cpuCenti: recordInt(e.cpuCenti) };
+}
+
+/** A record's `earlier`, rebuilt: the valid entries, the newest `EARLIER_MAX` of them, or null when none is. */
+export function recordedEarlier(list) {
+	if (!Array.isArray(list)) return null;
+	const kept = list.map(earlierEntry).filter((e) => e !== null).slice(-EARLIER_MAX);
+	return kept.length > 0 ? kept : null;
+}

package/src/run-history.mjs

@@ -6,7 +6,12 @@ import { resolveBackendName } from "./backend-registry.mjs";
 import { isForgeKind, targetSeparator } from "./forges.mjs";
 import { MODEL_REF_PATTERN as USAGE_ID_PATTERN } from "./model-ref.mjs";
 import { isProjectId } from "./project-id.mjs";
-import { recordedJobSize } from "./job-size.mjs";
+import { HOST_CPUS_MAX, HOST_MEMORY_MAX_MIB, HOST_SLOTS_MAX, recordedJobSize } from "./job-size.mjs";
+import { waitArmed } from "./wait-for.mjs";
+import { scheduledForMillis } from "./repeat-slot.mjs";
+import { EARLIER_MAX, recordedEarlier } from "./run-earlier.mjs";
+
+export { EARLIER_MAX, recordedEarlier };
 
 /**
  * Durable per-run history.
@@ -569,7 +574,7 @@ function rebuildUsage(u) {
  * default to `null` when the outcome does not carry them, so the record shape is stable whether or not
  * the source reports those fields.
  */
-export function buildRecord({ job, result, error, startedAt, endedAt, host = null, defaultBackend = null, project = null, size = null }) {
+export function buildRecord({ job, result, error, startedAt, endedAt, host = null, defaultBackend = null, project = null, size = null, capacity = null, earlier = null }) {
 	const data = job.data ?? {};
 	const kind = data.kind ?? job.name;
 	const source = result ?? error ?? {};
@@ -728,7 +733,114 @@ export function buildRecord({ job, result, error, startedAt, endedAt, host = nul
 		// refused: an off or unknown dimension never refuses by itself); `hostShare` an integer or null. Present only on
 		// the `job-size-exceeds-host` and `-share` records; null on every other.
 		hostBudget: recordedHostBudget(source.hostBudget),
+		// When the job became ELIGIBLE to run (issue #599, INT-RUN-HISTORY-FILE-CONTRACT), so `startedAt - queuedAt` is how
+		// long it waited for a slot. Additive, nullable, an explicit literal, TAIL position after `hostBudget` on the same
+		// contract. `job.timestamp + opts.delay`, not `job.timestamp` alone: BullMQ's job scheduler creates the next cron
+		// job when the current one runs, stamped with that moment and delayed until it is due (job-scheduler.js
+		// `getNextJobOpts`), so the timestamp alone made a daily trigger read as a day of waiting. `opts.delay` is the
+		// delay the job was ADDED with and survives a `moveToDelayed` (a pause, a deferral) and a retry, while `job.delay`
+		// is rewritten by both (measured against bullmq 5.80.4 on Valkey, worker/test/queued-at.integration.test.mjs), so
+		// a pause window and a deferral count as waiting, which they are. Null for a job held on `run.waitFor` (a wait its
+		// trigger asked for is not a wait for capacity) and for a retry (its wait would include the earlier attempt). An
+		// ISO string or null, a number's rendering, so PII-free.
+		queuedAt: queuedAtOf(job),
+		// What this host offered when the job took its slot (issue #599, INT-RUN-HISTORY-FILE-CONTRACT): `{ slots, memMiB,
+		// cpuCenti, cpus }`, the live PI_CONCURRENCY, the host budget as `hostBudget` above maps it, and the CPUs the OS
+		// reports. Additive, nullable, an explicit literal REBUILT here (`recordedCapacity`), TAIL position after `queuedAt`.
+		// Passed in by the processor only once the job is ADMITTED (index.mjs, where `startedAt` is set), so it is also
+		// the record's own answer to "did this run hold a slot": null on every refusal before one, which is what
+		// lets a capacity report tell a run from a refusal without guessing from the wall time (capacity.mjs).
+		capacity: recordedCapacity(capacity),
+		// The slot time of this job's EARLIER attempts (issue #599, INT-RUN-HISTORY-FILE-CONTRACT), which a retry's record
+		// would otherwise erase: it overwrites the same file and the same mirror key. `[{ host, startedAt, endedAt, memMiB,
+		// cpuCenti }]`, at most `EARLIER_MAX`, newest last, read by the processor from the previous record BEFORE this one
+		// replaces it (`earlierFrom`) and passed in. Additive, nullable, an explicit literal REBUILT here, TAIL position after
+		// `capacity`. A worker name, two ISO instants and two integers per entry: never the earlier attempt's dollars,
+		// tokens or usage, so every cost reader, which reads the top-level fields only, counts nothing twice. Null on a
+		// first attempt and when the previous record held no slot or could not be read.
+		earlier: recordedEarlier(earlier),
+		// Whether the FIRST attempt stalled and this pickup re-ran it (issue #599): BullMQ raised `stalledCounter`, not
+		// `attemptsMade`, so that pickup wrote no record and its slot time is in none. A stall after a failed attempt is a
+		// retry like any other (its earlier attempt's record is carried in `earlier`), so it is false there. A boolean,
+		// TAIL position after `earlier`, so the capacity report can say how many runs that is.
+		stalledRepick: Number.isInteger(job.stalledCounter) && job.stalledCounter > 0 && !(Number.isInteger(job.attemptsMade) && job.attemptsMade > 0),
+	};
+}
+
+/**
+ * Of two copies of one job's record (this host's file and the fleet's copy), the one a retry replaces: the HIGHER
+ * `attempt` (a retry on another host writes its own copy there, so either side can be the newer attempt), an exact tie
+ * broken by the later `endedAt` (run-mirror.mjs `mergeRuns`' rule), a remaining tie by the first given (the local file).
+ * Either may be null; a non-object is none.
+ */
+export function newerRecord(a, b) {
+	const ok = (r) => r !== null && typeof r === "object" && !Array.isArray(r);
+	if (!ok(a)) return ok(b) ? b : null;
+	if (!ok(b)) return a;
+	const attempt = (r) => (Number.isInteger(r.attempt) ? r.attempt : 0);
+	if (attempt(a) !== attempt(b)) return attempt(a) > attempt(b) ? a : b;
+	const end = (r) => {
+		const t = Date.parse(r.endedAt ?? r.startedAt ?? "");
+		return Number.isFinite(t) ? t : -Infinity;
+	};
+	return end(b) > end(a) ? b : a;
+}
+
+/**
+ * What a retry's record carries forward from the record it is about to replace: that record's own `earlier`, then its
+ * own slot interval when it held a slot (`capacity` an object), each rebuilt; null when there is nothing.
+ */
+export function earlierFrom(previous) {
+	if (previous === null || typeof previous !== "object" || Array.isArray(previous)) return null;
+	const carried = Array.isArray(previous.earlier) ? previous.earlier : [];
+	const own = previous.capacity !== null && typeof previous.capacity === "object" && !Array.isArray(previous.capacity) ? [{ host: previous.host, startedAt: previous.startedAt, endedAt: previous.endedAt, memMiB: previous.size?.memMiB, cpuCenti: previous.size?.cpuCenti }] : [];
+	return recordedEarlier([...carried, ...own]);
+}
+
+/**
+ * When a job became eligible to run, as an ISO string, or null: `job.timestamp` plus the delay it was added with
+ * (`opts.delay`, at least 0). Null for a job armed with `run.waitFor` (its wait is its trigger's own), for a retry
+ * (`attemptsMade` above 0) or a pickup after a stall (`stalledCounter` above 0), and for any input that is not a
+ * finite, representable instant.
+ */
+export function queuedAtOf(job) {
+	if (waitArmed(job?.data)) return null;
+	// A RETRY's wait would include its earlier attempt and that attempt's run (the add's moment is kept across attempts),
+	// so only a first attempt says how long the job waited for a slot. `attemptsMade` counts attempts finished before.
+	if (Number.isInteger(job?.attemptsMade) && job.attemptsMade > 0) return null;
+	// A pickup after a STALL is the same: BullMQ raises `stalledCounter`, not `attemptsMade`, and the add's moment is kept.
+	if (Number.isInteger(job?.stalledCounter) && job.stalledCounter > 0) return null;
+	// A job scheduler's job carries its exact slot in its id. BullMQ stamps `timestamp` and computes `delay` from two
+	// separate clock reads, so their sum can land a few milliseconds past the slot (measured in CI).
+	const slot = scheduledForMillis(job?.id);
+	if (Number.isSafeInteger(slot) && slot >= 0 && slot <= 8.64e15) return new Date(slot).toISOString();
+	const timestamp = job?.timestamp;
+	const delay = job?.opts?.delay ?? 0;
+	if (!Number.isSafeInteger(timestamp) || timestamp < 0 || typeof delay !== "number" || !Number.isFinite(delay)) return null;
+	const at = timestamp + Math.max(0, Math.trunc(delay));
+	// 8.64e15 is the last instant a Date can hold; past it `toISOString` throws, and this function must not.
+	return at <= 8.64e15 ? new Date(at).toISOString() : null;
+}
+
+/** A record's non-negative safe integer, else null. */
+const recordInt = (v) => (Number.isSafeInteger(v) && v >= 0 ? v : null);
+/** A record's budget dimension: an integer, `"off"` (the budget's `Infinity`, or the word itself), else null. */
+const recordBudget = (v) => (v === Infinity || v === "off" ? "off" : recordInt(v));
+
+/**
+ * A run's capacity as a record carries it: `{ slots, memMiB, cpuCenti, cpus }`, else null. `slots` and `cpus` are
+ * positive integers or null (unknown); each budget dimension is `recordedHostBudget`'s mapping, so `"off"` and null
+ * keep meaning "not limited" and "not known".
+ */
+export function recordedCapacity(value) {
+	if (value === null || typeof value !== "object" || Array.isArray(value)) return null;
+	// Each held to what a host can have (job-size.mjs `HOST_*_MAX`): past it a value is a fault, recorded as unknown.
+	const upTo = (max, read) => (v) => {
+		const n = read(v);
+		return typeof n === "number" && n > max ? null : n;
 	};
+	const positive = (v) => (Number.isSafeInteger(v) && v >= 1 ? v : null);
+	return { slots: upTo(HOST_SLOTS_MAX, positive)(value.slots), memMiB: upTo(HOST_MEMORY_MAX_MIB, recordBudget)(value.memMiB), cpuCenti: upTo(HOST_CPUS_MAX * 100, recordBudget)(value.cpuCenti), cpus: upTo(HOST_CPUS_MAX, positive)(value.cpus) };
 }
 
 /**
@@ -739,9 +851,7 @@ export function buildRecord({ job, result, error, startedAt, endedAt, host = nul
  */
 export function recordedHostBudget(value) {
 	if (value === null || typeof value !== "object" || Array.isArray(value)) return null;
-	const int = (v) => (Number.isSafeInteger(v) && v >= 0 ? v : null);
-	const budget = (v) => (v === Infinity || v === "off" ? "off" : int(v));
-	return { memMiB: budget(value.memMiB), cpuCenti: budget(value.cpuCenti), hostShare: int(value.hostShare) };
+	return { memMiB: recordBudget(value.memMiB), cpuCenti: recordBudget(value.cpuCenti), hostShare: recordInt(value.hostShare) };
 }
 
 /** What became of a collected plan. */