@edgehero/pi-dispatch 4.0.1 → 4.2.0

package/src/run-history.mjs

@@ -6,7 +6,12 @@ import { resolveBackendName } from "./backend-registry.mjs";
 import { isForgeKind, targetSeparator } from "./forges.mjs";
 import { MODEL_REF_PATTERN as USAGE_ID_PATTERN } from "./model-ref.mjs";
 import { isProjectId } from "./project-id.mjs";
-import { recordedJobSize } from "./job-size.mjs";
+import { HOST_CPUS_MAX, HOST_MEMORY_MAX_MIB, HOST_SLOTS_MAX, recordedJobSize } from "./job-size.mjs";
+import { waitArmed } from "./wait-for.mjs";
+import { scheduledForMillis } from "./repeat-slot.mjs";
+import { EARLIER_MAX, recordedEarlier } from "./run-earlier.mjs";
+
+export { EARLIER_MAX, recordedEarlier };
 
 /**
  * Durable per-run history.
@@ -159,8 +164,15 @@ export function parseExitTurns(text) {
  * it could not enforce before a call. The two refusals are live from the first image that knows the variables.
  * `cost-cap` is live from the image that declares `costCap`, whose runner carries the cost guard, and
  * `model-not-allowed` from the image that declares `modelPolicy`, whose runner carries the model guard.
+ *
+ * Then the runner's exit-2 refusals of a job's MCP servers (`MCP_RUNNER_REASONS`), each before the prompt. They read as
+ * `runner-policy` until they were named here, which hid an operator's fix (a server that answers 401, a command the
+ * image lacks) behind budget wording. `mcp-config-invalid` is also the pickup's FREE refusal of an overlay mcp.json it
+ * cannot read whole; that one returns `budgetReserved: false` and pages nobody, as the free `model-not-allowed` does.
+ * Live from the first image that declares `mcp`: an image built before the worker named them prints the same line.
  */
-export const RUNNER_POLICY_REASONS = new Set(["provider-auth-refused", "cost-cap", "model-not-allowed", "cost-cap-unenforceable", "model-policy-unenforceable"]);
+export const MCP_RUNNER_REASONS = Object.freeze(["mcp-unavailable", "mcp-config-invalid", "mcp-config-mismatch", "mcp-secret-missing", "mcp-command-missing", "mcp-shadowed"]);
+export const RUNNER_POLICY_REASONS = new Set(["provider-auth-refused", "cost-cap", "model-not-allowed", "cost-cap-unenforceable", "model-policy-unenforceable", ...MCP_RUNNER_REASONS]);
 
 /**
  * The reason off the decisive exit line (`decisiveExitLine`), or `null`: a member of `RUNNER_POLICY_REASONS`
@@ -569,7 +581,7 @@ function rebuildUsage(u) {
  * default to `null` when the outcome does not carry them, so the record shape is stable whether or not
  * the source reports those fields.
  */
-export function buildRecord({ job, result, error, startedAt, endedAt, host = null, defaultBackend = null, project = null, size = null }) {
+export function buildRecord({ job, result, error, startedAt, endedAt, host = null, defaultBackend = null, project = null, size = null, capacity = null, earlier = null }) {
 	const data = job.data ?? {};
 	const kind = data.kind ?? job.name;
 	const source = result ?? error ?? {};
@@ -728,7 +740,114 @@ export function buildRecord({ job, result, error, startedAt, endedAt, host = nul
 		// refused: an off or unknown dimension never refuses by itself); `hostShare` an integer or null. Present only on
 		// the `job-size-exceeds-host` and `-share` records; null on every other.
 		hostBudget: recordedHostBudget(source.hostBudget),
+		// When the job became ELIGIBLE to run (issue #599, INT-RUN-HISTORY-FILE-CONTRACT), so `startedAt - queuedAt` is how
+		// long it waited for a slot. Additive, nullable, an explicit literal, TAIL position after `hostBudget` on the same
+		// contract. `job.timestamp + opts.delay`, not `job.timestamp` alone: BullMQ's job scheduler creates the next cron
+		// job when the current one runs, stamped with that moment and delayed until it is due (job-scheduler.js
+		// `getNextJobOpts`), so the timestamp alone made a daily trigger read as a day of waiting. `opts.delay` is the
+		// delay the job was ADDED with and survives a `moveToDelayed` (a pause, a deferral) and a retry, while `job.delay`
+		// is rewritten by both (measured against bullmq 5.80.4 on Valkey, worker/test/queued-at.integration.test.mjs), so
+		// a pause window and a deferral count as waiting, which they are. Null for a job held on `run.waitFor` (a wait its
+		// trigger asked for is not a wait for capacity) and for a retry (its wait would include the earlier attempt). An
+		// ISO string or null, a number's rendering, so PII-free.
+		queuedAt: queuedAtOf(job),
+		// What this host offered when the job took its slot (issue #599, INT-RUN-HISTORY-FILE-CONTRACT): `{ slots, memMiB,
+		// cpuCenti, cpus }`, the live PI_CONCURRENCY, the host budget as `hostBudget` above maps it, and the CPUs the OS
+		// reports. Additive, nullable, an explicit literal REBUILT here (`recordedCapacity`), TAIL position after `queuedAt`.
+		// Passed in by the processor only once the job is ADMITTED (index.mjs, where `startedAt` is set), so it is also
+		// the record's own answer to "did this run hold a slot": null on every refusal before one, which is what
+		// lets a capacity report tell a run from a refusal without guessing from the wall time (capacity.mjs).
+		capacity: recordedCapacity(capacity),
+		// The slot time of this job's EARLIER attempts (issue #599, INT-RUN-HISTORY-FILE-CONTRACT), which a retry's record
+		// would otherwise erase: it overwrites the same file and the same mirror key. `[{ host, startedAt, endedAt, memMiB,
+		// cpuCenti }]`, at most `EARLIER_MAX`, newest last, read by the processor from the previous record BEFORE this one
+		// replaces it (`earlierFrom`) and passed in. Additive, nullable, an explicit literal REBUILT here, TAIL position after
+		// `capacity`. A worker name, two ISO instants and two integers per entry: never the earlier attempt's dollars,
+		// tokens or usage, so every cost reader, which reads the top-level fields only, counts nothing twice. Null on a
+		// first attempt and when the previous record held no slot or could not be read.
+		earlier: recordedEarlier(earlier),
+		// Whether the FIRST attempt stalled and this pickup re-ran it (issue #599): BullMQ raised `stalledCounter`, not
+		// `attemptsMade`, so that pickup wrote no record and its slot time is in none. A stall after a failed attempt is a
+		// retry like any other (its earlier attempt's record is carried in `earlier`), so it is false there. A boolean,
+		// TAIL position after `earlier`, so the capacity report can say how many runs that is.
+		stalledRepick: Number.isInteger(job.stalledCounter) && job.stalledCounter > 0 && !(Number.isInteger(job.attemptsMade) && job.attemptsMade > 0),
+	};
+}
+
+/**
+ * Of two copies of one job's record (this host's file and the fleet's copy), the one a retry replaces: the HIGHER
+ * `attempt` (a retry on another host writes its own copy there, so either side can be the newer attempt), an exact tie
+ * broken by the later `endedAt` (run-mirror.mjs `mergeRuns`' rule), a remaining tie by the first given (the local file).
+ * Either may be null; a non-object is none.
+ */
+export function newerRecord(a, b) {
+	const ok = (r) => r !== null && typeof r === "object" && !Array.isArray(r);
+	if (!ok(a)) return ok(b) ? b : null;
+	if (!ok(b)) return a;
+	const attempt = (r) => (Number.isInteger(r.attempt) ? r.attempt : 0);
+	if (attempt(a) !== attempt(b)) return attempt(a) > attempt(b) ? a : b;
+	const end = (r) => {
+		const t = Date.parse(r.endedAt ?? r.startedAt ?? "");
+		return Number.isFinite(t) ? t : -Infinity;
+	};
+	return end(b) > end(a) ? b : a;
+}
+
+/**
+ * What a retry's record carries forward from the record it is about to replace: that record's own `earlier`, then its
+ * own slot interval when it held a slot (`capacity` an object), each rebuilt; null when there is nothing.
+ */
+export function earlierFrom(previous) {
+	if (previous === null || typeof previous !== "object" || Array.isArray(previous)) return null;
+	const carried = Array.isArray(previous.earlier) ? previous.earlier : [];
+	const own = previous.capacity !== null && typeof previous.capacity === "object" && !Array.isArray(previous.capacity) ? [{ host: previous.host, startedAt: previous.startedAt, endedAt: previous.endedAt, memMiB: previous.size?.memMiB, cpuCenti: previous.size?.cpuCenti }] : [];
+	return recordedEarlier([...carried, ...own]);
+}
+
+/**
+ * When a job became eligible to run, as an ISO string, or null: `job.timestamp` plus the delay it was added with
+ * (`opts.delay`, at least 0). Null for a job armed with `run.waitFor` (its wait is its trigger's own), for a retry
+ * (`attemptsMade` above 0) or a pickup after a stall (`stalledCounter` above 0), and for any input that is not a
+ * finite, representable instant.
+ */
+export function queuedAtOf(job) {
+	if (waitArmed(job?.data)) return null;
+	// A RETRY's wait would include its earlier attempt and that attempt's run (the add's moment is kept across attempts),
+	// so only a first attempt says how long the job waited for a slot. `attemptsMade` counts attempts finished before.
+	if (Number.isInteger(job?.attemptsMade) && job.attemptsMade > 0) return null;
+	// A pickup after a STALL is the same: BullMQ raises `stalledCounter`, not `attemptsMade`, and the add's moment is kept.
+	if (Number.isInteger(job?.stalledCounter) && job.stalledCounter > 0) return null;
+	// A job scheduler's job carries its exact slot in its id. BullMQ stamps `timestamp` and computes `delay` from two
+	// separate clock reads, so their sum can land a few milliseconds past the slot (measured in CI).
+	const slot = scheduledForMillis(job?.id);
+	if (Number.isSafeInteger(slot) && slot >= 0 && slot <= 8.64e15) return new Date(slot).toISOString();
+	const timestamp = job?.timestamp;
+	const delay = job?.opts?.delay ?? 0;
+	if (!Number.isSafeInteger(timestamp) || timestamp < 0 || typeof delay !== "number" || !Number.isFinite(delay)) return null;
+	const at = timestamp + Math.max(0, Math.trunc(delay));
+	// 8.64e15 is the last instant a Date can hold; past it `toISOString` throws, and this function must not.
+	return at <= 8.64e15 ? new Date(at).toISOString() : null;
+}
+
+/** A record's non-negative safe integer, else null. */
+const recordInt = (v) => (Number.isSafeInteger(v) && v >= 0 ? v : null);
+/** A record's budget dimension: an integer, `"off"` (the budget's `Infinity`, or the word itself), else null. */
+const recordBudget = (v) => (v === Infinity || v === "off" ? "off" : recordInt(v));
+
+/**
+ * A run's capacity as a record carries it: `{ slots, memMiB, cpuCenti, cpus }`, else null. `slots` and `cpus` are
+ * positive integers or null (unknown); each budget dimension is `recordedHostBudget`'s mapping, so `"off"` and null
+ * keep meaning "not limited" and "not known".
+ */
+export function recordedCapacity(value) {
+	if (value === null || typeof value !== "object" || Array.isArray(value)) return null;
+	// Each held to what a host can have (job-size.mjs `HOST_*_MAX`): past it a value is a fault, recorded as unknown.
+	const upTo = (max, read) => (v) => {
+		const n = read(v);
+		return typeof n === "number" && n > max ? null : n;
 	};
+	const positive = (v) => (Number.isSafeInteger(v) && v >= 1 ? v : null);
+	return { slots: upTo(HOST_SLOTS_MAX, positive)(value.slots), memMiB: upTo(HOST_MEMORY_MAX_MIB, recordBudget)(value.memMiB), cpuCenti: upTo(HOST_CPUS_MAX * 100, recordBudget)(value.cpuCenti), cpus: upTo(HOST_CPUS_MAX, positive)(value.cpus) };
 }
 
 /**
@@ -739,9 +858,7 @@ export function buildRecord({ job, result, error, startedAt, endedAt, host = nul
  */
 export function recordedHostBudget(value) {
 	if (value === null || typeof value !== "object" || Array.isArray(value)) return null;
-	const int = (v) => (Number.isSafeInteger(v) && v >= 0 ? v : null);
-	const budget = (v) => (v === Infinity || v === "off" ? "off" : int(v));
-	return { memMiB: budget(value.memMiB), cpuCenti: budget(value.cpuCenti), hostShare: int(value.hostShare) };
+	return { memMiB: recordBudget(value.memMiB), cpuCenti: recordBudget(value.cpuCenti), hostShare: recordInt(value.hostShare) };
 }
 
 /** What became of a collected plan. */

package/src/run-mirror.mjs

@@ -39,6 +39,158 @@ import { UNREADABLE_RECORD } from "./run-history.mjs";
 /** The index: sanitized jobId -> the run's end (or start) in millis. */
 export const RUNS_INDEX = "runs:index";
 
+/**
+ * The fleet's HISTORY HORIZON (issue #599): a one-member ZSET (`trim`) whose score is the newest instant before which a
+ * writer has trimmed runs out of the index. Every writer trims the SHARED index by its OWN retention (and the shared
+ * count cap), so a host with a short `PI_LOG_RETENTION_DAYS` removes every host's older runs; a reader cannot know
+ * that from its own settings, and without this key it read the gap as idle time. Raised only, never lowered (`ZADD
+ * GT`), so concurrent writers agree on the latest cut without a lock.
+ */
+export const RUNS_HORIZON = "runs:horizon";
+/** The horizon's one member. */
+export const RUNS_HORIZON_MEMBER = "trim";
+
+/**
+ * WHEN THIS MIRROR STARTED (issue #599): a hash, `at` the instant in millis from which the index holds every run that
+ * ended, and `member` a run of the index that vouches for it. Nothing above says it: the reader's other bounds (the
+ * depth, the horizon, the cap, an expired body) all live in keys that a flushed or replaced Valkey, or an index that
+ * expired, loses together with the runs, and the next write then recreates an index that claimed the whole window, so a
+ * peer read every lost run as idle.
+ *
+ * A start is TRUSTED only while its `member` is in the index. The start can outlive its index (a worker of 4.0.1 sets
+ * the index's expiry to its own retention while the start keeps the deepest window; a pruning reader can remove every
+ * member), and an older worker can then recreate the index without knowing this key: a member that is still there is
+ * the one proof that the index the start describes is the index that exists. Rejected: also trusting it while
+ * `runs:horizon` is at or past it, since the horizon outlives a lost index just the same.
+ *   - The write that finds no index starts it NOW (the write's time), vouched for by the run it adds.
+ *   - A write that finds an index whose start it cannot trust (none, or its member is gone) starts it at the index's
+ *     oldest run before this write, vouched for by that run: the most the index can say, so time before it reads as
+ *     missing. A trusted start is kept.
+ *   - Whatever removes the vouching run while the start is trusted (a trim, `PRUNE_SCRIPT`) hands it to the oldest run
+ *     that remains, `at` unchanged, and deletes the start with the last run. So no trim moves `at`, earlier or later.
+ * It expires with the index (one `PEXPIRE` each, in the same script). A reader that finds no trusted start starts the
+ * mirror at the index's oldest run.
+ */
+export const RUNS_SINCE = "runs:since";
+
+/**
+ * Lua shared by the three scripts (KEYS[1] the index, KEYS[3] the start): whether the start is a hash, whether its member
+ * is held, and handing a trusted start on once its member is removed. A start that is not a hash (a string from a
+ * development build, or a hand SET) is read as absent, and the writing scripts delete it first (`sinceClean`), so a
+ * stray value can never make HGET fail with WRONGTYPE and stop the mirror.
+ */
+const SINCE_LUA = `
+local function sinceIsHash()
+	local t = redis.call("TYPE", KEYS[3])
+	return (t.ok or t) == "hash"
+end
+local function sinceClean()
+	if redis.call("EXISTS", KEYS[3]) == 1 and not sinceIsHash() then redis.call("DEL", KEYS[3]) end
+end
+local function sinceHeld()
+	if not sinceIsHash() then return false end
+	local member = redis.call("HGET", KEYS[3], "member")
+	return member and redis.call("ZSCORE", KEYS[1], member) and true or false
+end
+local function sinceHandOn()
+	local first = redis.call("ZRANGE", KEYS[1], 0, 0)
+	if first[1] then redis.call("HSET", KEYS[3], "member", first[1]) else redis.call("DEL", KEYS[3]) end
+end
+`;
+
+/**
+ * The writer's index write and trim, in one script so the member, the start (`RUNS_SINCE`) and the horizon it raises
+ * are atomic (issue #599). KEYS: the index, the horizon and the start; ARGV: the age cutoff in millis, the count cap,
+ * the expiry all three keys get, the horizon's member, the run's score, its id, and the write's time in millis.
+ *
+ * - The start is checked and set as `RUNS_SINCE` says, from the index as it was before this write, and the member is
+ *   added (`ZADD`).
+ * - By age (scores below the cutoff, exclusive) and by count (all but the newest `cap`). A trim that REMOVED something
+ *   raises the horizon with `ZADD GT`: to the cutoff, or after a count trim to the oldest score that remains, which is
+ *   conservative (a removed run ended at or before it). Nothing removed, nothing written: a fleet whose writers trim
+ *   nothing keeps no horizon. A trim that removed the start's member hands the start on.
+ * - The keys expire after the DEEPEST window any reader asks for (`mirrorWindowMs(0)`), never the writer's own. The
+ *   index's members are trimmed by score anyway; a short-retention writer that set the shared index's TTL to its own
+ *   window made the whole index expire on a quiet day, every peer's history with it and no horizon to say so. The
+ *   expiry still rolls with traffic: a fleet that runs nothing for that long has nothing to show.
+ */
+export const TRIM_SCRIPT = `${SINCE_LUA}
+sinceClean()
+local oldest = redis.call("ZRANGE", KEYS[1], 0, 0, "WITHSCORES")
+if not oldest[2] then
+	redis.call("DEL", KEYS[3])
+	redis.call("HSET", KEYS[3], "at", ARGV[7], "member", ARGV[6])
+elseif not (redis.call("HGET", KEYS[3], "at") and sinceHeld()) then
+	redis.call("DEL", KEYS[3])
+	redis.call("HSET", KEYS[3], "at", string.format("%d", tonumber(oldest[2])), "member", oldest[1])
+end
+redis.call("ZADD", KEYS[1], ARGV[5], ARGV[6])
+redis.call("PEXPIRE", KEYS[3], ARGV[3])
+local removed = redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", "(" .. ARGV[1])
+local horizon = nil
+if removed > 0 then horizon = tonumber(ARGV[1]) end
+if redis.call("ZREMRANGEBYRANK", KEYS[1], 0, -tonumber(ARGV[2]) - 1) > 0 then
+	local kept = redis.call("ZRANGE", KEYS[1], 0, 0, "WITHSCORES")
+	if kept[2] then
+		local score = tonumber(kept[2])
+		if horizon == nil or score > horizon then horizon = score end
+	end
+end
+if horizon then
+	redis.call("ZADD", KEYS[2], "GT", string.format("%d", horizon), ARGV[4])
+	redis.call("PEXPIRE", KEYS[2], ARGV[3])
+	if not sinceHeld() then sinceHandOn() end
+end
+redis.call("PEXPIRE", KEYS[1], ARGV[3])
+return horizon and string.format("%d", horizon) or false
+`;
+
+/**
+ * A pruning reader's removal of members whose bodies are gone (`readMirroredRuns`), in one script so the cut is
+ * recorded with it (issue #599). The run a removed member stood for is no longer shown, so the time up to it is no
+ * longer whole: the horizon is raised (`ZADD GT`) to the highest score removed, as `TRIM_SCRIPT` raises it for a trim,
+ * and a trusted start whose member is removed is handed on. Without it, the reader's "expired body" bound was erased by
+ * the very read that saw it, and the time before it read as idle. KEYS: the index, the horizon and the start; ARGV: the
+ * horizon's expiry, its member, then the ids. Returns the highest score removed, or false.
+ */
+export const PRUNE_SCRIPT = `${SINCE_LUA}
+sinceClean()
+-- Handed on only when the start was trusted BEFORE this removal: a start whose run was already gone (an older worker
+-- recreated the index under it) must not be revalidated by handing it to a run of the new index.
+local held = sinceHeld()
+local top = nil
+for i = 3, #ARGV do
+	local score = redis.call("ZSCORE", KEYS[1], ARGV[i])
+	if score then
+		redis.call("ZREM", KEYS[1], ARGV[i])
+		score = tonumber(score)
+		if top == nil or score > top then top = score end
+	end
+end
+if top then
+	redis.call("ZADD", KEYS[2], "GT", string.format("%d", top), ARGV[2])
+	redis.call("PEXPIRE", KEYS[2], ARGV[1])
+	if held and not sinceHeld() then sinceHandOn() end
+end
+return top and string.format("%d", top) or false
+`;
+
+/**
+ * The capacity reader's one snapshot of the index (issue #599): its size, its oldest score, the horizon, the start's
+ * `at` and whether its member is held, and the members that ended after ARGV[2] (exclusive) with their scores, newest
+ * first. One script, so a flush or a write between these reads cannot mix two states into one report. KEYS: the index,
+ * the horizon and the start; ARGV: the horizon's member and the window's start.
+ */
+export const READ_SCRIPT = `${SINCE_LUA}
+local size = redis.call("ZCARD", KEYS[1])
+if size == 0 then return {0} end
+local oldest = redis.call("ZRANGE", KEYS[1], 0, 0, "WITHSCORES")
+local horizon = redis.call("ZSCORE", KEYS[2], ARGV[1])
+local at = sinceIsHash() and redis.call("HGET", KEYS[3], "at") or false
+local range = redis.call("ZREVRANGEBYSCORE", KEYS[1], "+inf", "(" .. ARGV[2], "WITHSCORES")
+return {size, oldest[2] or false, horizon, at, sinceHeld() and 1 or 0, range}
+`;
+
 /** One run's own bytes. */
 export const runRecordKey = (sanitizedJobId) => `runs:rec:${sanitizedJobId}`;
 
@@ -106,17 +258,13 @@ export function makeRunMirror({ redis, retentionDays, now = () => Date.now(), lo
 				const score = Number.isFinite(at) ? at : now();
 				const body = JSON.stringify(record);
 				await bounded(redis.set(runRecordKey(sanitizedJobId), body, "PX", windowMs), timeoutMs);
-				await bounded(redis.zadd(RUNS_INDEX, score, sanitizedJobId), timeoutMs);
-				// Trimmed by the WRITER, twice: by age, and by count. Two `ZREMRANGE`s against a run that
-				// took minutes is free, and it means no reader has to pay for a backlog it did not create.
-				await bounded(redis.zremrangebyscore(RUNS_INDEX, "-inf", `(${now() - windowMs}`), timeoutMs);
-				await bounded(redis.zremrangebyrank(RUNS_INDEX, 0, -indexMax - 1), timeoutMs);
-				// ROLLING expiry, deliberately unlike `budget.mjs`'s set-once rule and deliberately like
-				// `pi-dispatch:sched-stalls:<schedulerId>`. A budget window must not be pushed forward by traffic or a busy
-				// day never resets; an ACTIVITY index should roll with traffic, because that is what it
-				// describes. A fleet that stops running jobs loses its index one window later, which is
-				// correct: there is nothing left to show.
-				await bounded(redis.pexpire(RUNS_INDEX, windowMs), timeoutMs);
+				// Indexed, and trimmed by the WRITER, twice: by age, and by count, with the mirror's start (`RUNS_SINCE`) and
+				// the fleet's horizon written by the same script (`TRIM_SCRIPT`), so no crash or timeout can land between a
+				// member and the record of where the index starts, or between a cut and the record of it. A script, not a
+				// MULTI, because both depend on what the index held: the start on whether it existed, the horizon on what a
+				// trim REMOVED and what remains.
+				const writtenMs = now();
+				await bounded(redis.eval(TRIM_SCRIPT, 3, RUNS_INDEX, RUNS_HORIZON, RUNS_SINCE, writtenMs - windowMs, indexMax, mirrorWindowMs(0), RUNS_HORIZON_MEMBER, score, sanitizedJobId, writtenMs), timeoutMs);
 				warned = false;
 				return true;
 			} catch (err) {
@@ -180,7 +328,8 @@ export async function readMirroredRuns(redis, { limit = 50, sinceMs = 0, now = (
 	}
 	if (stale.length > 0) {
 		try {
-			await bounded(redis.zrem(RUNS_INDEX, ...stale), timeoutMs);
+			// Removed with the cut recorded (`PRUNE_SCRIPT`): the time up to a pruned run is no longer whole.
+			await bounded(redis.eval(PRUNE_SCRIPT, 3, RUNS_INDEX, RUNS_HORIZON, RUNS_SINCE, mirrorWindowMs(0), RUNS_HORIZON_MEMBER, ...stale), timeoutMs);
 		} catch {
 			// best-effort: a straggler in the index costs one skipped row next time, never a wrong one
 		}

package/src/schedules.mjs

@@ -182,7 +182,7 @@ function normalizeCronSchedule({ on, run }, path, existsSync, fleet) {
 	// `portfolio` (issue #505) is conditional like `command`, so an unflagged schedule grows no key, and it rides at JOB
 	// level, never inside `trigger`: `trigger` is copied verbatim into `/job/event.json`, and whether a job may send a
 	// budget plan is the worker's decision to re-check against the live file, not a fact for the agent to read.
-	const data = { kind: "local", folder: run.folder, flow: run.flow, task: run.task, ...(run.command !== undefined && { command: run.command }), provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages: run.packages, image: run.image, ...(run.backend !== undefined && { backend: run.backend }), ...(run.excludeTools !== undefined && { excludeTools: run.excludeTools }), ...(run.models !== undefined && { models: run.models }), ...(run.maxCostUsd !== undefined && { maxCostUsd: run.maxCostUsd }), ...(run.skillsDir !== undefined && { skillsDir: run.skillsDir }), ...(run.secrets !== undefined && { secrets: run.secrets }), ...(run.secretsProfile !== undefined && { secretsProfile: run.secretsProfile }), resume: run.resume, ...(run.portfolio !== undefined && { portfolio: run.portfolio }), trigger: { id: on.id, pattern: on.pattern } };
+	const data = { kind: "local", folder: run.folder, flow: run.flow, task: run.task, ...(run.command !== undefined && { command: run.command }), provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages: run.packages, image: run.image, ...(run.backend !== undefined && { backend: run.backend }), ...(run.excludeTools !== undefined && { excludeTools: run.excludeTools }), ...(run.mcp !== undefined && { mcp: run.mcp }), ...(run.models !== undefined && { models: run.models }), ...(run.maxCostUsd !== undefined && { maxCostUsd: run.maxCostUsd }), ...(run.skillsDir !== undefined && { skillsDir: run.skillsDir }), ...(run.secrets !== undefined && { secrets: run.secrets }), ...(run.secretsProfile !== undefined && { secretsProfile: run.secretsProfile }), resume: run.resume, ...(run.portfolio !== undefined && { portfolio: run.portfolio }), trigger: { id: on.id, pattern: on.pattern } };
 	// Retention only; the deterministic repeat:<id>:<millis> jobId supplies dedup, so no jobId here, and
 	// scheduler jobs are not retried (DES-CRON-VIA-BULLMQ-SCHEDULER) so no attempts/backoff.
 	const opts = { removeOnComplete: { age: 24 * 3600 }, removeOnFail: { age: 7 * 24 * 3600 } };

package/src/size-suggest.mjs

@@ -42,7 +42,8 @@
  *   A size is rounded UP to a step: 256m up to 2g, 512m up to 8g, then 1g.
  * CPUs: fewer than `SUGGEST_MIN_SAMPLES` runs is not enough runs; the p95 cores used (CPU time over wall time) below
  * 0.4x the current cpus lowers to 1.25x that p95, never below 1.25x the window's largest cores used, nor 0.25;
- * otherwise it fits. A FACT (no call) rides along when the median run was throttled more than 25% of its wall time.
+ * otherwise it fits. A FACT (no call) rides along when the median run was throttled more than 25% of its wall time,
+ * read over the runs that report throttled time and only when at least `SUGGEST_MIN_SAMPLES` of them do.
  * The wall time is the record's pickup-to-end span, which includes the clone, so cores used read slightly LOW.
  *
  * THE CAP. A raise never goes past `cap` (this host's budget per dimension, or where that is off or unknown the host's
@@ -119,9 +120,10 @@ const isCount = (v) => Number.isSafeInteger(v) && v >= 0;
 /**
  * One record as a suggestion reads it, or null when it is not evidence: `{ at, size, memPeak, oom, wallUsec, cpuUsec,
  * throttledUsec, memFullUsec }`. `memPeak` (bytes, clamped to the record's own limit) is null when absent or not a
- * count; `wallUsec` is null unless the span is readable; `cpuUsec` and `throttledUsec` are null unless both can be read
- * with a wall time (CPU time clamped to the CPU ceiling over the wall, throttled time to the wall); `memFullUsec` is
- * null unless it can be read with a wall time.
+ * count; `wallUsec` is null unless the span is readable; `cpuUsec` is null unless it can be read with a wall time (CPU
+ * time clamped to the CPU ceiling over the wall), and `throttledUsec` likewise on its own (clamped to the wall): cgroup v2
+ * writes `throttled_usec` in `cpu.stat` only where the cpu controller is enabled, so a run without it is still a CPU
+ * sample, and only the throttle fact leaves it out. `memFullUsec` is null unless it can be read with a wall time.
  */
 function evidenceOf(record, project, nowMs) {
 	if (record === null || typeof record !== "object" || Array.isArray(record)) return null;
@@ -136,11 +138,10 @@ function evidenceOf(record, project, nowMs) {
 	const start = typeof record.startedAt === "string" ? Date.parse(record.startedAt) : NaN;
 	const wallMs = Number.isFinite(start) ? at - start : NaN;
 	const wallUsec = Number.isSafeInteger(wallMs) && wallMs > 0 && wallMs <= SUGGEST_MAX_WALL_MS ? wallMs * 1000 : null;
-	let cpu = { cpuUsec: null, throttledUsec: null };
-	if (wallUsec !== null && isCount(r.cpuUsec) && isCount(r.throttledUsec)) {
-		const cpuCap = (wallUsec * JOB_CPUS_CEILING_CENTI) / 100;
-		cpu = { cpuUsec: Math.min(r.cpuUsec, cpuCap), throttledUsec: Math.min(r.throttledUsec, wallUsec) };
-	}
+	const cpu = {
+		cpuUsec: wallUsec !== null && isCount(r.cpuUsec) ? Math.min(r.cpuUsec, (wallUsec * JOB_CPUS_CEILING_CENTI) / 100) : null,
+		throttledUsec: wallUsec !== null && isCount(r.cpuUsec) && isCount(r.throttledUsec) ? Math.min(r.throttledUsec, wallUsec) : null,
+	};
 	// unclamped: it is only ever compared (exactly, in BigInt) with 1% of the wall, and a stall past the wall is above it
 	const memFullUsec = wallUsec !== null && isCount(r.memFullUsec) ? r.memFullUsec : null;
 	if (memPeak === null && cpu.cpuUsec === null) return null;
@@ -174,7 +175,8 @@ function suggestMemory(runs, current, cap) {
 	const largestOom = oomSizes.length > 0 ? Math.max(...oomSizes) : null;
 	// at the limit (90% of it or more: peak x 10 >= limit x 9) AND stalled for memory more than 1% of the wall.
 	const pressured = relevant.filter((e) => !productAbove(e.size.memMiB * MIB, 9, e.memPeak, 10) && e.memFullUsec !== null && productAbove(e.memFullUsec, 100, e.wallUsec, 1)).length;
-	const evidence = { samples, p95MiB: p95 === null ? null : Math.ceil(p95 / MIB), maxPeakMiB: maxPeak === null ? null : Math.ceil(maxPeak / MIB), ooms, largestOomMiB: largestOom, pressured };
+	// `smaller`: the measured runs below the current size, which a chart draws and this verdict does not count.
+	const evidence = { samples, smaller: measured.length - samples, p95MiB: p95 === null ? null : Math.ceil(p95 / MIB), maxPeakMiB: maxPeak === null ? null : Math.ceil(maxPeak / MIB), ooms, largestOomMiB: largestOom, pressured };
 	const fact = pressured > 0 ? "pressure" : null;
 	const out = (reason, suggested, more = {}) => ({ current, suggested: suggested === null || suggested === current ? null : suggested, reason, evidence, fact, held: null, wanted: null, ...more });
 	if (ooms > 0) {
@@ -198,12 +200,17 @@ function suggestCpus(runs, current) {
 	const samples = relevant.length;
 	// ordered by the exact fraction, never a float: a/b < c/d  <=>  a x d < c x b.
 	const byFraction = (num) => (x, y) => (productAbove(num(y), x.wallUsec, num(x), y.wallUsec) ? -1 : productAbove(num(x), y.wallUsec, num(y), x.wallUsec) ? 1 : 0);
-	const throttle = samples > 0 ? rank([...relevant].sort(byFraction((e) => e.throttledUsec)), 50) : null;
+	// The throttle fact reads only the runs whose cgroup reported throttled time (see `evidenceOf`), and needs as many of
+	// them as a suggestion needs runs: one throttled run among twenty is not "the median run".
+	const throttled = relevant.filter((e) => e.throttledUsec !== null);
+	const throttle = throttled.length >= SUGGEST_MIN_SAMPLES ? rank([...throttled].sort(byFraction((e) => e.throttledUsec)), 50) : null;
 	const busy = samples > 0 ? rank([...relevant].sort(byFraction((e) => e.cpuUsec)), 95) : null;
 	const busiest = measured.length > 0 ? [...measured].sort(byFraction((e) => e.cpuUsec)).at(-1) : null;
 	const coresOf = (e) => (e === null ? null : Math.ceil((e.cpuUsec * 100) / e.wallUsec));
 	const evidence = {
 		samples,
+		smaller: measured.length - samples,
+		throttledSamples: throttled.length,
 		p95CoresCenti: coresOf(busy),
 		maxCoresCenti: coresOf(busiest),
 		throttledPct: throttle === null ? null : Math.round((throttle.throttledUsec * 100) / throttle.wallUsec),
@@ -211,7 +218,7 @@ function suggestCpus(runs, current) {
 	const out = (reason, suggested, fact = null) => ({ current, suggested: suggested === null || suggested === current ? null : suggested, reason, evidence, fact, held: null, wanted: null });
 	if (samples < SUGGEST_MIN_SAMPLES) return out("not-enough-runs", null);
 	// throttled more than 25% of the wall time (throttled x 4 > wall): a fact about the host's ceiling, never a call.
-	const fact = productAbove(throttle.throttledUsec, 4, throttle.wallUsec, 1) ? "ceiling" : null;
+	const fact = throttle !== null && productAbove(throttle.throttledUsec, 4, throttle.wallUsec, 1) ? "ceiling" : null;
 	// p95 cores below 0.4 x cpus: cpuUsec / wall < 0.4 x current / 100, that is cpuUsec x 1000 < 4 x current x wall.
 	if (productAbove(4 * current, busy.wallUsec, busy.cpuUsec, 1000)) {
 		const lower = Math.max(roundCpusUp(quarterMoreCenti(busy)), roundCpusUp(quarterMoreCenti(busiest)));
@@ -399,6 +406,19 @@ export const coresText = (centi) => `${formatCpus(centi)} core${centi === 100 ?
 /** CPUs in words: `1 CPU`, `0.5 CPUs`, `2 CPUs`. */
 export const cpusText = (centi) => `${formatCpus(centi)} CPU${centi === 100 ? "" : "s"}`;
 
+/**
+ * A "not enough runs" verdict in words for the panel's PROJECTS view and the dispatch_limit_edit preview (the insights
+ * page restates it; doctor says the same counts through `suggestionEvidence`): how many runs at the current size or
+ * larger it counted, and how many smaller ones it did not, which for memory the chart beside it draws:
+ * `not enough runs at 4g (0; 20 at smaller sizes)`. `sizeText` formats the dimension's size.
+ */
+export function notEnoughRunsText(dim, sizeText) {
+	const e = dim?.evidence ?? {};
+	const samples = Number.isSafeInteger(e.samples) ? e.samples : 0;
+	const smaller = Number.isSafeInteger(e.smaller) && e.smaller > 0 ? `; ${e.smaller} at smaller sizes` : "";
+	return `not enough runs at ${sizeText(dim.current)} (${samples}${smaller})`;
+}
+
 /**
  * The evidence of a suggestion in words, for doctor, the panel and the insights page alike: `{ memory, cpu, memoryHeld,
  * memoryFact, cpuFact }`, each a short clause (empty when it does not apply), such as `2 runs ended oom-killed (the
@@ -415,12 +435,12 @@ export function suggestionEvidence(suggestion) {
 	const cores = `p95 ${coresText(ce.p95CoresCenti ?? 0)} used, largest ${coresText(ce.maxCoresCenti ?? 0)}, over ${plural(ce.samples, "run")}`;
 	const memWords = {
 		"oom-killed": `${plural(me.ooms, "run")} ended oom-killed (the largest size killed ${formatMemory(me.largestOomMiB ?? 0)})`,
-		"not-enough-runs": `${me.samples} of the ${SUGGEST_MIN_SAMPLES} runs with measurements it needs`,
+		"not-enough-runs": `${me.samples} of the ${SUGGEST_MIN_SAMPLES} runs with measurements at ${formatMemory(m.current ?? 0)} or larger it needs${me.smaller > 0 ? `, ${me.smaller} more at smaller sizes` : ""}`,
 		oversized: peaks,
 		fits: peaks,
 	};
 	const cpuWords = {
-		"not-enough-runs": `${ce.samples} of the ${SUGGEST_MIN_SAMPLES} runs with measurements it needs`,
+		"not-enough-runs": `${ce.samples} of the ${SUGGEST_MIN_SAMPLES} runs with measurements at ${cpusText(c.current ?? 0)} or more it needs${ce.smaller > 0 ? `, ${ce.smaller} more at smaller sizes` : ""}`,
 		underused: cores,
 		fits: cores,
 	};