@edgehero/pi-dispatch 4.3.0 → 5.0.0

package/src/held-settle.mjs

@@ -0,0 +1,266 @@
+import { isFinalOutcome } from "./inbox-settle.mjs";
+import { enqueueForgeJob, QUEUE } from "./queue.mjs";
+import { runIdOf } from "./recovery-queue.mjs";
+import { sanitizeJobId } from "./run-history.mjs";
+import * as heldModule from "./held-replies.mjs";
+
+/**
+ * The worker's side of held replies (issue #638, DES-AWAIT-INPUT-ENDS-THE-JOB; the keys and the scripts are
+ * held-replies.mjs's): a keeper that keeps an awaitInput run reachable while it runs, the take inside the park that
+ * turns what it held into at most one resumed job, and the final settle that drops what a run that did not park held.
+ *
+ * WHY THE WORKER AND NOT THE RECEIVER DECIDES. The receiver only knows that a reply met a run that is still going; it
+ * cannot know whether that run will park (the agent may finish, fail, or be cancelled after asking). The worker learns
+ * it in the park sequence, or at the run's final outcome, so that is where a held reply becomes a job or is dropped. Dropping is today's
+ * coalescing exactly: a reply that met a run which then did NOT park was always deduplicated against it, and still is.
+ */
+
+/** The keeper's refresh period, the steer relay's: 2 s against a 30 s open key. */
+export const AWAIT_KEEPER_POLL_MS = 2_000;
+
+/**
+ * How far back before the park a held reply may have been written and still be taken as the answer (issue #638 review,
+ * M1). The keeper registers the run for its whole pickup, so without a bound a phrase-free comment made ten minutes
+ * into a long run, before the agent had asked anything, would be taken as "the answer" to a question posted later.
+ * Five minutes covers the seconds between the agent's question and the park with a wide margin; an older entry is
+ * dropped silently, which is the coalescing it always got. Residual, named: a comment written in the last five minutes
+ * before the question and not meant as an answer is taken as one (the resumed agent reads it as a reply and may ask
+ * again).
+ */
+export const HELD_LOOKBACK_MS = 5 * 60 * 1000;
+
+/**
+ * The one comment the replies after the first get, however many there are (fixed words, no mention, no quote of any
+ * reply). The first reply is the answer the parked run resumes with; a run asks for one answer, and starting a second
+ * resumed job from the same parked session would race the first for it.
+ */
+export const HELD_EXTRA_REPLY = "This run was waiting for one answer and took the first reply it was given. If this comment adds to it, comment again once the run has replied.";
+
+/** The comment when the held answer could not be started (an enqueue that failed): the reply is answered, never lost silently. */
+export const HELD_RESUME_FAILED_REPLY = "This run was waiting for an answer and could not start from the reply it was given. Comment again to answer it.";
+
+const codeOf = (e) => (typeof e?.code === "string" && e.code !== "" ? e.code : typeof e?.error === "string" ? e.error : "error");
+const targetNumber = (t) => (typeof t?.number === "number" ? t.number : typeof t?.number === "string" && /^[1-9][0-9]{0,15}$/.test(t.number) ? Number(t.number) : null);
+
+/** Whether a processor result is a run that PARKED: the completed branch's `reason: "awaiting-input"` (processor.mjs). */
+export function isParkedOutcome(result) {
+	return result !== null && typeof result === "object" && result.outcome === "completed" && result.reason === "awaiting-input";
+}
+
+/**
+ * `makeAwaitKeeper({ redis, job, host, now, log, pollMs, held })` -> `{ start(), stop() }`: keeps an awaitInput run
+ * live on its await target for the whole pickup (registered at start, refreshed every `pollMs`), so a reply in the
+ * seconds between the agent's question and the park can be held. Started by the processor only for a job whose
+ * `data.awaitInput === true` and whose target a forge reply can reach; stopped by the park's take, else at the settle
+ * (index.mjs).
+ *
+ * STOP LEAVES THE KEYS TO EXPIRE rather than deleting them: the settle's close removes them at a final outcome, and an
+ * attempt that will be retried (an InfraRetry with attempts left) keeps taking a held reply for the open key's 30 s,
+ * after which the receiver answers as it always did. Never throws; faults are logged once per step, by code.
+ */
+export function makeAwaitKeeper({ redis, job, host, now = Date.now, log = () => {}, pollMs = AWAIT_KEEPER_POLL_MS, held = heldModule } = {}) {
+	const runId = runIdOf(job);
+	const targetKey = held.awaitTargetKeyOf(job?.data ?? null);
+	const jobId = sanitizeJobId(job?.id);
+	let timer = null;
+	let running = null;
+	let stopped = false;
+	const said = new Set();
+	const fault = (step, code) => {
+		if (said.has(step)) return;
+		said.add(step);
+		log("held_keeper_fault", { jobId, step, code });
+	};
+	const refresh = async (step) => {
+		try {
+			const r = await held.refreshAwaitLive({ redis, runId, targetKey, host, now });
+			if (r?.error) fault(step, codeOf(r));
+		} catch (e) {
+			fault(step, codeOf(e));
+		}
+	};
+	const tick = () => {
+		if (running || stopped) return running ?? Promise.resolve();
+		running = refresh("refresh").finally(() => {
+			running = null;
+		});
+		return running;
+	};
+	return {
+		targetKey,
+		async start() {
+			if (timer !== null || stopped || targetKey === null) return;
+			// A closed list under this run id belongs to a run that already settled (a console retry keeps the job id).
+			const reset = await held.resetClosedHeld({ redis, runId });
+			if (reset?.error) fault("reset", codeOf(reset));
+			await refresh("register");
+			if (stopped) return;
+			timer = setInterval(() => void tick(), pollMs);
+			timer.unref?.();
+		},
+		async stop() {
+			stopped = true;
+			if (timer !== null) clearInterval(timer);
+			timer = null;
+			try {
+				await running;
+			} catch {}
+		},
+	};
+}
+
+/**
+ * `makeHeldSettle({ redis, queues, comment, log, now, enqueue, held })` -> `settleHeld({ job, runId, result, error,
+ * queueName })`, with `settleHeld.take({ job, runId, queueName })`: what an awaitInput run's held replies become.
+ *
+ * THE TAKE, INSIDE THE PARK (issue #638 review, H1). The processor's parked branch calls `take` after it has set the
+ * marker and released the window and before its comment (processor.mjs, the park sequence). `closeHeld` first (one
+ * script: no reply can be held after it, and every reply held before it is in its reply); only the FIRST close acts:
+ *   - entries older than HELD_LOOKBACK_MS before now are dropped silently (`held_replies_dropped`, `stale`);
+ *   - the first recent entry whose fallback names this run's forge, repo, target type and number (it was held under
+ *     them; one that does not is skipped with `held_reply_mismatch` and the next is tried) is enqueued as the resumed job
+ *     WITH the semantic window (`enqueueForgeJob`), on the queue that ran the run (a host queue stays on its host). The
+ *     window is the point: the park released this run's window a moment before, so the taken reply now holds it, and
+ *     every later answer (a phrase-free reply the marker routes, a phrase reply) is coalesced against that one job
+ *     instead of starting a second resumed run from the same parked transcript. An add the window swallowed against
+ *     ANOTHER job (an answer that came in between the release and the take, which is then the run) is not taken;
+ *     against THIS run's own job (the release failed) it is not taken either, and is answered as not started;
+ *   - every other recent entry gets ONE fixed comment, HELD_EXTRA_REPLY; when nothing could be started at all, one
+ *     HELD_RESUME_FAILED_REPLY instead (also when the close itself failed: an answer may have been held and lost).
+ * The take's close also raises the target's parked marker. Returns `{ resumed, answered, dropped, taken }`; the
+ * processor posts HELD_TAKEN_COMMENT instead of the parked comment when `taken` is set ("here" or "other").
+ *
+ * THE SETTLE, AT THE FINAL OUTCOME (the inbox settle's three sites and `isFinalOutcome`), is the backstop. After a take
+ * the list is closed and it acts on nothing (`first: false`), and when the take's own close failed it only drops
+ * (`afterTake`). Otherwise: a parked outcome whose take never ran (an unwired seam) gets the take's logic; ANY other outcome drops every held reply, which is exactly the coalescing a reply
+ * into a still-running run always got. That includes an operator cancel acknowledged before the result: the record says
+ * the run was cancelled, not parked, so its held answers are dropped (review, L2).
+ *
+ * Logs, ids and counts only: `held_reply_resumed { jobId, runId, n, queue }`, `held_replies_dropped { runId, n,
+ * stale? }`, `held_reply_mismatch { runId }`, `held_reply_not_taken { runId, survivingJobId }`, `held_reply_resume_failed
+ * { runId, queue, code }`, `held_settle_failed { jobId, code }`. Never throws; the counts or null.
+ */
+export function makeHeldSettle({ redis, queues, comment = async () => {}, log = () => {}, now = Date.now, enqueue = enqueueForgeJob, held = heldModule } = {}) {
+	const queueFor = (name) => {
+		if (typeof name === "string" && name.startsWith(`${QUEUE}@`) && typeof queues?.host === "function") return { queue: queues.host(name), name };
+		return { queue: queues?.shared?.(), name: QUEUE };
+	};
+	const answer = async (job, text) => {
+		try {
+			await comment({ ...job.data, id: job.id }, text);
+		} catch {}
+	};
+	// The same forge, repo, target type and number, and (review round 2) the same flow and command: the held fallback was
+	// held under this run's target, and one that would run different work is no answer to this run's question.
+	const sameTarget = (fields, d) => !((fields?.kind !== undefined && fields.kind !== d.kind) || fields?.repo !== d.repo || typeof fields?.target?.type !== "string" || fields.target.type !== d.target?.type || targetNumber(fields.target) === null || targetNumber(fields.target) !== targetNumber(d.target) || fields.flow !== d.flow || fields.command !== d.command);
+	const close = async (job, runId, { parked = false } = {}) => {
+		const closed = await held.closeHeld({ redis, runId, targetKey: held.awaitTargetKeyOf(job.data), parked });
+		if (!closed || closed.error) {
+			log("held_settle_failed", { jobId: sanitizeJobId(job.id), code: codeOf(closed) });
+			return null;
+		}
+		return { entries: Array.isArray(closed.entries) ? closed.entries : [], first: closed.first === true };
+	};
+
+	// The take's body, on entries a first close returned.
+	const start = async ({ job, runId, entries, queueName }) => {
+		// `taken`: "here" when this take started the resumed job, "other" when the window showed another job already is
+		// the answer (one that got in between the release and the take), null otherwise. Either word means the run has
+		// its resumed job, so the processor says so and a late steer message must not start a second one (review round 2).
+		const counts = { resumed: 0, answered: 0, dropped: 0, taken: null };
+		if (entries.length === 0) return counts;
+		const rid = sanitizeJobId(runId);
+		const t = typeof now === "function" ? now() : now;
+		const recent = entries.filter((e) => Number.isSafeInteger(e?.at) && e.at >= t - HELD_LOOKBACK_MS);
+		const stale = entries.length - recent.length;
+		if (stale > 0) {
+			counts.dropped = stale;
+			log("held_replies_dropped", { runId: rid, n: stale, stale });
+		}
+		if (recent.length === 0) return counts;
+		const route = queueFor(queueName ?? job.queueName);
+		let takenAt = -1;
+		let failed = false;
+		let tried = false;
+		for (let i = 0; i < recent.length; i++) {
+			const fields = recent[i].fallback;
+			if (!sameTarget(fields, job.data)) {
+				log("held_reply_mismatch", { runId: rid });
+				continue;
+			}
+			tried = true;
+			try {
+				if (!route.queue) throw Object.assign(new Error("no queue"), { code: "no-queue" });
+				// The queue's enqueue timestamp is later than the receiver's trusted receipt. Keep that receipt
+				// outside the payload so a held answer does not acquire artificial human waiting time.
+				const r = await enqueue(route.queue, job.data.kind, { ...fields, replyReceivedAt: recent[i].at });
+				if (r?.deduplicated === true) {
+					// Swallowed by the window: another job holds it (an answer between the release and the take, which is
+					// then the resumed run), or this run's own (its release failed, so nothing was started).
+					log("held_reply_not_taken", { runId: rid, survivingJobId: sanitizeJobId(r.survivingJobId) });
+					// The run's own job id, or its run id for a #645 recovery whose original holds the window.
+					failed = r.survivingJobId === job.id || r.survivingJobId === runId;
+					if (!failed) counts.taken = "other";
+				} else {
+					takenAt = i;
+					counts.resumed = 1;
+					counts.taken = "here";
+					log("held_reply_resumed", { jobId: sanitizeJobId(r?.jobId), runId: rid, n: recent.length, queue: route.name });
+				}
+			} catch (e) {
+				log("held_reply_resume_failed", { runId: rid, queue: route.name, code: codeOf(e) });
+				failed = true;
+			}
+			break;
+		}
+		const rest = recent.length - (takenAt >= 0 ? 1 : 0);
+		if (rest > 0) {
+			// "Took the first reply" is true when one was taken here or by the job that holds the window; otherwise the
+			// replies were not started at all and say so.
+			await answer(job, takenAt >= 0 || (tried && !failed) ? HELD_EXTRA_REPLY : HELD_RESUME_FAILED_REPLY);
+			counts.answered = rest;
+		}
+		return counts;
+	};
+
+	// `afterTake`: the park's take ran for this outcome (the processor says so). A take whose close FAILED already told the
+	// person nothing was started (HELD_RESUME_FAILED_REPLY), so a backstop close that succeeds afterwards must not start a
+	// job anyway and contradict it (review round 2): after a take, the backstop only ever drops.
+	async function settleHeld({ job, runId = runIdOf(job), result = null, error = null, queueName = null, afterTake = false } = {}) {
+		try {
+			if (job?.data?.awaitInput !== true) return null;
+			if (!isFinalOutcome({ job, result, error })) return null;
+			if (runId === null || runId === undefined || runId === "") return null;
+			const closed = await close(job, runId);
+			if (closed === null) return null;
+			if (!closed.first || closed.entries.length === 0) return { resumed: 0, answered: 0, dropped: 0, taken: null };
+			if (isParkedOutcome(result) && afterTake !== true) return await start({ job, runId, entries: closed.entries, queueName });
+			log("held_replies_dropped", { runId: sanitizeJobId(runId), n: closed.entries.length });
+			return { resumed: 0, answered: 0, dropped: closed.entries.length, taken: null };
+		} catch (e) {
+			log("held_settle_failed", { jobId: sanitizeJobId(job?.id ?? ""), code: codeOf(e) });
+			return null;
+		}
+	}
+
+	settleHeld.take = async function takeHeld({ job, runId = runIdOf(job), queueName = null } = {}) {
+		try {
+			if (job?.data?.awaitInput !== true) return null;
+			if (runId === null || runId === undefined || runId === "") return null;
+			// `parked`: the close also raises the target's parked marker (held-replies.mjs), so a reply delivered after the run
+			// left its target is routed as a parked reply rather than dropped for the label it raced.
+			const closed = await close(job, runId, { parked: true });
+			if (closed === null) {
+				// L1: the close failed, so a reply may have been held and will never be seen: say so once.
+				await answer(job, HELD_RESUME_FAILED_REPLY);
+				return null;
+			}
+			if (!closed.first) return { resumed: 0, answered: 0, dropped: 0, taken: null };
+			return await start({ job, runId, entries: closed.entries, queueName });
+		} catch (e) {
+			log("held_settle_failed", { jobId: sanitizeJobId(job?.id ?? ""), code: codeOf(e) });
+			return null;
+		}
+	};
+	return settleHeld;
+}

package/src/hook-runs.mjs

@@ -0,0 +1,188 @@
+/**
+ * Display history, never a notification queue (INT-HOOK-RUNS-KEY, #775). Deleting hook:* changes no hook or job.
+ * Each completion appends one bounded, versioned record; the key expires seven days after the last append.
+ * The small memory drain absorbs simultaneous completions without making a hook wait for Valkey. It never retries:
+ * a failed append is lost and named. An operation that timed out still owns the drain until it ACTUALLY settles,
+ * because ioredis queues commands while offline and a timeout alone cannot bound that queue.
+ */
+import { publishedJobId, LIVE_JOB_ID_RE } from "./live-jobs.mjs";
+import { WORKER_NAME_RE } from "./worker-name.mjs";
+import { RECORD_REASON_RE } from "./record-reason.mjs";
+
+export const HOOK_HOSTS = "hook:hosts";
+export const hookRunsKey = (name) => `hook:h:${name}`;
+export const HOOK_RUN_VERSION = 1;
+export const HOOK_RUNS_MAX = 20;
+export const HOOK_RUN_BYTES = 1024;
+export const HOOK_HOSTS_MAX = 64;
+export const HOOK_PENDING_MAX = 64;
+export const HOOK_RUNS_TTL_MS = 7 * 24 * 60 * 60 * 1000;
+/** The portable union, not this reader's platform: a Linux worker may be read from macOS or Windows. */
+export const HOOK_SIGNALS = Object.freeze(["SIGABRT", "SIGALRM", "SIGBUS", "SIGCHLD", "SIGCLD", "SIGCONT", "SIGEMT", "SIGFPE", "SIGHUP", "SIGILL", "SIGINFO", "SIGINT", "SIGIO", "SIGIOT", "SIGKILL", "SIGLOST", "SIGPIPE", "SIGPOLL", "SIGPROF", "SIGPWR", "SIGQUIT", "SIGSEGV", "SIGSTKFLT", "SIGSTOP", "SIGSYS", "SIGTERM", "SIGTRAP", "SIGTSTP", "SIGTTIN", "SIGTTOU", "SIGUNUSED", "SIGURG", "SIGUSR1", "SIGUSR2", "SIGVTALRM", "SIGWINCH", "SIGXCPU", "SIGXFSZ", "SIGBREAK"]);
+export const HOOK_DETAILS = Object.freeze(["exit", "timeout", "spawn", "unresolvable", "argv-unusable", "signal-unknown", ...HOOK_SIGNALS.map((signal) => `signal-${signal}`)]);
+const instant = (value) => Number.isSafeInteger(value) && value >= 0;
+// Windows reports the full unsigned process status (including NTSTATUS), not POSIX's low eight bits.
+const codeOf = (value) => Number.isInteger(value) && value >= 0 && value <= 0xffffffff ? value : null;
+const outcomeOf = (value) => value === "failed" || value === "policy" ? value : null;
+const reasonOf = (value) => typeof value === "string" && RECORD_REASON_RE.test(value) ? value : "infra";
+
+export function buildHookRun(fields, at) {
+	return { v: HOOK_RUN_VERSION, at, jobId: publishedJobId(fields?.jobId), outcome: outcomeOf(fields?.outcome), reason: reasonOf(fields?.reason), detail: HOOK_DETAILS.includes(fields?.detail) ? fields.detail : "signal-unknown", code: codeOf(fields?.code) };
+}
+
+/** Another host's text is projected again, including records passed directly to the model boundary. */
+export function parseHookRun(raw) {
+	if (typeof raw !== "string" || Buffer.byteLength(raw) > HOOK_RUN_BYTES) return null;
+	let value;
+	try { value = JSON.parse(raw); } catch { return null; }
+	if (!value || value.v !== HOOK_RUN_VERSION || !instant(value.at) || !(value.jobId === null || typeof value.jobId === "string" && LIVE_JOB_ID_RE.test(value.jobId)) || !(value.outcome === null || outcomeOf(value.outcome)) || typeof value.reason !== "string" || !RECORD_REASON_RE.test(value.reason) || !HOOK_DETAILS.includes(value.detail) || !(value.code === null || codeOf(value.code) === value.code)) return null;
+	return { v: HOOK_RUN_VERSION, at: value.at, jobId: value.jobId, outcome: value.outcome, reason: value.reason, detail: value.detail, code: value.code };
+}
+
+function bounded(promise, timeoutMs = 2000) {
+	return new Promise((resolve, reject) => {
+		const timer = setTimeout(() => reject(Error("timeout")), timeoutMs);
+		Promise.resolve(promise).then((value) => { clearTimeout(timer); resolve(value); }, (error) => { clearTimeout(timer); reject(error); });
+	});
+}
+const validName = (name) => typeof name === "string" && WORKER_NAME_RE.test(name);
+
+/** No message from a client, record, executable or environment reaches this diagnostic. */
+export function makeHookRunsPublisher({ redis, name, now = Date.now, log = () => {}, timeoutMs = 2000 }) {
+	const pending = [];
+	let draining = false;
+	let closed = false;
+	let pruneAt = null;
+	let pruneCursor = "0";
+	const note = (reason) => { try { log("hook_runs_publish_failed", { reason }); } catch {} };
+	const prune = async (at) => {
+		if (closed || pruneAt !== null && at - pruneAt < 15_000) return;
+		pruneAt = at;
+		const found = await redis.sscan(HOOK_HOSTS, pruneCursor, "COUNT", HOOK_HOSTS_MAX);
+		if (!Array.isArray(found) || typeof found[0] !== "string" || !Array.isArray(found[1])) return;
+		pruneCursor = found[0];
+		const names = found[1].slice(0, HOOK_HOSTS_MAX).filter((member) => validName(member) && member !== name);
+		if (closed || !names.length) return;
+		const pipe = redis.pipeline();
+		// Checking and removing together cannot erase an index entry another host has just republished.
+		for (const member of names) pipe.eval("if redis.call('EXISTS', KEYS[1]) == 0 then return redis.call('SREM', KEYS[2], ARGV[1]) end return 0", 2, hookRunsKey(member), HOOK_HOSTS, member);
+		const result = await pipe.exec();
+		if (!Array.isArray(result) || result.length !== names.length || result.some((row) => !Array.isArray(row) || row[0])) throw Error("unavailable");
+	};
+	const drain = async () => {
+		draining = true;
+		while (!closed && pending.length) {
+			const record = pending.shift();
+			let said = false;
+			const fail = () => { if (!said && !closed) { said = true; note("unavailable"); } };
+			const timer = setTimeout(fail, timeoutMs);
+			timer.unref?.();
+			try {
+				const key = hookRunsKey(name);
+				const tx = redis.multi().lpush(key, JSON.stringify(record)).ltrim(key, 0, HOOK_RUNS_MAX - 1).pexpire(key, HOOK_RUNS_TTL_MS).sadd(HOOK_HOSTS, name);
+				const answer = await tx.exec();
+				if (!Array.isArray(answer) || answer.length !== 4 || answer.some((row) => !Array.isArray(row) || row[0])) fail();
+				else await prune(record.at);
+			} catch { fail(); }
+			finally { clearTimeout(timer); }
+		}
+		draining = false;
+	};
+	return {
+		publish(fields) {
+			if (closed) return;
+			if (!validName(name)) { note("invalid-host"); return; }
+			if (pending.length + (draining ? 1 : 0) >= HOOK_PENDING_MAX) { note("full"); return; }
+			let at;
+			try { at = now(); } catch { note("invalid-time"); return; }
+			if (!instant(at)) { note("invalid-time"); return; }
+			pending.push(buildHookRun(fields, at));
+			if (!draining) void drain();
+		},
+		close() { closed = true; pending.length = 0; },
+	};
+}
+
+/**
+ * Bounded discovery: live names first, then at most four SSCAN pages, no SMEMBERS. COUNT is a hint, so even a large
+ * response is cut before issuing row commands. SCARD reports an incomplete index walk without calling it empty.
+ * A lost index still reads known live hosts; an unreadable index makes that answer partial, never complete.
+ */
+async function hostNames(redis, names, timeoutMs, stopped) {
+	const live = [...new Set(names.filter(validName))].sort();
+	const wanted = new Set(live.slice(0, HOOK_HOSTS_MAX));
+	const indexed = new Set();
+	const members = new Set();
+	let total = 0;
+	let partial = live.length > HOOK_HOSTS_MAX;
+	let indexFailed = false;
+	let cursor = "0";
+	try {
+		total = await bounded(redis.scard(HOOK_HOSTS), timeoutMs);
+		if (!Number.isSafeInteger(total) || total < 0) throw Error("invalid");
+		if (total > 0) for (let page = 0; page < 4 && !stopped(); page++) {
+			const result = await bounded(redis.sscan(HOOK_HOSTS, cursor, "COUNT", HOOK_HOSTS_MAX), timeoutMs);
+			if (!Array.isArray(result) || typeof result[0] !== "string" || !Array.isArray(result[1])) throw Error("invalid");
+			cursor = result[0];
+			for (const name of result[1].slice(0, HOOK_HOSTS_MAX * 4)) {
+				members.add(name);
+				if (validName(name)) { indexed.add(name); if (wanted.size < HOOK_HOSTS_MAX) wanted.add(name); }
+			}
+			if (cursor === "0" || wanted.size === HOOK_HOSTS_MAX) break;
+		}
+		partial ||= cursor !== "0" || total !== members.size || indexed.size !== members.size;
+	} catch { partial = true; indexFailed = true; }
+	// With an incomplete walk the unseen members may overlap the omitted live names. Count only the union we SAW,
+	// explicitly a floor; adding the two remainders would count a large fleet's same hosts twice.
+	const more = Math.max(0, new Set([...live, ...indexed]).size - wanted.size);
+	const moreUnknown = indexFailed || cursor !== "0" || total !== members.size;
+	return { names: [...wanted], more, moreUnknown, partial: partial || more > 0 };
+}
+
+/** Readers write nothing. Expired index members are pruned by the next publisher, not by a console request. */
+export async function readHookRuns(redis, { names = [], timeoutMs = 2000 } = {}) {
+	let stopped = false;
+	// Register the WHOLE-read timer before issuing any command. Per-command bounds alone can turn four slow
+	// discovery pages and one row read into ten seconds; late settlement must never start follow-up commands.
+	const work = Promise.resolve().then(() => readHookRunsInner(redis, names, timeoutMs, () => stopped));
+	try { return await bounded(work, timeoutMs); }
+	catch { return { hosts: [], more: 0, moreUnknown: true, partial: true }; }
+	finally { stopped = true; }
+}
+
+async function readHookRunsInner(redis, names, timeoutMs, stopped) {
+	const selected = await hostNames(redis, names, timeoutMs, stopped);
+	if (stopped()) return { hosts: [], more: selected.more, moreUnknown: true, partial: true };
+	const hosts = await Promise.all(selected.names.map(async (name) => {
+		try {
+			const key = hookRunsKey(name);
+			const [raw, ttl] = await Promise.all([bounded(redis.lrange(key, 0, HOOK_RUNS_MAX), timeoutMs), bounded(redis.pttl(key), timeoutMs)]);
+			if (!Array.isArray(raw) || raw.length > HOOK_RUNS_MAX || !Number.isSafeInteger(ttl)) throw Error("invalid");
+			if (raw.length === 0 && ttl === -2) return { name, runs: [], absent: true };
+			if (ttl <= 0 || ttl > HOOK_RUNS_TTL_MS || raw.length === 0) throw Error("invalid");
+			const runs = raw.map(parseHookRun);
+			if (runs.some((record) => record === null)) throw Error("invalid");
+			return { name, runs };
+		} catch { return { name, runs: [], unreadable: true }; }
+	}));
+	return { hosts: hosts.sort((a, b) => a.name.localeCompare(b.name)), more: selected.more, moreUnknown: selected.moreUnknown, partial: selected.partial || hosts.some((host) => host.unreadable) };
+}
+
+/** Public display/model DTO. Recomputing counts prevents a polluted DTO inventing a successful hook history. */
+export function hookRunsView(value) {
+	const hosts = [];
+	const input = Array.isArray(value?.hosts) ? value.hosts : [];
+	const omitted = Math.max(0, input.length - HOOK_HOSTS_MAX);
+	let partial = value?.partial !== false || omitted > 0;
+	const seen = new Set();
+	for (const host of input.slice(0, HOOK_HOSTS_MAX)) {
+		if (!validName(host?.name) || seen.has(host.name)) { partial = true; continue; }
+		seen.add(host.name);
+		const raw = Array.isArray(host.runs) ? host.runs : [];
+		const runs = raw.slice(0, HOOK_RUNS_MAX).map((run) => { try { return parseHookRun(JSON.stringify(run)); } catch { return null; } });
+		const unreadable = host.unreadable === true || !Array.isArray(host.runs) || raw.length > HOOK_RUNS_MAX || runs.some((run) => !run);
+		const counts = unreadable ? null : Object.fromEntries([...new Set(runs.map((run) => run.detail))].map((detail) => [detail, runs.filter((run) => run.detail === detail).length]));
+		hosts.push({ name: host.name, unreadable, absent: !unreadable && (host.absent === true || runs.length === 0), runs: unreadable ? [] : runs, counts });
+	}
+	return { hosts, more: (Number.isSafeInteger(value?.more) && value.more >= 0 ? value.more : 0) + omitted, moreUnknown: value?.moreUnknown === true, partial: partial || hosts.some((host) => host.unreadable) };
+}

package/src/host-budget.mjs

@@ -45,8 +45,8 @@ export const HOST_BUDGET_KEYS = Object.freeze({
 });
 
 /**
- * How often a job the budget deferred asks again. Its own value, like every re-check cadence, because nothing records WHY
- * a job sits in the delayed set and the wake instant is the only evidence there is: distinct from the scope re-check
+ * How often a job the budget deferred asks again. Its own value, like every re-check cadence, so the wake instants of the
+ * different deferrals stay apart (the reason itself is recorded, `defer-reasons.mjs`): distinct from the scope re-check
  * (5 s), the endpoint re-check (7 s), the wait throttle floor (11 s) and the supersede re-ask (15 s). A test keeps them
  * apart. Short, because a hold keeps the room meanwhile: the wait is the latency of a freed slot, not a queue position.
  */
@@ -306,7 +306,7 @@ export function rankHolds(waiters, ledger, minJobsOf = () => 0) {
 
 /**
  * THE ADMISSION RULE (PURE): may `ask` (`{ id, project, memMiB, cpuCenti }`) start now? `{ ok: true }`, or `{ ok: false,
- * why }` with `why` `share` (its project would hold more than its `hostShare` of the budget) or `budget` (what runs, plus
+ * why }` with `why` `share` (its project would hold more than its `hostShare` of the budget) or `memory`, `cpu`, `count` (what runs, plus
  * `ask`, plus every hold ranked above it, does not fit in memory, in CPU or in the job COUNT). `holds` is `rankHolds`'
  * order; a job that holds no hold has every hold above it. The share is judged first, so a job its own project's share
  * stops is not reported as waiting for the budget (and so does not hold room it could not use).
@@ -328,7 +328,10 @@ export function admit({ budget, ledger, holds, ask, shareOf = () => null }) {
 	const counted = [...ledger, ask, ...(rank === -1 ? holds : holds.slice(0, rank))];
 	const need = sumOf(counted);
 	const count = Number.isSafeInteger(budget.count) ? budget.count : null;
-	if (!fits(need.memMiB, budget.memMiB) || !fits(need.cpuCenti, budget.cpuCenti) || !fits(counted.length, count)) return { ok: false, why: "budget" };
+	// A stable order explains the first blocked dimension without changing admission or ranked holds.
+	if (!fits(need.memMiB, budget.memMiB)) return { ok: false, why: "memory" };
+	if (!fits(need.cpuCenti, budget.cpuCenti)) return { ok: false, why: "cpu" };
+	if (!fits(counted.length, count)) return { ok: false, why: "count" };
 	return { ok: true };
 }
 
@@ -643,7 +646,7 @@ export function makeHostBudget({ settings, jobDefault = { memMiB: 4096, cpuCenti
 		/**
 		 * THE GATE, synchronous: `{ admitted: true }` with the hold taken under `ticket`, or `{ admitted: false, why }`:
 		 * `unseeded` (the boot listing of the job's `venue` is not read yet) and `running-here` (the ledger already holds
-		 * this id, or an orphan carries the container `name` this pickup will use) make no waiter; `budget` and `share`
+		 * this id, or an orphan carries the container `name` this pickup will use) make no waiter; dimension refusals and `share`
 		 * keep (or make) the job a waiter. `getState` is the job's own, kept for `verify`.
 		 */
 		gate({ id, ticket = null, project = null, size, venue = null, name = null, getState = null, limits = scopedLimits() }) {

package/src/host-clone.mjs

@@ -0,0 +1,72 @@
+/** Shared host-clone primitives. The credential helper lives outside every agent-reachable tree (#771). */
+import { execFile } from "node:child_process";
+import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, win32 } from "node:path";
+import { promisify } from "node:util";
+import { GIT_SAFE_CONFIG, storeGitEnv } from "./git-hardening.mjs";
+
+const exec = promisify(execFile);
+export const HOST_CLONE_FLAGS = Object.freeze([...GIT_SAFE_CONFIG, "-c", "protocol.ext.allow=never", "-c", "credential.helper=", "--no-pager"]);
+export const hardenedCloneArgs = (args) => [...HOST_CLONE_FLAGS, ...args];
+
+export function createAskpassHelper() {
+	const dir = mkdtempSync(join(tmpdir(), "pi-askpass-"));
+	const path = join(dir, "askpass.sh");
+	try {
+		writeFileSync(path, "#!/bin/sh\nprintf '%s' \"$GIT_ASKPASS_TOKEN\"\n", { mode: 0o700 });
+		chmodSync(path, 0o700);
+		return { dir, path };
+	} catch (error) {
+		rmSync(dir, { recursive: true, force: true });
+		throw error;
+	}
+}
+
+export async function defaultHostGit(cwd, args, { env, signal, timeout } = {}) {
+	const { stdout } = await exec("git", args, { cwd, encoding: "utf8", maxBuffer: 16 * 1024 * 1024, env, ...(timeout ? { timeout } : {}), ...(signal ? { signal } : {}) });
+	return stdout;
+}
+
+/** The determinate missing-ref class, shared with the job preparer's existing refusal. */
+export function isCloneRefGone(error) {
+	const text = `${error?.stderr ?? ""}\n${error?.message ?? ""}`.toLowerCase();
+	return ["couldn't find remote ref", "not our ref", "unadvertised object", "did not send all necessary objects", "reference is not a tree"].some((part) => text.includes(part));
+}
+
+/** storeGitEnv's trust switches are shared; Windows keeps absolute drive/UNC tool paths with its own separator. */
+export function hostCloneEnvironment({ home, path, platform = process.platform }) {
+	const clean = storeGitEnv({ home, path });
+	delete clean.GIT_DIR;
+	if (platform === "win32") clean.PATH = String(path ?? "").split(";").filter((entry) => win32.isAbsolute(entry)).join(";");
+	return clean;
+}
+
+/**
+ * A credential-free checkout after the one authenticated fetch. No materialisation, prompt, session or agent inputs.
+ * The empty helper home and allowlisted environment also stop a repository's attributes selecting a host filter.
+ */
+export async function cloneHostWorkspace({ workspace, remote, ref, token, env = process.env, git = defaultHostGit, askpass = createAskpassHelper, platform = process.platform, signal } = {}) {
+	const url = new URL(remote);
+	if (!["https:", "http:"].includes(url.protocol) || url.username || url.password || url.search || url.hash) throw new Error("sandbox clone requires a credential-free configured http(s) remote");
+	const helper = askpass();
+	try {
+		const home = join(helper.dir, "home");
+		mkdirSync(home, { mode: 0o700 });
+		mkdirSync(workspace, { recursive: true, mode: 0o700 });
+		// env-internal PATH: use only absolute host tool directories, as the checkpoint-store Git does.
+		const clean = hostCloneEnvironment({ home, path: env.PATH, platform });
+		// env-internal GIT_SSL_CAINFO: an operator's private CA trust input, never repository content or a container env.
+		if (typeof env.GIT_SSL_CAINFO === "string") clean.GIT_SSL_CAINFO = env.GIT_SSL_CAINFO;
+		const run = (args, net = false) => git(workspace, hardenedCloneArgs(args), { timeout: 120_000, env: net ? { ...clean, GIT_ASKPASS: helper.path, GIT_ASKPASS_TOKEN: token } : clean, ...(signal ? { signal } : {}) });
+		await run(["init"]);
+		await run(["remote", "add", "origin", remote]);
+		await run(["fetch", "--depth=1", "--no-tags", "--no-recurse-submodules", "origin", ref], true);
+		await run(["checkout", "--detach", "FETCH_HEAD"]);
+		const sha = String(await run(["rev-parse", "HEAD"])).trim();
+		if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(sha)) throw new Error("sandbox clone returned no commit id");
+		return { workspace, sha };
+	} finally {
+		rmSync(helper.dir, { recursive: true, force: true });
+	}
+}