@edgehero/pi-dispatch 4.3.0 → 5.0.0

package/src/cancel-cli.mjs

@@ -8,6 +8,8 @@
  *   held on run.waitFor  -> the shared removeHeldJob sequence (hold keys first, then the job); no record
  *                           (the dispatch_wait_cancel rule).
  *   delayed / queued     -> job.remove(); no record, same rule.
+ *                           Both through `removeQueuedJob` (cancel-state.mjs, issue #737), the sequence the
+ *                           console's cancel-queued shares, which also deletes the removed job's defer record.
  *   active               -> a `cancel:req` key + a brief ack poll (cancel-state.mjs), because the abort can
  *                           only be raised by the process that holds the job. No ack within the window is a
  *                           NAMED failure, never a silent one: the job may be on a host that is down, or on
@@ -22,17 +24,29 @@
  * job's attempt count.
  */
 
-import { cancelReqKey, ranBefore, removeHeldJob, requestCancel } from "./cancel-state.mjs";
-import { jobKey } from "./wait-state.mjs";
+import { cancelReqKey, ranBefore, removeQueuedJob, requestCancel } from "./cancel-state.mjs";
 
 /** Same budget the kill switch gives the host registry before acting on what the keyspace alone says. */
 const FLEET_READ_TIMEOUT_MS = 2_000;
 
+/**
+ * A sentence another module wrote (BullMQ's, a client's, Valkey's own: of any length, holding any bytes), as a line of
+ * this command carries it to stderr, which is a terminal: every control character a terminal acts on (C0, DEL and C1:
+ * an ESC sequence, a CSI with no ESC, a bell, a carriage return that overwrites the line) made a space, then cut at 200
+ * characters. The console carries such text by the same substitution and the same cut (`boundedText` in the admin's
+ * job-actions.mjs, over its `scrubControls`), which a test holds this to on the class a terminal executes. The
+ * console's class is wider (it also substitutes what draws as nothing, or as a blank that is not a space, for a
+ * reader), and it is asked of the admin's own width table, which the worker does not import.
+ */
+export function boundedText(text) {
+	return String(text ?? "no reason given").replace(/[\u0000-\u001f\u007f-\u009f]/g, " ").slice(0, 200);
+}
+
 /**
  * Run the cancel. Returns the process exit code. Every collaborator is a seam with the production default,
  * so tests drive states and races without a queue; `write` is the stdout seam cli.mjs already injects.
  */
-export async function runCancel(jobId, url, { write = (chunk) => process.stdout.write(chunk), errWrite = (chunk) => process.stderr.write(chunk), redisFn, queueFn, parseConnectionFn, readLiveHostsFn, discoverHostQueuesFn, fleetQueueNamesFn, unionQueueNamesFn, ackTimeoutMs = 10_000, pollMs = 250, sleep, refusalFn } = {}) {
+export async function runCancel(jobId, url, { write = (chunk) => process.stdout.write(chunk), errWrite = (chunk) => process.stderr.write(chunk), redisFn, queueFn, parseConnectionFn, readLiveHostsFn, discoverHostQueuesFn, fleetQueueNamesFn, unionQueueNamesFn, ackTimeoutMs = 10_000, pollMs = 250, sleep, refusalFn, touchStore, touchOutbox = null, now = Date.now } = {}) {
 	const fail = (message) => {
 		errWrite(`error: ${message}\n`);
 		return 1;
@@ -98,45 +112,44 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 			}
 		}
 		if (!job) {
-			return fail(`no job ${jobId} in ${names.length} queue(s)${blind ? ` [registry unreadable: ${blind} — a host that is not registered may still hold it]` : ""}`);
+			return fail(`no job ${jobId} in ${names.length} queue(s)${blind ? ` [registry unreadable: ${boundedText(blind)} — a host that is not registered may still hold it]` : ""}`);
 		}
 
-		// Held FIRST (see the header). The hash tells held apart from plain-delayed; `since` is the clock
-		// that makes it a real hold rather than a counter key.
-		const hash = await probe.hgetall(jobKey(jobId)).catch(() => null);
-		if (hash?.since) {
-			const res = await removeHeldJob({ redis: probe, queue: owner, jobId });
-			if (res.ok) {
-				// A hold consumes no attempt (`skipAttempt`), so a held job that ran is a retry held again by its conditions.
-				write(`cancelled ${jobId}: it was held on its wait condition, and ${ranBefore(res)}\n`);
-				return 0;
-			}
-			return fail(`could not cancel held job ${jobId}: ${res.invalid}`);
+		// Held FIRST, then a job waiting to start: the shared sequence (`removeQueuedJob`, cancel-state.mjs), which the
+		// console's cancel-queued runs too. It judges the hold by the `wait:job` hash's `since`, the clock that makes it a
+		// real hold rather than a counter key, and deletes the removed job's defer record. This verb's reach is its own:
+		// it removes a job waiting to retry as well, which the console refuses.
+		const queued = await removeQueuedJob({ redis: probe, queue: owner, job, jobId, touchStore, touchOutbox, now });
+		if (queued.ok && queued.touchWarning) errWrite("warning: the job was removed; touch tracking is incomplete. Do not repeat the cancellation.\n");
+		if (queued.ok && queued.held) {
+			// A hold consumes no attempt (`skipAttempt`), so a held job that ran is a retry held again by its conditions.
+			write(`cancelled ${jobId}: it was held on its wait condition, and ${ranBefore(queued)}\n`);
+			return 0;
 		}
-
-		let state = await job.getState().catch(() => "unknown");
-		if (state === "delayed" || state === "waiting" || state === "prioritized" || state === "paused") {
-			try {
-				await job.remove();
-				write(`removed ${jobId} (${state}): ${ranBefore(job)}\n`);
-				return 0;
-			} catch (error) {
-				// The one race worth handling by name: picked up between getState and remove. remove() throws
-				// on a locked job, so re-read and fall through to the active path rather than failing an
-				// operator whose job is now exactly the case the active path exists for.
-				state = await job.getState().catch(() => "unknown");
-				// The re-read state and the throw's own words together, labelled apart: the error came from
-				// the remove attempt, the state from after it, and gluing them unlabelled read as one
-				// mismatched diagnosis (review finding).
-				if (state !== "active") return fail(`could not remove ${jobId} — it is now ${state}; the remove failed with: ${error.message}`);
-			}
+		// A removal that failed after the hold's keys were deleted changed something: the job lost its hold and was not removed,
+		// so the line says so and how to finish, never only the error (which reads as "nothing happened"). Every error a
+		// line carries is bounded (`boundedText`): it is another module's words, of any length and with any bytes.
+		if (queued.invalid !== undefined && queued.cleared === true) {
+			return fail(`could not cancel held job ${jobId}: its hold was cleared, but the job itself could not be removed (${boundedText(queued.invalid)}); it may still be queued, now without its hold. Run \`pi-dispatch cancel ${jobId}\` again to remove or stop it`);
+		}
+		if (queued.invalid !== undefined) return fail(`could not cancel held job ${jobId}: ${boundedText(queued.invalid)}`);
+		if (queued.ok) {
+			write(`removed ${jobId} (${queued.state}): ${ranBefore(job)}\n`);
+			return 0;
 		}
+		const state = queued.state;
+		// The one race worth handling by name: picked up between getState and remove. remove() throws on a locked job, so
+		// the state is re-read and an active job falls through to the active path rather than failing an operator whose
+		// job is now exactly the case the active path exists for. Otherwise the re-read state and the throw's own words
+		// together, labelled apart: the error came from the remove attempt, the state from after it, and gluing them
+		// unlabelled read as one mismatched diagnosis.
+		if (queued.raced && state !== "active") return fail(`could not remove ${jobId} — it is now ${state}; the remove failed with: ${boundedText(queued.error.message)}`);
 
 		if (state === "active") {
 			requested = true;
 			const res = await requestCancel({ redis: probe, jobId, ackTimeoutMs, pollMs, ...(sleep ? { sleep } : {}) });
 			if (res.ack !== undefined) {
-				write(`cancel accepted by ${res.ack === "" ? "the worker" : res.ack} — stopping the container (allow ~30s); the run record will say operator-cancel\n`);
+				write(`cancel accepted by ${res.ack === "" ? "the worker" : boundedText(res.ack)} — stopping the container (allow ~30s); the run record will say operator-cancel\n`);
 				return 0;
 			}
 			// Gate round 2 of PR #479: the worker stops its cancel poll before it holds or retries a job, so a request that
@@ -165,7 +178,7 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 			]);
 			clearTimeout(bound);
 		}
-		return fail(error?.valkeyRefused ? error.message : `could not reach Valkey at ${(await import("./connection.mjs")).urlShown(url)}: ${(await import("./valkey-auth.mjs")).valkeyDownHint(url)}\n  ${error.message}`);
+		return fail(error?.valkeyRefused ? error.message : `could not reach Valkey at ${(await import("./connection.mjs")).urlShown(url)}: ${(await import("./valkey-auth.mjs")).valkeyDownHint(url)}\n  ${boundedText(error?.message)}`);
 	} finally {
 		probe.disconnect?.();
 		for (const q of queues) await q.close().catch(() => {});

package/src/cancel-state.mjs

@@ -1,3 +1,17 @@
+import { makeTouchStore, touchScopeOf } from "./touch-state.mjs";
+import { trackTouchAction } from "./touch-outbox.mjs";
+import { isProjectId } from "./project-id.mjs";
+
+const isWorkload = (job) => job.data?.kind !== "question-grouping" && !["zero-touch-grouping", "answer-draft"].includes(job.data?.purpose);
+
+function stopTouch(job, queue, jobId, now) {
+	const data = job.data ?? {};
+	const owner = /^pi-jobs@([A-Za-z0-9._-]{1,64})$/.exec(queue?.name ?? "")?.[1] ?? null;
+	// A shared-queue local job has no owning machine yet. Hashing an empty host manufactures a different
+	// folder identity from its eventual run, so leave that attribution unknown rather than claiming a mismatch.
+	const knownScope = /^[a-f0-9]{64}$/.test(data.touchScopeId ?? "") || !(data.kind === "local" || data.folder) || owner !== null;
+	return { type: "stop", runId: jobId, at: now(), ...(knownScope ? { scopeId: touchScopeOf(data, owner) } : {}), ...(isProjectId(data.projectId) ? { project: data.projectId } : {}), nonce: `stop:${jobId}` };
+}
 /**
  * The `cancel:` keyspace: how an operator's cancel reaches the worker that owns a running job (issue #287).
  *
@@ -24,8 +38,19 @@
  * silently loses an operator's edit. This is a transient, attended, TTL-bounded one-shot whose loss is
  * REPORTED: the requester is watching the ack poll, and a request that reaches nobody comes back as a
  * named timeout, never as silence.
+ *
+ * BESIDE IT, THE `defer:` KEYSPACE (issue #737), which this module deletes from and never writes:
+ *
+ *   defer:job:<jobId>         HASH, why the worker put a waiting job back on the delayed set
+ *                             (`defer-reasons.mjs` holds the fields and their rules, `defer-write.mjs` writes
+ *                             it, with its own TTL). `removeQueuedJob` deletes it once the job is removed, so a
+ *                             removed job leaves no reason behind. The TTL is the backstop for a delete that
+ *                             fails, and a record without its job is never shown: every reader starts from
+ *                             the queue's own sets.
  */
 
+import { deferKey } from "./defer-reasons.mjs";
+import { dropKey, dropModelAttempt, modelAttemptIdOf } from "./model-attempts.mjs";
 import { HELD_SET, jobKey, leaseKey } from "./wait-state.mjs";
 
 /** How long an unanswered request may sit before redis reaps it (the requester deletes it sooner). */
@@ -36,6 +61,21 @@ export const CANCEL_ACK_TTL_MS = 5 * 60_000;
 
 export const cancelReqKey = (jobId) => `cancel:req:${jobId}`;
 export const cancelAckKey = (jobId) => `cancel:ack:${jobId}`;
+// Separate keys make an older worker fail closed: it cannot mistake an agent request for a human's.
+export const agentCancelReqKey = (jobId) => `cancel:agent:req:${jobId}`;
+export const agentCancelAckKey = (jobId) => `cancel:agent:ack:${jobId}`;
+
+const cancelRequestKeyOf = (jobId, interventionOrigin) => (interventionOrigin === "agent" ? agentCancelReqKey(jobId) : cancelReqKey(jobId));
+
+/**
+ * Takes back a request whose answer was lost, for a caller that goes on to report a failure (the console's fleet reader; the
+ * CLI's catch does the same under its own bound). A SET with no answer can still land: a Valkey that stopped answering holds
+ * it on its socket and runs it when it wakes, so the DEL goes right behind it, in order. Not awaited: on the same outage it
+ * has its own bound to wait out, and the caller already has its answer.
+ */
+export function withdrawCancel({ redis, jobId, interventionOrigin = "human" }) {
+	Promise.resolve().then(() => redis.del(cancelRequestKeyOf(jobId, interventionOrigin))).catch(() => {});
+}
 
 /**
  * Ask whichever worker holds `jobId` to cancel it, and wait briefly for the answer.
@@ -48,17 +88,32 @@ export const cancelAckKey = (jobId) => `cancel:ack:${jobId}`;
  *
  * `pollMs`/`sleep` are injectable so tests never wait on a wall clock.
  */
-export async function requestCancel({ redis, jobId, ackTimeoutMs = 10_000, pollMs = 250, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) }) {
-	await redis.set(cancelReqKey(jobId), "operator-cancel", "PX", CANCEL_REQ_TTL_MS);
+export async function requestCancel({ redis, jobId, interventionOrigin = "human", ackTimeoutMs = 10_000, pollMs = 250, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) }) {
+	const requestKey = cancelRequestKeyOf(jobId, interventionOrigin);
+	const ackKey = interventionOrigin === "agent" ? agentCancelAckKey(jobId) : cancelAckKey(jobId);
+	await redis.set(requestKey, interventionOrigin === "agent" ? "agent-cancel" : "operator-cancel", "PX", CANCEL_REQ_TTL_MS);
 	for (let waited = 0; ; waited += pollMs) {
-		const host = await redis.get(cancelAckKey(jobId));
+		let host;
+		try {
+			host = await redis.get(ackKey);
+		} catch (err) {
+			// The request went out and its answer is what was lost, so no caller may say nothing was changed.
+			if (err && typeof err === "object") err.valkeyMayStillRun = true;
+			throw err;
+		}
 		if (host !== null && host !== undefined) return { ack: host };
 		if (waited + pollMs > ackTimeoutMs) break;
 		await sleep(pollMs);
 	}
-	// Best-effort: losing this DEL to a blip leaves the 60s TTL as the backstop, and the worker-side GET
-	// racing it by one tick is a named residual (the operator is told to re-check `status`).
-	await redis.del(cancelReqKey(jobId)).catch(() => {});
+	// Losing this DEL leaves the request standing for its 60s TTL, where a worker can still read it, so that is no "nothing was
+	// changed" and it rejects, marked, as a lost answer does. The worker-side GET racing the DEL by one tick is a named residual
+	// (the operator is told to re-check `status`).
+	try {
+		await redis.del(requestKey);
+	} catch (err) {
+		if (err && typeof err === "object") err.valkeyMayStillRun = true;
+		throw err;
+	}
 	return { timeout: true };
 }
 
@@ -80,18 +135,42 @@ export function ranBefore(job) {
 		: `it made ${made} attempts before and was waiting to retry; whatever those attempts recorded stays, and cancel writes none`;
 }
 
+/**
+ * Whether the hold is still there, read back after its delete failed: its CLOCK (`since`), the field that makes the hash
+ * a hold, since the worker's own counters can re-create the key without it. False when it cannot be read.
+ */
+async function holdInPlace(redis, jobId) {
+	if (typeof redis?.hexists !== "function") return false;
+	try {
+		return (await redis.hexists(jobKey(jobId), "since")) === 1;
+	} catch {
+		return false;
+	}
+}
+
 /**
  * Remove a job that is held on a `run.waitFor` condition: hold keys first, then the job (issue #230's
  * sequence, issue #287's shared home). This is the inner body of the admin's `cancelHeldJob`, moved here
  * so the CLI's held path and the panel's are ONE sequence rather than two copies that can drift -- the
  * read-model's own rule is that key derivations are imported from the worker, never re-implemented.
  *
- * Returns `{ ok: true, jobId, attemptsMade }` or `{ invalid: <reason> }`; never throws. Writes no run record of its
- * own (INT-RUN-HISTORY-FILE-CONTRACT records terminal states of runs, and a removed hold is none); `attemptsMade` is
- * what the job had done before, for the caller's sentence (`ranBefore`, issue #477): a job held again on a retry has
- * made attempts, and whatever they recorded stays.
+ * Returns `{ ok: true, jobId, attemptsMade }`, `{ invalid: <reason> }` (refused, or failed before anything was
+ * deleted: nothing was changed), or `{ invalid: <reason>, cleared: true }` when it failed once the hold's first key had
+ * been sent for deletion (most often `remove` refusing a job a worker locked in between): the hold is gone, or partly
+ * gone, while the job's own removal failed, which a caller must never word as "nothing was changed". A first delete
+ * that failed answers `{ invalid }` only when the hold is read back in place afterwards (Valkey refused the command:
+ * NOPERM, READONLY, MISCONF); a hold not read back, or a read that fails, keeps the mark, since a refusal can answer a
+ * resend of a delete that already ran. Never throws.
+ * Writes no run record of its own (INT-RUN-HISTORY-FILE-CONTRACT records terminal states of runs, and a removed hold
+ * is none); `attemptsMade` is what the job had done before, for the caller's sentence (`ranBefore`, issue #477): a job
+ * held again on a retry has made attempts, and whatever they recorded stays. Once the job is removed, its model attempt
+ * key goes too (`dropModelAttempt`, issue #812: bounded by `timeoutMs`, best-effort), since no retry is left to read it.
+ * `dropDefer` (removeQueuedJob's) deletes the job's defer record beside it, both sent before either is waited for, so the
+ * two cost a caller one bound, as they do for a waiting job, rather than two in a row.
  */
-export async function removeHeldJob({ redis, queue, jobId }) {
+export async function removeHeldJob({ redis, queue, jobId, timeoutMs = 2_000, dropDefer = false, touchStore = undefined, touchOutbox = null, now = Date.now, interventionOrigin = "human" }) {
+	// Set as the first hold key's delete is sent, before it is answered: a delete lost in transit may still have landed.
+	let cleared = false;
 	try {
 		const hash = await redis.hgetall(jobKey(jobId));
 		// A hold is a hash with a CLOCK, not merely a non-empty hash: the worker's own counters create this
@@ -110,16 +189,135 @@ export async function removeHeldJob({ redis, queue, jobId }) {
 		// The hold goes FIRST. If `remove` throws (an active job is locked) or a caller's timeout fires
 		// mid-sequence, an orphaned hash would keep a row on the panel for a job that no longer exists; an
 		// orphaned JOB is merely a job that still runs, which is the state the operator was already in.
-		await redis.del(jobKey(jobId));
+		cleared = true;
+		try {
+			await redis.del(jobKey(jobId));
+		} catch (err) {
+			// A failed DEL may still have run: ioredis resends an unanswered command after a reconnect, and the refusal
+			// (an ACL without DEL, a read-only replica, a failed save) can be the answer to that resend. So the hold is
+			// read back, and only a hold still in place makes "nothing was changed" true; a read that fails keeps the mark.
+			if (await holdInPlace(redis, jobId)) cleared = false;
+			throw err;
+		}
 		await redis.srem(HELD_SET, jobId).catch(() => {});
 		if (hash.dedupId) {
 			const holder = await redis.get(leaseKey(hash.dedupId));
 			if (holder === jobId) await redis.del(leaseKey(hash.dedupId));
 		}
-		await job.remove();
+		const store = touchStore === undefined ? (typeof redis?.eval === "function" ? makeTouchStore({ redis, now }) : null) : touchStore;
+		const removal = interventionOrigin === "agent" ? (await job.remove(), { ok: true }) : await trackTouchAction({ store, outbox: touchOutbox, event: stopTouch(job, queue, jobId, now), act: async () => { await job.remove(); return { ok: true }; }, succeeded: () => !(job.attemptsMade > 0) && isWorkload(job) });
+		await Promise.all([dropDefer ? dropDeferRecord(redis, jobId, timeoutMs) : null, dropAttemptOf(redis, queue, jobId, timeoutMs)]);
 		// Issue #477: what the job had done, for the caller's sentence (`ranBefore`); read off the job just removed.
-		return { ok: true, jobId, attemptsMade: Number.isInteger(job.attemptsMade) && job.attemptsMade > 0 ? job.attemptsMade : 0 };
+		return { ok: true, jobId, attemptsMade: Number.isInteger(job.attemptsMade) && job.attemptsMade > 0 ? job.attemptsMade : 0, ...(removal.touchWarning ? { touchWarning: removal.touchWarning } : {}) };
 	} catch (err) {
-		return { invalid: err?.message ?? String(err) };
+		const invalid = err?.message ?? String(err);
+		return cleared ? { invalid, cleared: true } : { invalid };
 	}
 }
+
+/**
+ * The states a job that has not started waits in, by the names `job.getState()` answers with: what `removeQueuedJob`
+ * removes. `paused` is the CLI's own list since issue #287; BullMQ 5.80.4 reads a job in a paused queue's list as
+ * `waiting`, so on it that job is removed under that word.
+ */
+export const QUEUED_STATES = Object.freeze(["delayed", "waiting", "prioritized", "paused"]);
+
+/**
+ * Remove a job that waits to start (issue #737): the ONE sequence `pi-dispatch cancel` and the console's cancel-queued
+ * run, so the two cannot drift. Who may be removed is the caller's rule, not this function's: the CLI removes a job
+ * waiting to retry too, and the console refuses one that has ever started. `job` is the job the caller found on `queue`.
+ *
+ *   held on run.waitFor (its `wait:job` hash has `since`)  -> `removeHeldJob`, hold keys first, then the job;
+ *   delayed, waiting, prioritized or paused (`QUEUED_STATES`) -> `job.remove()`;
+ *   any other state                                      -> nothing is changed.
+ *
+ * A HOLD IS JUDGED FIRST: a held job IS delayed, and the plain path would strand its `wait:*` keys as a panel row for a
+ * job that no longer exists. A hash that cannot be read is not a hold, as the CLI has always read it. `state` is the
+ * state the caller already read, or null to read it here.
+ *
+ * `guard` (optional; the CLI passes none) is a caller's last check, asked with `job` right before the removal is sent:
+ * before `removeHeldJob` for a held job, and after the state check, before `remove()`, otherwise. It answers null to go
+ * on, or what to refuse with; a guard that throws refuses too. BullMQ's remove checks only the job's lock and its
+ * scheduler, so a caller whose rule is about what the job has DONE (the console's never-started rule) reads that again
+ * here, one round trip before the remove (for a held job, before `removeHeldJob`'s own reads and deletes), rather than
+ * at its own earlier check.
+ *
+ * Once the job is removed, its defer record (`defer:job:<jobId>`) is deleted too: bounded by `timeoutMs`, best-effort,
+ * and only after a removal that succeeded, so a job that stays keeps its reason. Its model attempt key goes the same way
+ * (issue #812), beside the record, so the two deletes share one bound: here for a waiting job, inside `removeHeldJob`
+ * (`dropDefer`) for a held one. Returns, and never throws:
+ *   `{ ok: true, held: true, jobId, attemptsMade }`         a held job, removed;
+ *   `{ invalid }`                                            `removeHeldJob`'s own refusal of a held job, nothing changed;
+ *   `{ invalid, cleared: true }`                             `removeHeldJob` failed after its hold's keys were deleted: the
+ *                                                            hold is gone and the job's own removal failed (never "nothing
+ *                                                            was changed");
+ *   `{ refused, error? }`                                    the guard's answer (`refused`), or its throw (`refused` null,
+ *                                                            `error` the throw); nothing was sent;
+ *   `{ notQueued: true, state }`                             not waiting to start (`active`, `completed`, `unknown`...);
+ *   `{ ok: true, held: false, jobId, state, attemptsMade }`  a waiting job, removed;
+ *   `{ raced: true, state, error }`                          `remove()` threw (most often: a worker took the job between
+ *                                                            the read and the remove, or it is a job scheduler's next
+ *                                                            run, which BullMQ refuses to remove); `state` is read again.
+ * `attemptsMade` is the job's own count for the caller's sentence (`ranBefore`), 0 when it is not a positive whole number.
+ */
+export async function removeQueuedJob({ redis, queue, job, jobId, state = null, timeoutMs = 2_000, guard = null, touchStore = undefined, touchOutbox = null, now = Date.now, interventionOrigin = "human" }) {
+	const hash = await Promise.resolve()
+		.then(() => redis.hgetall(jobKey(jobId)))
+		.catch(() => null);
+	if (hash?.since) {
+		const stop = await guarded(guard, job);
+		if (stop !== null) return stop;
+		const res = await removeHeldJob({ redis, queue, jobId, timeoutMs, dropDefer: true, touchStore, touchOutbox, now, interventionOrigin });
+		if (!res.ok) return res.cleared === true ? { invalid: res.invalid, cleared: true } : { invalid: res.invalid };
+		return { ok: true, held: true, jobId: res.jobId, attemptsMade: res.attemptsMade, ...(res.touchWarning ? { touchWarning: res.touchWarning } : {}) };
+	}
+	if (!job) return { notQueued: true, state: "unknown" };
+	const current = typeof state === "string" ? state : await stateOf(job);
+	if (!QUEUED_STATES.includes(current)) return { notQueued: true, state: current };
+	const stop = await guarded(guard, job);
+	if (stop !== null) return stop;
+	let removal;
+	try {
+		const store = touchStore === undefined ? (typeof redis?.eval === "function" ? makeTouchStore({ redis, now }) : null) : touchStore;
+		removal = interventionOrigin === "agent" ? (await job.remove(), { ok: true }) : await trackTouchAction({ store, outbox: touchOutbox, event: stopTouch(job, queue, jobId, now), act: async () => { await job.remove(); return { ok: true }; }, succeeded: () => !(job.attemptsMade > 0) && isWorkload(job) });
+	} catch (error) {
+		return { raced: true, state: await stateOf(job), error };
+	}
+	await Promise.all([dropDeferRecord(redis, jobId, timeoutMs), dropAttemptOf(redis, queue, jobId, timeoutMs)]);
+	return { ok: true, held: false, jobId, state: current, attemptsMade: Number.isInteger(job.attemptsMade) && job.attemptsMade > 0 ? job.attemptsMade : 0, ...(removal.touchWarning ? { touchWarning: removal.touchWarning } : {}) };
+}
+
+/** `removeQueuedJob`'s guard asked: null to go on (no guard, or a null answer), else `{ refused }`, or `{ refused: null, error }` for a throw. */
+async function guarded(guard, job) {
+	if (typeof guard !== "function") return null;
+	try {
+		const answer = await guard(job);
+		return answer === null || answer === undefined ? null : { refused: answer };
+	} catch (error) {
+		return { refused: null, error };
+	}
+}
+
+/** A job's state, "unknown" where it cannot be read, as the CLI has always read it. */
+function stateOf(job) {
+	return Promise.resolve()
+		.then(() => job.getState())
+		.catch(() => "unknown");
+}
+
+/**
+ * Delete a removed job's model attempt key (issue #812), under the id the processor wrote it with: the job's queue's name
+ * and its id. A queue handle without a name (BullMQ's always has one) names no key, so nothing is sent.
+ */
+async function dropAttemptOf(redis, queue, jobId, timeoutMs) {
+	if (typeof queue?.name !== "string" || queue.name === "") return;
+	await dropModelAttempt(redis, modelAttemptIdOf(queue.name, jobId), timeoutMs);
+}
+
+/**
+ * Delete a removed job's defer record, bounded and best-effort (model-attempts.mjs `dropKey`, the one such delete): the
+ * record's own TTL is the backstop for a delete that fails, and a record whose job is gone is never read.
+ */
+function dropDeferRecord(redis, jobId, timeoutMs) {
+	return dropKey(redis, deferKey(jobId), timeoutMs);
+}

package/src/capabilities.mjs

@@ -96,6 +96,41 @@ export function parseCaps(raw) {
 	return out;
 }
 
+/**
+ * The implicit profile, `DEFAULT_SECRETS_PROFILE` in `secrets.mjs`, copied rather than imported for the
+ * reason the charset above is: that module spawns resolvers and this one is loaded by the receiver. A test
+ * holds the two equal.
+ */
+export const IMPLICIT_SECRET_PROFILE = "default";
+
+/**
+ * The profile a job's secrets resolve through, or undefined when it asked for none. `secretsArmed`'s rule
+ * (`secrets.mjs`), spelled again here for the same reason as the constant: an explicit profile, or a
+ * non-empty `secrets` map, which then resolves through the implicit `default` (issue #704). Before #704 only
+ * the explicit field was a need, so a trigger that never wrote `run.secretsProfile` was never routed and took
+ * the coin flip this module exists to remove.
+ *
+ * UPGRADE THE WORKERS BEFORE THE RECEIVER (and the console, whose re-run routes by this rule too). The router
+ * only ever picks a host that ADVERTISES every need, so the new need never sends a job to a host that cannot
+ * serve it. But an older worker advertises `s:default` only when its ENV declares it: one whose `default`
+ * lives in the overlay serves it without saying so. For a job with only this need that costs nothing (no
+ * advertiser is abstention 3, the shared queue it used before). For a job with a SECOND need it can regress:
+ * `run.secrets` without a profile plus a `w:ci` wait, an older host A declaring `w:ci` and serving `default`
+ * from its overlay, host B declaring neither. Before #704 the job needed `[w:ci]` and was routed to A; under
+ * a new receiver it needs `[s:default, w:ci]`, no host advertises both, and the shared queue lets B pop it
+ * and refuse. Once every worker runs this release, A advertises its overlay `default` and the job is routed
+ * to A again. An older receiver computes no such need and routes as before.
+ */
+// Exported (issue #712) for the run record's `secretsProfile` (run-history.mjs), which must name the profile this rule
+// routes by and `resolveSecrets` resolves through, and must not import secrets.mjs, which spawns.
+export function secretProfileNeeded(job) {
+	const named = job?.secretsProfile;
+	if (typeof named === "string" && named !== "") return named;
+	const secrets = job?.secrets;
+	if (secrets !== null && typeof secrets === "object" && Object.keys(secrets).length > 0) return IMPLICIT_SECRET_PROFILE;
+	return undefined;
+}
+
 /**
  * What a JOB needs, from the same two fields the worker's own refusals read.
  *
@@ -105,7 +140,7 @@ export function parseCaps(raw) {
  */
 export function jobNeeds(job) {
 	const needs = new Set();
-	const secret = token(CAP_SECRET, job?.secretsProfile);
+	const secret = token(CAP_SECRET, secretProfileNeeded(job));
 	if (secret) needs.add(secret);
 	if (Array.isArray(job?.waitFor)) {
 		for (const condition of job.waitFor) {

package/src/capacity-cli.mjs

@@ -16,11 +16,15 @@ import { parseArgs } from "node:util";
 import { CAPACITY_WINDOWS, computeCapacity } from "./capacity.mjs";
 import { readCapacityRecords } from "./capacity-records.mjs";
 import { formatCpus, formatMemory } from "./job-size.mjs";
+import { RUN_MIRROR_MAX_RUNS_MAX, RUN_MIRROR_MAX_RUNS_MIN, runMirrorMaxRunsOf } from "./retention.mjs";
 
 /** How long the CLI waits on the host registry, the kill switch's budget. */
 const FLEET_READ_TIMEOUT_MS = 2_000;
-/** The deployment keys this command reads beside the Valkey URL. */
-export const CAPACITY_ENV_KEYS = Object.freeze(["PI_LOGS_DIR", "PI_LOG_RETENTION_DAYS", "PI_WORKER_NAME"]);
+/**
+ * The deployment keys this command reads beside the Valkey URL. `PI_RUN_MIRROR_MAX_RUNS` (issue #761) is the run index's
+ * cap, which tells an index cut by count from one that is only large (`readMirrorWindow`).
+ */
+export const CAPACITY_ENV_KEYS = Object.freeze(["PI_LOGS_DIR", "PI_LOG_RETENTION_DAYS", "PI_WORKER_NAME", "PI_RUN_MIRROR_MAX_RUNS"]);
 
 /**
  * Run the command. Returns the exit code. Every collaborator is a seam with the production default.
@@ -45,7 +49,7 @@ export async function runCapacity(args, { env = process.env, write = (chunk) =>
 	if (deployment.problem) return fail(deployment.problem);
 	const facts = await deploymentFacts(deployment.env);
 	if (facts.problem) return fail(facts.problem);
-	const { logsDir, retentionDays, localHost } = facts;
+	const { logsDir, retentionDays, localHost, runMirrorMaxRuns } = facts;
 
 	const { killSwitchValkeyUrls, makeRedisClient, urlShown } = await import("./connection.mjs");
 	const picked = (pickUrls ?? killSwitchValkeyUrls)({ env, flagUrl: parsed.values["valkey-url"] ?? null });
@@ -75,7 +79,7 @@ export async function runCapacity(args, { env = process.env, write = (chunk) =>
 		// report's clock: a row's age decides how far its running jobs count (`LIVE_FRESH_MS`).
 		const [fleet, read] = await Promise.all([
 			redis ? (readLiveHostsFn ?? readLiveHosts)(redis, { now, timeoutMs: FLEET_READ_TIMEOUT_MS, prune: false }).catch((error) => ({ unreachable: error?.message ?? "registry unreadable" })) : { unreachable: null },
-			readCapacityRecords({ redis, logsDir, sinceMs: windowStartMs, nowMs, retentionDays, localHost, noMirrorReason: refusedNote, timeoutMs: FLEET_READ_TIMEOUT_MS, ...(fs ? { fs } : {}) }),
+			readCapacityRecords({ redis, logsDir, sinceMs: windowStartMs, nowMs, retentionDays, localHost, indexMax: runMirrorMaxRuns, noMirrorReason: refusedNote, timeoutMs: FLEET_READ_TIMEOUT_MS, ...(fs ? { fs } : {}) }),
 		]);
 		if (named && (fleet.unreachable || read.mirrorState.startsWith("unreachable"))) return fail(`could not read Valkey at ${urlShown(url)}: ${fleet.unreachable ? `host registry ${fleet.unreachable}` : `run mirror ${read.mirrorState}`}`);
 		const coverage = { ...read.coverage, reason: [read.coverage.reason, fleet.unreachable ? `host registry unreadable (${fleet.unreachable})` : null].filter(Boolean).join("; ") || null };
@@ -100,20 +104,23 @@ export function onlyHost(report, host) {
 	if (!known.includes(host)) return { unknown: known };
 	const shown = report.hosts.find((h) => h.name === host);
 	const { fromMs, source: _source, truncated, ...counts } = shown.coverage;
-	return { report: { ...report, hosts: [shown], coverage: { ...report.coverage, fromMs, truncated, ...counts, historyNotShared: report.coverage.historyNotShared.filter((n) => n === host) } } };
+	return { report: { ...report, fleet: undefined, hosts: [shown], coverage: { ...report.coverage, fromMs, truncated, ...counts, historyNotShared: report.coverage.historyNotShared.filter((n) => n === host) } } };
 }
 
 /**
  * What this command reads of the deployment: the logs directory (config.mjs `logsDirPath`, the worker's rule), the
- * retention (30 days unless set; 0 keeps the files) and this host's name (`PI_WORKER_NAME`, else the hostname), or
- * `{ problem }` for a retention the worker would refuse to boot with.
+ * retention (30 days unless set; 0 keeps the files), this host's name (`PI_WORKER_NAME`, else the hostname) and the run
+ * index's cap (`runMirrorMaxRuns`, `PI_RUN_MIRROR_MAX_RUNS`, 5,000 unless set; issue #761), or `{ problem }` for a
+ * retention or a cap the worker would refuse to boot with.
  */
 export async function deploymentFacts(env) {
 	const raw = env.PI_LOG_RETENTION_DAYS;
 	if (raw !== undefined && raw !== "" && !/^\d{1,6}$/.test(raw)) return { problem: `PI_LOG_RETENTION_DAYS must be a non-negative integer (got ${JSON.stringify(raw)})` };
+	const cap = runMirrorMaxRunsOf(env.PI_RUN_MIRROR_MAX_RUNS);
+	if (cap.problem) return { problem: `PI_RUN_MIRROR_MAX_RUNS must be an integer ${RUN_MIRROR_MAX_RUNS_MIN}-${RUN_MIRROR_MAX_RUNS_MAX} (got ${JSON.stringify(env.PI_RUN_MIRROR_MAX_RUNS)})` };
 	const { logsDirPath, defaultWorkerName, WORKER_NAME_RE } = await import("./config.mjs");
 	const named = env.PI_WORKER_NAME;
-	return { logsDir: logsDirPath(env), retentionDays: raw === undefined || raw === "" ? 30 : Number(raw), localHost: typeof named === "string" && WORKER_NAME_RE.test(named) ? named : defaultWorkerName() };
+	return { logsDir: logsDirPath(env), retentionDays: raw === undefined || raw === "" ? 30 : Number(raw), localHost: typeof named === "string" && WORKER_NAME_RE.test(named) ? named : defaultWorkerName(), runMirrorMaxRuns: cap.value };
 }
 
 /** A span in the largest whole units that say it: `40s`, `6m`, `2h 5m`, `3d 4h`. */
@@ -164,6 +171,7 @@ const CONTROL = /[\u0000-\u0009\u000b-\u001f\u007f-\u009f]/g;
 /** The human report: one block per host, then what the fleet's history covers. Control characters stripped. */
 export function capacityText(report, { since }) {
 	const lines = [];
+	if (report.fleet) lines.push(...fleetFacts(report.fleet));
 	for (const h of report.hosts) lines.push(...hostLines(h, since, report.coverage));
 	if (report.hosts.length === 0) lines.push(`${capitalized(noRunText(report.coverage, `in the last ${since}`))}.`);
 	lines.push(...coverageLines(report.coverage));
@@ -373,3 +381,15 @@ export function coverageLines(cov) {
 
 /** The fleet's history cut by the run mirror (`coverage.truncated`), as the CLI's coverage line says it. */
 export const TRUNCATED_NOTE = "history truncated: the run mirror holds nothing older for at least one host, so its earlier time is counted as neither busy nor idle";
+
+/** Shared fleet headline wording for the CLI and the Spend capacity view. */
+export function fleetFacts(fleet) {
+	const pct = (value) => value === null ? "unknown" : percentText(value);
+	const wait = (value) => value === null ? "no run recorded one" : durationText(value);
+	return [
+		`Fleet busy share of slot time: ${pct(fleet.busyPerMille)}${fleet.incomplete ? " (coverage incomplete)" : ""}`,
+		`Fleet peak: ${fleet.peak} jobs at once${fleet.peakAtMs === null ? "" : ` at ${new Date(fleet.peakAtMs).toISOString()}`}`,
+		`Fleet wait for a slot: p95 ${wait(fleet.waits.p95Ms)}, median ${wait(fleet.waits.p50Ms)}`,
+		`Fleet promised memory at the busiest hour: ${pct(fleet.promisedMemPerMille)}`,
+	];
+}