@edgehero/pi-dispatch 4.3.1 → 5.1.0

package/src/index.mjs

@@ -1,28 +1,42 @@
+import { resolveModelRoles } from "./model-roles.mjs";
+import { SubscriptionWait } from "./subscription-wait.mjs";
+import { zeroRatedFor } from "./reserve-estimate.mjs";
+import { isTransientOverlayRead } from "./model-catalog.mjs";
+import { checkFailoverCandidate, markModelCooldown, readModelAttempt, selectFailoverJob, writeModelAttempt } from "./model-failover.mjs";
+import { modelAttemptIdOf } from "./model-attempts.mjs";
 import { availableParallelism } from "node:os";
 import { DelayedError, UnrecoverableError, Worker } from "bullmq";
 import { assertJudgedConnection, onValkeyError } from "./connection.mjs";
 import { jobContainerName } from "./backend-local.mjs";
 import { scrubCredentials } from "./redact.mjs";
+import { cleanTouch } from "./touch-state.mjs";
 import { BACKEND_NOT_REGISTERED } from "./backend-registry.mjs";
-import { CANCEL_ACK_TTL_MS, cancelAckKey, cancelReqKey } from "./cancel-state.mjs";
-import { InfraRetry, NETNS_KEEPER_CRASH_LOOP, NETNS_KEEPER_NOT_HOLDING, TERMINAL_COMMENTS, runJob } from "./processor.mjs";
+import { CANCEL_ACK_TTL_MS, cancelAckKey, cancelReqKey, agentCancelReqKey, agentCancelAckKey } from "./cancel-state.mjs";
+import { InfraRetry, NETNS_KEEPER_CRASH_LOOP, NETNS_KEEPER_NOT_HOLDING, NewerReleaseHold, TERMINAL_COMMENTS, runJob, checkDraftAnswerSource } from "./processor.mjs";
+import { CANCELLED_BEFORE_START_COMMENT, SIZE_REFUSAL_COMMENTS } from "./comment-tables.mjs";
 import { NETNS_KEEPER_YOUNG_HOLD_MAX_MS, netnsKeeperCrashLoopSentence, netnsKeeperLoopAgainSentence } from "./netns-keeper.mjs";
 import { PODMAN_RESTART_HOLD_EXPIRED, PODMAN_RESTART_HOLD_MAX_MS, PODMAN_RESTART_HOLD_RECHECK_MS } from "./runtime-observations.mjs";
-import { earlierFrom, targetFor } from "./run-history.mjs";
+import { earlierFrom, recordOfThisJob, recordReadable, targetFor } from "./run-history.mjs";
 import { isPerMachineHost } from "./backends.mjs";
 import { hash16 } from "./fleet-lease.mjs";
 import { endpointsForModel } from "./model-endpoints.mjs";
 import { splitModelEntry } from "./model-ref.mjs";
 import { effectiveCostCapMicros } from "./money.mjs";
-import { dollarWindowCaps } from "./dollar-budget.mjs";
+import { dollarWindowCaps, valkeyEndpointTag } from "./dollar-budget.mjs";
 import { concurrencyFor, dollarCapsFor, makeInFlight, modelDollarRows, projectDollarCapsFor, projectRowFor, rowScopeFor, scopedLedgers } from "./scoped-limits.mjs";
 import { memberScopeOf, projectOf } from "./projects.mjs";
 import { resolveJobSize } from "./job-size.mjs";
 import { BUDGET_RECHECK_MS, HOST_BUDGET_TICK_MS, NEVER_FITS_RECHECK_MS, makeHostBudget } from "./host-budget.mjs";
 import { governedDollars } from "./allocation.mjs";
 import { WAIT_AFTER_MAX_DEFAULT_MS, WAIT_INTERVAL_FLOOR_MS, afterMs, unreadableConditions, waitArmed, waitBackoffMs, waitLabel, waitProfileNames } from "./wait-for.mjs";
-import { makeWaitState } from "./wait-state.mjs";
+import { answerOf, makeWaitState } from "./wait-state.mjs";
+import { makeDeferTo } from "./defer-write.mjs";
+import { jobDataNewer } from "./job-data-version.mjs";
 import { releaseSemanticWindow } from "./queue.mjs";
+import { runIdOf } from "./recovery-queue.mjs";
+import { isFinalOutcome, settleInbox } from "./inbox-settle.mjs";
+import { makeAwaitKeeper } from "./held-settle.mjs";
+import { isProjectId } from "./project-id.mjs";
 
 
 export const QUEUE = "pi-jobs";
@@ -49,11 +63,18 @@ export const REJECTED_SEEN_MAX = 1000;
 // docker daemon bounds any herd by its own concurrency, and a contended wake just re-defers.
 export const SCOPE_BUSY_RECHECK_MS = 5_000;
 
+// The newer-release re-check (issue #761, DES-STORE-VERSION-WHEN-NEEDED): a job this worker cannot read, or one whose
+// settings, triggers or pause windows file a newer pi-dispatch wrote, waits for this host's upgrade. Nothing here can
+// end that wait but an upgrade or a rollback of the file, so it asks rarely: ten minutes keeps a fleet mid-upgrade
+// from spinning on jobs it cannot run, and an upgraded worker picks the job up on its next look.
+export const NEWER_RELEASE_RECHECK_MS = 600_000;
+
 // The endpoint-busy re-check (issue #503): a job whose model server has every slot held re-tests on a fixed
 // cadence, for the scope re-check's reason (a held slot has no natural "until"). Its own value rather than a
-// borrow of 5s, because nothing records WHY a job sits in the delayed set and the wake instant is the only
-// evidence an operator has: 7s is distinct from the scope re-check, the wait throttle floor and the supersede
-// re-ask, and a test keeps all four apart. Short, because a local model run is often short too.
+// borrow of 5s: 7s is distinct from the scope re-check, the wait throttle floor and the supersede re-ask, and a
+// test keeps all four apart. Since issue #737 every deferral records its reason (`defer-write.mjs`), and the
+// distinct instants stay the evidence for a reader older than that record, and for a job whose record could not
+// be written. Short, because a local model run is often short too.
 export const ENDPOINT_BUSY_RECHECK_MS = 7_000;
 
 // How long a job waits before re-asking whether a target's holder is still alive (issue #230). Reached only
@@ -73,9 +94,10 @@ const THROTTLE_ALARM = 5;
 
 // The floor under a throttled or aborted re-ask. Its own constant rather than a borrow of
 // SUPERSEDE_RECHECK_MS, which documents an unrelated concern. Deliberately NOT 5s: that is
-// SCOPE_BUSY_RECHECK_MS, and INT-WAIT-PROFILES-CONTRACT rests on wait deferrals being distinguishable from
-// scope deferrals by wake instant -- nothing records WHY a job sits in the delayed set, so the instants are
-// the only evidence there is. A test pins the two apart.
+// SCOPE_BUSY_RECHECK_MS, and INT-WAIT-PROFILES-CONTRACT has wait deferrals distinguishable from scope
+// deferrals by wake instant. The defer record (`defer-write.mjs`, issue #737) now says why a job sits in the
+// delayed set; the instants remain the evidence for a reader older than it and for a record that could not be
+// written. A test pins the two apart.
 const THROTTLE_FLOOR_MS = 11_000;
 
 /**
@@ -87,6 +109,13 @@ const THROTTLE_FLOOR_MS = 11_000;
  */
 const ABORT_GRACE_MS = 30_000;
 
+/**
+ * The longest `boundAfterAbort` waits for its give-up hook (the host budget hold, and since issue #644 the live
+ * publisher's finish, itself bounded at 3 s plus 5 s) before it resolves anyway: a hook that hangs must not undo the
+ * bound this function exists for.
+ */
+const GIVE_UP_FINISH_MS = 10_000;
+
 /**
  * Resolve `run` normally, but stop waiting once the abort has fired and the grace has passed.
  *
@@ -110,9 +139,21 @@ function boundAfterAbort(run, signal, job, log, graceMs = ABORT_GRACE_MS, onStop
 				if (settled) return;
 				settled = true;
 				log("stop_did_not_take", { job: job.id, graceMs });
-				// Issue #596, phase 2: the container may still run, so its host budget hold must outlive this job.
-				onStopDidNotTake();
-				resolve({ code: 137, aborted: true, turns: null, tokens: null, session: null, usage: null, context: null, exitReason: null });
+				// Issue #596, phase 2: the container may still run, so its host budget hold must outlive this job. Issue
+				// #644: the hook also ends the attempt's live publisher, and is awaited (bounded) BEFORE this resolves,
+				// because the processor removes the job directory the publisher's last read needs once it has the result.
+				let wait = null;
+				Promise.race([
+					Promise.resolve()
+						.then(onStopDidNotTake)
+						.catch(() => {}),
+					new Promise((done) => {
+						wait = setTimeout(done, GIVE_UP_FINISH_MS);
+						wait.unref?.();
+					}),
+				])
+					.finally(() => clearTimeout(wait))
+					.then(() => resolve({ code: 137, aborted: true, turns: null, tokens: null, session: null, usage: null, context: null, exitReason: null }));
 			}, graceMs);
 			// A boot-blocking handle is not wanted here: the worker should be able to exit if everything else
 			// has finished, and this timer only matters while a job is still in flight.
@@ -145,7 +186,9 @@ function boundAfterAbort(run, signal, job, log, graceMs = ABORT_GRACE_MS, onStop
  * runs inside runJob against the freshly passed caps (CONST-BUDGET-BEFORE-TOKENS).
  *
  * The read happens once per pickup, right after the scope gate and before the model endpoint gate (issue #503),
- * because that gate needs the effective provider and model, which the overlay can supply.
+ * because that gate needs the effective provider and model, which the overlay can supply. One more read sits at the
+ * processor's head (issue #761) and asks only whether a newer pi-dispatch wrote the file; every value a job runs with
+ * still comes from the pickup's read.
  */
 /**
  * A job's endpoint set (issues #503, #502): the endpoints its main model AND every model on its effective allowed
@@ -218,6 +261,11 @@ export function makeRunningJobs() {
 			entries.set(next, { ...entry });
 			return next;
 		},
+		/** Update only this pickup: a retry with the same job id has a different key. */
+		update(key, patch) {
+			const entry = entries.get(key);
+			if (entry) entries.set(key, { ...entry, ...patch });
+		},
 		/** Removes one pickup's entry; true only the first time. */
 		remove(key) {
 			return entries.delete(key);
@@ -228,13 +276,44 @@ export function makeRunningJobs() {
 	};
 }
 
-export function makeProcessor({ cancelJob, stopContainer, containerName = (job) => jobContainerName(job.id), redis, getSettings, hostBudget = null, abortGraceMs = ABORT_GRACE_MS, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], projects = () => [], jobSizeEnv = {}, allocation = null, inFlight = makeInFlight(), hostBound = null, checkLease = null, scopeLease = null, endpointSlots = makeInFlight(), endpointLease = null, modelEndpoints = null, overlayModels = () => null, endpointSetFor = mainModelEndpoints, deps, recordRun = () => {}, settledRecord = null, previousRecord = null, timeoutMs = JOB_TIMEOUT_MS, cancelPollMs = 2_000, cancelStopBoundMs = CANCEL_STOP_BOUND_MS, hostName = "", multiHost = false, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, checkTimeoutMs = () => 10_000, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random, cpus = availableParallelism, runningJobs = makeRunningJobs() }) {
+// A free pre-pickup refusal needs its own project snapshot, before settlement can await a live configuration edit.
+// It never enters pickup, whose one snapshot must stay after lost-lock checks and immediately before pause gates.
+function makeBeforePickupRecorder({ projects, settleSteerBeforePickup, recordRun, now }) {
+	return async (job, boundary) => {
+		const project = projectOf(job.data, projects());
+		const result = { ...boundary, budgetReserved: false };
+		await settleSteerBeforePickup({ job, result });
+		await recordRun({ job, result, project, startedAt: new Date(now()).toISOString(), endedAt: new Date(now()).toISOString() });
+		return result;
+	};
+}
+
+export function makeProcessor({ cancelJob, stopContainer, containerName = (job) => jobContainerName(job.id), redis, getSettings, readModelRoles = () => ({}), hostBudget = null, abortGraceMs = ABORT_GRACE_MS, applyConcurrency = () => {}, pauseUntil = () => null, pauseWindowsNewer = () => false, triggersNewer = () => false, scopedLimits = () => [], projects = () => [], jobSizeEnv = {}, allocation = null, inFlight = makeInFlight(), hostBound = null, checkLease = null, scopeLease = null, endpointSlots = makeInFlight(), endpointLease = null, modelEndpoints = null, overlayModels = () => null, endpointSetFor = mainModelEndpoints, deps, recordRun = () => {}, settledRecord = null, previousRecord = null, timeoutMs = JOB_TIMEOUT_MS, cancelPollMs = 2_000, cancelStopBoundMs = CANCEL_STOP_BOUND_MS, hostName = "", multiHost = false, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, checkTimeoutMs = () => 10_000, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random, cpus = availableParallelism, runningJobs = makeRunningJobs(), valkeyTag = null }) {
 	// The lost-lock gate's rejection lines, said ONCE per job id, stall count and attempt. The gate runs before every
 	// deferral (pause window, wait, scope or endpoint busy), and BullMQ never resets a job's stall count, so a stalled
 	// scheduled job whose record is refused meets the gate again on every deferred pickup: a 30-minute scope-busy hold
 	// is about 360 of them. The verdict cannot change between them (same record, same attempt), so one line says it.
 	// Bounded, oldest out first, so a long-lived worker holds at most REJECTED_SEEN_MAX keys.
 	const rejectedSeen = new Set();
+	// Watch producer refusals finish before pickup and cannot carry a steering session.
+	const recordWatchOutcome = recordRun;
+	// A free boundary refusal happens before pickup creates a keeper or takes a host slot. Its existing inbox and
+	// held replies still need a final close before the record, just as every terminal path inside pickup does.
+	const settleSteerBeforePickup = async ({ job, result }) => {
+		let held = null;
+		if (job.data?.awaitInput === true && typeof deps?.settleHeld === "function") {
+			try { held = await deps.settleHeld({ job, result, afterTake: false }); } catch {}
+		}
+		if (job.data?.steer !== true) return;
+		const steered = await settleInbox({ redis, job, result, log: deps?.log ?? (() => {}), lateContext: { heldResumed: Number(held?.resumed) > 0 }, ...(typeof deps?.onLateForge === "function" ? { onLateForge: deps.onLateForge } : {}) });
+		if (steered) result.steered = steered;
+	};
+	const recordBeforePickup = makeBeforePickupRecorder({ projects, settleSteerBeforePickup, recordRun, now });
+	// THE ONE DEFERRAL (issue #737, `defer-write.mjs`): every gate below that puts the job back on the delayed set goes
+	// through `deferTo`, which records the gate's reason in `defer:job:<id>` (bounded, fail-open) and then makes the move
+	// and the `DelayedError` throw the gates always made; `noteRetry` records an infrastructure retry's backoff before its
+	// throw. Built once per processor, so its log episode (one line per outage, not per wake) spans every job it defers.
+	const { deferTo, noteRetry } = makeDeferTo({ redis, DelayedError, now, hostName, log: (event, fields) => deps?.log?.(event, fields) });
 	const firstRejection = (key) => {
 		if (rejectedSeen.has(key)) return false;
 		rejectedSeen.add(key);
@@ -266,51 +345,187 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			};
 			const record = await settledRecord(job.id, { attempt, since: job.timestamp, onReject });
 			if (record) {
+				// A record a NEWER pi-dispatch wrote (issue #761) settles the attempt by its stable core, as `recordVerdict`
+				// judged it, and is replayed by that core alone: its `reason`, `exitCode`, `turns`, `tokens` and
+				// `budgetReserved` may mean something this build does not know, and the completed listener pages and logs on
+				// them. So the job ends as the record's outcome with every other field null: `pagesAsPolicy` needs a reason
+				// it knows and so pages nobody, and the `job_completed` line names the outcome alone.
+				if (!recordReadable(record)) {
+					deps?.log?.("job_lost_lock_after_completion", { jobId: job.id, outcome: record.outcome });
+					return { outcome: record.outcome, reason: null, exitCode: null, turns: null, tokens: null, budgetReserved: null };
+				}
 				deps?.log?.("job_lost_lock_after_completion", { jobId: job.id, outcome: record.outcome, ...(record.reason ? { reason: record.reason } : {}) });
 				return { outcome: record.outcome, reason: record.reason ?? null, exitCode: record.exitCode ?? null, turns: record.turns ?? null, tokens: record.tokens ?? null, budgetReserved: record.budgetReserved ?? null };
 			}
 		}
 
+		// THE RUN'S INBOX AT ITS FINAL OUTCOME (issue #646, inbox-settle.mjs). A steer-armed job's inbox is closed, and what
+		// became of its messages counted, IMMEDIATELY BEFORE EVERY RECORD WRITE in this function, so the record's `steered`
+		// is the inbox's final word and a message can no longer be pushed into a run whose outcome is being written. ONE
+		// helper, called on the line before each `recordRun`/`recordAfterGate` (a bolt in steer-settle.test.mjs refuses a
+		// record site without it), which itself decides whether the outcome is final: an InfraRetry with attempts left and
+		// a #645 recovery handoff leave the inbox open for the next attempt. It decorates the result (or the error) with
+		// `steered` and the attempt's `steerSkipped`, which the record rebuilds; never throws, and a non-steer job costs
+		// nothing. `steerRan` is what this attempt's runContainer said about steering (null before any container).
+		let steerRan = null;
+		// Issue #712: this attempt's named live-transcript skip (runContainer's `liveSkipped`), null before any container.
+		// Carried onto the result here, at the one call every record site makes (the bolt above holds them to it), rather
+		// than at each of the processor's return statements, so no record site can be given it and another not.
+		let liveRan = null;
+		// Issue #712: the OpenShell venues' policy facts (runContainer's `openshell`, backend-openshell.mjs), carried the same
+		// way and for the same reason; the record rebuilds them (run-history.mjs `recordedOpenshell`), so nothing is trusted here.
+		let openshellRan = null;
+		let selectedEffective = null;
+		let providerFailureRan = null;
+		let modelRolesRan = null;
+		const carryLiveSkip = (target) => {
+			if (target === null || typeof target !== "object") return;
+			try {
+				if (liveRan !== null && target.liveSkipped === undefined) target.liveSkipped = liveRan;
+				if (openshellRan !== null && target.openshell === undefined) target.openshell = openshellRan;
+				if (selectedEffective?.modelRole) target.modelRole = selectedEffective.modelRole;
+				if (selectedEffective?.modelRole) { target.provider = selectedEffective.provider; target.model = selectedEffective.model; }
+				if (selectedEffective?.modelWhy) { target.modelWhy = selectedEffective.modelWhy; target.provider = selectedEffective.provider; target.model = selectedEffective.model; }
+				if (providerFailureRan) target.providerFailure = providerFailureRan;
+				if (modelRolesRan) target.modelRoles = modelRolesRan;
+				// Every attempt replaces the same record. Carry intervention fallback and uncertainty from the
+				// prior attempt, or a later cold start would erase the only proof of an earlier resumed answer.
+				// Only from a record this build can read (issue #761): a newer one's touches are outside its stable core and
+				// may mean something else, and this attempt's record would present them as this build's own. And only from
+				// this job's own previous attempt's record (`priorAttemptRecord` passed `recordOfThisJob` at the pickup's
+				// read, issue #811): an older job's interventions under a reused id are not this run's.
+				const prior = recordReadable(priorAttemptRecord) ? priorAttemptRecord : null;
+				if (prior?.touchLineageSkipped === "source-unavailable") target.touchLineageSkipped = "source-unavailable";
+				const touches = [...(Array.isArray(prior?.touches) ? prior.touches : []), ...(Array.isArray(target.touches) ? target.touches : [])].map(cleanTouch).filter(Boolean);
+				if (touches.length) target.touches = [...new Map(touches.map((event) => [event.id, event])).values()].sort((a, b) => a.at - b.at || a.id.localeCompare(b.id)).slice(0, 16);
+			} catch {}
+		};
+		const settleSteer = async ({ result = null, error = null } = {}) => {
+			const held = await settleHeldReplies({ result, error });
+			carryLiveSkip(result ?? error);
+			if (job.data?.steer !== true) return;
+			// Issue #638: a run both steer-armed and awaitInput can end parked with a held reply AND a late steer message.
+			// Only one of them becomes the resumed job: the held reply, settled first, and the late messages are then
+			// answered (`heldResumed`), never a second job for the same parked run.
+			const steered = await settleInbox({ redis, job, result, error, log: deps?.log ?? (() => {}), lateContext: { heldResumed: Number(held?.resumed) > 0 || Number(heldTaken?.resumed) > 0 || heldTaken?.taken === "other" }, ...(typeof deps?.onLateForge === "function" ? { onLateForge: deps.onLateForge } : {}) });
+			const skipped = typeof steerRan?.steerSkipped === "string" ? steerRan.steerSkipped : null;
+			const target = result ?? error;
+			if (target === null || typeof target !== "object") return;
+			try {
+				if (steered) target.steered = steered;
+				if (skipped !== null) target.steerSkipped = skipped;
+			} catch {}
+		};
+
+		// HELD REPLIES AT THE SAME FINAL OUTCOME (issue #638, held-settle.mjs), called first by `settleSteer` so that every
+		// record site the bolt above holds to the steer settle settles an awaitInput run's held replies too: one call per
+		// site, so no site can be given one and not the other. At a final outcome the keeper stops BEFORE the close, so no
+		// refresh in flight can put the run back on its target after the close removed it. `deps.settleHeld` is start.mjs's
+		// `makeHeldSettle`; unwired, nothing is held and nothing is settled. Never throws.
+		let awaitKeeper = null;
+		// What the park's take started (issue #638 review, H1: the take runs INSIDE the park sequence, before this settle),
+		// kept so the steer settle can tell its late messages a held reply already became the resumed job.
+		let heldTaken = null;
+		// Whether the take ran at all: the backstop settle then only drops (a take whose close failed already answered).
+		let heldTakeRan = false;
+		const takeHeldReplies = async () => {
+			if (job.data?.awaitInput !== true || typeof deps?.takeHeld !== "function") return null;
+			heldTakeRan = true;
+			await awaitKeeper?.stop();
+			heldTaken = await deps.takeHeld({ job });
+			return heldTaken;
+		};
+		const settleHeldReplies = async ({ result = null, error = null } = {}) => {
+			if (job.data?.awaitInput !== true) return;
+			try {
+				if (!isFinalOutcome({ job, result, error })) return;
+				await awaitKeeper?.stop();
+				if (typeof deps?.settleHeld === "function") return await deps.settleHeld({ job, result, error, afterTake: heldTakeRan });
+			} catch {}
+			return null;
+		};
+
 		// THE EARLIER ATTEMPTS' SLOT TIME (issue #599): a retry, or a pickup after a stall, writes its record over the one
 		// before it (one file and one mirror key per job id), so that record is read NOW, before anything here can replace
 		// it, and its slot interval (with the ones it carried) rides every record this pickup writes (`earlier`). Only on
 		// such a pickup, so a first attempt reads nothing; bounded by the reader, and a fault reads as none.
+		// The record counts only when it is THIS job's previous attempt's (`recordOfThisJob`, the rule the lost-lock checks
+		// share; issue #811): ids are reused (a lost queue, a hand redelivery), and a record an older job left under the
+		// same id would hand this job that job's slot time (`earlier`), its interventions (the touches), and a
+		// never-started verdict the failover retry would start over from the primary on (`retryNeverStarted`). The
+		// previous attempt is `attemptsMade` after a failure, which BullMQ counts, or this pickup's own number,
+		// `attemptsMade + 1`, after a stall, which it does not (a scheduled job's stall puts it back to wait with only its
+		// `stalledCounter` raised). Both are kept, the higher winning, rather than one chosen from `stalledCounter`:
+		// BullMQ never lowers that count, so a job that stalled once and later failed would look for the stalled
+		// attempt's number, miss its own failed record, and a failover retry with no model attempt key would then be
+		// refused `model-failover-no-candidate` (a comment, a used-up delivery, a disarmed one-shot). What that leaves: a
+		// stall that wrote no record leaves the attempt before it to answer, which is still this job's own.
+		// Each side (this host's file, the fleet's copy) is held to the rule BEFORE the higher attempt is kept
+		// (`previousRecord`'s `keep`), so a stale record at a higher attempt cannot hide this job's own on the other side,
+		// and the answer is judged again after, so a reader that ignores `keep` cannot hand the stale record back. The count
+		// is read as `buildRecord` reads it for the record's own `attempt`, so the two cannot disagree.
+		const attemptsBefore = Number.isInteger(job.attemptsMade) && job.attemptsMade >= 0 ? job.attemptsMade : 0;
+		const ownPrior = (record) => [attemptsBefore, attemptsBefore + 1].some((attempt) => recordOfThisJob(record, { jobId: job.id, attempt, since: job.timestamp }));
 		let earlier = null;
+		let priorAttemptRecord = null;
 		if (typeof previousRecord === "function" && ((Number.isInteger(job.attemptsMade) && job.attemptsMade > 0) || Number(job.stalledCounter) > 0)) {
 			try {
-				earlier = earlierFrom(await previousRecord(job.id));
+				const read = await previousRecord(job.id, { keep: ownPrior });
+				priorAttemptRecord = ownPrior(read) ? read : null;
+				earlier = earlierFrom(priorAttemptRecord);
 			} catch {
+				priorAttemptRecord = null;
 				earlier = null;
 			}
 		}
 
 		// Scoped pause windows (REQ-SCOPED-PAUSE-WINDOWS): if this job's folder/repo is inside an active pause
 		// window, DEFER it to the window end via BullMQ's delayed set -- the job keeps its identity/dedup and
-		// auto-resumes when re-picked. This is FIRST, before the kill timer, the settings read, and the budget
-		// reservation, so a deferred job arms no timer, reserves no slot, and spends nothing
-		// (CONST-BUDGET-BEFORE-TOKENS). `moveToDelayed` needs the worker's `token`; the `> now + 1s` guard keeps
-		// a boundary tick from busy-deferring. A thrown `DelayedError` is how BullMQ learns the job was deferred
-		// (worker.js recognises it) rather than completed or failed.
+		// auto-resumes when re-picked. This is FIRST of the deferral gates, before the kill timer, the settings read, and
+		// the budget reservation, so a deferred job arms no timer, reserves no slot, and spends nothing
+		// (CONST-BUDGET-BEFORE-TOKENS). `deferTo` records the reason, then moves the job with the worker's `token` and
+		// throws the `DelayedError` by which BullMQ learns the job was deferred (worker.js recognises it) rather than
+		// completed or failed; the `> now + 1s` guard keeps a boundary tick from busy-deferring.
 		// One clock snapshot for both the window lookup and the guard, so they cannot disagree across the
 		// call (and a test can inject a fixed clock). `now` defaults to the real wall clock in production.
 		const nowMs = now();
-		const until = pauseUntil(job.data, nowMs);
-		if (until && until > nowMs + 1000) {
-			await job.moveToDelayed(until, token);
-			throw new DelayedError();
+		// The job's project (issue #499, INT-PROJECTS-FILE-CONTRACT), resolved ONCE here from one read of the projects ref
+		// (an in-memory ref, so the read is free), and for the limits snapshot's reason below: the gate, the ledger and the
+		// record agree for this attempt, whatever an operator does to projects.json mid-run. A retry or a deferral is a new
+		// pickup and resolves again. Above the pause gate since issue #737, so every deferral's record names it, the pause
+		// window's included. Every record from the never-fits check on carries it (the refusal there by hand, every one
+		// past the wait gate through `recordAfterGate`); the wait gate's refusals carry none and are resolved from the live
+		// ref (start.mjs). An id or null, never a name.
+		const pickupProjects = projects();
+		const grouping = job.data?.purpose === "zero-touch-grouping" ? job.data.questionGrouping : null;
+		const sourceProject = grouping?.project;
+		// The classifier has an empty internal workspace, so folder membership would bill it to "other".
+		// Its private runtime resolved the source's historical project; reject a deleted binding before any
+		// reserve instead of silently charging another project's allowance.
+		if (grouping && (sourceProject !== null && (!isProjectId(sourceProject) || !pickupProjects.some((row) => row.id === sourceProject)))) {
+			const result = { outcome: "policy", reason: "question-group-config-changed", budgetReserved: false };
+			await settleSteer({ result });
+			await recordRun({ job, result, startedAt: new Date(nowMs).toISOString(), endedAt: new Date(nowMs).toISOString(), project: null });
+			return result;
+		}
+		const project = grouping ? sourceProject : projectOf(job.data, pickupProjects);
+		// BullMQ marks this pickup active before entering here. The canary takes its exclusive
+		// lease before proving all active lists empty, so either side of that race sees the other.
+		if (await deps?.canaryBusy?.()) await deferTo(job, token, nowMs + SCOPE_BUSY_RECHECK_MS, "canary", { project });
+		// Issue #761: the pause windows file on disk was written by a newer pi-dispatch. The windows the gate below reads
+		// are the LAST GOOD ones, and the newer file may pause this very job (a field this build would drop narrows when
+		// a window applies), so judging it by them would run a job the operator paused. Every job waits for this host's
+		// upgrade or a readable file instead, right before the pause gate and so before every hold and spend.
+		if (pauseWindowsNewer() === true) {
+			deps?.log?.("newer_release_hold", { jobId: job.id, what: "the pause windows file", delayMs: NEWER_RELEASE_RECHECK_MS });
+			await deferTo(job, token, nowMs + NEWER_RELEASE_RECHECK_MS, "newer-release", { project });
 		}
+		const until = pauseUntil(job.data, nowMs);
+		if (until && until > nowMs + 1000) await deferTo(job, token, until, "pause-window", { project });
 
-		// The limits snapshot, the project and the size, read here, right after the pause gate and ABOVE the wait gate, so
-		// the never-fits check below can refuse a size before a job waits (issue #596, gate round 1 of phase 2).
+		// The limits snapshot and the size, read here, right after the pause gate and ABOVE the wait gate, so the
+		// never-fits check below can refuse a size before a job waits (issue #596, gate round 1 of phase 2).
 		const limits = scopedLimits();
-		// The job's project (issue #499, INT-PROJECTS-FILE-CONTRACT), resolved ONCE here from one read of the projects ref,
-		// beside the limits snapshot and for its reason: the gate, the ledger and the record agree for this attempt,
-		// whatever an operator does to projects.json mid-run. A retry or a deferral is a new pickup and resolves again.
-		// Every record from the never-fits check on carries it (the refusal there by hand, every one past the wait gate
-		// through `recordAfterGate`); the wait gate's refusals carry none and are resolved from the live ref (start.mjs).
-		// An id or null, never a name.
-		const pickupProjects = projects();
-		const project = projectOf(job.data, pickupProjects);
 		// THE JOB'S SIZE (issue #596, `job-size.mjs`), resolved ONCE here from the same limits snapshot and pickup project
 		// every gate below reads: the project row's memory and CPUs, else the deployment's PI_JOB_MEMORY and PI_JOB_CPUS
 		// (`jobSizeEnv`, validated at boot), else 4g and 2. It reaches the container as an ARGUMENT (runJob's `jobSize`,
@@ -353,12 +568,12 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 					const result = { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, budgetReserved: false, hostBudget: { memMiB: budgetNow.memMiB, cpuCenti: budgetNow.cpuCenti, hostShare: share } };
 					// Above the wait gate, so through the recorder's own arguments rather than the bound one below it: the
 					// same pickup project and size, by hand once.
+					await settleSteer({ result });
 					recordRun({ job, result, startedAt: at, endedAt: new Date().toISOString(), project, size, earlier });
 					return result;
 				}
 				deps?.log?.("job_size_never_fits_here_deferred", { jobId: job.id, project, misfit, delayMs: NEVER_FITS_RECHECK_MS, ...sizeFields });
-				await job.moveToDelayed(nowMs + NEVER_FITS_RECHECK_MS, token);
-				throw new DelayedError();
+				await deferTo(job, token, nowMs + NEVER_FITS_RECHECK_MS, "size-never-fits", { project });
 			}
 		}
 
@@ -395,6 +610,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// resolver path or a vault topology -- `secret-profile-unknown` sets that rule.
 				if (sentence && deps?.comment) await Promise.resolve(deps.comment(job.data, sentence)).catch(() => {});
 				const result = { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, budgetReserved: false };
+				await settleSteer({ result });
 				recordRun({ job, result, startedAt: at, endedAt: new Date().toISOString(), earlier });
 				return result;
 			};
@@ -464,14 +680,12 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 					// for both; refusing would drop a delivery over a holder that may be gone. So decide
 					// nothing: re-defer briefly and ask again once the probe can answer.
 					deps?.log?.("wait_supersede_unverified", { jobId: job.id, heldBy: claim.holder ?? null, delayMs: SUPERSEDE_RECHECK_MS });
-					await job.moveToDelayed(nowMs + SUPERSEDE_RECHECK_MS, token);
-					throw new DelayedError();
+					await deferTo(job, token, nowMs + SUPERSEDE_RECHECK_MS, "wait-supersede", { project, holder: claim.holder });
 				}
 				if (claim.tookOverFrom) deps?.log?.("wait_lease_taken_over", { jobId: job.id, from: claim.tookOverFrom });
-				await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: holdUntil });
+				await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: holdUntil, afterMs: holdUntil });
 				deps?.log?.("wait_deferred", { jobId: job.id, until: new Date(holdUntil).toISOString(), label: waitLabel(job.data) });
-				await job.moveToDelayed(holdUntil, token);
-				throw new DelayedError();
+				await deferTo(job, token, holdUntil, "wait-after", { project });
 			}
 
 			// TIER 2: the polled conditions. Last, because it is the only part of this gate that spawns a
@@ -497,7 +711,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 					// the lease, so a job that never wins one has no `since`, never reaches the maximum, and
 					// re-wakes forever with no record and no bound. There is no deciding check to run first
 					// here -- that is the whole condition -- so the bound applies directly.
-					await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: nowMs + maxWaitMs() });
+					await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: nowMs + maxWaitMs(), afterMs: holdUntil });
 					if (held >= maxWaitMs()) {
 						return await refuseWait("wait-expired", "wait_expired", { reason: "max-wait-unchecked", denials, heldForMs: held }, `Gave up waiting: this deployment could not run the check often enough to answer within the maximum wait. Not run.`);
 					}
@@ -507,8 +721,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 					const wait = Math.max(THROTTLE_FLOOR_MS, Math.floor(waitBackoffMs(intervalMs(), held) / 4));
 					const delay = wait + Math.floor(wait * 0.1 * random());
 					deps?.log?.("wait_check_throttled", { jobId: job.id, delayMs: delay, slots });
-					await job.moveToDelayed(nowMs + delay, token);
-					throw new DelayedError();
+					await deferTo(job, token, nowMs + delay, "wait-throttled", { project });
 				}
 
 				// Declared outside the try below because the branches AFTER it read both.
@@ -537,12 +750,11 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 						checkSlots.release(WAIT_CHECK_KEY);
 						const denials = await waitState.noteThrottle(job.id, { denied: true });
 						if (denials === THROTTLE_ALARM) deps?.log?.("wait_capacity_exceeded", { jobId: job.id, denials, slots: checkSlotCount(), where: "fleet", hint: "raise PI_WAIT_CHECK_SLOTS or PI_CONCURRENCY, lengthen PI_WAIT_INTERVAL_MS, or hold fewer jobs" });
-						await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: nowMs + maxWaitMs() });
+						await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: nowMs + maxWaitMs(), afterMs: holdUntil });
 						const wait = Math.max(THROTTLE_FLOOR_MS, Math.floor(waitBackoffMs(intervalMs(), held) / 4));
 						const delay = wait + Math.floor(wait * 0.1 * random());
 						deps?.log?.("wait_check_throttled", { jobId: job.id, delayMs: delay, slots: checkSlotCount(), where: "fleet" });
-						await job.moveToDelayed(nowMs + delay, token);
-						throw new DelayedError();
+						await deferTo(job, token, nowMs + delay, "wait-throttled", { project });
 					}
 				}
 				// THE LEASE IS HELD FROM THE `tryAcquire` ABOVE, so every exit from here down must release it.
@@ -563,8 +775,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 					}
 					if (claim.retry) {
 						deps?.log?.("wait_supersede_unverified", { jobId: job.id, heldBy: claim.holder ?? null, delayMs: SUPERSEDE_RECHECK_MS });
-						await job.moveToDelayed(nowMs + SUPERSEDE_RECHECK_MS, token);
-						throw new DelayedError();
+						await deferTo(job, token, nowMs + SUPERSEDE_RECHECK_MS, "wait-supersede", { project, holder: claim.holder });
 					}
 
 					await waitState.noteThrottle(job.id, { denied: false }); // granted: the run of denials ends here
@@ -574,6 +785,11 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 					for (const profile of profiles) {
 						checked = profile;
 						verdict = await deps.checkWait(profile, targetFor(job.data?.kind, job.data), { signal });
+						// What this check said, for a console to show beside the hold (issue #737), in the gate's own words;
+						// nothing for a check that said nothing about the condition. Fail-open. Optional-called: an injected
+						// wait state written before the answers existed records none.
+						const answer = answerOf(verdict);
+						if (answer !== null) await waitState.noteAnswer?.(job.id, { profile, answer });
 						if (verdict?.profileUnknown || verdict?.verdict !== "go") break;
 					}
 				} finally {
@@ -602,8 +818,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 					// fault, or a rolling deploy would spend a job's whole budget on its own restarts and then
 					// blame the operator's script for it.
 					deps?.log?.("wait_check_aborted", { jobId: job.id, profile: checked });
-					await job.moveToDelayed(nowMs + THROTTLE_FLOOR_MS, token);
-					throw new DelayedError();
+					await deferTo(job, token, nowMs + THROTTLE_FLOOR_MS, "wait-aborted", { project });
 				}
 
 				if (verdict?.verdict === "hold") {
@@ -644,10 +859,9 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 					const jittered = base + Math.floor(base * 0.1 * random());
 					const remaining = Math.max(0, maxWaitMs() - held);
 					const delay = Math.max(1000, Math.min(jittered, remaining));
-					await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: nowMs + delay });
+					await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: nowMs + delay, afterMs: holdUntil });
 					deps?.log?.("wait_deferred", { jobId: job.id, profile: checked, fault, heldForMs: held, delayMs: delay });
-					await job.moveToDelayed(nowMs + delay, token);
-					throw new DelayedError();
+					await deferTo(job, token, nowMs + delay, "wait-hold", { project });
 				}
 
 				// FAIL CLOSED on anything that is not literally go. Everything above tests for a specific
@@ -661,8 +875,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 					deps?.log?.("wait_check_unintelligible", { jobId: job.id, profile: checked });
 					await waitState.noteCheck(job.id, { fault: true });
 					const base = waitBackoffMs(intervalMs(), held);
-					await job.moveToDelayed(nowMs + base + Math.floor(base * 0.1 * random()), token);
-					throw new DelayedError();
+					await deferTo(job, token, nowMs + base + Math.floor(base * 0.1 * random()), "wait-unclear", { project });
 				}
 
 				// Every profile answered go. Record the last check so the count bound sees it.
@@ -684,9 +897,10 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 		// not burn re-check wakes) and after the wait gate (a job holding until tomorrow must not sit on a
 		// folder while it does), and STRICTLY above the `try` below, like the pause gate and for the same two
 		// reasons: a DelayedError thrown inside the try would be converted to UnrecoverableError by the
-		// catch, and a moveToDelayed rejection here must escape RAW into BullMQ's normal failed-attempt
-		// handling exactly as the pause gate's does (inside the try it would become a permanent failure
-		// plus a failure record for what was a transient blip). The limits snapshot is read ONCE here and
+		// catch, and a move rejection here must escape RAW into BullMQ's normal failed-attempt handling
+		// exactly as the pause gate's does (inside the try it would become a permanent failure plus a failure
+		// record for what was a transient blip). `deferTo` rethrows a move's rejection unchanged after
+		// removing the reason it wrote for it, so that holds through it. The limits snapshot is read ONCE here and
 		// shared with `scopedLedgers` below, so the gate and the money ledger cannot disagree mid-job.
 		// tryAcquire is a synchronous check-and-increment -- no await between read and take, so Node's
 		// single thread makes it atomic at any concurrency -- and the local-folder limit is a structural 1
@@ -727,6 +941,8 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 		// This pickup's entry in the running jobs (issue #599, phase 2), null until it is admitted below.
 		let runningKey = null;
 		let stopDidNotTake = false;
+		// Issue #644: the attempt's live publisher finish, handed over by runContainer's `onLive` (null until it is open).
+		let finishLive = null;
 		let name;
 		let venue;
 		// THE SCOPE HOLDS, in acquire order (issue #499 part B): the repo or folder slot, then the project slot. Each is
@@ -739,12 +955,13 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 		};
 		// THE ENDPOINT HOLDS (issue #503), `{ id, fleet }` each, in id order.
 		const endpointHolds = [];
-		const releaseEndpointHolds = () => {
-			const taken = endpointHolds.splice(0).reverse();
+		const releaseEndpointHolds = (subset = null) => {
+			const taken = subset === null ? endpointHolds.splice(0).reverse() : subset.splice(0).reverse();
 			for (const hold of taken) endpointSlots.release(hold.id);
 			return Promise.all(taken.map((hold) => hold.fleet?.release?.()));
 		};
-		const releaseAllHolds = ({ orphan = false } = {}) => {
+		const releaseAllHolds = ({ orphan = false, candidateEndpoints = null } = {}) => {
+			if (candidateEndpoints !== null) return releaseEndpointHolds(candidateEndpoints);
 			// The running-jobs entry first (issue #599, phase 2): this pickup no longer runs the job, orphan or not.
 			if (runningKey !== null) {
 				runningJobs.remove(runningKey);
@@ -770,8 +987,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 		if (hostBound && !hostBudget) {
 			if (!hostBound.slots.tryAcquire(HOST_SLOT_KEY, hostBound.limit())) {
 				deps?.log?.("host_busy_deferred", { jobId: job.id, delayMs: SCOPE_BUSY_RECHECK_MS });
-				await job.moveToDelayed(nowMs + SCOPE_BUSY_RECHECK_MS, token);
-				throw new DelayedError();
+				await deferTo(job, token, nowMs + SCOPE_BUSY_RECHECK_MS, "host-slot", { project });
 			}
 			hostHeld = true;
 		}
@@ -800,29 +1016,33 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 		// `capacity` (issue #599) rides the same binding, and is null until the job is admitted below (where `startedAt` is
 		// set): every record of a refusal before that point says, by its null, that the job never held a slot.
 		let capacity = null;
-		const recordAfterGate = (args) => recordRun({ ...args, project, size, capacity, earlier });
+		let pickupMaxTurns = null;
+		const recordAfterGate = (args) => recordRun({ ...args, project, size, capacity, earlier, maxTurns: pickupMaxTurns });
 
 		// The MATCHED ROW's scope keys both the in-process slot and the fleet lease (issue #498), the same string
 		// `budgetCapsFor` hashes below and the boot sweeper hashes from the file: a qualified `github:acme/web` row holds
 		// GitHub jobs only, a bare `acme/web` row holds every forge's under the key it always had. With no row it is the
 		// job's canonical scope, so the folder mutex is keyed exactly as before.
 		const scope = rowScopeFor(job.data, limits);
-		// Give every hold back (scope and host), then defer. Every scope-gate deferral goes through here.
-		const deferScope = async (fields) => {
+		// Give every hold back (scope and host), then defer. Every scope-gate deferral goes through here, with the reason
+		// its caller names (issue #737); `chained` names the parent as its holder, the run whose scope it waits for.
+		const deferScope = async (why, fields) => {
 			// Every hold goes back before we defer, the host slot too: `makeInFlight().release` is not idempotent, so a
 			// slot held across a deferral would be a slot this machine never gets back.
 			await releaseAllHolds();
 			deps?.log?.(fields.event, { jobId: job.id, kind: job.data?.kind === "local" ? "local" : "forge", delayMs: SCOPE_BUSY_RECHECK_MS, ...fields.extra });
-			await job.moveToDelayed(nowMs + SCOPE_BUSY_RECHECK_MS, token);
-			throw new DelayedError();
+			await deferTo(job, token, nowMs + SCOPE_BUSY_RECHECK_MS, why, { project, ...(why === "chained" ? { holder: job.data.parentJobId } : {}) });
 		};
+		// The repo or folder slot's reason. A chained child (its parent enqueued it before the parent's finally gave the
+		// folder back) waits on that parent's run; any other job on the scope's own limit, by its kind.
+		const scopeWhy = typeof job.data?.parentJobId === "string" ? "chained" : job.data?.kind === "local" ? "scope-folder" : "scope-repo";
 		if (scope) {
 			const ceiling = concurrencyFor(job.data, limits);
 			if (!inFlight.tryAcquire(scope, ceiling)) {
 				// Optional-chained: makeProcessor gives `deps` no default and bare wirings pass deps: {}.
 				// The scope itself stays out of the log line (no-pii-in-logs -- a local scope is a full
 				// host path); the delayed count and the job id are what an operator needs to see it.
-				await deferScope({ event: "scope_busy_deferred" });
+				await deferScope(scopeWhy, { event: "scope_busy_deferred" });
 			}
 			const repoHold = { key: scope, fleet: null };
 			scopeHolds.push(repoHold);
@@ -843,7 +1063,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			// matching row, so a deployment with no scoped-limits file issues no command at all.
 			if (scopeLease && job.data?.kind !== "local" && Number.isFinite(ceiling)) {
 				repoHold.fleet = await scopeLease.acquire(job.id, { slots: ceiling, keyArgs: [hash16(scope)] });
-				if (!repoHold.fleet) await deferScope({ event: "scope_busy_deferred", extra: { where: "fleet" } });
+				if (!repoHold.fleet) await deferScope(scopeWhy, { event: "scope_busy_deferred", extra: { where: "fleet" } });
 			}
 		}
 
@@ -857,13 +1077,13 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 		const projectRow = projectRowFor(limits, project);
 		if (projectRow && Number.isSafeInteger(projectRow.concurrent)) {
 			if (!inFlight.tryAcquire(projectRow.scope, projectRow.concurrent)) {
-				await deferScope({ event: "scope_busy_deferred", extra: { ledger: "project" } });
+				await deferScope("scope-project", { event: "scope_busy_deferred", extra: { ledger: "project" } });
 			}
 			const projectHold = { key: projectRow.scope, fleet: null };
 			scopeHolds.push(projectHold);
 			if (scopeLease) {
 				projectHold.fleet = await scopeLease.acquire(job.id, { slots: projectRow.concurrent, keyArgs: [hash16(projectRow.scope)] });
-				if (!projectHold.fleet) await deferScope({ event: "scope_busy_deferred", extra: { ledger: "project", where: "fleet" } });
+				if (!projectHold.fleet) await deferScope("scope-project", { event: "scope_busy_deferred", extra: { ledger: "project", where: "fleet" } });
 			}
 		}
 
@@ -881,6 +1101,145 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			settingsThrew = true;
 			settingsError = error;
 		}
+		// Issue #761: an overlay written by a newer pi-dispatch is not a broken file, it is one this build cannot read. The
+		// settings-overlay-invalid refusal below is for a broken file; this job instead waits for this host's upgrade (or a
+		// readable file), deferred `newer-release`: a refusal would use up a forge delivery and spend a one-shot over a file
+		// nobody got wrong. HERE, right after the read and above every gate that reads the settings, and outside the main
+		// `try` (a DelayedError inside it would become an UnrecoverableError), every hold given back first, as each gate's
+		// deferral does. Nothing is settled, recorded, minted or reserved. The processor's head asks the same question
+		// before any branch runs, so a file that was newer then never gets here; this one covers a file that turned newer
+		// between that read and this one.
+		if (!settingsThrew && settings?.invalid && settings.newer === true) {
+			await releaseAllHolds();
+			deps?.log?.("newer_release_hold", { jobId: job.id, what: "the settings file", delayMs: NEWER_RELEASE_RECHECK_MS });
+			await deferTo(job, token, now() + NEWER_RELEASE_RECHECK_MS, "newer-release", { project });
+		}
+
+		// Failover selection precedes endpoint leases and every paid gate. Capture failures here and report them
+		// inside the ordinary record/release try below; no early throw may strand the scope slots already held.
+		let selectionError = null;
+		let selectionRefused = null;
+		let selectionWait = null;
+		let failoverSnapshot = null;
+		let chosenModel = null;
+		// Set by the selection when a retry can be judged only by a record a newer pi-dispatch wrote (issue #761): the job
+		// then waits below, outside this block's `try`, instead of being refused.
+		let priorRecordNewer = false;
+		const modelAttemptId = modelAttemptIdOf(job.queueName ?? QUEUE, job.id);
+		if (!settingsThrew && !settings?.invalid) {
+			try {
+				let effective = resolveModelRoles(effectiveJobOf(job.data, settings, deps?.allowedModels ?? null, (event, fields) => deps?.log?.(event, { jobId: job.id, ...fields })), job.data.watch || job.data.answerDraft ? {} : await readModelRoles(), project, { authoredProvider: job.data.provider });
+				selectedEffective = effective;
+				if (effective.fallbackModels !== undefined || (effective.recover === true && effective.recoveryOf)) {
+					const endpoints = modelEndpoints?.() ?? [];
+					let models;
+					try { models = overlayModels(); }
+					catch (error) {
+						if (error?.piDispatchConfig || error?.overlayLink || error?.overlayNotAFile || (typeof error?.code === "string" && !isTransientOverlayRead(error.code))) {
+							error.modelSelectionRefused = true;
+						}
+						throw error;
+					}
+					failoverSnapshot = { endpoints, models, set: [] };
+					let recoveryModel = null;
+					if (effective.recover === true && effective.recoveryOf) {
+						if (!deps?.workspaceCheckpointer) { const error = new Error("the recovery store is absent"); error.modelSelectionReason = "recovery-checkpoint-absent"; throw error; }
+						const read = await deps.workspaceCheckpointer.readManifest(effective.recoveryOf);
+						if (read?.ok && read.manifest?.provider && read.manifest?.model) recoveryModel = `${read.manifest.provider}/${read.manifest.model}`;
+						// Issue #761: a checkpoint a newer pi-dispatch wrote is not a missing one. The recovery waits for a worker
+						// that reads it (`newer-release`, deferred below as the processor's own pickup would), never refused here,
+						// where the refusal would use up its one attempt over a checkpoint nothing is wrong with.
+						else if (read?.newer === true) throw new NewerReleaseHold("the run's checkpoint");
+						// A missing checkpoint is refused by runJob's existing free gate, never replaced by a fresh model.
+						else { const error = new Error("the recovery model is absent"); error.modelSelectionReason = "recovery-checkpoint-absent"; throw error; }
+						const ref = splitModelEntry(recoveryModel);
+						if (!ref) { const error = new Error("the recovery model is invalid"); error.modelSelectionReason = "recovery-checkpoint-absent"; throw error; }
+						effective = { ...effective, provider: ref.provider, model: ref.model };
+					}
+					const retry = Number(job.attemptsMade) > 0 || Number(job.stalledCounter) > 0;
+					// Issue #761: a retry whose previous attempt's record a newer pi-dispatch wrote, with no model attempt key to
+					// lock to. That record's `reason`, `budgetReserved` and `exitCode` are outside its stable core, so it cannot
+					// say the attempt never started a container, and the selection would refuse the retry
+					// (`model-failover-no-candidate`) rather than start over from the primary. That refusal comments, records,
+					// uses up a forge delivery and disarms a one-shot over work nothing is wrong with; the job waits instead, as
+					// it does for every store this build cannot read, until a host that reads the record takes it or a later
+					// record replaces it. Exactly that case and no other: a model attempt key keeps the lock to its model, and a
+					// record this build can read keeps today's answers. The key is read here only in that case (the selection
+					// reads it again when it is there), and a throw is the selection's own read failing, handled below as such.
+					// The record is the pickup's (`priorAttemptRecord`), which counts only when it is this job's previous
+					// attempt's (`recordOfThisJob`, `ownPrior` at the pickup's read): one an older job left under a reused id, or an
+					// earlier attempt's, says nothing about the attempt being retried, so the retry is judged as one with no
+					// record, never held by it. That read held each side to the rule before the higher attempt was kept, so a
+					// stale record at a higher attempt on one side hides nothing on the other, and one read serves the hold,
+					// `retryNeverStarted` below, `earlier` and the touches alike (issue #811).
+					const waitMayApply = effective.fallbackModels !== undefined && retry && !recoveryModel;
+					priorRecordNewer = waitMayApply && priorAttemptRecord !== null && !recordReadable(priorAttemptRecord) && (await readModelAttempt(redis, modelAttemptId)) === null;
+					const selection = priorRecordNewer ? {} : effective.fallbackModels === undefined ? { job: effective } : await selectFailoverJob(effective, {
+						redis, id: modelAttemptId, retry,
+						// Read from a record this build can read only (issue #761): a newer one's `reason`, `budgetReserved` and
+						// `exitCode` are outside its stable core. Such a record is no usable prior record, so the retry goes by the
+						// model attempt key alone, locked to the model it names (with no key the job waited above), never started
+						// over from the primary on a record this build cannot vouch for. Nor on another job's (issue #811): the
+						// record is this job's previous attempt's or none, so an older job's `container-never-started` under a
+						// reused id cannot start this retry over from the primary after an attempt that may have spent.
+						recoveryModel, retryNeverStarted: recordReadable(priorAttemptRecord) && priorAttemptRecord.reason === "container-never-started" && priorAttemptRecord.budgetReserved === false && priorAttemptRecord.exitCode === null, models, limits, now, log: deps?.log,
+						reservationFor: (candidate) => zeroRatedFor(candidate, { modelEndpoints: failoverSnapshot, builtinModel: deps?.builtinModel }).zeroRated ? 0 : candidate.maxCostMicros,
+						subscriptionCheck: deps?.subscriptionRuntime ? (candidate) => deps.subscriptionRuntime.check(candidate) : null,
+						checkCandidate: (candidate) => deps?.checkFailoverCandidate ? deps.checkFailoverCandidate(candidate, { models, endpoints }) : checkFailoverCandidate(candidate, { models, endpoints, credential: deps?.checkProviderCredential, lookup: deps?.builtinModel }),
+						acquireEndpoints: async (candidate) => {
+							const set = [...new Map(endpointSetFor({ models, job: candidate, endpoints }).map((endpoint) => [endpoint.id, endpoint])).values()].sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
+							const held = [];
+							for (const endpoint of set) {
+								if (!endpointSlots.tryAcquire(endpoint.id, endpoint.slots)) {
+									await releaseAllHolds({ candidateEndpoints: held });
+									return false;
+								}
+								let fleet = null;
+								try { if (endpointLease && !isPerMachineHost(endpoint.host)) fleet = await endpointLease.acquire(job.id, { slots: endpoint.slots, keyArgs: [hash16(endpoint.id)] }); }
+								catch (error) { held.push({ id: endpoint.id, fleet: null }); await releaseAllHolds({ candidateEndpoints: held }); throw error; }
+								held.push({ id: endpoint.id, fleet });
+								if (endpointLease && !isPerMachineHost(endpoint.host) && !fleet) {
+									await releaseAllHolds({ candidateEndpoints: held });
+									return false;
+								}
+								if (fleet?.degraded) deps?.log?.("endpoint_lease_degraded", { jobId: job.id, endpoint: endpoint.id });
+							}
+							endpointHolds.push(...held);
+							failoverSnapshot.set = set;
+							return true;
+						},
+					});
+					selectedEffective = selection.job ?? null;
+					chosenModel = selection.ref ?? null;
+					if (chosenModel) await writeModelAttempt(redis, modelAttemptId, { ref: chosenModel, modelWhy: selectedEffective.modelWhy });
+					selectionRefused = selection.refused ?? null;
+					selectionWait = selection.wait ? selection : null;
+				}
+				if (selectedEffective && !selectionRefused && !selectionWait && deps?.subscriptionRuntime) {
+					const subscription = await deps.subscriptionRuntime.check(selectedEffective);
+					if (!subscription.ok) {
+						if (subscription.unavailable) throw new Error("subscription state unavailable");
+						if (subscription.wait) selectionWait = subscription;
+						else selectionRefused = subscription.reason;
+					}
+				}
+			} catch (error) {
+				if (error?.modelSelectionReason) selectionRefused = error.modelSelectionReason;
+				else if (error?.modelSelectionRefused) selectionRefused = "model-failover-no-candidate";
+				else selectionError = error instanceof InfraRetry ? error : new InfraRetry("the model candidates could not be read", { cause: error, reason: "container-never-started", budgetReserved: false });
+			}
+		}
+		// The selection's two deferrals, through one exit that gives every hold back first: a retry whose previous record a
+		// newer pi-dispatch wrote (issue #761, above; the selection did not run, so it named no wait), or a wait the
+		// selection or the subscription check named.
+		if (priorRecordNewer || selectionWait) {
+			await releaseAllHolds();
+			if (priorRecordNewer) {
+				deps?.log?.("newer_release_hold", { jobId: job.id, what: "the previous attempt's run record", delayMs: NEWER_RELEASE_RECHECK_MS });
+				await deferTo(job, token, now() + NEWER_RELEASE_RECHECK_MS, "newer-release", { project });
+			}
+			await deferTo(job, token, selectionWait.until, selectionWait.wait, { project });
+		}
 
 		// THE MODEL ENDPOINT GATE (issue #503, INT-MODEL-ENDPOINTS-FILE-CONTRACT). A declared local model server
 		// has a fixed number of parallel slots, and nothing else bounds how many jobs pile onto it: three per host
@@ -909,8 +1268,8 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 		// One snapshot per pickup (the endpoints, the overlay models, the derived set), handed to runJob as
 		// `modelEndpoints` so a later gate reads the same declaration this one leased against. With no endpoints
 		// declared the overlay is not even read, and the job touches nothing new: no command, no key, no field.
-		let endpointSnapshot = null;
-		if (modelEndpoints && !settingsThrew && !settings?.invalid) {
+		let endpointSnapshot = failoverSnapshot;
+		if (failoverSnapshot === null && !selectionError && !selectionRefused && modelEndpoints && !settingsThrew && !settings?.invalid) {
 			let endpoints = [];
 			let models = null;
 			let set = [];
@@ -941,7 +1300,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 					// ONE hold per endpoint id: a set naming an endpoint twice would take its slot and then wait on itself,
 					// forever on `slots: 1`. Deduplicated before the sort, so the order rule sees each endpoint once.
 					const byId = new Map();
-					for (const e of endpointSetFor({ models, job: effectiveJobOf(job.data, settings, deps?.allowedModels ?? null), endpoints })) if (!byId.has(e.id)) byId.set(e.id, e);
+					for (const e of endpointSetFor({ models, job: selectedEffective ?? effectiveJobOf(job.data, settings, deps?.allowedModels ?? null), endpoints })) if (!byId.has(e.id)) byId.set(e.id, e);
 					set = [...byId.values()].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
 				}
 			} catch (err) {
@@ -971,8 +1330,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				if (where !== null) {
 					await releaseAllHolds();
 					deps?.log?.("endpoint_busy_deferred", { jobId: job.id, endpoint: endpoint.id, where, delayMs: ENDPOINT_BUSY_RECHECK_MS });
-					await job.moveToDelayed(nowMs + ENDPOINT_BUSY_RECHECK_MS, token);
-					throw new DelayedError();
+					await deferTo(job, token, nowMs + ENDPOINT_BUSY_RECHECK_MS, "endpoint", { project, endpoint: endpoint.id });
 				}
 				endpointHolds.push({ id: endpoint.id, fleet });
 			}
@@ -1005,14 +1363,14 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				await releaseAllHolds();
 				budgetState.budgetDeferred = true;
 				deps?.log?.("host_budget_deferred", { jobId: job.id, project, why: verdict.why, rank: verdict.rank, memMiB: size.memMiB, cpuCenti: size.cpuCenti, delayMs: BUDGET_RECHECK_MS });
-				await job.moveToDelayed(nowMs + BUDGET_RECHECK_MS, token);
-				throw new DelayedError();
+				await deferTo(job, token, nowMs + BUDGET_RECHECK_MS, "host-budget", { project, budgetWhy: verdict.why });
 			}
 			budgetHeld = true;
 		}
 
 		let startedAt;
 		let timer;
+		let subscriptionDeadlineAt = null;
 		let cancelPoll;
 		let cancelPolling = false;
 		// The poll tick in flight, KEPT (gate round 2 of PR #479), so the error handler can stop the poll and wait for it
@@ -1069,7 +1427,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			capacity = capacityNow();
 			// This host runs the job from here on (issue #599, phase 2): its entry, at the same instant as `startedAt`, given
 			// back by `releaseAllHolds` on every exit. `add` is total, so this block still cannot throw.
-			runningKey = runningJobs.add({ id: job.id, project, memMiB: size.memMiB, cpuCenti: size.cpuCenti, at: Date.parse(startedAt) });
+			runningKey = runningJobs.add({ id: job.id, project, memMiB: size.memMiB, cpuCenti: size.cpuCenti, at: Date.parse(startedAt), holdMicros: 0, holdAt: Date.parse(startedAt) });
 			// The producer of the name both boot reapers sweep by substring. Built from the shared prefix
 			// rather than typed here, so a rename cannot land in the producer and not in the sweeps (#227).
 			// From the VENUE that will build the container, not from the local adapter reached for directly:
@@ -1105,6 +1463,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				if (err?.code !== BACKEND_NOT_REGISTERED) throw err;
 				name = null;
 			}
+			subscriptionDeadlineAt = now() + timeoutMs;
 			timer = setTimeout(() => {
 				// BullMQ has no per-job kill timer; this is ours. cancelJob raises the AbortSignal.
 				Promise.resolve(cancelJob(job.id, "job-timeout-30m")).catch(() => {});
@@ -1146,7 +1505,9 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 					cancelPolling = true;
 					cancelTick = (async () => {
 						try {
-							const req = await redis.get(cancelReqKey(job.id));
+							const humanReq = await redis.get(cancelReqKey(job.id));
+							const agentReq = humanReq == null ? await redis.get(agentCancelReqKey(job.id)) : null;
+							const req = humanReq ?? agentReq;
 							if (req === null || req === undefined) return;
 							// Stopped while this read was in flight: the outcome is being decided, so this request is left
 							// unanswered (its requester's timeout re-reads the job), never acknowledged after the fact.
@@ -1154,12 +1515,12 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 							// Bound to this worker (the injection comment in createWorker): `false` means the job
 							// left the tracked map in this same instant, in which case the finally below clears
 							// this interval anyway and the request's TTL reaps the key.
-							const took = await Promise.resolve(cancelJob(job.id, "operator-cancel")).catch(() => false);
+							const took = await Promise.resolve(cancelJob(job.id, agentReq === null ? "operator-cancel" : "agent-cancel")).catch(() => false);
 							if (took === false) return;
 							// Ack first, then consume the request: losing the DEL to a blip costs one redundant
 							// re-read, losing the ack costs the operator a false "nobody answered".
-							await redis.set(cancelAckKey(job.id), hostName, "PX", CANCEL_ACK_TTL_MS).catch(() => {});
-							await redis.del(cancelReqKey(job.id)).catch(() => {});
+							await redis.set(agentReq === null ? cancelAckKey(job.id) : agentCancelAckKey(job.id), hostName, "PX", CANCEL_ACK_TTL_MS).catch(() => {});
+							await redis.del(agentReq === null ? cancelReqKey(job.id) : agentCancelReqKey(job.id)).catch(() => {});
 							clearInterval(cancelPoll);
 						} catch {
 							// Fail open: the next tick re-asks.
@@ -1180,9 +1541,27 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			throw error;
 		}
 
+		// HELD REPLIES (issue #638, DES-AWAIT-INPUT-ENDS-THE-JOB): an awaitInput run is live on its await target from here,
+		// after the pause, size, wait, scope, endpoint and host-slot gates have admitted it and right before the settings
+		// overlay check and `runJob` (so before the mint and the container), until the park's take or the settle stop it,
+		// so a reply posted between the agent's question and the park is held instead of being deduplicated against this
+		// run or dropped for the missing marker. Only with a wired settle (unwired, a held reply would never be answered)
+		// and a Valkey client that can hold one. Never throws.
+		if (job.data?.awaitInput === true && typeof deps?.settleHeld === "function" && typeof redis?.multi === "function" && typeof redis?.eval === "function") {
+			awaitKeeper = makeAwaitKeeper({ redis, job, host: hostName || "worker", now, log: deps?.log ?? (() => {}) });
+			await awaitKeeper.start().catch(() => {});
+		}
+
 		try {
 			// The read itself happened once, above the endpoint gate; its throw lands HERE, where it always did.
 			if (settingsThrew) throw settingsError;
+			if (selectionError) throw selectionError;
+			if (selectionRefused) {
+				const result = { outcome: "policy", reason: selectionRefused, exitCode: null, turns: null, tokens: null, budgetReserved: false };
+				await settleSteer({ result });
+				recordAfterGate({ job, result, startedAt, endedAt: new Date().toISOString() });
+				return result;
+			}
 			if (settings.invalid) {
 				// A present-but-invalid overlay is a POLICY refusal, RETURNED (never thrown) so BullMQ marks the
 				// job completed and does not retry a file that can never parse (CONST-RETRY-INFRA-ONLY). Resolved
@@ -1192,6 +1571,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// refusals are the others): each is decided before or during the settings read, so no honest effective
 				// value exists yet -- buildRecord defaults both null.
 				const result = { outcome: "policy", reason: "settings-overlay-invalid", exitCode: null, turns: null, tokens: null, budgetReserved: false };
+				await settleSteer({ result });
 				recordAfterGate({ job, result, startedAt, endedAt: new Date().toISOString() });
 				return result;
 			}
@@ -1208,7 +1588,8 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			// values reserveBudget checks, never when it runs.
 			// The one call that logs a malformed queued cap (#501): the endpoint gate above computes the same job and
 			// stays quiet, so it is reported once per pickup.
-			const effectiveJob = effectiveJobOf(job.data, settings, deps?.allowedModels ?? null, (event, fields) => deps?.log?.(event, { jobId: job.id, ...fields }));
+			const effectiveJob = selectedEffective ?? effectiveJobOf(job.data, settings, deps?.allowedModels ?? null, (event, fields) => deps?.log?.(event, { jobId: job.id, ...fields }));
+			pickupMaxTurns = effectiveJob.maxTurns;
 
 			// THE ENVELOPE (issue #504 part B, DES-DELEGATED-ALLOCATION-INSIDE-ENVELOPE). With one loaded, every job is governed:
 			// its project's, or `_other`'s, share of the applied split narrows the dollar ledgers below, read ONCE here
@@ -1236,7 +1617,12 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			}
 
 			const result = await runJob(effectiveJob, {
+				queueJobId: String(job.id),
+				deploymentMaxCostUsd: settings.maxCostUsd,
+				rawApprovalJob: job.data,
 				redis,
+				// Issue #761: the tag of the Valkey the dollar counters are in, for the record's `dollars.reservedIn`.
+				valkeyTag,
 				// The three spend windows (week/month null when disabled) and the soft-hold band, resolved this
 				// job-start under overlay > env. reserveBudget checks them before the container.
 				caps: { day: settings.dailyCap, week: settings.weeklyCap, month: settings.monthlyCap },
@@ -1262,6 +1648,9 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// folder it RESOLVES. The processor asks which project the resolved folder belongs to, from the same projects
 				// snapshot, and refuses before any reserve when it is another project's.
 				pickupProject: project,
+				// Issue #704 review: a bundle profile this host lacks defers a job on a multi-host worker's SHARED queue
+				// (another host may serve it), the never-fits gate's predicate above; elsewhere it refuses.
+				bundleProfileDeferrable: multiHost && (job.queueName ?? QUEUE) === QUEUE,
 				folderProject: (folder) => projectOf({ kind: "local", folder }, pickupProjects),
 				// Issue #596: the size resolved at pickup above, for the retained run's manifest and the container's argv.
 				jobSize: size,
@@ -1281,12 +1670,19 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// runJob context is unchanged.
 				...(endpointSnapshot ? { modelEndpoints: endpointSnapshot } : {}),
 				...deps,
+				...(deps?.subscriptionRuntime ? {
+					prepareSubscriptionAuth: async (j) => {
+						try { return await deps.subscriptionRuntime.prepare(j, { deadlineAt: subscriptionDeadlineAt, signal, attemptId: `${modelAttemptId}:${job.attemptsMade ?? 0}:${job.stalledCounter ?? 0}` }); }
+						catch (cause) { throw new InfraRetry("subscription authentication could not be prepared", { cause, reason: "container-never-started", budgetReserved: false }); }
+					},
+					validateSubscriptionAuth: (snapshots) => deps.subscriptionRuntime.validate(snapshots),
+				} : {}),
 				// Issue #502: the skew check must see `models` as the job ARRIVED, not as `effectiveJobOf` filled it from
 				// PI_ALLOWED_MODELS, or a trigger list a stale receiver dropped would read as present and the job would run on
 				// the deployment's list. Only when the wiring supplies the check, so a bare processor keeps the default.
 				// The same for `maxCostUsd` (#501): `effectiveJobOf` keeps the arrived value under its own key and writes the
 				// resolved cap as `maxCostMicros`, but it is read off `job.data` here too, so no later fill can hide a drop.
-				...(deps?.checkWaitSkew ? { checkWaitSkew: (j, ...rest) => deps.checkWaitSkew({ ...j, models: job.data?.models, maxCostUsd: job.data?.maxCostUsd }, ...rest) } : {}),
+				...(deps?.checkWaitSkew ? { checkWaitSkew: (j, ...rest) => deps.checkWaitSkew({ ...j, models: job.data?.models, fallbackModels: job.data?.fallbackModels, maxCostUsd: job.data?.maxCostUsd }, ...rest) } : {}),
 				// #227. BOUNDED AFTER THE ABORT, and this is what makes `abortable` an honest declaration.
 				//
 				// `makeRunContainer`'s promise settles ONLY on the docker child's `close` or `error`. Nothing
@@ -1318,16 +1714,45 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// left for a cancel to stop. A cancel acknowledged before that moment raced the exit and lost nothing it was
 				// promised: the run is recorded as that cancel (the existing path, "partial work may exist") even when the
 				// container happened to exit on its own, because the operator was told the record would say operator-cancel.
-				runContainer: (ctx) =>
-					boundAfterAbort(deps.runContainer({ ...ctx, name, signal }), signal, job, deps.log ?? (() => {}), abortGraceMs, () => {
+				runContainer: async (ctx) => {
+					if (chosenModel) {
+						try { await writeModelAttempt(redis, modelAttemptId, { ref: chosenModel, modelWhy: selectedEffective.modelWhy }); }
+						catch (cause) { throw new InfraRetry("the model attempt could not be recorded", { cause, reason: "container-never-started" }); }
+					}
+					// `jobId` (issue #643) names the live transcript stream. Since issue #645 it is the RUN id (`runIdOf`): a recovery
+					// (`.rN`) continues its run's stream, progress key and status comment, while its record stays its own.
+					// Issue #644: `onLive` hands over the attempt's publisher finish, so giving up on a container whose stop did not
+					// take also ends its progress key (outcome `unknown`: nothing saw it exit) instead of leaving its heartbeat to
+					// rewrite the key as running for as long as the worker lives.
+					return boundAfterAbort(deps.runContainer({ ...ctx, name, signal, jobId: runIdOf(job), onLive: (finish) => (finishLive = finish) }), signal, job, deps.log ?? (() => {}), abortGraceMs, () => {
 						stopDidNotTake = true;
+						return finishLive?.("unknown");
 					}).then(async (r) => {
+						// Only signed, matching terminal metadata can permit a later cross-model replay. The pessimistic
+						// marker written before spawn survives crashes and tools that terminate the runner.
+						if (r?.exitAuth === "verified" && r?.exitLineCode === r?.code && r?.aborted !== true) modelRolesRan = r.modelRoles ?? null;
+						if (r?.exitAuth === "verified" && r?.exitLineCode === r?.code && r?.code === 1 && r?.aborted !== true) {
+							providerFailureRan = r.providerFailure ?? null;
+							try {
+								if (chosenModel) await writeModelAttempt(redis, modelAttemptId, { ref: chosenModel, modelWhy: selectedEffective.modelWhy, safe: r.toolsStarted === false && r.replaySafe === true && !!r.providerFailure, failure: r.providerFailure });
+								if (deps?.subscriptionRuntime) await deps.subscriptionRuntime.observe(effectiveJob, r, ctx.subscriptionAuth ?? []);
+								if (r.providerFailure) await markModelCooldown(redis, `${effectiveJob.provider}/${effectiveJob.model}`, r.providerFailure, { now });
+							} catch (cause) {
+								// Issue #800: a verified line was read, and the processor never sees this result, so the throw carries what it said.
+								throw new InfraRetry("the provider failure could not be recorded", { cause, reason: "container-infra-exit", exitCode: r.code, turns: r.turns, tokens: r.tokens, usage: r.usage, provider: effectiveJob.provider, model: effectiveJob.model, loaded: r.loaded });
+							}
+						}
+						// Issue #646: what this attempt said about steering, for the record's `steerSkipped` (settleSteer above).
+						steerRan = r && typeof r === "object" ? { steerSkipped: r.steerSkipped ?? null } : null;
+						liveRan = r && typeof r === "object" && typeof r.liveSkipped === "string" ? r.liveSkipped : null;
+						openshellRan = r && typeof r === "object" && r.openshell !== null && typeof r.openshell === "object" ? r.openshell : null;
 						await stopCancelPoll();
-						const raced = r && r.aborted !== true && signal.aborted === true && signal.reason === "operator-cancel";
+						const raced = r && r.aborted !== true && signal.aborted === true && ["operator-cancel", "agent-cancel"].includes(signal.reason);
 						if (raced) deps.log?.("cancel_acked_as_container_exited", { jobId: job.id, exitCode: r.code ?? null });
 						const run = raced ? { ...r, aborted: true } : r;
 						return run?.aborted ? { ...run, abortReason: typeof signal.reason === "string" ? signal.reason : null } : run;
-					}),
+					});
+				},
 				// REQ-TRIGGER-SECRETS. The resolver runs INSIDE the 30-minute kill timer armed above, so it has
 				// to be abortable for the same reason runContainer does: a resolver blocking on an unreachable
 				// vault would otherwise hold its slot until its own timeout, and an abort landing mid-resolution
@@ -1339,6 +1764,9 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// tunables above. It is bound here rather than at construction for the reason the overlay exists:
 				// an operator who declares a profile in the panel must not have to restart the worker.
 				...(deps.resolveSecrets ? { resolveSecrets: (j) => deps.resolveSecrets(j, { signal, overlayProfiles: settings.secretProfiles ?? {} }) } : {}),
+				// Issue #704: the bundle seam, bound the same way and for the same two reasons: the read runs inside the kill
+				// timer, so it takes the signal, and the overlay half of the bundle profiles is this job-start's settings read.
+				...(deps.secretBundles ? { secretBundles: { lookup: (profile) => deps.secretBundles.lookup(profile, { overlayBundles: settings.secretBundles ?? {} }), resolve: (j, binding) => deps.secretBundles.resolve(j, binding, { signal, overlayBundles: settings.secretBundles ?? {} }) } } : {}),
 				// collectChain (INT-OUTBOX-CONTRACT) reads the completed parent's REAL BullMQ job: its `.id`
 				// (the parent id children carry) and `.data` (kind/chainDepth). runJob's own `job` is the
 				// effectiveJob -- a spread of job.data with no `.id`/`.data` -- so inject the real wrapper here,
@@ -1350,6 +1778,9 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// unconditionally, AFTER the `...deps` spread like the injections around it: whether the key may outlive a park
 				// is not a choice a wiring gets to make by omission.
 				releaseParkedWindow: () => releaseSemanticWindow(job, deps?.log),
+				// Issue #638: the held replies, taken at the park on the REAL job (its id is the run's, its `queueName` the queue
+				// the resumed job goes to), bound unconditionally like the release; it takes nothing unless wired.
+				takeHeldReplies: () => takeHeldReplies().catch(() => null),
 				// The plan collector (issue #505) the same way and for the same reason: the writer it records is the REAL job's
 				// `.id`, and the authority it checks is `.data` as queued. The pickup's `portfolio` decision rides in `ctx`.
 				...(deps.collectPlan ? { collectPlan: (ctx) => deps.collectPlan({ ...ctx, job }) } : {}),
@@ -1368,6 +1799,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// The processor tests inject `prepareWorkspace` directly and never see this wrapper, which is
 				// why nothing caught it (REQ-RESUMABLE-SESSION).
 				...(deps.prepareWorkspace ? { prepareWorkspace: (j, t, opts) => deps.prepareWorkspace(j, t, { ...opts, queueJobId: job.id }) } : {}),
+				replyReceivedAt: Number.isSafeInteger(job.data?.replyReceivedAt) && job.data.replyReceivedAt >= 0 ? job.data.replyReceivedAt : Number.isSafeInteger(job.timestamp) ? job.timestamp : null,
 				// The one-shot pre-spend check (issue #231) needs the REAL BullMQ job's `.id` to excuse this
 				// delivery's own earlier attempt -- runJob's effectiveJob has no `.id`, prepareWorkspace's
 				// own injection above states why, and this one mirrors it. Omitted when unwired so a bare
@@ -1376,6 +1808,15 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// this one no options today, so there is nothing to lose yet -- and the day it does, a
 				// replacing wrapper would lose it in the same silence.
 				...(deps.checkOnceSpent ? { checkOnceSpent: (j, opts) => deps.checkOnceSpent(j, { ...opts, queueJobId: job.id }) } : {}),
+				// Issue #645: a recovery is enqueued from the REAL BullMQ job (its `.id` is the run id of an original, its `.name`
+				// the add's name, its `.data` what the recovery copies), which only this scope holds; runJob's effectiveJob has
+				// none of them. Bound like collectChain above, and omitted when unwired so runJob keeps its inert default.
+				...(deps.enqueueRecovery ? { enqueueRecovery: (args) => deps.enqueueRecovery({ ...args, job }) } : {}),
+				// The jobs running on this host (`{ list() }`, ids and admission instants), so a recovery whose base job still runs
+				// HERE is delayed rather than started beside it. The processor's own ledger, unless a wiring hands one in.
+				runningJobs: deps.runningJobs ?? runningJobs,
+				// Observability follows the actual reservation, never the configured cap before admission.
+				onDollarHold: (hold) => runningJobs.update?.(runningKey, { holdMicros: hold?.amountMicros ?? 0, holdAt: hold?.at ?? Date.parse(startedAt) }),
 			});
 			// Gate round 4 of PR #479: stopped again as runJob returns, before anything is recorded (a result with no
 			// container never passed the exit stop above). A cancel that lands after is never acknowledged, and its
@@ -1384,10 +1825,11 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			// the cancel the operator was told of, the result it replaced named in the log.
 			await stopCancelPoll();
 			let outcome = result;
-			if (signal.aborted === true && signal.reason === "operator-cancel" && result?.reason !== "operator-cancel") {
+			if (signal.aborted === true && ["operator-cancel", "agent-cancel"].includes(signal.reason) && result?.reason !== signal.reason) {
 				deps.log?.("cancel_acked_before_result", { jobId: job.id, was: `${result?.outcome}/${result?.reason ?? ""}` });
-				outcome = { ...result, outcome: "policy", reason: "operator-cancel" };
+				outcome = { ...result, outcome: "policy", reason: signal.reason };
 			}
+			await settleSteer({ result: outcome });
 			recordAfterGate({ job, result: outcome, startedAt, endedAt: new Date().toISOString() });
 			return outcome;
 		} catch (error) {
@@ -1395,8 +1837,9 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			// delayed set WITHOUT spending an attempt, the pause gate's move, since a service held up past one retry backoff
 			// failed a job that heals by itself (measured). Bounded by the hold's own start, stored on the job because a
 			// deferral leaves `attemptsMade` alone and a worker restart must not reset it; past the bound it fails for good
-			// with its own reason token, whose comment names the restart. Nothing is recorded per deferral: the job never
-			// started, and one record a minute for an hour would bury the run history.
+			// with its own reason token, whose comment names the restart. No run record is written per deferral (its defer
+			// record says why it waits, and is replaced by the next): the job never started, and one run record a minute
+			// for an hour would bury the run history.
 			//
 			// Gate round 3: the hour counts only while the worker kept checking. A check that comes more than two recheck
 			// periods after the last one (the queue was paused, `pi-dispatch pause`, or no worker ran) starts the hold
@@ -1415,22 +1858,25 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			// so it survives the restart.
 			// `readPending: false` asks only whether the poll ALREADY acknowledged a cancel (the signal), and leaves a request it
 			// had not read unanswered.
+			let cancellationReason = signal?.reason === "agent-cancel" ? "agent-cancel" : "operator-cancel";
 			const operatorCancelled = async ({ readPending = true } = {}) => {
-				if (signal?.aborted === true) return signal.reason === "operator-cancel";
+				if (signal?.aborted === true) return ["operator-cancel", "agent-cancel"].includes(signal.reason);
 				if (!readPending) return false;
 				if (typeof redis?.get !== "function") return false;
 				let req = null;
 				try {
 					req = await redis.get(cancelReqKey(job.id));
+					if (req == null) { req = await redis.get(agentCancelReqKey(job.id)); if (req != null) cancellationReason = "agent-cancel"; }
 				} catch {
 					return false;
 				}
 				if (req === null || req === undefined) return false;
-				await Promise.resolve(redis.set?.(cancelAckKey(job.id), hostName, "PX", CANCEL_ACK_TTL_MS)).catch(() => {});
-				await Promise.resolve(redis.del?.(cancelReqKey(job.id))).catch(() => {});
+				await Promise.resolve(redis.set?.(cancellationReason === "agent-cancel" ? agentCancelAckKey(job.id) : cancelAckKey(job.id), hostName, "PX", CANCEL_ACK_TTL_MS)).catch(() => {});
+				await Promise.resolve(redis.del?.(cancellationReason === "agent-cancel" ? agentCancelReqKey(job.id) : cancelReqKey(job.id))).catch(() => {});
 				return true;
 			};
-			// A retry that already spent (a container that ran and exited as infra) keeps what it spent on the record, and
+			// A retry that already spent (a container that ran and exited as infra) keeps what it spent on the record (and, issue
+			// #644, where its run got to, `progress`, which the processor put on the error beside `resources`), and
 			// says the processor's own operator-cancel sentence; one that never started says it was cancelled before it did.
 			// The session rides along as the aborted-container path keeps it (`mergeSession`, already applied to the throw).
 			// A NON-retryable error (gate round 3) is not known to have started nothing, so it says the processor's own
@@ -1438,9 +1884,10 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			const endCancelled = async ({ retryable }) => {
 				const spent = error?.budgetReserved === true;
 				const beforeStart = retryable && !spent;
-				const result = { outcome: "policy", reason: "operator-cancel", exitCode: spent ? (error.exitCode ?? null) : null, turns: spent ? (error.turns ?? null) : null, tokens: spent ? (error.tokens ?? null) : null, ...(spent && error.usage ? { usage: error.usage } : {}), provider: error?.provider ?? null, model: error?.model ?? null, session: error?.session ?? null, budgetReserved: retryable ? spent : (error?.budgetReserved ?? null), ...(error?.dollars ? { dollars: error.dollars } : {}), ...(spent && error?.resources ? { resources: error.resources } : {}) };
+				const result = { outcome: "policy", reason: cancellationReason, exitCode: spent ? (error.exitCode ?? null) : null, turns: spent ? (error.turns ?? null) : null, tokens: spent ? (error.tokens ?? null) : null, ...(spent && error.usage ? { usage: error.usage } : {}), provider: error?.provider ?? null, model: error?.model ?? null, session: error?.session ?? null, budgetReserved: retryable ? spent : (error?.budgetReserved ?? null), ...(error?.dollars ? { dollars: error.dollars } : {}), ...(spent && error?.resources ? { resources: error.resources } : {}), ...(spent && error?.progress ? { progress: error.progress } : {}), ...(error?.loaded ? { loaded: error.loaded } : {}) };
 				deps?.log?.("job_cancelled_instead_of_retry", { jobId: job.id, spent, retryable, ...(retryable ? {} : { failure: scrubCredentials(String(error?.message ?? error)).slice(0, 300) }) });
-				if (deps?.comment) await Promise.resolve(deps.comment(job.data, beforeStart ? CANCELLED_BEFORE_START_COMMENT : TERMINAL_COMMENTS["operator-cancel"])).catch(() => {});
+				if (deps?.comment) await Promise.resolve(deps.comment(job.data, beforeStart && cancellationReason !== "agent-cancel" ? CANCELLED_BEFORE_START_COMMENT : TERMINAL_COMMENTS[cancellationReason])).catch(() => {});
+				await settleSteer({ result });
 				recordAfterGate({ job, result, startedAt, endedAt: new Date().toISOString() });
 				return result;
 			};
@@ -1461,7 +1908,13 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			// other recorded. A retryable error reads that request too, since the job would otherwise run again.
 			await stopCancelPoll();
 			const retryable = error instanceof InfraRetry;
-			if (await operatorCancelled({ readPending: retryable })) return await endCancelled({ retryable });
+			if (await operatorCancelled({ readPending: retryable || error instanceof SubscriptionWait })) return await endCancelled({ retryable });
+			if (error instanceof SubscriptionWait) {
+				await releaseAllHolds();
+				if (error.wait === "subscription-auth") await deferTo(job, token, error.until, "subscription-auth", { project });
+				if (error.wait === "subscription-quota") await deferTo(job, token, error.until, "subscription-quota", { project });
+				if (error.wait === "subscription-owner") await deferTo(job, token, error.until, "subscription-owner", { project });
+			}
 			if (error?.holdUntilRestart === true) {
 				const heldAt = now();
 				const last = job.data?.podmanRestartHoldLastMs;
@@ -1470,13 +1923,47 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				if (heldAt - since < PODMAN_RESTART_HOLD_MAX_MS) {
 					await job.updateData({ ...job.data, podmanRestartHoldSinceMs: since, podmanRestartHoldLastMs: heldAt });
 					deps?.log?.("podman_restart_hold", { jobId: job.id, heldForMs: heldAt - since, delayMs: PODMAN_RESTART_HOLD_RECHECK_MS, reason: String(error.message).slice(0, 300) });
-					await job.moveToDelayed(heldAt + PODMAN_RESTART_HOLD_RECHECK_MS, token);
-					throw new DelayedError();
+					await deferTo(job, token, heldAt + PODMAN_RESTART_HOLD_RECHECK_MS, "podman-restart", { project });
 				}
 				const expired = Object.assign(new UnrecoverableError(`held ${Math.round((heldAt - since) / 60_000)} min for rootful Podman's service to restart, and it did not: ${error.message}`), { reason: PODMAN_RESTART_HOLD_EXPIRED, provider: error.provider ?? null, model: error.model ?? null, budgetReserved: false });
+				await settleSteer({ error: expired });
 				recordAfterGate({ job, error: expired, startedAt, endedAt: new Date().toISOString() });
 				throw expired;
 			}
+			// Issue #645: a recovery job whose base attempt is still running on this host waits for it (`RecoveryWaitHold`, a
+			// never-started InfraRetry carrying `recoveryWaitMs`). A recovery job has ONE attempt, so a plain throw would fail
+			// it while the run is alive; the delay costs no attempt. It ends when the base settles, or at the run's recovery
+			// deadline, which the host-lost sweep enforces.
+			// Issue #704 review: a secret bundle profile this host does not serve (or declares twice), on a multi-host
+			// worker's SHARED queue (`SecretBundleProfileHold`): handed back for the never-fits gate's recheck without
+			// spending an attempt, so a host that declares the profile can take it. On a single host, or for a job on a
+			// host's own queue, the processor refuses it instead (`secret-bundle-profile-unknown` / `-ambiguous`). Like
+			// the never-fits deferral it waits for as long as no host serves it, and doctor fails on every host that
+			// takes the project's jobs without the profile.
+			//
+			// RAISED IN THE PROCESSOR, not beside the never-fits gate above, deliberately (review round 2 asked): the
+			// decision needs the per-job settings read (the overlay half of the bundle profiles), which comes after the
+			// wait gate and the scope acquire, and the skip rules (chained, manual, opted-out, a changed trigger) that
+			// must run first live in the processor's free gate. Hoisting the lookup alone would judge a job the processor
+			// then skips, and would read the overlay a second time per pickup. The cost of staying is that each recheck
+			// re-runs the gates above it (a held wait is re-asked, the scope is re-acquired, the image and egress
+			// preflights re-run), all free reads, once a minute per held job.
+			if (error?.bundleProfileHold === true) {
+				deps?.log?.("secret_bundle_profile_hold", { jobId: job.id, delayMs: NEVER_FITS_RECHECK_MS, reason: typeof error.reason === "string" ? error.reason : null });
+				await deferTo(job, token, now() + NEVER_FITS_RECHECK_MS, "bundle-profile", { project });
+			}
+			if (Number.isFinite(error?.recoveryWaitMs) && error.recoveryWaitMs > 0) {
+				deps?.log?.("recovery_wait_hold", { jobId: job.id, delayMs: error.recoveryWaitMs });
+				await deferTo(job, token, now() + error.recoveryWaitMs, "recovery-wait", { project, holder: job.data?.recoveryOf });
+			}
+			// Issue #761: a free gate of the processor read something a newer pi-dispatch wrote (`NewerReleaseHold`: the live
+			// triggers file, for the one-shot, skew and approval checks, or a recovery's checkpoint), before the mint and every
+			// reserve. Deferred like the recovery hold above, without spending an attempt, and with nothing settled, recorded or
+			// posted: the job waits for a worker that reads the file. The log names the store, never the error's words.
+			if (error?.newerRelease === true) {
+				deps?.log?.("newer_release_hold", { jobId: job.id, what: typeof error.what === "string" ? error.what : null, delayMs: NEWER_RELEASE_RECHECK_MS });
+				await deferTo(job, token, now() + NEWER_RELEASE_RECHECK_MS, "newer-release", { project });
+			}
 			// Issue #476: a job whose egress preflight found the rootless network keeper running on its own bridge but younger
 			// than the minimum age is HELD until it is old enough (one `moveToDelayed`, no attempt), the move the hold above
 			// makes, since a keeper merely young is not a keeper that does not hold: every joint start of the stack read it
@@ -1493,20 +1980,23 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				if (!restarted && heldMs < NETNS_KEEPER_YOUNG_HOLD_MAX_MS) {
 					await job.updateData({ ...job.data, netnsKeeperHoldSinceMs: keeperHold.since, netnsKeeperHoldLastMs: keeperHold.at, netnsKeeperHoldStartedMs: young.startedMs });
 					deps?.log?.("netns_keeper_young_hold", { jobId: job.id, keeperAgeMs: young.ageMs, heldForMs: heldMs, delayMs: young.waitMs });
-					await job.moveToDelayed(keeperHold.at + young.waitMs, token);
-					throw new DelayedError();
+					await deferTo(job, token, keeperHold.at + young.waitMs, "netns-keeper", { project });
 				}
 				const was = keeperHold.startedMs ?? young.startedMs;
 				await markKeeperLoop(job, was === young.startedMs ? [was] : [was, young.startedMs]);
-				const loop = new InfraRetry(netnsKeeperCrashLoopSentence({ was, now: young.startedMs, heldMs, remedy: error.keeperRemedy }), { reason: NETNS_KEEPER_CRASH_LOOP, provider: error.provider ?? null, model: error.model ?? null, budgetReserved: false });
+				const loop = new InfraRetry(netnsKeeperCrashLoopSentence({ was, now: young.startedMs, heldMs, remedy: error.keeperRemedy }), { reason: NETNS_KEEPER_CRASH_LOOP, provider: error.provider ?? null, model: error.model ?? null, budgetReserved: false, loaded: error.loaded });
+				await settleSteer({ error: loop });
 				recordAfterGate({ job, error: loop, startedAt, endedAt: new Date().toISOString() });
+				await noteRetry(job, { project });
 				throw loop;
 			}
 			if (error?.reason === NETNS_KEEPER_NOT_HOLDING && keeperHold.startedMs !== null) {
 				// The keeper this job was waiting on is no longer running on its bridge: it died young, the loop's other face.
 				await markKeeperLoop(job, [keeperHold.startedMs]);
-				const loop = new InfraRetry(netnsKeeperCrashLoopSentence({ was: keeperHold.startedMs, now: null, problem: error.keeperProblem ?? null, heldMs: keeperHold.at - keeperHold.since, remedy: error.keeperRemedy }), { reason: NETNS_KEEPER_CRASH_LOOP, provider: error.provider ?? null, model: error.model ?? null, budgetReserved: false });
+				const loop = new InfraRetry(netnsKeeperCrashLoopSentence({ was: keeperHold.startedMs, now: null, problem: error.keeperProblem ?? null, heldMs: keeperHold.at - keeperHold.since, remedy: error.keeperRemedy }), { reason: NETNS_KEEPER_CRASH_LOOP, provider: error.provider ?? null, model: error.model ?? null, budgetReserved: false, loaded: error.loaded });
+				await settleSteer({ error: loop });
 				recordAfterGate({ job, error: loop, startedAt, endedAt: new Date().toISOString() });
+				await noteRetry(job, { project });
 				throw loop;
 			}
 			// A LATER ATTEMPT OF A JOB THAT SAW THE LOOP (gate of PR #479). The loop's retry comes after the queue's 60 s
@@ -1516,12 +2006,20 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			// failure of this job after it is still named the loop.
 			const loopSeen = job.data?.netnsKeeperLoopSeen;
 			if (error?.reason === NETNS_KEEPER_NOT_HOLDING && Array.isArray(loopSeen) && loopSeen.length > 0) {
-				const loop = new InfraRetry(netnsKeeperLoopAgainSentence({ seen: loopSeen, problem: error.keeperProblem ?? null, remedy: error.keeperRemedy }), { reason: NETNS_KEEPER_CRASH_LOOP, provider: error.provider ?? null, model: error.model ?? null, budgetReserved: false });
+				const loop = new InfraRetry(netnsKeeperLoopAgainSentence({ seen: loopSeen, problem: error.keeperProblem ?? null, remedy: error.keeperRemedy }), { reason: NETNS_KEEPER_CRASH_LOOP, provider: error.provider ?? null, model: error.model ?? null, budgetReserved: false, loaded: error.loaded });
+				await settleSteer({ error: loop });
 				recordAfterGate({ job, error: loop, startedAt, endedAt: new Date().toISOString() });
+				await noteRetry(job, { project });
 				throw loop;
 			}
+			await settleSteer({ error });
 			recordAfterGate({ job, error, startedAt, endedAt: new Date().toISOString() });
-			if (error instanceof InfraRetry) throw error; // retryable: BullMQ retries per attempts
+			if (error instanceof InfraRetry) {
+				// Retryable: BullMQ retries per attempts, after its backoff. The retry's reason is recorded first (issue #737),
+				// since BullMQ delays the job itself after this throw, where no gate of ours can say why.
+				await noteRetry(job, { project });
+				throw error;
+			}
 			// A non-retryable, non-infra error (our bug) must not retry forever. UnrecoverableError
 			// records it as failed-and-distinct in the queue's failed set without a retry.
 			throw new UnrecoverableError(error.message);
@@ -1539,16 +2037,119 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			clearTimeout(timer);
 			clearInterval(cancelPoll);
 			signal.removeEventListener("abort", onAbort);
+			// After the settle (issue #638): a final outcome stopped it already; an attempt that will be retried leaves its
+			// keys to their 30 s expiry, so a reply in that gap is still held.
+			await awaitKeeper?.stop().catch(() => {});
 		}
 	};
 	return async function processor(job, token, signal) {
-		if (!hostBudget) return pickup(job, token, signal, { budgetDeferred: false });
+		// First, before any gate reads the data: a job queued by a newer pi-dispatch, whose fields this build would drop
+		// or misread, waits for a worker that can read it (issue #761). A deferral, never a refusal: a refusal would use up
+		// a forge delivery (its id deduplicates a redelivery for `FORGE_JOB_RETENTION_DAYS`) and spend a one-shot
+		// trigger, and its record would be built from data this build cannot read. Nothing is spent, recorded or posted.
+		if (jobDataNewer(job.data)) {
+			deps?.log?.("newer_release_hold", { jobId: job.id, what: "job data", delayMs: NEWER_RELEASE_RECHECK_MS });
+			await deferTo(job, token, now() + NEWER_RELEASE_RECHECK_MS, "newer-release", {});
+		}
+		// Issue #761: the triggers file on disk was written by a newer pi-dispatch (start.mjs's reload refused it and keeps
+		// the last good schedules). ONE gate for every job, right after the job data's and before every branch below,
+		// rather than a patch in each reader: the file is read live by many of them (the fork policy, the secret bundles,
+		// the model roles, the answer rules, the question grouping, the approvals, the watch rules, the one-shot and skew
+		// checks), and each would otherwise run on its last good copy or refuse. Waiting for this host's upgrade is the one
+		// answer that neither misreads the file nor loses the work. A cron job minted from the last good schedules waits
+		// here too, so no schedule the newer file changed runs as it was. Free, before anything is read, held or spent.
+		if (triggersNewer() === true) {
+			deps?.log?.("newer_release_hold", { jobId: job.id, what: "the triggers file", delayMs: NEWER_RELEASE_RECHECK_MS });
+			await deferTo(job, token, now() + NEWER_RELEASE_RECHECK_MS, "newer-release", {});
+		}
+		// Issue #761: the settings file was written by a newer pi-dispatch. Asked HERE, with the same `getSettings` the
+		// pickup reads, because several refusals run before the pickup's own read (the question grouping's, the never-fits
+		// size, the wait gate's) and each comments, records and disarms a one-shot: one gate makes "every job defers" true
+		// for all of them, where a check in each branch would be one more place to forget. Only the newer answer is acted
+		// on. A throw falls through to the pickup, which reads again and handles it as it always did; nothing is held yet,
+		// so nothing is given back.
+		let headSettings = null;
+		try { headSettings = await getSettings(); } catch {}
+		if (headSettings?.invalid && headSettings.newer === true) {
+			deps?.log?.("newer_release_hold", { jobId: job.id, what: "the settings file", delayMs: NEWER_RELEASE_RECHECK_MS });
+			await deferTo(job, token, now() + NEWER_RELEASE_RECHECK_MS, "newer-release", {});
+		}
+		const sourceRefusal = await checkDraftAnswerSource(job.data, deps);
+		if (sourceRefusal) {
+			return recordBeforePickup(job, sourceRefusal);
+		}
+		if (job.data?.kind === "answer-draft" || job.data?.answerDraft) {
+			if (!deps.answerDraftRuntime) return { outcome: "policy", reason: "answer-draft-unavailable", budgetReserved: false };
+			const resolved = await deps.answerDraftRuntime.resolve(job.data, job.id);
+			// Issue #761: the runtime read a triggers file a newer pi-dispatch wrote (a write that landed before the watcher's
+			// reload said so) and answered before writing anything: the job waits as at the head, without using an attempt.
+			if (typeof resolved.newer === "string") {
+				deps?.log?.("newer_release_hold", { jobId: job.id, what: resolved.newer, delayMs: NEWER_RELEASE_RECHECK_MS });
+				await deferTo(job, token, now() + NEWER_RELEASE_RECHECK_MS, "newer-release", {});
+			}
+			if (resolved.result) {
+				await recordWatchOutcome({ job, result: resolved.result, startedAt: new Date(now()).toISOString(), endedAt: new Date(now()).toISOString() });
+				return resolved.result;
+			}
+			job.data = resolved.data;
+		}
+		if (job.data?.kind === "zero-touch-grouping" || job.data?.questionGrouping) {
+			if (!deps.questionGroupRuntime) return { outcome: "policy", reason: "question-group-unavailable", budgetReserved: false };
+			const resolved = await deps.questionGroupRuntime.resolve(job.data, job.id);
+			// Issue #761: the runtime read a triggers or settings file a newer pi-dispatch wrote (one that turned newer after
+			// the head's checks) and answered before setting any status: the job waits as at the head, never refused.
+			if (typeof resolved.newer === "string") {
+				deps?.log?.("newer_release_hold", { jobId: job.id, what: resolved.newer, delayMs: NEWER_RELEASE_RECHECK_MS });
+				await deferTo(job, token, now() + NEWER_RELEASE_RECHECK_MS, "newer-release", {});
+			}
+			if (resolved.result) {
+				await recordWatchOutcome({ job, result: resolved.result, startedAt: new Date(now()).toISOString(), endedAt: new Date(now()).toISOString() });
+				return resolved.result;
+			}
+			job.data = resolved.data;
+		}
+		if (job.data?.kind === "watch" || job.data?.watch) {
+			if (!deps.watchRuntime) return { outcome: "policy", reason: "watch-unavailable" };
+			// Issue #761: the watch runtime judges its own pause window below (`pauseUntil`, from the last good windows), and
+			// it takes the finding, or files the issue, as soon as that says the target is not paused, before pickup's pause
+			// gate ever runs. A pause windows file written by a newer pi-dispatch may pause the target, so the job waits for
+			// this host's upgrade first, before anything is taken, as every other job does at the pause gate.
+			if (pauseWindowsNewer() === true) {
+				deps?.log?.("newer_release_hold", { jobId: job.id, what: "the pause windows file", delayMs: NEWER_RELEASE_RECHECK_MS });
+				await deferTo(job, token, now() + NEWER_RELEASE_RECHECK_MS, "newer-release", {});
+			}
+			const resolved = await deps.watchRuntime.resolve(job.data, job.id);
+			// Issue #761: the runtime read the triggers file or the settings a newer pi-dispatch wrote (a write that landed
+			// before the watcher's reload said so), and answered BEFORE taking the finding: the job waits as at the head.
+			if (typeof resolved.newer === "string") {
+				deps?.log?.("newer_release_hold", { jobId: job.id, what: resolved.newer, delayMs: NEWER_RELEASE_RECHECK_MS });
+				await deferTo(job, token, now() + NEWER_RELEASE_RECHECK_MS, "newer-release", {});
+			}
+			if (resolved.deferUntil) await deferTo(job, token, resolved.deferUntil, "pause-window");
+			if (resolved.result) {
+				await recordWatchOutcome({ job, result: resolved.result, startedAt: new Date(now()).toISOString(), endedAt: new Date(now()).toISOString() });
+				return resolved.result;
+			}
+			job.data = resolved.data;
+		}
+		// Kept workspaces can hold an arbitrary prepared ref; they never become agent workspaces (#771).
+		const sandboxBoundary = await deps.checkSandboxWorkspace?.(job.data);
+		if (sandboxBoundary?.unavailable) throw new InfraRetry("the local workspace sandbox boundary could not be read");
+		if (sandboxBoundary?.outcome === "policy") {
+			return recordBeforePickup(job, sandboxBoundary);
+		}
+		if (!hostBudget) {
+			const result = await pickup(job, token, signal, { budgetDeferred: false });
+			if (job.data?.answerDraft && result?.outcome !== "completed") await deps.answerDraftRuntime?.refused(job.data, result).catch(() => {});
+			return result;
+		}
 		// The pickup's TICKET: the ledger entry this pickup takes carries it, and only this pickup's release or
 		// orphan can act on that entry, so a second pickup of the same job id (a stalled scheduled job handed back while
 		// its first attempt still runs here) can never give back the first's hold.
 		const budgetState = { budgetDeferred: false, ticket: hostBudget.enter(job.id) };
 		try {
 			const result = await pickup(job, token, signal, budgetState);
+			if (job.data?.answerDraft && result?.outcome !== "completed") await deps.answerDraftRuntime?.refused(job.data, result).catch(() => {});
 			hostBudget.forget(job.id);
 			return result;
 		} catch (error) {
@@ -1569,19 +2170,9 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
  */
 export const CANCEL_STOP_BOUND_MS = 1_000;
 
-/**
- * The forge comments for the two never-fits refusals (issue #596, phase 2). GENERIC on purpose: never the size, the
- * budget or the project, which are operator configuration an issue author can act on none of. The worker log, the run
- * record and doctor name both sizes. There is no fleet refusal (gate round 1 of phase 2): a job on the shared
- * queue waits for a host it fits on.
- */
-export const SIZE_REFUSAL_COMMENTS = Object.freeze({
-	"job-size-exceeds-host": "Refused: this job's size is larger than the worker host's job budget, so it could never start there. No container was started and nothing was spent. Ask the operator to lower this project's job size or raise the host budget. Not run.",
-	"job-size-exceeds-share": "Refused: this job's size is larger than the share of the worker host's job budget its project may use, so it could never start there. No container was started and nothing was spent. Ask the operator to lower this project's job size or raise its host share. Not run.",
-});
-
-/** The comment for a job the operator cancelled before it started, where it would have been held or retried (gate of PR #479). */
-export const CANCELLED_BEFORE_START_COMMENT = "Stopped: the operator cancelled this run before it started. Nothing was spent. Not retried.";
+// The never-fits refusals' comments (SIZE_REFUSAL_COMMENTS, issue #596) and the cancelled-before-start one live in
+// comment-tables.mjs since issue #710, re-exported here under the names they always had.
+export { CANCELLED_BEFORE_START_COMMENT, SIZE_REFUSAL_COMMENTS };
 
 /**
  * Store the keeper starts a crash loop was seen at on the job (gate of PR #479), outside the hold's 30 s window, so a
@@ -1608,7 +2199,7 @@ export function keeperHoldState(job, at) {
 	return { at, since, startedMs };
 }
 
-export function createWorker({ connection, name, stopContainer, containerName, hostQueue = null, checkLease = null, scopeLease = null, checkTimeoutMs, concurrency, getSettings, redis, deps, recordRun, settledRecord = null, previousRecord = null, limiter, pauseUntil, scopedLimits, projects, jobSizeEnv = {}, allocation = null, inFlight = makeInFlight(), waitState, afterMaxMs, checkSlots = makeInFlight(), checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, hostSlots = makeInFlight(), endpointSlots = makeInFlight(), endpointLease = null, modelEndpoints = null, overlayModels, extraClosers = [], hostBudget: hostBudgetOptions = null, runningJobs = makeRunningJobs() }) {
+export function createWorker({ connection, name, stopContainer, containerName, hostQueue = null, checkLease = null, scopeLease = null, checkTimeoutMs, concurrency, getSettings, readModelRoles, redis, deps, recordRun, settledRecord = null, previousRecord = null, limiter, pauseUntil, pauseWindowsNewer, triggersNewer, scopedLimits, projects, jobSizeEnv = {}, allocation = null, inFlight = makeInFlight(), waitState, afterMaxMs, checkSlots = makeInFlight(), checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, hostSlots = makeInFlight(), endpointSlots = makeInFlight(), endpointLease = null, modelEndpoints = null, overlayModels, extraClosers = [], hostBudget: hostBudgetOptions = null, runningJobs = makeRunningJobs() }) {
 	// One Worker per queue name (issue #57). A host-affine job -- one whose folder, secret resolver or wait
 	// check lives on THIS machine -- is enqueued to `pi-jobs@<name>` rather than filtered for at pickup,
 	// because BullMQ has no selective pop and the put-it-back alternative does not work: promotion out of
@@ -1651,6 +2242,10 @@ export function createWorker({ connection, name, stopContainer, containerName, h
 	for (const queueName of names) {
 		let worker; // referenced by cancelJob/applyConcurrency before assignment; only called later, so the TDZ is fine
 		const processor = makeProcessor({
+			// Issue #761: the endpoint tag of the Valkey this worker's clients dial (`valkeyEndpointTag`), from the judged
+			// connection's own options, the address `parseConnection` read off the pinned URL, so it is the tag doctor computes
+			// from the same URL. Null for a connection with none, which writes no `dollars.reservedIn`.
+			valkeyTag: valkeyEndpointTag(connection),
 			// Bound to THIS worker: a job on the host queue is cancelled by the worker draining that queue,
 			// and the shared handle could not reach it.
 			cancelJob: (id, reason) => worker.cancelJob(id, reason),
@@ -1666,6 +2261,7 @@ export function createWorker({ connection, name, stopContainer, containerName, h
 			containerName,
 			redis,
 			getSettings,
+			readModelRoles,
 			// Issue #596, phase 2: the ONE host budget. `multiHost` is whether this worker declared a fleet (a host queue):
 			// without one it declares no fleet and no other host is there to wait for, so a size that never fits here is refused rather than
 			// deferred for a host that does not exist (gate round 2 of phase 2).
@@ -1679,6 +2275,12 @@ export function createWorker({ connection, name, stopContainer, containerName, h
 				for (const w of workers) if (w.concurrency !== n) w.concurrency = n;
 			},
 			pauseUntil,
+			// Issue #761: whether the pause windows file on disk was written by a newer pi-dispatch (start.mjs keeps it beside
+			// the last good windows). Every job defers `newer-release` at the pause gate while it is.
+			pauseWindowsNewer,
+			// Issue #761: whether the triggers file on disk was written by a newer pi-dispatch (start.mjs keeps it beside the
+			// running schedules). Every job defers `newer-release` at the processor's head while it is.
+			triggersNewer,
 			// SHARED across both workers, and that sharing is the point rather than an optimisation: the
 			// folder mutex, the per-scope ceiling and the wait-check lease all bound the HOST, so two
 			// independent maps would double every one of them exactly as two Workers double concurrency.
@@ -1767,8 +2369,11 @@ export function createWorker({ connection, name, stopContainer, containerName, h
 		// what a stranded loop leaks is the rest of the list: `registry.close()` is the DEL that keeps a
 		// stopped host from lingering as a ghost peer for its full TTL, and a ghost peer with a stale
 		// `fpCron` is what makes a later `reconcileGated` refuse a legitimate reconcile. The comment above
-		// promised this isolation before the code delivered it (issue #295). It bounds nothing, though: a
-		// closer that never settles still blocks exit, which no closer here does.
+		// promised this isolation before the code delivered it (issue #295). The loop itself bounds nothing:
+		// a closer that never settles still blocks exit, so each closer bounds its own wait. The queues
+		// `start.mjs` hands over did not until issue #813: each one's close waited for a start that never
+		// comes when nothing answers VALKEY_URL, so this stop waited for its supervisor's kill. They now wait
+		// a bounded time for the start and for the close, then disconnect (`closeQueueSettled`, `settleCloseOf`).
 		//
 		// Read LATE and deliberately: `start.mjs` pushes its live-edit watchers into this array AFTER handing
 		// it over, because they are armed after the boot reconcile. Anything here that snapshots or copies