@edgehero/pi-dispatch 1.5.0 → 1.6.1

package/src/index.mjs

@@ -2,7 +2,10 @@ import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 import { DelayedError, UnrecoverableError, Worker } from "bullmq";
 import { InfraRetry, runJob } from "./processor.mjs";
+import { targetFor } from "./run-history.mjs";
 import { budgetCapsFor, canonicalScope, concurrencyFor, makeInFlight } from "./scoped-limits.mjs";
+import { WAIT_AFTER_MAX_DEFAULT_MS, WAIT_INTERVAL_FLOOR_MS, afterMs, unreadableConditions, waitArmed, waitBackoffMs, waitLabel, waitProfileNames } from "./wait-for.mjs";
+import { makeWaitState } from "./wait-state.mjs";
 
 const exec = promisify(execFile);
 
@@ -16,6 +19,28 @@ export const JOB_TIMEOUT_MS = 30 * 60 * 1000; // REQ-JOB-TIMEOUT-30M
 // docker daemon bounds any herd by its own concurrency, and a contended wake just re-defers.
 export const SCOPE_BUSY_RECHECK_MS = 5_000;
 
+// How long a job waits before re-asking whether a target's holder is still alive (issue #230). Reached only
+// when the liveness probe could not answer, which is a redis or queue fault rather than a normal state, so
+// this is a short retry rather than a cadence: the job is deciding nothing and holding nothing while it
+// waits, and the fault it is waiting out is usually seconds long.
+export const SUPERSEDE_RECHECK_MS = 15_000;
+
+// The one key the check lease counts under. A single global counter rather than one per profile: what it
+// bounds is this worker's wall-clock spent answering questions, and that is shared whatever is being asked.
+const WAIT_CHECK_KEY = "wait-check";
+
+// How many consecutive lease denials one job absorbs before the deployment is told its checking capacity is
+// short. Logged ONCE per run of denials rather than per wake: an alarm that repeats every re-check is the
+// always-on amber this project rejects elsewhere, and the operator only needs telling once per episode.
+const THROTTLE_ALARM = 5;
+
+// The floor under a throttled or aborted re-ask. Its own constant rather than a borrow of
+// SUPERSEDE_RECHECK_MS, which documents an unrelated concern. Deliberately NOT 5s: that is
+// SCOPE_BUSY_RECHECK_MS, and INT-WAIT-PROFILES-CONTRACT rests on wait deferrals being distinguishable from
+// scope deferrals by wake instant -- nothing records WHY a job sits in the delayed set, so the instants are
+// the only evidence there is. A test pins the two apart.
+const THROTTLE_FLOOR_MS = 11_000;
+
 /**
  * Build the BullMQ processor.
  *
@@ -35,7 +60,7 @@ export const SCOPE_BUSY_RECHECK_MS = 5_000;
  * The overlay changes which values the spend caps take, never when they are checked -- reserveBudget still
  * runs inside runJob against the freshly passed caps (CONST-BUDGET-BEFORE-TOKENS).
  */
-export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now() }) {
+export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
 	return async function processor(job, token, signal) {
 		// Scoped pause windows (REQ-SCOPED-PAUSE-WINDOWS): if this job's folder/repo is inside an active pause
 		// window, DEFER it to the window end via BullMQ's delayed set -- the job keeps its identity/dedup and
@@ -53,9 +78,295 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 			throw new DelayedError();
 		}
 
+		// The wait gate (issue #230, REQ-WAIT-FOR). THIRD: after the pause gate, because a paused job must
+		// not burn a wait evaluation any more than it burns a scope re-check, and BEFORE the scope acquire,
+		// because a job that is going to sit until tomorrow morning must not hold the folder mutex while it
+		// does. Strictly above the `try` for the two reasons the gates below it document.
+		//
+		// The order WITHIN the gate is determinate-refusals-then-holds, which is CONST-BUDGET-BEFORE-TOKENS'
+		// shape applied to time rather than to money: a condition this deployment can never answer must be
+		// refused now, not after a day of waiting.
+		//
+		// On throwing above the `try`: an exception here escapes into BullMQ's normal failed-attempt handling,
+		// which is WANTED for `moveToDelayed` (the scope gate below gives the argument: a transient rejection
+		// must stay a transient failure rather than becoming a permanent one) and unwanted everywhere else. So
+		// the state and comment seams fail open by construction, and `recordRun` is relied on not to throw --
+		// its writer swallows fs errors by contract, which is the same reliance the settings-overlay refusal
+		// below already makes.
+		if (waitArmed(job.data)) {
+			// The supersede identity: the queue's semantic key PLUS the trigger that produced this job.
+			// The semantic key alone is `repo<sep>number:flow`, which two DIFFERENT triggers on one target and
+			// flow legitimately share -- a label rule that waits a day and a comment rule that waits a minute
+			// would coalesce, and the second would be refused with a message claiming they wait on "the same
+			// conditions" when they do not. Adding the raw trigger index makes the key mean one intent.
+			const matchedIndex = job.data?.trigger?.matched?.index;
+			const dedupId = job.deduplicationId ? `${job.deduplicationId}#${Number.isInteger(matchedIndex) ? matchedIndex : "?"}` : null;
+			const refuseWait = async (reason, logEvent, fields, sentence) => {
+				// The INJECTED clock, like both gates above: a record whose timestamps ignore the test clock
+				// is a record no test of this gate can assert about.
+				const at = new Date(now()).toISOString();
+				await waitState.release(job.id, { dedupId });
+				deps?.log?.(logEvent, { jobId: job.id, ...fields });
+				// The comment names the FIELD and the operator's own words for the condition, never a
+				// resolver path or a vault topology -- `secret-profile-unknown` sets that rule.
+				if (sentence && deps?.comment) await Promise.resolve(deps.comment(job.data, sentence)).catch(() => {});
+				const result = { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, budgetReserved: false };
+				recordRun({ job, result, startedAt: at, endedAt: new Date().toISOString() });
+				return result;
+			};
+
+			// EVERY condition must be one this worker understands, checked before anything else. The loader
+			// refuses an unknown condition, but the loader is a DIFFERENT PROCESS: `job.data.waitFor` arrives
+			// over Redis from the receiver, and this whole feature exists because receiver-worker version
+			// skew is real. `makeCheckWaitSkew` closes the backward direction (the file has conditions the
+			// job arrived without); this closes the forward one (a newer receiver enqueues a condition shape
+			// this worker cannot read). Without it the gate would fall through, log `wait_cleared`, and run
+			// the job -- asserting in the log that conditions cleared which it never evaluated, which is the
+			// same undetectable paid run the backward check exists to stop.
+			// A sibling that was held on this same target may already have cleared it. Checked FIRST, because
+			// it is free and determinate, and because the window it closes is one no lease can: two jobs
+			// holding through an outage that outlives their leases would each wake, find no holder, and run.
+			if (dedupId) {
+				const satisfiedBy = await waitState.satisfiedBy(dedupId);
+				if (satisfiedBy && satisfiedBy !== job.id) {
+					return await refuseWait("wait-superseded", "wait_superseded", { satisfiedBy }, "Another delivery for this target already finished waiting on the same conditions. Not run.");
+				}
+			}
+
+			const unreadable = unreadableConditions(job.data);
+			if (unreadable.length > 0) {
+				// Its OWN token, not `wait-skew`. Both are version skew, and the REMEDIES are opposites --
+				// upgrade the receiver there, upgrade the worker here -- so one token in a durable record
+				// would tell an operator that something is out of step and not which way to move.
+				return await refuseWait("wait-unreadable", "refused_wait_unreadable", { conditions: unreadable.length }, `Refused: this job carries ${unreadable.length} wait condition${unreadable.length === 1 ? "" : "s"} this worker cannot read, so it cannot honour them. The worker is older than the service that enqueued this job. Not run.`);
+			}
+
+			// A `profile` condition needs a checker, and with none wired NOTHING can answer it. Refused rather
+			// than ignored: a wait the deployment cannot perform must not read as a wait that passed.
+			const profiles = waitProfileNames(job.data);
+			// Declared-ness is a table lookup, so it belongs with the other free refusals rather than inside
+			// the check. Without it here, `[{after: "<tomorrow>"}, {profile: "typo"}]` holds for a day and
+			// THEN refuses -- which is the exact sentence the ordering rule above promises will not happen.
+			const undeclared = deps?.waitProfileDeclared ? profiles.find((name) => !deps.waitProfileDeclared(name)) : undefined;
+			if (undeclared !== undefined) {
+				return await refuseWait("wait-profile-unknown", "wait_profile_unknown", { profile: undeclared }, `Waiting on \`${undeclared}\` is not something this deployment can answer: no such wait profile is declared here. Not run.`);
+			}
+			if (profiles.length > 0 && !deps?.checkWait) {
+				return await refuseWait("wait-profile-unknown", "wait_profile_unknown", { profile: profiles[0] }, `Waiting on \`${profiles[0]}\` is not something this deployment can answer. Not run.`);
+			}
+
+			const holdUntil = afterMs(job.data); // named apart from the pause gate's `until` above, which it would otherwise shadow
+			// An instant further out than the ceiling is refused at FIRST pickup rather than held toward:
+			// holding for a month to then refuse tells the operator nothing they could not have been told now.
+			if (holdUntil !== null && holdUntil - nowMs > afterMaxMs()) {
+				return await refuseWait("wait-after-beyond-max", "wait_after_beyond_max", { delayMs: holdUntil - nowMs }, `The \`after\` instant is further out than this deployment allows a job to wait. Not run.`);
+			}
+
+			// The pause gate's boundary guard, for its reason: a tick landing on the instant must run rather
+			// than busy-defer to a moment already past.
+			if (holdUntil !== null && holdUntil > nowMs + 1000) {
+				// `isJobLive` is what stops a vanished holder's lease becoming a tombstone that refuses this
+				// target for the rest of the hold. Optional: an unwired probe means the holder cannot be
+				// checked, which ADMITS and says so -- one duplicate run beats one dropped delivery, which is
+				// `OQ-027`'s call ("one wasted vault read beats one dropped job") on this feature's terms.
+				const claim = await waitState.claim(job.id, { dedupId, untilMs: holdUntil, isLive: deps?.isJobLive });
+				if (claim.heldBy) {
+					// Another delivery for this same target and flow is already holding. Both would clear
+					// together and both would be paid, which is the accumulation the acceptance forbids.
+					return await refuseWait("wait-superseded", "wait_superseded", { heldBy: claim.heldBy }, "Another delivery for this target is already waiting on the same conditions. Not run.");
+				}
+				if (claim.retry) {
+					// The holder could not be checked. Holding anyway would put two jobs on one target and pay
+					// for both; refusing would drop a delivery over a holder that may be gone. So decide
+					// nothing: re-defer briefly and ask again once the probe can answer.
+					deps?.log?.("wait_supersede_unverified", { jobId: job.id, heldBy: claim.holder ?? null, delayMs: SUPERSEDE_RECHECK_MS });
+					await job.moveToDelayed(nowMs + SUPERSEDE_RECHECK_MS, token);
+					throw new DelayedError();
+				}
+				if (claim.tookOverFrom) deps?.log?.("wait_lease_taken_over", { jobId: job.id, from: claim.tookOverFrom });
+				await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: holdUntil });
+				deps?.log?.("wait_deferred", { jobId: job.id, until: new Date(holdUntil).toISOString(), label: waitLabel(job.data) });
+				await job.moveToDelayed(holdUntil, token);
+				throw new DelayedError();
+			}
+
+			// TIER 2: the polled conditions. Last, because it is the only part of this gate that spawns a
+			// process -- the free refusals above it are free, and the free hold above it is free.
+			if (profiles.length > 0) {
+				const held = (await waitState.heldForMs(job.id)) ?? 0;
+				const counted = await waitState.counters(job.id);
+
+				// One check at a time, process-wide, and never the worker's last free slot. This is the bound
+				// that keeps a wait from starving the paid work it is waiting for: slots x timeout is the most
+				// wall-clock a worker can spend answering questions instead of running jobs. Computed against
+				// the LIVE concurrency rather than the boot value, because the overlay can lower it.
+				const slots = Math.min(checkSlotCount(), Math.max(1, concurrencyNow() - 1));
+				if (!checkSlots.tryAcquire(WAIT_CHECK_KEY, slots)) {
+					// Denials are counted, and a run of them is the ONE symptom the capacity bound has. The
+					// lease deliberately caps how much wall-clock this worker spends checking; being at that
+					// cap constantly means demand exceeds it, which the issue's own economics say arrives
+					// silently -- paid jobs starve behind checks that spend nothing and nothing says why.
+					const denials = await waitState.noteThrottle(job.id, { denied: true });
+					if (denials === THROTTLE_ALARM) deps?.log?.("wait_capacity_exceeded", { jobId: job.id, denials, slots, hint: "raise PI_WAIT_CHECK_SLOTS or PI_CONCURRENCY, lengthen PI_WAIT_INTERVAL_MS, or hold fewer jobs" });
+					// A starved job still needs a CLOCK and a CEILING, or the lease turns into the very
+					// starvation it exists to bound: without this the hold is stamped only on a wake that won
+					// the lease, so a job that never wins one has no `since`, never reaches the maximum, and
+					// re-wakes forever with no record and no bound. There is no deciding check to run first
+					// here -- that is the whole condition -- so the bound applies directly.
+					await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: nowMs + maxWaitMs() });
+					if (held >= maxWaitMs()) {
+						return await refuseWait("wait-expired", "wait_expired", { reason: "max-wait-unchecked", denials, heldForMs: held }, `Gave up waiting: this deployment could not run the check often enough to answer within the maximum wait. Not run.`);
+					}
+					// Denied. Re-ask at a fraction of the cadence rather than the full backoff (which would
+					// turn one lost coin-flip into a fifteen-minute penalty) or a flat few seconds (which at
+					// scale is a herd). Jittered, so a fleet of denied jobs does not return together.
+					const wait = Math.max(THROTTLE_FLOOR_MS, Math.floor(waitBackoffMs(intervalMs(), held) / 4));
+					const delay = wait + Math.floor(wait * 0.1 * random());
+					deps?.log?.("wait_check_throttled", { jobId: job.id, delayMs: delay, slots });
+					await job.moveToDelayed(nowMs + delay, token);
+					throw new DelayedError();
+				}
+
+				// Declared outside the try below because the branches AFTER it read both.
+				let verdict = null;
+				let checked = null;
+				// THE LEASE IS HELD FROM THE `tryAcquire` ABOVE, so every exit from here down must release it.
+				// The try opens here and not at the check loop, which is where it used to open: the supersede
+				// claim sits between the two, and BOTH of its exits leave -- one returns `wait-superseded`,
+				// the other re-defers and throws -- so a claim that refused or could not be verified walked
+				// out holding the slot. At the shipped default of one slot that wedged every wait check on
+				// the worker until it restarted, and the symptom was silent in the worst way: held jobs kept
+				// throttling and eventually recorded `wait-expired` with `max-wait-unchecked`, which blames
+				// the deployment's capacity for a slot this gate leaked.
+				try {
+					// Claimed BEFORE the check, not after: a second delivery for an already-held target is a free
+					// determinate refusal, and paying for a subprocess first inverts the free-before-costly rule
+					// this gate's own header invokes. Tier 1 already claims in this order.
+					const claim = await waitState.claim(job.id, { dedupId, untilMs: nowMs + waitBackoffMs(intervalMs(), held), isLive: deps?.isJobLive });
+					if (claim.heldBy) {
+						return await refuseWait("wait-superseded", "wait_superseded", { heldBy: claim.heldBy }, "Another delivery for this target is already waiting on the same conditions. Not run.");
+					}
+					if (claim.retry) {
+						deps?.log?.("wait_supersede_unverified", { jobId: job.id, heldBy: claim.holder ?? null, delayMs: SUPERSEDE_RECHECK_MS });
+						await job.moveToDelayed(nowMs + SUPERSEDE_RECHECK_MS, token);
+						throw new DelayedError();
+					}
+
+					await waitState.noteThrottle(job.id, { denied: false }); // granted: the run of denials ends here
+					// Sequential, in the operator's writing order: the resolver's reason applies unchanged --
+					// naming the first condition that did not clear is what makes a held row readable, and a
+					// parallel fan-out would blame whichever lost the race on any given wake.
+					for (const profile of profiles) {
+						checked = profile;
+						verdict = await deps.checkWait(profile, targetFor(job.data?.kind, job.data), { signal });
+						if (verdict?.profileUnknown || verdict?.verdict !== "go") break;
+					}
+				} finally {
+					checkSlots.release(WAIT_CHECK_KEY);
+				}
+
+				if (verdict?.unusableTarget) {
+					// Determinate and unfixable by waiting: the job's own target is a shape no check can be
+					// handed. It belongs with the refusals, not the holds -- holding would spend the fault
+					// budget and then blame the operator's script for a value it was never given.
+					return await refuseWait("wait-unreadable", "refused_wait_unreadable", { profile: checked }, `Refused: this job's target cannot be handed to a wait check, so \`${checked}\` can never be asked. Not run.`);
+				}
+				if (verdict?.profileUnknown) {
+					return await refuseWait("wait-profile-unknown", "wait_profile_unknown", { profile: verdict.profileUnknown }, `Waiting on \`${verdict.profileUnknown}\` is not something this deployment can answer: no such wait profile is declared here. Not run.`);
+				}
+				if (verdict?.verdict === "refuse") {
+					// Exit 2: the check says this will NEVER clear. Terminal by the protocol's own words, and
+					// distinct from every "not yet" above it.
+					return await refuseWait("wait-refused", "wait_refused", { profile: checked, heldForMs: held }, `The check \`${checked}\` reports this will never clear. Not run.`);
+				}
+
+				if (verdict?.aborted) {
+					// The worker is stopping or this job was cancelled. Nothing was learned and nothing is
+					// owed: re-defer at once rather than at the full backoff, and count neither a check nor a
+					// fault, or a rolling deploy would spend a job's whole budget on its own restarts and then
+					// blame the operator's script for it.
+					deps?.log?.("wait_check_aborted", { jobId: job.id, profile: checked });
+					await job.moveToDelayed(nowMs + THROTTLE_FLOOR_MS, token);
+					throw new DelayedError();
+				}
+
+				if (verdict?.verdict === "hold") {
+					const fault = verdict.fault === true;
+					await waitState.noteCheck(job.id, { fault });
+					const faults = fault ? counted.faults + 1 : 0;
+
+					// A check that never answers is a broken script, not a slow condition, and OQ-030 is why
+					// this bound exists: most CLIs exit 1 for everything, so without it a typo would hold for
+					// the whole maximum wait and then blame the CONDITION rather than the check.
+					if (faults >= maxFaults()) {
+						return await refuseWait("wait-unanswerable", "wait_unanswerable", { profile: checked, faults }, `The check \`${checked}\` could not answer ${faults} times in a row. Not run.`);
+					}
+
+					// BOTH terminal bounds are tested AFTER the check and never before it, so a condition that
+					// cleared on the deciding wake runs instead of being recorded as never having cleared.
+					// Without that ordering the backoff's own quantisation makes "cleared at t+1s, declared
+					// never-cleared at t+900s" a structural lie in the durable record and in a public comment.
+					//
+					// The count bound reads `checks + 1` because this wake's check has just run: the job gets
+					// exactly `maxChecks` checks, the last of which is the deciding one. Putting it before the
+					// check instead -- so the act of testing the bound could not exceed it -- was the obvious
+					// spelling, and it silently made this whole guarantee untrue at every shipped default,
+					// because the count bound is the one that fires first there.
+					if (counted.checks + 1 >= maxChecks()) {
+						return await refuseWait("wait-expired", "wait_expired", { reason: "max-checks", checks: counted.checks + 1, profile: checked, heldForMs: held }, `Gave up waiting on \`${checked}\` after ${counted.checks + 1} checks. Not run.`);
+					}
+					if (held >= maxWaitMs()) {
+						return await refuseWait("wait-expired", "wait_expired", { reason: "max-wait", profile: checked, heldForMs: held }, `Gave up waiting on \`${checked}\`. Not run.`);
+					}
+
+					// Clamped to what is LEFT of the budget, never just the cadence. Without this an hourly
+					// interval under a fifteen-minute maximum holds for the full hour -- 400% of the bound the
+					// operator configured -- because the ceiling is only tested when a wake arrives, and the
+					// cadence decides when that is. The two knobs are independent `positiveInt`s and nothing
+					// cross-validates them, so the clamp is what makes the smaller one actually bind.
+					const base = waitBackoffMs(intervalMs(), held);
+					const jittered = base + Math.floor(base * 0.1 * random());
+					const remaining = Math.max(0, maxWaitMs() - held);
+					const delay = Math.max(1000, Math.min(jittered, remaining));
+					await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: nowMs + delay });
+					deps?.log?.("wait_deferred", { jobId: job.id, profile: checked, fault, heldForMs: held, delayMs: delay });
+					await job.moveToDelayed(nowMs + delay, token);
+					throw new DelayedError();
+				}
+
+				// FAIL CLOSED on anything that is not literally go. Everything above tests for a specific
+				// shape and falls through otherwise, and "otherwise" at this gate means STARTING A PAID
+				// CONTAINER -- so an `undefined`, a `null`, a `{}`, a mis-cased "GO" or a bare string from a
+				// checker would run the job silently, with no record field and no log line to distinguish it
+				// from a job whose check said yes. The shipped checker is total, and that is exactly the
+				// reasoning `unreadableConditions` above rejects: this is a dependency-injection seam, and a
+				// seam's guarantees are the caller's to enforce.
+				if (verdict?.verdict !== "go") {
+					deps?.log?.("wait_check_unintelligible", { jobId: job.id, profile: checked });
+					await waitState.noteCheck(job.id, { fault: true });
+					const base = waitBackoffMs(intervalMs(), held);
+					await job.moveToDelayed(nowMs + base + Math.floor(base * 0.1 * random()), token);
+					throw new DelayedError();
+				}
+
+				// Every profile answered go. Record the last check so the count bound sees it.
+				await waitState.noteCheck(job.id, { fault: false });
+			}
+
+			// Only a job that actually HELD has cleared. Without the check this line fires on the first
+			// pickup of a job whose instant had already passed, and again on every scope-busy re-check
+			// afterwards -- asserting a wait ended that never began.
+			const heldForMs = await waitState.heldForMs(job.id);
+			// Say so before releasing: a sibling held on this target must find the answer, not an empty lease.
+			if (heldForMs !== null) await waitState.markSatisfied(job.id, { dedupId });
+			await waitState.release(job.id, { dedupId });
+			if (heldForMs !== null) deps?.log?.("wait_cleared", { jobId: job.id, label: waitLabel(job.data), heldForMs });
+		}
+
 		// Per-scope concurrency and the one-job-per-folder mutex (issue #242,
-		// INT-SCOPED-LIMITS-FILE-CONTRACT). SECOND, after the pause gate (a paused job must not burn
-		// re-check wakes) and STRICTLY above the `try` below, like the pause gate and for the same two
+		// INT-SCOPED-LIMITS-FILE-CONTRACT). LAST of the three gates, after the pause gate (a paused job must
+		// not burn re-check wakes) and after the wait gate (a job holding until tomorrow must not sit on a
+		// folder while it does), and STRICTLY above the `try` below, like the pause gate and for the same two
 		// reasons: a DelayedError thrown inside the try would be converted to UnrecoverableError by the
 		// catch, and a moveToDelayed rejection here must escape RAW into BullMQ's normal failed-attempt
 		// handling exactly as the pause gate's does (inside the try it would become a permanent failure
@@ -123,7 +434,9 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 				// job completed and does not retry a file that can never parse (CONST-RETRY-INFRA-ONLY). Resolved
 				// before runJob, so no budget slot is reserved and no container starts (CONST-BUDGET-BEFORE-TOKENS).
 				// recordRun leaves the durable settings-overlay-invalid trace for the admin extension.
-				// No provider/model here, alone among the terminal results: the overlay is the thing that failed to parse, so no honest effective value exists -- buildRecord defaults both null.
+				// No provider/model here, as in every result this function returns from ABOVE the try (the wait gate's
+				// refusals are the others): each is decided before or during the settings read, so no honest effective
+				// value exists yet -- buildRecord defaults both null.
 				const result = { outcome: "policy", reason: "settings-overlay-invalid", exitCode: null, turns: null, tokens: null, budgetReserved: false };
 				recordRun({ job, result, startedAt, endedAt: new Date().toISOString() });
 				return result;
@@ -210,7 +523,7 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 	};
 }
 
-export function createWorker({ connection, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, scopedLimits, inFlight, extraClosers = [] }) {
+export function createWorker({ connection, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, scopedLimits, inFlight, waitState, afterMaxMs, checkSlots, checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, extraClosers = [] }) {
 	let worker; // referenced by cancelJob/applyConcurrency before assignment; only called later, so the TDZ is fine
 	const processor = makeProcessor({
 		cancelJob: (id, reason) => worker.cancelJob(id, reason),
@@ -228,6 +541,21 @@ export function createWorker({ connection, concurrency, getSettings, redis, deps
 		// shape means one per daemon).
 		scopedLimits,
 		inFlight,
+		// Issue #230. Undefined pass-throughs take makeProcessor's own defaults (a wait state over the same
+		// redis client, and the shared 30-day `after` ceiling), so a bare wiring behaves like a wired one.
+		waitState,
+		afterMaxMs,
+		// Issue #230, the polled tier. `concurrencyNow` reads the LIVE slot count rather than the boot value,
+		// because the overlay can lower it through `dispatch_set` and a check must never take the last free
+		// slot from a paid job. Late-bound over `worker` exactly as `applyConcurrency` is, and for the same
+		// reason: the value it needs does not exist until the Worker is constructed.
+		checkSlots,
+		checkSlotCount,
+		concurrencyNow: concurrencyNow ?? (() => worker?.concurrency ?? concurrency),
+		intervalMs,
+		maxWaitMs,
+		maxChecks,
+		maxFaults,
 		deps,
 		recordRun,
 	});

package/src/processor.mjs

@@ -54,6 +54,9 @@ export async function runJob(job, deps) {
 		// exactly as before, and the gate below only calls it for a job whose matched rule was a
 		// one-shot, so the default is never a probe running on every delivery.
 		checkOnceSpent = async () => ({ ok: true }),
+		// Issue #230. Admit-everything by default, like checkOnceSpent above and for its reason: an
+		// unwired seam must not refuse, and the wiring is what turns the check on.
+		checkWaitSkew = async () => ({ ok: true }),
 		// REQ-EGRESS-ALLOWLIST. Default admits everything, so a wiring that omits it behaves exactly as a
 		// deployment with no egress policy does -- which is also what the real factory returns when unarmed.
 		egressPreflight = async () => ({ ok: true }),
@@ -155,6 +158,28 @@ export async function runJob(job, deps) {
 			}
 		}
 
+		// The wait-skew check (issue #230), second on the ladder and for the first one's reasons: the same
+		// file read, free, determinate, credential-less, and pre-spend. It answers a question no other layer
+		// can: does the AUTHORED trigger carry wait conditions this job arrived without? That happens when a
+		// service below the version floor dropped the field as an unknown key, and the resulting run is
+		// byte-identical to a correct one everywhere it is recorded -- so this refusal is the only thing
+		// standing between a stale receiver and a paid job that ran when the operator wrote "wait".
+		{
+			const skew = await checkWaitSkew(job);
+			if (skew.skewed) {
+				// Named for the operator, not the payload: how many conditions were authored, never what
+				// they say. The fix is a version, so the comment says which one.
+				// The message names BOTH causes, because the more likely one is not a version at all. In the
+				// compose topology the receiver's single-file `:ro` mount pins a dead inode, so an operator
+				// who ADDS `waitFor` to an existing rule gets this refusal on every delivery from a service
+				// that is perfectly up to date and merely holding an older copy of the file. Naming only the
+				// version would send them looking for an upgrade they do not need.
+				await comment(job, `Refused: this trigger declares ${skew.conditions} wait condition${skew.conditions === 1 ? "" : "s"}, but the job reached the worker without them, which means it would have run immediately. Either a service in this deployment is below the version that carries the field, or one is still running against an older copy of the triggers file and needs restarting. Not run.`);
+				log("refused_wait_skew", { triggerIndex: job.trigger?.matched?.index ?? null, conditions: skew.conditions });
+				return { outcome: "policy", reason: "wait-skew", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false };
+			}
+		}
+
 		// The job image must exist on THIS host before anything else happens. Free, determinate and
 		// credential-less, so it precedes the mint, the clone and the reservation: a host that cannot run the
 		// image refuses without minting a credential it will not use, cloning a repo it will not read, or

package/src/queue.mjs

@@ -134,7 +134,7 @@ export async function enqueueGitLabJob(queue, fields) {
  * window, replicas never coalesce against each other, and an unflagged job's dedup id is the same string it
  * has always been.
  */
-export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, command, trigger, provider, model, maxTurns, packages, image, skillsDir, instructions, resume, secrets, secretsProfile, replica, replicas }) {
+export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, command, trigger, provider, model, maxTurns, packages, image, skillsDir, instructions, resume, secrets, secretsProfile, waitFor, replica, replicas }) {
 	const jobId = forgeDeliveryJobId(kind, trigger?.deliveryId, replica);
 	// `packages` (whether to load the operator-staged pi packages) and `image` (which container image to run)
 	// come off the MATCHED trigger (INT-TRIGGERS-FILE-CONTRACT / REQ-GLOBAL-PI-OVERLAY) and land on `data`
@@ -179,6 +179,16 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
 		// is copied verbatim into /job/event.json, which an agent reads.
 		...(secrets !== undefined && { secrets }),
 		...(secretsProfile !== undefined && { secretsProfile }),
+		// Issue #230. The conditions the worker holds this job on, carried so the PICKUP gate can read them:
+		// that gate runs above the per-job settings read and never re-parses the triggers file for its terms.
+		// At JOB level, and here that placement is a correctness requirement rather than a convention --
+		// `trigger` is copied VERBATIM into /job/event.json (prepare-local.mjs), so a `trigger.waitFor` would
+		// hand the agent the operator's own gate. Conditional like every field above, so an unflagged job's
+		// data keeps exactly the keys it has today. The dedup options below are deliberately NOT widened for
+		// a waiting job: that key carries no trigger identity and outlives the job it was set for, so a
+		// longer window would suppress an unflagged sibling's deliveries and go on suppressing them after
+		// this job finished. Coalescing a held target is the worker's `wait:` keyspace's job instead.
+		...(waitFor !== undefined && { waitFor }),
 		// Conditional for the same reason packages/image/resume are: an unflagged job's data must keep
 		// exactly the keys it has today. `replica` is this job's 1-based index and `replicas` the set size;
 		// both are integers, so the run record they land in stays PII-free by construction.

package/src/run-history.mjs

@@ -114,7 +114,34 @@ export function parseExitTurns(text) {
  * Read-only telemetry, exactly like `parseExitTurns`: NEVER throws and MUST NOT feed exit-code or retry
  * classification (INT-RUNNER-EXIT-CODE-PROTOCOL). A malformed or non-object `tokens` (or one missing a
  * numeric `total`) is `null`, never a partial that could poison the daily token counter.
+ *
+ * The admitted object is REBUILT through `rebuildTokens` rather than returned as it arrived. See that
+ * function for why: the pass-through it replaces is what made this comment's "integer token counts and
+ * numeric cost only" false one level below `buildRecord`'s literal.
+ */
+/**
+ * The CLOSED `session.reason` enum, verbatim from `INT-RUN-HISTORY-FILE-CONTRACT`. Three producers write
+ * this field (resolve, runner, promote) and the contract has always called the set closed; until this
+ * list existed, nothing enforced it and the runner's half was an unchecked string. Kept here rather than
+ * beside the store because this module is where the container's copy is admitted, and an enum that lives
+ * anywhere but the admission point is a comment, not a check.
  */
+const SESSION_REASONS = new Set([
+	"resumed",
+	"absent",
+	"expired",
+	"conversation-too-old",
+	"resume-chain-too-long",
+	"context-too-full",
+	"too-large",
+	"unparseable",
+	"not-a-regular-file",
+	"pi-version-changed",
+	"locked",
+	"promote-failed",
+	"disabled",
+]);
+
 /**
  * The runner's `session` object off the exit line: `{ resumed: <bool>, reason: "<enum>" }` or null when
  * the container died before emitting one (REQ-RESUMABLE-SESSION).
@@ -125,7 +152,8 @@ export function parseExitTurns(text) {
  * without both numbers it is indistinguishable from an ordinary cold start. A feature that fails open
  * must still say that it did.
  *
- * PII-free by construction: a boolean and a fixed enum. No key, no branch name, no path.
+ * PII-free by construction: a boolean and a fixed enum. No key, no branch name, no path -- and since the
+ * `SESSION_REASONS` check below, that sentence is enforced rather than merely intended.
  */
 export function parseExitSession(text) {
 	if (typeof text !== "string") return null;
@@ -137,7 +165,14 @@ export function parseExitSession(text) {
 		if (parsed?.event !== "exit") continue;
 		const sess = parsed?.session;
 		if (sess && typeof sess === "object" && !Array.isArray(sess) && typeof sess.resumed === "boolean") {
-			return { resumed: sess.resumed, reason: typeof sess.reason === "string" ? sess.reason : null };
+			// The reason is checked against the CLOSED enum, not merely against `typeof === "string"`, which
+			// is what it used to be. The container owns this value, so an unchecked string put an
+			// attacker-shapeable one into a record whose PII-free property rests on holding none -- while
+			// the comment above claimed "a boolean and a fixed enum". An unrecognised token reads as `null`
+			// (the runner said nothing this contract can represent) rather than being carried through: the
+			// enum is documented CLOSED in INT-RUN-HISTORY-FILE-CONTRACT, so a value outside it was already
+			// contract-violating and every consumer already handles null.
+			return { resumed: sess.resumed, reason: SESSION_REASONS.has(sess.reason) ? sess.reason : null };
 		}
 		return null;
 	}
@@ -176,6 +211,43 @@ export function parseExitContext(text) {
 	return null;
 }
 
+/**
+ * The keys the runner actually emits, in its own emission order: the metered snapshot
+ * (`image/runner/src/usage-meter.mjs` -> `snapshot`) plus the token-budget fallback
+ * (`image/runner/run-job.mjs` -> `pickTotals`, which sends the first four and `metered: false`). Order
+ * matters because it is what makes a conformant runner's object round-trip byte-identically through the
+ * rebuild below, so the record's bytes do not move for anyone running a real image.
+ */
+const TOKEN_KEYS = ["input", "output", "total", "cost", "metered", "rootTotal", "otherTotal", "looseTotal", "sessions", "calls", "unresolved", "unpriced"];
+
+/**
+ * Rebuild the billed totals from a closed key list rather than passing the container's object through.
+ *
+ * This function exists because the pass-through was a hole. `parseExitTokens` used to `return t`
+ * verbatim whenever `t.total` was a number, so any key the container invented -- a path, a branch name,
+ * a string it read out of the workspace -- rode into the durable record, and from there into anything
+ * that mirrors it. The record's PII-free-by-construction property held at `buildRecord`'s own level and
+ * NOT one level down, while this module's own comment claimed "integer token counts and numeric cost
+ * only". `parseExitUsage` already rebuilds for exactly this reason and says so; this is the sibling that
+ * did not, and the asymmetry was an oversight rather than a decision.
+ *
+ * A key the runner omitted stays OMITTED rather than becoming null: the fallback shape legitimately
+ * carries only five of the twelve, and a null there would read as "measured zero" for a number nobody
+ * measured. `typeof === "number"` rather than `Number.isFinite`, deliberately, so this narrows WHICH
+ * KEYS survive and never which objects are admitted -- the admission gate above is unchanged.
+ */
+function rebuildTokens(t) {
+	const out = {};
+	for (const key of TOKEN_KEYS) {
+		if (key === "metered") {
+			if (typeof t.metered === "boolean") out.metered = t.metered;
+		} else if (typeof t[key] === "number") {
+			out[key] = t[key];
+		}
+	}
+	return out;
+}
+
 export function parseExitTokens(text) {
 	if (typeof text !== "string") return null;
 	const lines = text.split("\n");
@@ -185,7 +257,7 @@ export function parseExitTokens(text) {
 		const parsed = parseTailLine(line);
 		if (parsed?.event !== "exit") continue;
 		const t = parsed?.tokens;
-		if (t && typeof t === "object" && !Array.isArray(t) && typeof t.total === "number") return t;
+		if (t && typeof t === "object" && !Array.isArray(t) && typeof t.total === "number") return rebuildTokens(t);
 		return null;
 	}
 	return null;
@@ -387,7 +459,12 @@ export function buildRecord({ job, result, error, startedAt, endedAt }) {
  * separator from the table, so the notation a forge uses is the notation its records carry -- and a forge
  * added later inherits a label rather than a null.
  */
-function targetFor(kind, data) {
+/*
+ * Exported since issue #230: a held job's panel row needs the same id-only label a run record carries, and
+ * the wait gate would otherwise re-derive it. Two spellings of "which issue is this" is how one of them
+ * starts carrying a title.
+ */
+export function targetFor(kind, data) {
 	if (kind === "local") return `local:${basename(data.folder ?? "")}`;
 	if (isForgeKind(kind)) return `${data.repo}${targetSeparator(kind, data.target?.type)}${data.target?.number}`;
 	return null;

package/src/service.mjs

@@ -972,6 +972,15 @@ async function doRestart(ctx, values) {
 			await ctx.sleep(2000);
 			({ active = 0 } = await queue.getJobCounts("active"));
 		}
+		// A HELD job is neither active nor waiting, so the loop above has just reported a drained queue with
+		// however many jobs still parked on `run.waitFor` (issue #230). They are safe -- a hold spends
+		// nothing, survives a restart and reserves no slot -- but the operator is upgrading, and those jobs
+		// will wake against the new version. Said plainly rather than left to be discovered, which is what
+		// this command would otherwise be doing: reporting a drained queue it cannot see all of.
+		const delayed = await queue.getJobCounts("delayed").then((c) => Number(c?.delayed ?? 0), () => 0);
+		if (delayed > 0) {
+			ctx.out(`note: ${delayed} job(s) sit in the delayed set (cron next-occurrences, retry backoff, quiet hours, or jobs held on run.waitFor). None is active, so none blocked this drain; they will wake against the new version.\n`);
+		}
 		const stopped = await doStop(ctx);
 		if (stopped !== 0) {
 			ctx.out("restart did not happen — the queue STAYS PAUSED; fix the service, then `pi-dispatch resume`.\n");