@edgehero/pi-dispatch 1.5.0 → 1.6.0

package/src/index.mjs

@@ -2,7 +2,10 @@ import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 import { DelayedError, UnrecoverableError, Worker } from "bullmq";
 import { InfraRetry, runJob } from "./processor.mjs";
+import { targetFor } from "./run-history.mjs";
 import { budgetCapsFor, canonicalScope, concurrencyFor, makeInFlight } from "./scoped-limits.mjs";
+import { WAIT_AFTER_MAX_DEFAULT_MS, WAIT_INTERVAL_FLOOR_MS, afterMs, unreadableConditions, waitArmed, waitBackoffMs, waitLabel, waitProfileNames } from "./wait-for.mjs";
+import { makeWaitState } from "./wait-state.mjs";
 
 const exec = promisify(execFile);
 
@@ -16,6 +19,28 @@ export const JOB_TIMEOUT_MS = 30 * 60 * 1000; // REQ-JOB-TIMEOUT-30M
 // docker daemon bounds any herd by its own concurrency, and a contended wake just re-defers.
 export const SCOPE_BUSY_RECHECK_MS = 5_000;
 
+// How long a job waits before re-asking whether a target's holder is still alive (issue #230). Reached only
+// when the liveness probe could not answer, which is a redis or queue fault rather than a normal state, so
+// this is a short retry rather than a cadence: the job is deciding nothing and holding nothing while it
+// waits, and the fault it is waiting out is usually seconds long.
+export const SUPERSEDE_RECHECK_MS = 15_000;
+
+// The one key the check lease counts under. A single global counter rather than one per profile: what it
+// bounds is this worker's wall-clock spent answering questions, and that is shared whatever is being asked.
+const WAIT_CHECK_KEY = "wait-check";
+
+// How many consecutive lease denials one job absorbs before the deployment is told its checking capacity is
+// short. Logged ONCE per run of denials rather than per wake: an alarm that repeats every re-check is the
+// always-on amber this project rejects elsewhere, and the operator only needs telling once per episode.
+const THROTTLE_ALARM = 5;
+
+// The floor under a throttled or aborted re-ask. Its own constant rather than a borrow of
+// SUPERSEDE_RECHECK_MS, which documents an unrelated concern. Deliberately NOT 5s: that is
+// SCOPE_BUSY_RECHECK_MS, and INT-WAIT-PROFILES-CONTRACT rests on wait deferrals being distinguishable from
+// scope deferrals by wake instant -- nothing records WHY a job sits in the delayed set, so the instants are
+// the only evidence there is. A test pins the two apart.
+const THROTTLE_FLOOR_MS = 11_000;
+
 /**
  * Build the BullMQ processor.
  *
@@ -35,7 +60,7 @@ export const SCOPE_BUSY_RECHECK_MS = 5_000;
  * The overlay changes which values the spend caps take, never when they are checked -- reserveBudget still
  * runs inside runJob against the freshly passed caps (CONST-BUDGET-BEFORE-TOKENS).
  */
-export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now() }) {
+export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
 	return async function processor(job, token, signal) {
 		// Scoped pause windows (REQ-SCOPED-PAUSE-WINDOWS): if this job's folder/repo is inside an active pause
 		// window, DEFER it to the window end via BullMQ's delayed set -- the job keeps its identity/dedup and
@@ -53,9 +78,286 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 			throw new DelayedError();
 		}
 
+		// The wait gate (issue #230, REQ-WAIT-FOR). THIRD: after the pause gate, because a paused job must
+		// not burn a wait evaluation any more than it burns a scope re-check, and BEFORE the scope acquire,
+		// because a job that is going to sit until tomorrow morning must not hold the folder mutex while it
+		// does. Strictly above the `try` for the two reasons the gates below it document.
+		//
+		// The order WITHIN the gate is determinate-refusals-then-holds, which is CONST-BUDGET-BEFORE-TOKENS'
+		// shape applied to time rather than to money: a condition this deployment can never answer must be
+		// refused now, not after a day of waiting.
+		//
+		// On throwing above the `try`: an exception here escapes into BullMQ's normal failed-attempt handling,
+		// which is WANTED for `moveToDelayed` (the scope gate below gives the argument: a transient rejection
+		// must stay a transient failure rather than becoming a permanent one) and unwanted everywhere else. So
+		// the state and comment seams fail open by construction, and `recordRun` is relied on not to throw --
+		// its writer swallows fs errors by contract, which is the same reliance the settings-overlay refusal
+		// below already makes.
+		if (waitArmed(job.data)) {
+			// The supersede identity: the queue's semantic key PLUS the trigger that produced this job.
+			// The semantic key alone is `repo<sep>number:flow`, which two DIFFERENT triggers on one target and
+			// flow legitimately share -- a label rule that waits a day and a comment rule that waits a minute
+			// would coalesce, and the second would be refused with a message claiming they wait on "the same
+			// conditions" when they do not. Adding the raw trigger index makes the key mean one intent.
+			const matchedIndex = job.data?.trigger?.matched?.index;
+			const dedupId = job.deduplicationId ? `${job.deduplicationId}#${Number.isInteger(matchedIndex) ? matchedIndex : "?"}` : null;
+			const refuseWait = async (reason, logEvent, fields, sentence) => {
+				// The INJECTED clock, like both gates above: a record whose timestamps ignore the test clock
+				// is a record no test of this gate can assert about.
+				const at = new Date(now()).toISOString();
+				await waitState.release(job.id, { dedupId });
+				deps?.log?.(logEvent, { jobId: job.id, ...fields });
+				// The comment names the FIELD and the operator's own words for the condition, never a
+				// resolver path or a vault topology -- `secret-profile-unknown` sets that rule.
+				if (sentence && deps?.comment) await Promise.resolve(deps.comment(job.data, sentence)).catch(() => {});
+				const result = { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, budgetReserved: false };
+				recordRun({ job, result, startedAt: at, endedAt: new Date().toISOString() });
+				return result;
+			};
+
+			// EVERY condition must be one this worker understands, checked before anything else. The loader
+			// refuses an unknown condition, but the loader is a DIFFERENT PROCESS: `job.data.waitFor` arrives
+			// over Redis from the receiver, and this whole feature exists because receiver-worker version
+			// skew is real. `makeCheckWaitSkew` closes the backward direction (the file has conditions the
+			// job arrived without); this closes the forward one (a newer receiver enqueues a condition shape
+			// this worker cannot read). Without it the gate would fall through, log `wait_cleared`, and run
+			// the job -- asserting in the log that conditions cleared which it never evaluated, which is the
+			// same undetectable paid run the backward check exists to stop.
+			// A sibling that was held on this same target may already have cleared it. Checked FIRST, because
+			// it is free and determinate, and because the window it closes is one no lease can: two jobs
+			// holding through an outage that outlives their leases would each wake, find no holder, and run.
+			if (dedupId) {
+				const satisfiedBy = await waitState.satisfiedBy(dedupId);
+				if (satisfiedBy && satisfiedBy !== job.id) {
+					return await refuseWait("wait-superseded", "wait_superseded", { satisfiedBy }, "Another delivery for this target already finished waiting on the same conditions. Not run.");
+				}
+			}
+
+			const unreadable = unreadableConditions(job.data);
+			if (unreadable.length > 0) {
+				// Its OWN token, not `wait-skew`. Both are version skew, and the REMEDIES are opposites --
+				// upgrade the receiver there, upgrade the worker here -- so one token in a durable record
+				// would tell an operator that something is out of step and not which way to move.
+				return await refuseWait("wait-unreadable", "refused_wait_unreadable", { conditions: unreadable.length }, `Refused: this job carries ${unreadable.length} wait condition${unreadable.length === 1 ? "" : "s"} this worker cannot read, so it cannot honour them. The worker is older than the service that enqueued this job. Not run.`);
+			}
+
+			// A `profile` condition needs a checker, and with none wired NOTHING can answer it. Refused rather
+			// than ignored: a wait the deployment cannot perform must not read as a wait that passed.
+			const profiles = waitProfileNames(job.data);
+			// Declared-ness is a table lookup, so it belongs with the other free refusals rather than inside
+			// the check. Without it here, `[{after: "<tomorrow>"}, {profile: "typo"}]` holds for a day and
+			// THEN refuses -- which is the exact sentence the ordering rule above promises will not happen.
+			const undeclared = deps?.waitProfileDeclared ? profiles.find((name) => !deps.waitProfileDeclared(name)) : undefined;
+			if (undeclared !== undefined) {
+				return await refuseWait("wait-profile-unknown", "wait_profile_unknown", { profile: undeclared }, `Waiting on \`${undeclared}\` is not something this deployment can answer: no such wait profile is declared here. Not run.`);
+			}
+			if (profiles.length > 0 && !deps?.checkWait) {
+				return await refuseWait("wait-profile-unknown", "wait_profile_unknown", { profile: profiles[0] }, `Waiting on \`${profiles[0]}\` is not something this deployment can answer. Not run.`);
+			}
+
+			const holdUntil = afterMs(job.data); // named apart from the pause gate's `until` above, which it would otherwise shadow
+			// An instant further out than the ceiling is refused at FIRST pickup rather than held toward:
+			// holding for a month to then refuse tells the operator nothing they could not have been told now.
+			if (holdUntil !== null && holdUntil - nowMs > afterMaxMs()) {
+				return await refuseWait("wait-after-beyond-max", "wait_after_beyond_max", { delayMs: holdUntil - nowMs }, `The \`after\` instant is further out than this deployment allows a job to wait. Not run.`);
+			}
+
+			// The pause gate's boundary guard, for its reason: a tick landing on the instant must run rather
+			// than busy-defer to a moment already past.
+			if (holdUntil !== null && holdUntil > nowMs + 1000) {
+				// `isJobLive` is what stops a vanished holder's lease becoming a tombstone that refuses this
+				// target for the rest of the hold. Optional: an unwired probe means the holder cannot be
+				// checked, which ADMITS and says so -- one duplicate run beats one dropped delivery, which is
+				// `OQ-027`'s call ("one wasted vault read beats one dropped job") on this feature's terms.
+				const claim = await waitState.claim(job.id, { dedupId, untilMs: holdUntil, isLive: deps?.isJobLive });
+				if (claim.heldBy) {
+					// Another delivery for this same target and flow is already holding. Both would clear
+					// together and both would be paid, which is the accumulation the acceptance forbids.
+					return await refuseWait("wait-superseded", "wait_superseded", { heldBy: claim.heldBy }, "Another delivery for this target is already waiting on the same conditions. Not run.");
+				}
+				if (claim.retry) {
+					// The holder could not be checked. Holding anyway would put two jobs on one target and pay
+					// for both; refusing would drop a delivery over a holder that may be gone. So decide
+					// nothing: re-defer briefly and ask again once the probe can answer.
+					deps?.log?.("wait_supersede_unverified", { jobId: job.id, heldBy: claim.holder ?? null, delayMs: SUPERSEDE_RECHECK_MS });
+					await job.moveToDelayed(nowMs + SUPERSEDE_RECHECK_MS, token);
+					throw new DelayedError();
+				}
+				if (claim.tookOverFrom) deps?.log?.("wait_lease_taken_over", { jobId: job.id, from: claim.tookOverFrom });
+				await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: holdUntil });
+				deps?.log?.("wait_deferred", { jobId: job.id, until: new Date(holdUntil).toISOString(), label: waitLabel(job.data) });
+				await job.moveToDelayed(holdUntil, token);
+				throw new DelayedError();
+			}
+
+			// TIER 2: the polled conditions. Last, because it is the only part of this gate that spawns a
+			// process -- the free refusals above it are free, and the free hold above it is free.
+			if (profiles.length > 0) {
+				const held = (await waitState.heldForMs(job.id)) ?? 0;
+				const counted = await waitState.counters(job.id);
+
+				// One check at a time, process-wide, and never the worker's last free slot. This is the bound
+				// that keeps a wait from starving the paid work it is waiting for: slots x timeout is the most
+				// wall-clock a worker can spend answering questions instead of running jobs. Computed against
+				// the LIVE concurrency rather than the boot value, because the overlay can lower it.
+				const slots = Math.min(checkSlotCount(), Math.max(1, concurrencyNow() - 1));
+				if (!checkSlots.tryAcquire(WAIT_CHECK_KEY, slots)) {
+					// Denials are counted, and a run of them is the ONE symptom the capacity bound has. The
+					// lease deliberately caps how much wall-clock this worker spends checking; being at that
+					// cap constantly means demand exceeds it, which the issue's own economics say arrives
+					// silently -- paid jobs starve behind checks that spend nothing and nothing says why.
+					const denials = await waitState.noteThrottle(job.id, { denied: true });
+					if (denials === THROTTLE_ALARM) deps?.log?.("wait_capacity_exceeded", { jobId: job.id, denials, slots, hint: "raise PI_WAIT_CHECK_SLOTS or PI_CONCURRENCY, lengthen PI_WAIT_INTERVAL_MS, or hold fewer jobs" });
+					// A starved job still needs a CLOCK and a CEILING, or the lease turns into the very
+					// starvation it exists to bound: without this the hold is stamped only on a wake that won
+					// the lease, so a job that never wins one has no `since`, never reaches the maximum, and
+					// re-wakes forever with no record and no bound. There is no deciding check to run first
+					// here -- that is the whole condition -- so the bound applies directly.
+					await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: nowMs + maxWaitMs() });
+					if (held >= maxWaitMs()) {
+						return await refuseWait("wait-expired", "wait_expired", { reason: "max-wait-unchecked", denials, heldForMs: held }, `Gave up waiting: this deployment could not run the check often enough to answer within the maximum wait. Not run.`);
+					}
+					// Denied. Re-ask at a fraction of the cadence rather than the full backoff (which would
+					// turn one lost coin-flip into a fifteen-minute penalty) or a flat few seconds (which at
+					// scale is a herd). Jittered, so a fleet of denied jobs does not return together.
+					const wait = Math.max(THROTTLE_FLOOR_MS, Math.floor(waitBackoffMs(intervalMs(), held) / 4));
+					const delay = wait + Math.floor(wait * 0.1 * random());
+					deps?.log?.("wait_check_throttled", { jobId: job.id, delayMs: delay, slots });
+					await job.moveToDelayed(nowMs + delay, token);
+					throw new DelayedError();
+				}
+
+				// Claimed BEFORE the check, not after: a second delivery for an already-held target is a free
+				// determinate refusal, and paying for a subprocess first inverts the free-before-costly rule
+				// this gate's own header invokes. Tier 1 already claims in this order.
+				const claim = await waitState.claim(job.id, { dedupId, untilMs: nowMs + waitBackoffMs(intervalMs(), held), isLive: deps?.isJobLive });
+				if (claim.heldBy) {
+					return await refuseWait("wait-superseded", "wait_superseded", { heldBy: claim.heldBy }, "Another delivery for this target is already waiting on the same conditions. Not run.");
+				}
+				if (claim.retry) {
+					deps?.log?.("wait_supersede_unverified", { jobId: job.id, heldBy: claim.holder ?? null, delayMs: SUPERSEDE_RECHECK_MS });
+					await job.moveToDelayed(nowMs + SUPERSEDE_RECHECK_MS, token);
+					throw new DelayedError();
+				}
+
+				await waitState.noteThrottle(job.id, { denied: false }); // granted: the run of denials ends here
+				let verdict = null;
+				let checked = null;
+				try {
+					// Sequential, in the operator's writing order: the resolver's reason applies unchanged --
+					// naming the first condition that did not clear is what makes a held row readable, and a
+					// parallel fan-out would blame whichever lost the race on any given wake.
+					for (const profile of profiles) {
+						checked = profile;
+						verdict = await deps.checkWait(profile, targetFor(job.data?.kind, job.data), { signal });
+						if (verdict?.profileUnknown || verdict?.verdict !== "go") break;
+					}
+				} finally {
+					checkSlots.release(WAIT_CHECK_KEY);
+				}
+
+				if (verdict?.unusableTarget) {
+					// Determinate and unfixable by waiting: the job's own target is a shape no check can be
+					// handed. It belongs with the refusals, not the holds -- holding would spend the fault
+					// budget and then blame the operator's script for a value it was never given.
+					return await refuseWait("wait-unreadable", "refused_wait_unreadable", { profile: checked }, `Refused: this job's target cannot be handed to a wait check, so \`${checked}\` can never be asked. Not run.`);
+				}
+				if (verdict?.profileUnknown) {
+					return await refuseWait("wait-profile-unknown", "wait_profile_unknown", { profile: verdict.profileUnknown }, `Waiting on \`${verdict.profileUnknown}\` is not something this deployment can answer: no such wait profile is declared here. Not run.`);
+				}
+				if (verdict?.verdict === "refuse") {
+					// Exit 2: the check says this will NEVER clear. Terminal by the protocol's own words, and
+					// distinct from every "not yet" above it.
+					return await refuseWait("wait-refused", "wait_refused", { profile: checked, heldForMs: held }, `The check \`${checked}\` reports this will never clear. Not run.`);
+				}
+
+				if (verdict?.aborted) {
+					// The worker is stopping or this job was cancelled. Nothing was learned and nothing is
+					// owed: re-defer at once rather than at the full backoff, and count neither a check nor a
+					// fault, or a rolling deploy would spend a job's whole budget on its own restarts and then
+					// blame the operator's script for it.
+					deps?.log?.("wait_check_aborted", { jobId: job.id, profile: checked });
+					await job.moveToDelayed(nowMs + THROTTLE_FLOOR_MS, token);
+					throw new DelayedError();
+				}
+
+				if (verdict?.verdict === "hold") {
+					const fault = verdict.fault === true;
+					await waitState.noteCheck(job.id, { fault });
+					const faults = fault ? counted.faults + 1 : 0;
+
+					// A check that never answers is a broken script, not a slow condition, and OQ-030 is why
+					// this bound exists: most CLIs exit 1 for everything, so without it a typo would hold for
+					// the whole maximum wait and then blame the CONDITION rather than the check.
+					if (faults >= maxFaults()) {
+						return await refuseWait("wait-unanswerable", "wait_unanswerable", { profile: checked, faults }, `The check \`${checked}\` could not answer ${faults} times in a row. Not run.`);
+					}
+
+					// BOTH terminal bounds are tested AFTER the check and never before it, so a condition that
+					// cleared on the deciding wake runs instead of being recorded as never having cleared.
+					// Without that ordering the backoff's own quantisation makes "cleared at t+1s, declared
+					// never-cleared at t+900s" a structural lie in the durable record and in a public comment.
+					//
+					// The count bound reads `checks + 1` because this wake's check has just run: the job gets
+					// exactly `maxChecks` checks, the last of which is the deciding one. Putting it before the
+					// check instead -- so the act of testing the bound could not exceed it -- was the obvious
+					// spelling, and it silently made this whole guarantee untrue at every shipped default,
+					// because the count bound is the one that fires first there.
+					if (counted.checks + 1 >= maxChecks()) {
+						return await refuseWait("wait-expired", "wait_expired", { reason: "max-checks", checks: counted.checks + 1, profile: checked, heldForMs: held }, `Gave up waiting on \`${checked}\` after ${counted.checks + 1} checks. Not run.`);
+					}
+					if (held >= maxWaitMs()) {
+						return await refuseWait("wait-expired", "wait_expired", { reason: "max-wait", profile: checked, heldForMs: held }, `Gave up waiting on \`${checked}\`. Not run.`);
+					}
+
+					// Clamped to what is LEFT of the budget, never just the cadence. Without this an hourly
+					// interval under a fifteen-minute maximum holds for the full hour -- 400% of the bound the
+					// operator configured -- because the ceiling is only tested when a wake arrives, and the
+					// cadence decides when that is. The two knobs are independent `positiveInt`s and nothing
+					// cross-validates them, so the clamp is what makes the smaller one actually bind.
+					const base = waitBackoffMs(intervalMs(), held);
+					const jittered = base + Math.floor(base * 0.1 * random());
+					const remaining = Math.max(0, maxWaitMs() - held);
+					const delay = Math.max(1000, Math.min(jittered, remaining));
+					await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: nowMs + delay });
+					deps?.log?.("wait_deferred", { jobId: job.id, profile: checked, fault, heldForMs: held, delayMs: delay });
+					await job.moveToDelayed(nowMs + delay, token);
+					throw new DelayedError();
+				}
+
+				// FAIL CLOSED on anything that is not literally go. Everything above tests for a specific
+				// shape and falls through otherwise, and "otherwise" at this gate means STARTING A PAID
+				// CONTAINER -- so an `undefined`, a `null`, a `{}`, a mis-cased "GO" or a bare string from a
+				// checker would run the job silently, with no record field and no log line to distinguish it
+				// from a job whose check said yes. The shipped checker is total, and that is exactly the
+				// reasoning `unreadableConditions` above rejects: this is a dependency-injection seam, and a
+				// seam's guarantees are the caller's to enforce.
+				if (verdict?.verdict !== "go") {
+					deps?.log?.("wait_check_unintelligible", { jobId: job.id, profile: checked });
+					await waitState.noteCheck(job.id, { fault: true });
+					const base = waitBackoffMs(intervalMs(), held);
+					await job.moveToDelayed(nowMs + base + Math.floor(base * 0.1 * random()), token);
+					throw new DelayedError();
+				}
+
+				// Every profile answered go. Record the last check so the count bound sees it.
+				await waitState.noteCheck(job.id, { fault: false });
+			}
+
+			// Only a job that actually HELD has cleared. Without the check this line fires on the first
+			// pickup of a job whose instant had already passed, and again on every scope-busy re-check
+			// afterwards -- asserting a wait ended that never began.
+			const heldForMs = await waitState.heldForMs(job.id);
+			// Say so before releasing: a sibling held on this target must find the answer, not an empty lease.
+			if (heldForMs !== null) await waitState.markSatisfied(job.id, { dedupId });
+			await waitState.release(job.id, { dedupId });
+			if (heldForMs !== null) deps?.log?.("wait_cleared", { jobId: job.id, label: waitLabel(job.data), heldForMs });
+		}
+
 		// Per-scope concurrency and the one-job-per-folder mutex (issue #242,
-		// INT-SCOPED-LIMITS-FILE-CONTRACT). SECOND, after the pause gate (a paused job must not burn
-		// re-check wakes) and STRICTLY above the `try` below, like the pause gate and for the same two
+		// INT-SCOPED-LIMITS-FILE-CONTRACT). LAST of the three gates, after the pause gate (a paused job must
+		// not burn re-check wakes) and after the wait gate (a job holding until tomorrow must not sit on a
+		// folder while it does), and STRICTLY above the `try` below, like the pause gate and for the same two
 		// reasons: a DelayedError thrown inside the try would be converted to UnrecoverableError by the
 		// catch, and a moveToDelayed rejection here must escape RAW into BullMQ's normal failed-attempt
 		// handling exactly as the pause gate's does (inside the try it would become a permanent failure
@@ -123,7 +425,9 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 				// job completed and does not retry a file that can never parse (CONST-RETRY-INFRA-ONLY). Resolved
 				// before runJob, so no budget slot is reserved and no container starts (CONST-BUDGET-BEFORE-TOKENS).
 				// recordRun leaves the durable settings-overlay-invalid trace for the admin extension.
-				// No provider/model here, alone among the terminal results: the overlay is the thing that failed to parse, so no honest effective value exists -- buildRecord defaults both null.
+				// No provider/model here, as in every result this function returns from ABOVE the try (the wait gate's
+				// refusals are the others): each is decided before or during the settings read, so no honest effective
+				// value exists yet -- buildRecord defaults both null.
 				const result = { outcome: "policy", reason: "settings-overlay-invalid", exitCode: null, turns: null, tokens: null, budgetReserved: false };
 				recordRun({ job, result, startedAt, endedAt: new Date().toISOString() });
 				return result;
@@ -210,7 +514,7 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 	};
 }
 
-export function createWorker({ connection, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, scopedLimits, inFlight, extraClosers = [] }) {
+export function createWorker({ connection, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, scopedLimits, inFlight, waitState, afterMaxMs, checkSlots, checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, extraClosers = [] }) {
 	let worker; // referenced by cancelJob/applyConcurrency before assignment; only called later, so the TDZ is fine
 	const processor = makeProcessor({
 		cancelJob: (id, reason) => worker.cancelJob(id, reason),
@@ -228,6 +532,21 @@ export function createWorker({ connection, concurrency, getSettings, redis, deps
 		// shape means one per daemon).
 		scopedLimits,
 		inFlight,
+		// Issue #230. Undefined pass-throughs take makeProcessor's own defaults (a wait state over the same
+		// redis client, and the shared 30-day `after` ceiling), so a bare wiring behaves like a wired one.
+		waitState,
+		afterMaxMs,
+		// Issue #230, the polled tier. `concurrencyNow` reads the LIVE slot count rather than the boot value,
+		// because the overlay can lower it through `dispatch_set` and a check must never take the last free
+		// slot from a paid job. Late-bound over `worker` exactly as `applyConcurrency` is, and for the same
+		// reason: the value it needs does not exist until the Worker is constructed.
+		checkSlots,
+		checkSlotCount,
+		concurrencyNow: concurrencyNow ?? (() => worker?.concurrency ?? concurrency),
+		intervalMs,
+		maxWaitMs,
+		maxChecks,
+		maxFaults,
 		deps,
 		recordRun,
 	});

package/src/processor.mjs

@@ -54,6 +54,9 @@ export async function runJob(job, deps) {
 		// exactly as before, and the gate below only calls it for a job whose matched rule was a
 		// one-shot, so the default is never a probe running on every delivery.
 		checkOnceSpent = async () => ({ ok: true }),
+		// Issue #230. Admit-everything by default, like checkOnceSpent above and for its reason: an
+		// unwired seam must not refuse, and the wiring is what turns the check on.
+		checkWaitSkew = async () => ({ ok: true }),
 		// REQ-EGRESS-ALLOWLIST. Default admits everything, so a wiring that omits it behaves exactly as a
 		// deployment with no egress policy does -- which is also what the real factory returns when unarmed.
 		egressPreflight = async () => ({ ok: true }),
@@ -155,6 +158,28 @@ export async function runJob(job, deps) {
 			}
 		}
 
+		// The wait-skew check (issue #230), second on the ladder and for the first one's reasons: the same
+		// file read, free, determinate, credential-less, and pre-spend. It answers a question no other layer
+		// can: does the AUTHORED trigger carry wait conditions this job arrived without? That happens when a
+		// service below the version floor dropped the field as an unknown key, and the resulting run is
+		// byte-identical to a correct one everywhere it is recorded -- so this refusal is the only thing
+		// standing between a stale receiver and a paid job that ran when the operator wrote "wait".
+		{
+			const skew = await checkWaitSkew(job);
+			if (skew.skewed) {
+				// Named for the operator, not the payload: how many conditions were authored, never what
+				// they say. The fix is a version, so the comment says which one.
+				// The message names BOTH causes, because the more likely one is not a version at all. In the
+				// compose topology the receiver's single-file `:ro` mount pins a dead inode, so an operator
+				// who ADDS `waitFor` to an existing rule gets this refusal on every delivery from a service
+				// that is perfectly up to date and merely holding an older copy of the file. Naming only the
+				// version would send them looking for an upgrade they do not need.
+				await comment(job, `Refused: this trigger declares ${skew.conditions} wait condition${skew.conditions === 1 ? "" : "s"}, but the job reached the worker without them, which means it would have run immediately. Either a service in this deployment is below the version that carries the field, or one is still running against an older copy of the triggers file and needs restarting. Not run.`);
+				log("refused_wait_skew", { triggerIndex: job.trigger?.matched?.index ?? null, conditions: skew.conditions });
+				return { outcome: "policy", reason: "wait-skew", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false };
+			}
+		}
+
 		// The job image must exist on THIS host before anything else happens. Free, determinate and
 		// credential-less, so it precedes the mint, the clone and the reservation: a host that cannot run the
 		// image refuses without minting a credential it will not use, cloning a repo it will not read, or

package/src/queue.mjs

@@ -134,7 +134,7 @@ export async function enqueueGitLabJob(queue, fields) {
  * window, replicas never coalesce against each other, and an unflagged job's dedup id is the same string it
  * has always been.
  */
-export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, command, trigger, provider, model, maxTurns, packages, image, skillsDir, instructions, resume, secrets, secretsProfile, replica, replicas }) {
+export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, command, trigger, provider, model, maxTurns, packages, image, skillsDir, instructions, resume, secrets, secretsProfile, waitFor, replica, replicas }) {
 	const jobId = forgeDeliveryJobId(kind, trigger?.deliveryId, replica);
 	// `packages` (whether to load the operator-staged pi packages) and `image` (which container image to run)
 	// come off the MATCHED trigger (INT-TRIGGERS-FILE-CONTRACT / REQ-GLOBAL-PI-OVERLAY) and land on `data`
@@ -179,6 +179,16 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
 		// is copied verbatim into /job/event.json, which an agent reads.
 		...(secrets !== undefined && { secrets }),
 		...(secretsProfile !== undefined && { secretsProfile }),
+		// Issue #230. The conditions the worker holds this job on, carried so the PICKUP gate can read them:
+		// that gate runs above the per-job settings read and never re-parses the triggers file for its terms.
+		// At JOB level, and here that placement is a correctness requirement rather than a convention --
+		// `trigger` is copied VERBATIM into /job/event.json (prepare-local.mjs), so a `trigger.waitFor` would
+		// hand the agent the operator's own gate. Conditional like every field above, so an unflagged job's
+		// data keeps exactly the keys it has today. The dedup options below are deliberately NOT widened for
+		// a waiting job: that key carries no trigger identity and outlives the job it was set for, so a
+		// longer window would suppress an unflagged sibling's deliveries and go on suppressing them after
+		// this job finished. Coalescing a held target is the worker's `wait:` keyspace's job instead.
+		...(waitFor !== undefined && { waitFor }),
 		// Conditional for the same reason packages/image/resume are: an unflagged job's data must keep
 		// exactly the keys it has today. `replica` is this job's 1-based index and `replicas` the set size;
 		// both are integers, so the run record they land in stays PII-free by construction.

package/src/run-history.mjs

@@ -387,7 +387,12 @@ export function buildRecord({ job, result, error, startedAt, endedAt }) {
  * separator from the table, so the notation a forge uses is the notation its records carry -- and a forge
  * added later inherits a label rather than a null.
  */
-function targetFor(kind, data) {
+/*
+ * Exported since issue #230: a held job's panel row needs the same id-only label a run record carries, and
+ * the wait gate would otherwise re-derive it. Two spellings of "which issue is this" is how one of them
+ * starts carrying a title.
+ */
+export function targetFor(kind, data) {
 	if (kind === "local") return `local:${basename(data.folder ?? "")}`;
 	if (isForgeKind(kind)) return `${data.repo}${targetSeparator(kind, data.target?.type)}${data.target?.number}`;
 	return null;

package/src/service.mjs

@@ -972,6 +972,15 @@ async function doRestart(ctx, values) {
 			await ctx.sleep(2000);
 			({ active = 0 } = await queue.getJobCounts("active"));
 		}
+		// A HELD job is neither active nor waiting, so the loop above has just reported a drained queue with
+		// however many jobs still parked on `run.waitFor` (issue #230). They are safe -- a hold spends
+		// nothing, survives a restart and reserves no slot -- but the operator is upgrading, and those jobs
+		// will wake against the new version. Said plainly rather than left to be discovered, which is what
+		// this command would otherwise be doing: reporting a drained queue it cannot see all of.
+		const delayed = await queue.getJobCounts("delayed").then((c) => Number(c?.delayed ?? 0), () => 0);
+		if (delayed > 0) {
+			ctx.out(`note: ${delayed} job(s) sit in the delayed set (cron next-occurrences, retry backoff, quiet hours, or jobs held on run.waitFor). None is active, so none blocked this drain; they will wake against the new version.\n`);
+		}
 		const stopped = await doStop(ctx);
 		if (stopped !== 0) {
 			ctx.out("restart did not happen — the queue STAYS PAUSED; fix the service, then `pi-dispatch resume`.\n");

package/src/start.mjs

@@ -22,9 +22,11 @@ import { makeCleanup, makeForgePreparers, makePrepareWorkspace } from "./prepare
 import { listRunningSandboxes } from "./sandbox.mjs";
 import { makeSandboxReaper } from "./sandbox-store.mjs";
 import { makeSessionStore } from "./session-store.mjs";
-import { makeCheckOnceSpent, makeDisarmOnce } from "./triggers-file.mjs";
+import { makeCheckOnceSpent, makeCheckWaitSkew, makeDisarmOnce } from "./triggers-file.mjs";
 import { loadPauseWindows, pauseUntilMs } from "./pause-windows.mjs";
 import { loadScopedLimits } from "./scoped-limits.mjs";
+import { makeWaitChecker } from "./wait-check.mjs";
+import { makeWaitState } from "./wait-state.mjs";
 import { makeQueue } from "./queue.mjs";
 import { makeRunContainer } from "./run-container.mjs";
 import { makeSecretsResolver } from "./secrets.mjs";
@@ -466,6 +468,18 @@ export async function startWorker(
 		// Issue #242: the scoped-limits snapshot the pickup gate and the scoped budget read, once per
 		// pickup, from the live-reloaded ref -- same next-job grain as pauseUntil above.
 		scopedLimits: () => scopedLimits.current,
+		// Issue #230. The `after` ceiling is read per pickup from config rather than frozen into the
+		// processor, so it is one value with one home; the wait state shares the budget's redis client
+		// because it describes the same delayed jobs that client already reasons about.
+		afterMaxMs: () => config.waitAfterMaxMs,
+		waitState: makeWaitState({ redis }),
+		// The polled tier's bounds, read per pickup from config so they are one value with one home. The
+		// slot count is a CEILING the gate clamps against the live concurrency, never the final number.
+		checkSlotCount: () => config.waitCheckSlots,
+		intervalMs: () => config.waitIntervalMs,
+		maxWaitMs: () => config.waitMaxMs,
+		maxChecks: () => config.waitMaxChecks,
+		maxFaults: () => config.waitMaxFaults,
 		deps: {
 			collectChain,
 			// The one-shot pre-spend check (issue #231): reads the same file the disarm writes, refuses
@@ -473,6 +487,35 @@ export async function startWorker(
 			// spending delivery is excused). In the compose topology this check is the once-enforcement
 			// layer, because the receiver's single-file :ro mount pins a dead inode until restart.
 			checkOnceSpent: makeCheckOnceSpent({ triggersPath: onceTriggersFile }),
+			// Issue #230. The same file and the same fail-open posture, but its own mtime-cached read: this one
+			// asks whether the AUTHORED entry declares wait conditions the job arrived without, which is how a
+			// service below the version floor turns a wait into a paid run nothing can tell from a correct
+			// one. In the compose topology the worker's read is the live inode while the receiver's is dead
+			// until restart, which is exactly the deployment where the skew happens.
+			checkWaitSkew: makeCheckWaitSkew({ triggersPath: onceTriggersFile }),
+			// Issue #230. Whether a job the supersede lease names is still in the queue. Without it a holder
+			// that vanished by any route except the clean one leaves a key that refuses every later delivery
+			// for that target until it expires -- and a refused forge delivery is gone, since no webhook
+			// resends it. `getJob` answers from the queue rather than from our own bookkeeping, so the two
+			// cannot agree with each other while both being wrong.
+			// REQ-WAIT-FOR's polled tier. Built here for the image and egress preflights' reason: one
+			// deployment value, one place, so the gate that refuses an undeclared profile and the spawn that
+			// runs it cannot disagree about which checks exist. The env-declared table is parsed once at boot
+			// (it is env, not overlay -- the gate reads its config above the per-job settings read).
+			// The free half of the profile check: whether this deployment declares the name at all. A table
+			// lookup, so it belongs with the gate's other free refusals rather than inside the subprocess.
+			waitProfileDeclared: (name) => typeof config.waitProfiles[name] === "string",
+			checkWait: makeWaitChecker({ profiles: config.waitProfiles, timeoutMs: config.waitCheckTimeoutMs, log }),
+			isJobLive: async (id) => {
+				const held = await runtimeQueue.getJob(id);
+				if (!held) return false;
+				// EXISTENCE IS NOT LIVENESS, and the difference decides whether a target stays deafened:
+				// `removeOnComplete`/`removeOnFail` keep a finished job's hash for 31 days, so a holder that
+				// can never wake again would answer "still waiting" for a month. Only a state it can still be
+				// picked up from counts.
+				const state = await held.getState();
+				return state === "delayed" || state === "waiting" || state === "active" || state === "prioritized" || state === "waiting-children";
+			},
 			// One deployment default, two consumers, adjacent by construction: the preflight that refuses a missing
 			// image BEFORE the budget slot, and the factory that puts it in the argv. Both resolve a trigger's own
 			// `run.image` through the same resolveJobImage, so the image that was checked is the image that runs.

package/src/triggers-file.mjs

@@ -376,6 +376,83 @@ export function makeCheckOnceSpent({ triggersPath, fs = nodeFs }) {
 	};
 }
 
+/**
+ * Detect the version skew `run.waitFor` opens (issue #230), pre-spend, from the worker's own file read.
+ *
+ * The hazard is `DES-TRIGGERS-UNIFIED-FILE`'s widening rule arriving somewhere it has never bitten. Unknown
+ * keys DROP, which for every previous field was a harmless no-op: an old parser meeting `run.image` gives
+ * you the default image and a job that ran. `waitFor` is the first field whose ABSENCE is destructive -- a
+ * receiver below the floor enqueues the job without it and the worker runs it UNHELD, producing a record, a
+ * panel row and a log line byte-identical to one that correctly waited. Success is the least detectable
+ * failure available, and the whole point of a wait is that running now is the destructive option.
+ *
+ * `docs/secrets.md`'s answer to the same skew is documentation plus a version floor, and it is not enough
+ * here for two reasons: a dropped secret surfaces as a 401 and an agent report that reads wrong, and
+ * `doctor` cannot see the receiver's installed version from the worker host, so its warning would fire on
+ * every deployment using the feature, forever -- the always-on amber the panel's own design rejects.
+ *
+ * So the worker checks the authored file itself. It already reads it per job for the one-shot gate, and in
+ * the compose topology this read is the authoritative one: the receiver's single-file `:ro` mount pins a
+ * dead inode until restart, which is precisely the deployment where the skew bites.
+ *
+ * FAIL-OPEN throughout, `readDisarmState`'s posture and for its reason: an unreadable or changed file means
+ * "run", because a broken read must never wedge every job. Only a positive, identity-confirmed mismatch --
+ * this entry authored conditions, this job carries none -- refuses.
+ */
+export function makeCheckWaitSkew({ triggersPath, fs = nodeFs }) {
+	// Cached by mtime, unlike `checkOnceSpent` which re-reads every time. The difference is which jobs each
+	// one runs for: the one-shot check is gated on `matched.once === true`, so it is rare by construction and
+	// its comment can call one read cheap. This one has to look at EVERY forge job, because the whole point
+	// is to catch a job that arrived WITHOUT the field, and there is nothing on such a job to narrow by. So a
+	// stat replaces a read-and-parse on the hot path. The residual is millisecond mtime granularity: two
+	// writes inside one millisecond would serve a stale parse for one job, which fails OPEN like every other
+	// uncertainty here.
+	let cache = null; // { mtimeMs, size, triggers }
+	const read = () => {
+		try {
+			const st = fs.statSync?.(triggersPath);
+			if (cache && st && cache.mtimeMs === st.mtimeMs && cache.size === st.size) return cache.triggers;
+			const raw = JSON.parse(fs.readFileSync(triggersPath, "utf8"));
+			const triggers = Array.isArray(raw?.triggers) ? raw.triggers : null;
+			if (st) cache = { mtimeMs: st.mtimeMs, size: st.size, triggers };
+			return triggers;
+		} catch {
+			cache = null;
+			return null;
+		}
+	};
+
+	return async function checkWaitSkew(job) {
+		if (typeof triggersPath !== "string" || triggersPath === "") return { ok: true };
+		// Cron and CLI jobs carry no matched index, and cannot carry `waitFor` at all.
+		const index = job?.trigger?.matched?.index;
+		if (!Number.isInteger(index)) return { ok: true };
+		// The field arrived. Whether its conditions are SATISFIED is the gate's business, not this check's.
+		if (Array.isArray(job?.waitFor) && job.waitFor.length > 0) return { ok: true };
+
+		const triggers = read();
+		if (triggers === null) return { ok: true };
+		const entry = triggers[index];
+		const authored = entry?.run?.waitFor;
+		if (!Array.isArray(authored) || authored.length === 0) return { ok: true };
+
+		// The identity guard readDisarmState keeps, and WIDER than flow alone. `triggerIndex` is a RAW array
+		// position, so an insertion, a reorder, or a stray `triggers.json` at the worker's cwd can all put a
+		// different trigger here -- and two rules sharing a flow are indistinguishable on flow alone, which
+		// made a false refusal reachable by ordinary editing. Kind and on-type are compared too, on
+		// `readDisarmState`'s precedent of checking the item number rather than trusting the index.
+		//
+		// Every mismatch folds to "run", never to a refusal: the true answer to "is this job missing
+		// conditions someone wrote for it?" is then unknown, and unknown must not refuse a paid delivery.
+		if (entry?.run?.flow !== job?.flow || entry?.run?.command !== job?.command) return { ok: true };
+		if (entry?.run?.kind !== job?.kind) return { ok: true };
+		const onType = job?.trigger?.matched?.type;
+		if (typeof onType === "string" && entry?.on?.type !== onType) return { ok: true };
+
+		return { skewed: true, conditions: authored.length };
+	};
+}
+
 export function readDisarmState({ triggersPath, index, number, flow, command, fs = nodeFs }) {
 	let raw;
 	try {