@edgehero/pi-dispatch 1.4.0 → 1.6.0

package/src/index.mjs

@@ -2,11 +2,44 @@ import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 import { DelayedError, UnrecoverableError, Worker } from "bullmq";
 import { InfraRetry, runJob } from "./processor.mjs";
+import { targetFor } from "./run-history.mjs";
+import { budgetCapsFor, canonicalScope, concurrencyFor, makeInFlight } from "./scoped-limits.mjs";
+import { WAIT_AFTER_MAX_DEFAULT_MS, WAIT_INTERVAL_FLOOR_MS, afterMs, unreadableConditions, waitArmed, waitBackoffMs, waitLabel, waitProfileNames } from "./wait-for.mjs";
+import { makeWaitState } from "./wait-state.mjs";
 
 const exec = promisify(execFile);
 
 export const QUEUE = "pi-jobs";
 export const JOB_TIMEOUT_MS = 30 * 60 * 1000; // REQ-JOB-TIMEOUT-30M
+// The scope-busy re-check (issue #242): a held scope has no natural "until" (the holder may run to
+// JOB_TIMEOUT_MS), so a deferred job re-tests on a fixed cadence. 5s keeps the worst case trivial
+// (<=360 wakes across a 30-minute hold, each ~1ms of synchronous predicate briefly occupying a slot)
+// while a same-folder CHAINED job -- enqueued by its parent before the parent's finally releases the
+// folder -- pays exactly one re-check, not fifteen seconds of dead air. No jitter: one worker per
+// docker daemon bounds any herd by its own concurrency, and a contended wake just re-defers.
+export const SCOPE_BUSY_RECHECK_MS = 5_000;
+
+// How long a job waits before re-asking whether a target's holder is still alive (issue #230). Reached only
+// when the liveness probe could not answer, which is a redis or queue fault rather than a normal state, so
+// this is a short retry rather than a cadence: the job is deciding nothing and holding nothing while it
+// waits, and the fault it is waiting out is usually seconds long.
+export const SUPERSEDE_RECHECK_MS = 15_000;
+
+// The one key the check lease counts under. A single global counter rather than one per profile: what it
+// bounds is this worker's wall-clock spent answering questions, and that is shared whatever is being asked.
+const WAIT_CHECK_KEY = "wait-check";
+
+// How many consecutive lease denials one job absorbs before the deployment is told its checking capacity is
+// short. Logged ONCE per run of denials rather than per wake: an alarm that repeats every re-check is the
+// always-on amber this project rejects elsewhere, and the operator only needs telling once per episode.
+const THROTTLE_ALARM = 5;
+
+// The floor under a throttled or aborted re-ask. Its own constant rather than a borrow of
+// SUPERSEDE_RECHECK_MS, which documents an unrelated concern. Deliberately NOT 5s: that is
+// SCOPE_BUSY_RECHECK_MS, and INT-WAIT-PROFILES-CONTRACT rests on wait deferrals being distinguishable from
+// scope deferrals by wake instant -- nothing records WHY a job sits in the delayed set, so the instants are
+// the only evidence there is. A test pins the two apart.
+const THROTTLE_FLOOR_MS = 11_000;
 
 /**
  * Build the BullMQ processor.
@@ -27,7 +60,7 @@ export const JOB_TIMEOUT_MS = 30 * 60 * 1000; // REQ-JOB-TIMEOUT-30M
  * The overlay changes which values the spend caps take, never when they are checked -- reserveBudget still
  * runs inside runJob against the freshly passed caps (CONST-BUDGET-BEFORE-TOKENS).
  */
-export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now() }) {
+export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
 	return async function processor(job, token, signal) {
 		// Scoped pause windows (REQ-SCOPED-PAUSE-WINDOWS): if this job's folder/repo is inside an active pause
 		// window, DEFER it to the window end via BullMQ's delayed set -- the job keeps its identity/dedup and
@@ -45,19 +78,345 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 			throw new DelayedError();
 		}
 
-		const startedAt = new Date().toISOString();
-		const name = `pi-job-${job.id}`;
-		const timer = setTimeout(() => {
-			// BullMQ has no per-job kill timer; this is ours. cancelJob raises the AbortSignal.
-			Promise.resolve(cancelJob(job.id, "job-timeout-30m")).catch(() => {});
-		}, timeoutMs);
+		// The wait gate (issue #230, REQ-WAIT-FOR). THIRD: after the pause gate, because a paused job must
+		// not burn a wait evaluation any more than it burns a scope re-check, and BEFORE the scope acquire,
+		// because a job that is going to sit until tomorrow morning must not hold the folder mutex while it
+		// does. Strictly above the `try` for the two reasons the gates below it document.
+		//
+		// The order WITHIN the gate is determinate-refusals-then-holds, which is CONST-BUDGET-BEFORE-TOKENS'
+		// shape applied to time rather than to money: a condition this deployment can never answer must be
+		// refused now, not after a day of waiting.
+		//
+		// On throwing above the `try`: an exception here escapes into BullMQ's normal failed-attempt handling,
+		// which is WANTED for `moveToDelayed` (the scope gate below gives the argument: a transient rejection
+		// must stay a transient failure rather than becoming a permanent one) and unwanted everywhere else. So
+		// the state and comment seams fail open by construction, and `recordRun` is relied on not to throw --
+		// its writer swallows fs errors by contract, which is the same reliance the settings-overlay refusal
+		// below already makes.
+		if (waitArmed(job.data)) {
+			// The supersede identity: the queue's semantic key PLUS the trigger that produced this job.
+			// The semantic key alone is `repo<sep>number:flow`, which two DIFFERENT triggers on one target and
+			// flow legitimately share -- a label rule that waits a day and a comment rule that waits a minute
+			// would coalesce, and the second would be refused with a message claiming they wait on "the same
+			// conditions" when they do not. Adding the raw trigger index makes the key mean one intent.
+			const matchedIndex = job.data?.trigger?.matched?.index;
+			const dedupId = job.deduplicationId ? `${job.deduplicationId}#${Number.isInteger(matchedIndex) ? matchedIndex : "?"}` : null;
+			const refuseWait = async (reason, logEvent, fields, sentence) => {
+				// The INJECTED clock, like both gates above: a record whose timestamps ignore the test clock
+				// is a record no test of this gate can assert about.
+				const at = new Date(now()).toISOString();
+				await waitState.release(job.id, { dedupId });
+				deps?.log?.(logEvent, { jobId: job.id, ...fields });
+				// The comment names the FIELD and the operator's own words for the condition, never a
+				// resolver path or a vault topology -- `secret-profile-unknown` sets that rule.
+				if (sentence && deps?.comment) await Promise.resolve(deps.comment(job.data, sentence)).catch(() => {});
+				const result = { outcome: "policy", reason, exitCode: null, turns: null, tokens: null, budgetReserved: false };
+				recordRun({ job, result, startedAt: at, endedAt: new Date().toISOString() });
+				return result;
+			};
+
+			// EVERY condition must be one this worker understands, checked before anything else. The loader
+			// refuses an unknown condition, but the loader is a DIFFERENT PROCESS: `job.data.waitFor` arrives
+			// over Redis from the receiver, and this whole feature exists because receiver-worker version
+			// skew is real. `makeCheckWaitSkew` closes the backward direction (the file has conditions the
+			// job arrived without); this closes the forward one (a newer receiver enqueues a condition shape
+			// this worker cannot read). Without it the gate would fall through, log `wait_cleared`, and run
+			// the job -- asserting in the log that conditions cleared which it never evaluated, which is the
+			// same undetectable paid run the backward check exists to stop.
+			// A sibling that was held on this same target may already have cleared it. Checked FIRST, because
+			// it is free and determinate, and because the window it closes is one no lease can: two jobs
+			// holding through an outage that outlives their leases would each wake, find no holder, and run.
+			if (dedupId) {
+				const satisfiedBy = await waitState.satisfiedBy(dedupId);
+				if (satisfiedBy && satisfiedBy !== job.id) {
+					return await refuseWait("wait-superseded", "wait_superseded", { satisfiedBy }, "Another delivery for this target already finished waiting on the same conditions. Not run.");
+				}
+			}
+
+			const unreadable = unreadableConditions(job.data);
+			if (unreadable.length > 0) {
+				// Its OWN token, not `wait-skew`. Both are version skew, and the REMEDIES are opposites --
+				// upgrade the receiver there, upgrade the worker here -- so one token in a durable record
+				// would tell an operator that something is out of step and not which way to move.
+				return await refuseWait("wait-unreadable", "refused_wait_unreadable", { conditions: unreadable.length }, `Refused: this job carries ${unreadable.length} wait condition${unreadable.length === 1 ? "" : "s"} this worker cannot read, so it cannot honour them. The worker is older than the service that enqueued this job. Not run.`);
+			}
+
+			// A `profile` condition needs a checker, and with none wired NOTHING can answer it. Refused rather
+			// than ignored: a wait the deployment cannot perform must not read as a wait that passed.
+			const profiles = waitProfileNames(job.data);
+			// Declared-ness is a table lookup, so it belongs with the other free refusals rather than inside
+			// the check. Without it here, `[{after: "<tomorrow>"}, {profile: "typo"}]` holds for a day and
+			// THEN refuses -- which is the exact sentence the ordering rule above promises will not happen.
+			const undeclared = deps?.waitProfileDeclared ? profiles.find((name) => !deps.waitProfileDeclared(name)) : undefined;
+			if (undeclared !== undefined) {
+				return await refuseWait("wait-profile-unknown", "wait_profile_unknown", { profile: undeclared }, `Waiting on \`${undeclared}\` is not something this deployment can answer: no such wait profile is declared here. Not run.`);
+			}
+			if (profiles.length > 0 && !deps?.checkWait) {
+				return await refuseWait("wait-profile-unknown", "wait_profile_unknown", { profile: profiles[0] }, `Waiting on \`${profiles[0]}\` is not something this deployment can answer. Not run.`);
+			}
+
+			const holdUntil = afterMs(job.data); // named apart from the pause gate's `until` above, which it would otherwise shadow
+			// An instant further out than the ceiling is refused at FIRST pickup rather than held toward:
+			// holding for a month to then refuse tells the operator nothing they could not have been told now.
+			if (holdUntil !== null && holdUntil - nowMs > afterMaxMs()) {
+				return await refuseWait("wait-after-beyond-max", "wait_after_beyond_max", { delayMs: holdUntil - nowMs }, `The \`after\` instant is further out than this deployment allows a job to wait. Not run.`);
+			}
+
+			// The pause gate's boundary guard, for its reason: a tick landing on the instant must run rather
+			// than busy-defer to a moment already past.
+			if (holdUntil !== null && holdUntil > nowMs + 1000) {
+				// `isJobLive` is what stops a vanished holder's lease becoming a tombstone that refuses this
+				// target for the rest of the hold. Optional: an unwired probe means the holder cannot be
+				// checked, which ADMITS and says so -- one duplicate run beats one dropped delivery, which is
+				// `OQ-027`'s call ("one wasted vault read beats one dropped job") on this feature's terms.
+				const claim = await waitState.claim(job.id, { dedupId, untilMs: holdUntil, isLive: deps?.isJobLive });
+				if (claim.heldBy) {
+					// Another delivery for this same target and flow is already holding. Both would clear
+					// together and both would be paid, which is the accumulation the acceptance forbids.
+					return await refuseWait("wait-superseded", "wait_superseded", { heldBy: claim.heldBy }, "Another delivery for this target is already waiting on the same conditions. Not run.");
+				}
+				if (claim.retry) {
+					// The holder could not be checked. Holding anyway would put two jobs on one target and pay
+					// for both; refusing would drop a delivery over a holder that may be gone. So decide
+					// nothing: re-defer briefly and ask again once the probe can answer.
+					deps?.log?.("wait_supersede_unverified", { jobId: job.id, heldBy: claim.holder ?? null, delayMs: SUPERSEDE_RECHECK_MS });
+					await job.moveToDelayed(nowMs + SUPERSEDE_RECHECK_MS, token);
+					throw new DelayedError();
+				}
+				if (claim.tookOverFrom) deps?.log?.("wait_lease_taken_over", { jobId: job.id, from: claim.tookOverFrom });
+				await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: holdUntil });
+				deps?.log?.("wait_deferred", { jobId: job.id, until: new Date(holdUntil).toISOString(), label: waitLabel(job.data) });
+				await job.moveToDelayed(holdUntil, token);
+				throw new DelayedError();
+			}
+
+			// TIER 2: the polled conditions. Last, because it is the only part of this gate that spawns a
+			// process -- the free refusals above it are free, and the free hold above it is free.
+			if (profiles.length > 0) {
+				const held = (await waitState.heldForMs(job.id)) ?? 0;
+				const counted = await waitState.counters(job.id);
+
+				// One check at a time, process-wide, and never the worker's last free slot. This is the bound
+				// that keeps a wait from starving the paid work it is waiting for: slots x timeout is the most
+				// wall-clock a worker can spend answering questions instead of running jobs. Computed against
+				// the LIVE concurrency rather than the boot value, because the overlay can lower it.
+				const slots = Math.min(checkSlotCount(), Math.max(1, concurrencyNow() - 1));
+				if (!checkSlots.tryAcquire(WAIT_CHECK_KEY, slots)) {
+					// Denials are counted, and a run of them is the ONE symptom the capacity bound has. The
+					// lease deliberately caps how much wall-clock this worker spends checking; being at that
+					// cap constantly means demand exceeds it, which the issue's own economics say arrives
+					// silently -- paid jobs starve behind checks that spend nothing and nothing says why.
+					const denials = await waitState.noteThrottle(job.id, { denied: true });
+					if (denials === THROTTLE_ALARM) deps?.log?.("wait_capacity_exceeded", { jobId: job.id, denials, slots, hint: "raise PI_WAIT_CHECK_SLOTS or PI_CONCURRENCY, lengthen PI_WAIT_INTERVAL_MS, or hold fewer jobs" });
+					// A starved job still needs a CLOCK and a CEILING, or the lease turns into the very
+					// starvation it exists to bound: without this the hold is stamped only on a wake that won
+					// the lease, so a job that never wins one has no `since`, never reaches the maximum, and
+					// re-wakes forever with no record and no bound. There is no deciding check to run first
+					// here -- that is the whole condition -- so the bound applies directly.
+					await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: nowMs + maxWaitMs() });
+					if (held >= maxWaitMs()) {
+						return await refuseWait("wait-expired", "wait_expired", { reason: "max-wait-unchecked", denials, heldForMs: held }, `Gave up waiting: this deployment could not run the check often enough to answer within the maximum wait. Not run.`);
+					}
+					// Denied. Re-ask at a fraction of the cadence rather than the full backoff (which would
+					// turn one lost coin-flip into a fifteen-minute penalty) or a flat few seconds (which at
+					// scale is a herd). Jittered, so a fleet of denied jobs does not return together.
+					const wait = Math.max(THROTTLE_FLOOR_MS, Math.floor(waitBackoffMs(intervalMs(), held) / 4));
+					const delay = wait + Math.floor(wait * 0.1 * random());
+					deps?.log?.("wait_check_throttled", { jobId: job.id, delayMs: delay, slots });
+					await job.moveToDelayed(nowMs + delay, token);
+					throw new DelayedError();
+				}
+
+				// Claimed BEFORE the check, not after: a second delivery for an already-held target is a free
+				// determinate refusal, and paying for a subprocess first inverts the free-before-costly rule
+				// this gate's own header invokes. Tier 1 already claims in this order.
+				const claim = await waitState.claim(job.id, { dedupId, untilMs: nowMs + waitBackoffMs(intervalMs(), held), isLive: deps?.isJobLive });
+				if (claim.heldBy) {
+					return await refuseWait("wait-superseded", "wait_superseded", { heldBy: claim.heldBy }, "Another delivery for this target is already waiting on the same conditions. Not run.");
+				}
+				if (claim.retry) {
+					deps?.log?.("wait_supersede_unverified", { jobId: job.id, heldBy: claim.holder ?? null, delayMs: SUPERSEDE_RECHECK_MS });
+					await job.moveToDelayed(nowMs + SUPERSEDE_RECHECK_MS, token);
+					throw new DelayedError();
+				}
+
+				await waitState.noteThrottle(job.id, { denied: false }); // granted: the run of denials ends here
+				let verdict = null;
+				let checked = null;
+				try {
+					// Sequential, in the operator's writing order: the resolver's reason applies unchanged --
+					// naming the first condition that did not clear is what makes a held row readable, and a
+					// parallel fan-out would blame whichever lost the race on any given wake.
+					for (const profile of profiles) {
+						checked = profile;
+						verdict = await deps.checkWait(profile, targetFor(job.data?.kind, job.data), { signal });
+						if (verdict?.profileUnknown || verdict?.verdict !== "go") break;
+					}
+				} finally {
+					checkSlots.release(WAIT_CHECK_KEY);
+				}
+
+				if (verdict?.unusableTarget) {
+					// Determinate and unfixable by waiting: the job's own target is a shape no check can be
+					// handed. It belongs with the refusals, not the holds -- holding would spend the fault
+					// budget and then blame the operator's script for a value it was never given.
+					return await refuseWait("wait-unreadable", "refused_wait_unreadable", { profile: checked }, `Refused: this job's target cannot be handed to a wait check, so \`${checked}\` can never be asked. Not run.`);
+				}
+				if (verdict?.profileUnknown) {
+					return await refuseWait("wait-profile-unknown", "wait_profile_unknown", { profile: verdict.profileUnknown }, `Waiting on \`${verdict.profileUnknown}\` is not something this deployment can answer: no such wait profile is declared here. Not run.`);
+				}
+				if (verdict?.verdict === "refuse") {
+					// Exit 2: the check says this will NEVER clear. Terminal by the protocol's own words, and
+					// distinct from every "not yet" above it.
+					return await refuseWait("wait-refused", "wait_refused", { profile: checked, heldForMs: held }, `The check \`${checked}\` reports this will never clear. Not run.`);
+				}
+
+				if (verdict?.aborted) {
+					// The worker is stopping or this job was cancelled. Nothing was learned and nothing is
+					// owed: re-defer at once rather than at the full backoff, and count neither a check nor a
+					// fault, or a rolling deploy would spend a job's whole budget on its own restarts and then
+					// blame the operator's script for it.
+					deps?.log?.("wait_check_aborted", { jobId: job.id, profile: checked });
+					await job.moveToDelayed(nowMs + THROTTLE_FLOOR_MS, token);
+					throw new DelayedError();
+				}
+
+				if (verdict?.verdict === "hold") {
+					const fault = verdict.fault === true;
+					await waitState.noteCheck(job.id, { fault });
+					const faults = fault ? counted.faults + 1 : 0;
+
+					// A check that never answers is a broken script, not a slow condition, and OQ-030 is why
+					// this bound exists: most CLIs exit 1 for everything, so without it a typo would hold for
+					// the whole maximum wait and then blame the CONDITION rather than the check.
+					if (faults >= maxFaults()) {
+						return await refuseWait("wait-unanswerable", "wait_unanswerable", { profile: checked, faults }, `The check \`${checked}\` could not answer ${faults} times in a row. Not run.`);
+					}
+
+					// BOTH terminal bounds are tested AFTER the check and never before it, so a condition that
+					// cleared on the deciding wake runs instead of being recorded as never having cleared.
+					// Without that ordering the backoff's own quantisation makes "cleared at t+1s, declared
+					// never-cleared at t+900s" a structural lie in the durable record and in a public comment.
+					//
+					// The count bound reads `checks + 1` because this wake's check has just run: the job gets
+					// exactly `maxChecks` checks, the last of which is the deciding one. Putting it before the
+					// check instead -- so the act of testing the bound could not exceed it -- was the obvious
+					// spelling, and it silently made this whole guarantee untrue at every shipped default,
+					// because the count bound is the one that fires first there.
+					if (counted.checks + 1 >= maxChecks()) {
+						return await refuseWait("wait-expired", "wait_expired", { reason: "max-checks", checks: counted.checks + 1, profile: checked, heldForMs: held }, `Gave up waiting on \`${checked}\` after ${counted.checks + 1} checks. Not run.`);
+					}
+					if (held >= maxWaitMs()) {
+						return await refuseWait("wait-expired", "wait_expired", { reason: "max-wait", profile: checked, heldForMs: held }, `Gave up waiting on \`${checked}\`. Not run.`);
+					}
+
+					// Clamped to what is LEFT of the budget, never just the cadence. Without this an hourly
+					// interval under a fifteen-minute maximum holds for the full hour -- 400% of the bound the
+					// operator configured -- because the ceiling is only tested when a wake arrives, and the
+					// cadence decides when that is. The two knobs are independent `positiveInt`s and nothing
+					// cross-validates them, so the clamp is what makes the smaller one actually bind.
+					const base = waitBackoffMs(intervalMs(), held);
+					const jittered = base + Math.floor(base * 0.1 * random());
+					const remaining = Math.max(0, maxWaitMs() - held);
+					const delay = Math.max(1000, Math.min(jittered, remaining));
+					await waitState.hold(job.id, { dedupId, target: targetFor(job.data?.kind, job.data), label: waitLabel(job.data), untilMs: nowMs + delay });
+					deps?.log?.("wait_deferred", { jobId: job.id, profile: checked, fault, heldForMs: held, delayMs: delay });
+					await job.moveToDelayed(nowMs + delay, token);
+					throw new DelayedError();
+				}
+
+				// FAIL CLOSED on anything that is not literally go. Everything above tests for a specific
+				// shape and falls through otherwise, and "otherwise" at this gate means STARTING A PAID
+				// CONTAINER -- so an `undefined`, a `null`, a `{}`, a mis-cased "GO" or a bare string from a
+				// checker would run the job silently, with no record field and no log line to distinguish it
+				// from a job whose check said yes. The shipped checker is total, and that is exactly the
+				// reasoning `unreadableConditions` above rejects: this is a dependency-injection seam, and a
+				// seam's guarantees are the caller's to enforce.
+				if (verdict?.verdict !== "go") {
+					deps?.log?.("wait_check_unintelligible", { jobId: job.id, profile: checked });
+					await waitState.noteCheck(job.id, { fault: true });
+					const base = waitBackoffMs(intervalMs(), held);
+					await job.moveToDelayed(nowMs + base + Math.floor(base * 0.1 * random()), token);
+					throw new DelayedError();
+				}
+
+				// Every profile answered go. Record the last check so the count bound sees it.
+				await waitState.noteCheck(job.id, { fault: false });
+			}
+
+			// Only a job that actually HELD has cleared. Without the check this line fires on the first
+			// pickup of a job whose instant had already passed, and again on every scope-busy re-check
+			// afterwards -- asserting a wait ended that never began.
+			const heldForMs = await waitState.heldForMs(job.id);
+			// Say so before releasing: a sibling held on this target must find the answer, not an empty lease.
+			if (heldForMs !== null) await waitState.markSatisfied(job.id, { dedupId });
+			await waitState.release(job.id, { dedupId });
+			if (heldForMs !== null) deps?.log?.("wait_cleared", { jobId: job.id, label: waitLabel(job.data), heldForMs });
+		}
+
+		// Per-scope concurrency and the one-job-per-folder mutex (issue #242,
+		// INT-SCOPED-LIMITS-FILE-CONTRACT). LAST of the three gates, after the pause gate (a paused job must
+		// not burn re-check wakes) and after the wait gate (a job holding until tomorrow must not sit on a
+		// folder while it does), and STRICTLY above the `try` below, like the pause gate and for the same two
+		// reasons: a DelayedError thrown inside the try would be converted to UnrecoverableError by the
+		// catch, and a moveToDelayed rejection here must escape RAW into BullMQ's normal failed-attempt
+		// handling exactly as the pause gate's does (inside the try it would become a permanent failure
+		// plus a failure record for what was a transient blip). The limits snapshot is read ONCE here and
+		// shared with `scopedCaps` below, so the gate and the money ledger cannot disagree mid-job.
+		// tryAcquire is a synchronous check-and-increment -- no await between read and take, so Node's
+		// single thread makes it atomic at any concurrency -- and the local-folder limit is a structural 1
+		// (concurrencyFor) with no file and no off-switch: the scheduler mints a cron trigger's next
+		// occurrence at pickup and promotes it on time alone, so a slow run overlaps its own successor
+		// (measured: 301ms of live container overlap through this very processor) unless this gate holds.
+		// Infinity-limited scopes still acquire, so release stays uniform for every scoped job.
+		const limits = scopedLimits();
+		const scope = canonicalScope(job.data);
+		let held = false;
+		if (scope) {
+			if (!inFlight.tryAcquire(scope, concurrencyFor(job.data, limits))) {
+				// Optional-chained: makeProcessor gives `deps` no default and bare wirings pass deps: {}.
+				// The scope itself stays out of the log line (no-pii-in-logs -- a local scope is a full
+				// host path); the delayed count and the job id are what an operator needs to see it.
+				deps?.log?.("scope_busy_deferred", { jobId: job.id, kind: job.data?.kind === "local" ? "local" : "forge", delayMs: SCOPE_BUSY_RECHECK_MS });
+				await job.moveToDelayed(nowMs + SCOPE_BUSY_RECHECK_MS, token);
+				throw new DelayedError();
+			}
+			held = true;
+		}
+
+		let startedAt;
+		let name;
+		let timer;
+		let onAbort;
+		try {
+			// Nothing between the acquire above and the main `try` below may throw unguarded: the releasing
+			// finally belongs to THAT try, so an unguarded throw here would leak the hold and wedge the
+			// scope until a worker restart. Nothing in this block CAN throw today (setTimeout and
+			// addEventListener on the bullmq-allocated controller are total at processor arity 3); the
+			// guard is structural, not observational.
+			startedAt = new Date().toISOString();
+			name = `pi-job-${job.id}`;
+			timer = setTimeout(() => {
+				// BullMQ has no per-job kill timer; this is ours. cancelJob raises the AbortSignal.
+				Promise.resolve(cancelJob(job.id, "job-timeout-30m")).catch(() => {});
+			}, timeoutMs);
 
-		// Abort (timeout OR shutdown) => stop the container. docker stop sends SIGTERM then SIGKILL
-		// after the grace period; the runner exits and runContainer returns/throws.
-		const onAbort = () => {
-			Promise.resolve(stopContainer(name)).catch(() => {});
-		};
-		signal.addEventListener("abort", onAbort, { once: true });
+			// Abort (timeout OR shutdown) => stop the container. docker stop sends SIGTERM then SIGKILL
+			// after the grace period; the runner exits and runContainer returns/throws.
+			onAbort = () => {
+				Promise.resolve(stopContainer(name)).catch(() => {});
+			};
+			signal.addEventListener("abort", onAbort, { once: true });
+		} catch (error) {
+			// Release and CLEAR the flag: this throw never reaches the main finally below, but a shared
+			// scope must never be releasable twice -- a double release frees another holder's slot.
+			if (held) {
+				inFlight.release(scope);
+				held = false;
+			}
+			clearTimeout(timer);
+			throw error;
+		}
 
 		try {
 			const settings = await getSettings();
@@ -66,7 +425,9 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 				// job completed and does not retry a file that can never parse (CONST-RETRY-INFRA-ONLY). Resolved
 				// before runJob, so no budget slot is reserved and no container starts (CONST-BUDGET-BEFORE-TOKENS).
 				// recordRun leaves the durable settings-overlay-invalid trace for the admin extension.
-				// No provider/model here, alone among the terminal results: the overlay is the thing that failed to parse, so no honest effective value exists -- buildRecord defaults both null.
+				// No provider/model here, as in every result this function returns from ABOVE the try (the wait gate's
+				// refusals are the others): each is decided before or during the settings read, so no honest effective
+				// value exists yet -- buildRecord defaults both null.
 				const result = { outcome: "policy", reason: "settings-overlay-invalid", exitCode: null, turns: null, tokens: null, budgetReserved: false };
 				recordRun({ job, result, startedAt, endedAt: new Date().toISOString() });
 				return result;
@@ -99,6 +460,10 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 				// The daily TOKEN cap (issue #25), same overlay > env resolution. Check-AFTER, so it gates the
 				// NEXT job on prior recorded spend; null => the daily token counter is disabled.
 				tokenCap: settings.dailyTokenCap,
+				// This job's scoped budget windows (issue #242), from the SAME limits snapshot the pickup
+				// gate above read -- one read per pickup, so gate and ledger agree for this job's whole
+				// life. Null when no row carries a money window for this scope.
+				scopedCaps: budgetCapsFor(job.data, limits),
 				...deps,
 				runContainer: (ctx) => deps.runContainer({ ...ctx, name, signal }),
 				// REQ-TRIGGER-SECRETS. The resolver runs INSIDE the 30-minute kill timer armed above, so it has
@@ -140,13 +505,16 @@ export function makeProcessor({ cancelJob, stopContainer, redis, getSettings, ap
 			// records it as failed-and-distinct in the queue's failed set without a retry.
 			throw new UnrecoverableError(error.message);
 		} finally {
+			// Release FIRST and never throw (release clamps at zero by construction): a throw here would
+			// mask the job's real error, and a missed release wedges the scope until a worker restart.
+			if (held) inFlight.release(scope);
 			clearTimeout(timer);
 			signal.removeEventListener("abort", onAbort);
 		}
 	};
 }
 
-export function createWorker({ connection, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, extraClosers = [] }) {
+export function createWorker({ connection, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, scopedLimits, inFlight, waitState, afterMaxMs, checkSlots, checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, extraClosers = [] }) {
 	let worker; // referenced by cancelJob/applyConcurrency before assignment; only called later, so the TDZ is fine
 	const processor = makeProcessor({
 		cancelJob: (id, reason) => worker.cancelJob(id, reason),
@@ -159,6 +527,26 @@ export function createWorker({ connection, concurrency, getSettings, redis, deps
 			if (Number.isInteger(n) && worker.concurrency !== n) worker.concurrency = n;
 		},
 		pauseUntil,
+		// Undefined pass-throughs take makeProcessor's own defaults (no limits; a fresh per-processor
+		// in-flight map -- one per worker process, which under DES-CONCURRENCY-3's one-worker-per-daemon
+		// shape means one per daemon).
+		scopedLimits,
+		inFlight,
+		// Issue #230. Undefined pass-throughs take makeProcessor's own defaults (a wait state over the same
+		// redis client, and the shared 30-day `after` ceiling), so a bare wiring behaves like a wired one.
+		waitState,
+		afterMaxMs,
+		// Issue #230, the polled tier. `concurrencyNow` reads the LIVE slot count rather than the boot value,
+		// because the overlay can lower it through `dispatch_set` and a check must never take the last free
+		// slot from a paid job. Late-bound over `worker` exactly as `applyConcurrency` is, and for the same
+		// reason: the value it needs does not exist until the Worker is constructed.
+		checkSlots,
+		checkSlotCount,
+		concurrencyNow: concurrencyNow ?? (() => worker?.concurrency ?? concurrency),
+		intervalMs,
+		maxWaitMs,
+		maxChecks,
+		maxFaults,
 		deps,
 		recordRun,
 	});

package/src/init.mjs

@@ -19,6 +19,11 @@ const EMPTY_PACKAGES = `${JSON.stringify({ packages: [] }, null, 2)}\n`;
 // Operator-declared subscription plans (issue #53), read by the admin extension only — never at job
 // time. Versioned because a newer file must fail loud, and that cannot be retrofitted into a v1 reader.
 const EMPTY_SUBSCRIPTIONS = `${JSON.stringify({ version: 1, subscriptions: [] }, null, 2)}\n`;
+// Scoped limits (issue #242): per repo/folder run caps and concurrency. Empty is inert -- and the
+// one-job-per-folder mutex for local jobs is code, not configuration, so it needs no scaffold line.
+// Versioned for the subscriptions reason, sharpened: this is enforcement config, and a silently
+// down-read newer file would be a silently widened spend limit.
+const EMPTY_SCOPED_LIMITS = `${JSON.stringify({ version: 1, limits: [] }, null, 2)}\n`;
 /**
  * The egress allowlist (REQ-EGRESS-ALLOWLIST): the hosts a job container may reach, one bare hostname per
  * line. Scaffolded with the three a job cannot work without, and NOT empty -- unlike every other scaffold
@@ -73,6 +78,7 @@ export function runInit(cwd = process.cwd(), deps = {}) {
 	scaffold(fs, results, join(cwd, "pause-windows.json"), EMPTY_PAUSE_WINDOWS, "empty pause-windows list");
 	scaffold(fs, results, join(cwd, "pi-packages.json"), EMPTY_PACKAGES, "empty pi package list (stage with import-pi --with-packages)");
 	scaffold(fs, results, join(cwd, "subscriptions.json"), EMPTY_SUBSCRIPTIONS, "empty subscription list (declare plan prices for the admin's cost analytics)");
+	scaffold(fs, results, join(cwd, "scoped-limits.json"), EMPTY_SCOPED_LIMITS, "empty scoped-limits list (per repo/folder caps; the folder mutex needs no file)");
 	scaffold(fs, results, join(cwd, "egress-allowlist.conf"), DEFAULT_EGRESS_ALLOWLIST, "egress allowlist (provider + forge + registry; the egress policy is on unless PI_EGRESS=0)");
 
 	for (const [verb, name, note] of results) {

package/src/processor.mjs

@@ -1,6 +1,7 @@
 import { lstatSync } from "node:fs";
 import { checkTokenCap, recordTokenSpend, releaseBudget, reserveBudget } from "./budget.mjs";
 import { configError } from "./config.mjs";
+import { scopeKeyPrefix } from "./scoped-limits.mjs";
 import { DEFAULT_SECRETS_PROFILE, secretsArmed } from "./secrets.mjs";
 import { EXIT_COMPLETED, EXIT_INFRA, EXIT_POLICY } from "./exit-code.mjs";
 
@@ -37,6 +38,12 @@ export async function runJob(job, deps) {
 		caps, // { day, week, month }; week/month null when that window is disabled (REQ-SPEND-CAPS-MULTI-WINDOW)
 		softHoldPct, // int 1-99 or null; the soft-hold band applied to every active window
 		tokenCap = null, // int or null; the daily TOKEN cap (issue #25). Check-AFTER, so it gates the NEXT job on prior spend
+		// { scope, caps: { day, week, month } } | null -- this job's scoped budget windows (issue #242,
+		// INT-SCOPED-LIMITS-FILE-CONTRACT), resolved by the wiring from the same watched-limits snapshot the
+		// pickup gate read. Null when the file is unset or the scope's row is concurrency-only; the default
+		// keeps an unwired processor byte-identical. The folder MUTEX does not live here -- it is the pickup
+		// gate's, pre-everything; this is only the money half.
+		scopedCaps = null,
 		recordSpend = recordTokenSpend, // injected so the post-container INCRBY is testable/stubbable
 		// (job) => { ok } | { missing: <ref> } | { unavailable: <ref> }. The pre-spend check that the image
 		// this job names is on this host (image-preflight.mjs). Default admits everything, so a wiring that
@@ -47,6 +54,9 @@ export async function runJob(job, deps) {
 		// exactly as before, and the gate below only calls it for a job whose matched rule was a
 		// one-shot, so the default is never a probe running on every delivery.
 		checkOnceSpent = async () => ({ ok: true }),
+		// Issue #230. Admit-everything by default, like checkOnceSpent above and for its reason: an
+		// unwired seam must not refuse, and the wiring is what turns the check on.
+		checkWaitSkew = async () => ({ ok: true }),
 		// REQ-EGRESS-ALLOWLIST. Default admits everything, so a wiring that omits it behaves exactly as a
 		// deployment with no egress policy does -- which is also what the real factory returns when unarmed.
 		egressPreflight = async () => ({ ok: true }),
@@ -124,6 +134,7 @@ export async function runJob(job, deps) {
 	let token = null;
 	let prepared = null;
 	let reserved = false;
+	let scopedReserved = false;
 
 	try {
 		// The one-shot pre-spend check (issue #231), FIRST on the ladder: one file read, cheaper than
@@ -147,6 +158,28 @@ export async function runJob(job, deps) {
 			}
 		}
 
+		// The wait-skew check (issue #230), second on the ladder and for the first one's reasons: the same
+		// file read, free, determinate, credential-less, and pre-spend. It answers a question no other layer
+		// can: does the AUTHORED trigger carry wait conditions this job arrived without? That happens when a
+		// service below the version floor dropped the field as an unknown key, and the resulting run is
+		// byte-identical to a correct one everywhere it is recorded -- so this refusal is the only thing
+		// standing between a stale receiver and a paid job that ran when the operator wrote "wait".
+		{
+			const skew = await checkWaitSkew(job);
+			if (skew.skewed) {
+				// Named for the operator, not the payload: how many conditions were authored, never what
+				// they say. The fix is a version, so the comment says which one.
+				// The message names BOTH causes, because the more likely one is not a version at all. In the
+				// compose topology the receiver's single-file `:ro` mount pins a dead inode, so an operator
+				// who ADDS `waitFor` to an existing rule gets this refusal on every delivery from a service
+				// that is perfectly up to date and merely holding an older copy of the file. Naming only the
+				// version would send them looking for an upgrade they do not need.
+				await comment(job, `Refused: this trigger declares ${skew.conditions} wait condition${skew.conditions === 1 ? "" : "s"}, but the job reached the worker without them, which means it would have run immediately. Either a service in this deployment is below the version that carries the field, or one is still running against an older copy of the triggers file and needs restarting. Not run.`);
+				log("refused_wait_skew", { triggerIndex: job.trigger?.matched?.index ?? null, conditions: skew.conditions });
+				return { outcome: "policy", reason: "wait-skew", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false };
+			}
+		}
+
 		// The job image must exist on THIS host before anything else happens. Free, determinate and
 		// credential-less, so it precedes the mint, the clone and the reservation: a host that cannot run the
 		// image refuses without minting a credential it will not use, cloning a repo it will not read, or
@@ -458,11 +491,51 @@ export async function runJob(job, deps) {
 			return { outcome: "policy", reason: "daily-token-cap", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
 		}
 
-		// Budget last-but-before-container. A refusal here spends nothing (no container starts). Reserves across
+		// Per-scope budget windows (issue #242, INT-SCOPED-LIMITS-FILE-CONTRACT): the NARROWER ledger
+		// reserves FIRST, so a noisy scope's refusals never consume a global slot -- the global INCR below
+		// runs only for jobs the scope admitted. Same atomic INCR, same refused-still-counts invariant,
+		// through budget.mjs's keyPrefix seam (dayKey/weekKey/monthKey under budget:s:<hash16>). softHoldPct
+		// is deliberately GLOBAL-ONLY: the band is one operator brake on overall spend, not a per-row knob;
+		// scoped windows are hard caps (DES-SCOPED-LIMITS-AND-FOLDER-MUTEX).
+		if (scopedCaps) {
+			// A redis fault BETWEEN this reserve and the global one below strands the scoped INCR with no
+			// run and no refund -- the pre-existing mid-reserve posture, shared with the global ledger's
+			// own partial-INCR seam; the compensating release below covers REFUSALS, not faults.
+			const scoped = await reserveBudget(redis, { caps: scopedCaps.caps, now, keyPrefix: scopeKeyPrefix(scopedCaps.scope) });
+			scopedReserved = true;
+			if (!scoped.allowed) {
+				const w = scoped.blockedWindow;
+				const win = scoped.windows[w];
+				// A local job's scope is a full host path and its "comment" is not dropped -- the wiring's
+				// local adapter LOGS the text (start.mjs forgeFor fallthrough) -- so the path must never
+				// enter the message; "this folder" is enough beside the jobId the adapter logs. A forge
+				// scope IS the repo the comment posts on, safe to name.
+				const scopeLabel = job.kind === "local" ? "this folder" : scopedCaps.scope;
+				await comment(job, `Over the ${w} run cap for ${scopeLabel} (${win.cap}). Not run.`);
+				// The scope rides the log as its 16-hex key, NEVER the raw string: a folder-scoped cap would
+				// put a full host path in the worker log against no-pii-in-logs (the record keeps only
+				// basename(folder) for the same reason). The admin recomputes the key from the configured
+				// scope to join it back.
+				log("over_scope_budget", { scopeKey: scopeKeyPrefix(scopedCaps.scope), window: w, reserved: win.reserved, cap: win.cap, kind: job.kind === "local" ? "local" : "forge" });
+				// budgetReserved false: the GLOBAL slot was never touched (scoped reserves first). The scoped
+				// counter did INCR and keeps it -- its own refused-reservation-still-counts, per ledger.
+				return { outcome: "policy", reason: "scope-cap", exitCode: null, turns: null, tokens: null, provider: job.provider ?? null, model: job.model ?? null, budgetReserved: false }; // return => not retried
+			}
+		}
+
+		// GLOBAL budget last-but-before-container. A refusal here spends nothing (no container starts). Reserves across
 		// every active window (day + optional week/month) and the soft-hold band in one atomic pass.
 		const budget = await reserveBudget(redis, { caps, softHoldPct, now });
 		reserved = true;
 		if (!budget.allowed) {
+			// The scoped reserve above committed before this global refusal -- give that slot back. Without
+			// this, an exhausted global window drains every arriving scope's own day/week/month counters
+			// with zero runs to show for it (a storm against a spent global daily cap would empty a repo's
+			// week by noon). The scoped ledger's refused-still-counts covers the SCOPE's own refusal above,
+			// never a refusal it did not issue.
+			if (scopedReserved && scopedCaps) {
+				await releaseBudget(redis, { caps: scopedCaps.caps, now, keyPrefix: scopeKeyPrefix(scopedCaps.scope) });
+			}
 			const w = budget.blockedWindow;
 			const win = budget.windows[w];
 			if (budget.reason === "soft-hold") {
@@ -561,8 +634,12 @@ export async function runJob(job, deps) {
 		// budgetReserved reflects whether a slot stays spent: false when never-started refunds below,
 		// true for a real container that ran and spent (exit-1 infra / unknown exit).
 		if (e instanceof InfraRetry) e.budgetReserved = reserved && e.reason !== "container-never-started";
-		if (reserved && e instanceof InfraRetry && e.reason === "container-never-started") {
-			await releaseBudget(redis, { caps, now });
+		if (e instanceof InfraRetry && e.reason === "container-never-started") {
+			// Both-or-neither (issue #242): a never-started container can only follow BOTH reserves (the
+			// scoped one precedes the global one, and the container follows both), so they refund
+			// together -- and a scoped refusal returned above without ever touching the global ledger.
+			if (reserved) await releaseBudget(redis, { caps, now });
+			if (scopedReserved && scopedCaps) await releaseBudget(redis, { caps: scopedCaps.caps, now, keyPrefix: scopeKeyPrefix(scopedCaps.scope) });
 		}
 		throw e;
 	} finally {