@edgehero/pi-dispatch 1.5.0 → 1.6.0

package/src/wait-for.mjs

@@ -0,0 +1,315 @@
+/**
+ * `run.waitFor`: holding a job until a condition clears (issue #230).
+ *
+ * Two tiers, split by WHO CAN ANSWER, and the split is not cosmetic: one tier is free and the other
+ * spawns a process.
+ *
+ *   { "after": "2026-09-01T09:00:00Z" }   tier 1 -- a wall-clock instant, answered in-process from the
+ *                                         clock alone. One exact `moveToDelayed`, no polling at all.
+ *   { "profile": "jira" }                 tier 2 -- an operator-declared executable answers go / not yet
+ *                                         / cannot tell / never. Polled, and every poll is bounded.
+ *
+ * This module holds the two halves that BOTH the loader and the gate must agree on, so neither side
+ * re-derives them: what a legal `after` instant is, and what `PI_WAIT_PROFILES` declares. The trigger-side
+ * validator lives in `triggers.mjs` beside its siblings (`validateSecretsProfile`'s split), and reads this
+ * file rather than restating it. The spawner that runs a profile will do the same when it lands; NOTHING
+ * evaluates a wait yet, and this module is the only part of the feature currently wired to anything.
+ *
+ * PURE, FS-FREE AND ENV-FREE, for `secret-profiles.mjs`' reason: `config.mjs` imports this module and
+ * `admin/build.mjs` inlines that chain into the published console. Whether a declared path EXISTS is asked
+ * once, on the worker that is about to spawn it.
+ *
+ * `configError` is a local copy rather than an import from `config.mjs`, which imports this module. The
+ * cycle would resolve (function declarations hoist) but would be a trap for the next reader -- the same
+ * duplication, for the same reason, is in `secret-profiles.mjs` and `env-allowlist.mjs`.
+ */
+
+import { isAbsolute } from "node:path";
+
+function configError(message) {
+	const error = new Error(message);
+	error.piDispatchConfig = true;
+	return error;
+}
+
+/**
+ * The condition keys this version understands. `exclusive` is deliberately ABSENT and is refused by name
+ * in `triggers.mjs`: issue #242 shipped an always-on one-job-per-folder mutex at this very gate, so the
+ * condition it asked for is structural now, and `scoped-limits.json` covers the wider per-scope case.
+ * Accepting-and-ignoring it would be the silent no-op this project refuses.
+ */
+export const WAIT_CONDITION_KEYS = ["after", "profile"];
+
+/**
+ * The ceiling on conditions per trigger. Four, on `SECRETS_MAX`'s reasoning rather than `REPLICAS_MAX`':
+ * this multiplies no spend, but every `profile` condition is a subprocess evaluated before the container,
+ * so an unbounded array is an unbounded slot occupancy. Four is generous for a conjunction a human wrote.
+ */
+export const WAIT_CONDITION_MAX = 4;
+
+/**
+ * The re-check cadence floor, and the ceiling the elapsed-derived backoff climbs to.
+ *
+ * CLAMPED UP, NEVER REFUSED, which is the poller's posture and its reason verbatim: "a typo'd `1` must not
+ * turn the harness into a hammer". Note what that does and does not cover -- `positiveInt` still refuses
+ * `0`, a negative, a fraction and a non-number at boot, so only a positive integer BELOW the floor is
+ * quietly raised. The floor is 30s rather than the poller's because there is no third party asking for
+ * politeness here; what it protects is this worker's own concurrency slots.
+ *
+ * The floor is also load-bearing in a second place, and lowering it would break that silently: it is what
+ * lets a held job be told apart from a scope deferral, whose re-check is `SCOPE_BUSY_RECHECK_MS` (5s). Any
+ * value at or below that would make the two indistinguishable by wake instant.
+ *
+ * `WAIT_INTERVAL_MAX_MS` is the ceiling the backoff climbs to, and it is a ceiling on the BACKOFF, never on
+ * the operator: the rule the backoff must implement is `min(max(base·2^k, base), max(WAIT_INTERVAL_MAX_MS,
+ * base))`, so an operator who deliberately configured an hourly cadence to save money keeps it. Writing the
+ * cap as a bare `min(..., 900_000)` is the obvious spelling and it silently turns their hour into fifteen
+ * minutes -- a 4x cost overrun, in the direction they were trying to avoid, from a knob this file documents
+ * as clamped UPWARD.
+ */
+export const WAIT_INTERVAL_FLOOR_MS = 30_000;
+export const WAIT_INTERVAL_MAX_MS = 900_000;
+
+/**
+ * The default ceiling on how far out an `after` instant may sit, and deliberately NOT the maximum hold.
+ *
+ * An `after` polls nothing: it is one exact `moveToDelayed` to an instant, self-terminating and costing
+ * nothing while it waits, so bounding it by the polling budget would refuse "hold this until the
+ * maintenance window next month" for a reason about subprocesses it never runs. Thirty days is a bound on
+ * how far ahead a REVIEWED FILE may schedule, not a bound on cost.
+ *
+ * A literal shared by `config.mjs` and the processor's own default, so a bare `makeProcessor` under test
+ * and a wired worker agree on what "too far" means.
+ */
+export const WAIT_AFTER_MAX_DEFAULT_MS = 30 * 24 * 3600 * 1000;
+
+/**
+ * How long to wait before the next check, given the configured base and how long this job has been held.
+ *
+ * Doubles every ten base periods and settles at `WAIT_INTERVAL_MAX_MS` -- **or at `base`, whichever is
+ * larger**. That `Math.max` is the whole reason this is a function and not a constant the caller clamps
+ * against: the obvious spelling, `Math.min(grown, WAIT_INTERVAL_MAX_MS)`, silently turns a deliberately
+ * configured hourly cadence into a fifteen-minute one, which is a 4x cost overrun in exactly the direction
+ * the operator was economising. Shipped with the constant so the rule cannot be re-derived wrongly later.
+ *
+ * DERIVED FROM ELAPSED, never stored: a deferral consumes no attempt (`moveToDelayed` passes
+ * `skipAttempt: true`), so `attemptsMade` cannot carry a retry count and nothing else may either. Reading
+ * the schedule off the clock makes it survive a worker restart for free.
+ */
+export function waitBackoffMs(baseMs, elapsedMs) {
+	const base = Number.isFinite(baseMs) && baseMs > 0 ? baseMs : WAIT_INTERVAL_FLOOR_MS;
+	const elapsed = Number.isFinite(elapsedMs) && elapsedMs > 0 ? elapsedMs : 0;
+	const grown = base * 2 ** Math.floor(elapsed / (10 * base));
+	return Math.min(Math.max(grown, base), Math.max(WAIT_INTERVAL_MAX_MS, base));
+}
+
+/**
+ * An ISO-8601 instant that carries its own zone: `Z`, `z`, or a numeric offset. Seconds and fractional
+ * seconds are optional.
+ *
+ * THE ZONE IS REQUIRED, and that is the whole point of not using bare `Date.parse`. `Date.parse` accepts
+ * "2026-09-01T09:00:00" and resolves it against the WORKER'S local zone, so the same reviewed file would
+ * hold a job for a different instant on a machine in Amsterdam than on one in UTC, silently, with nothing
+ * in the file to explain the difference. `pause-windows.mjs` solves the same problem the other way (an
+ * explicit `tz` field, defaulting to UTC) because its times recur; a one-shot instant can simply carry its
+ * own offset, so requiring one is cheaper than inventing a second timezone field.
+ *
+ * The TIME fields are range-bounded in the pattern rather than left to `Date.parse`, for the same reason the
+ * calendar is re-checked below: `Date.parse` reads `T24:00:00` as the next day's midnight, so an hour field
+ * nobody can write on a clock would silently produce a hold one day later than the file says. `:60` in the
+ * minute or second field is refused for the same reason (there are no leap seconds to honour here, and a
+ * value that rolls is worse than a value that refuses).
+ */
+const AFTER_INSTANT = /^\d{4}-\d{2}-\d{2}[Tt]([01]\d|2[0-3]):[0-5]\d(:[0-5]\d(\.\d+)?)?([Zz]|[+-]\d{2}:\d{2})$/;
+
+/**
+ * A profile name. The same charset `triggers.mjs` allows in `run.waitFor[].profile`, and a local copy for
+ * `secret-profiles.mjs`' stated reason: a name that failed here after passing there would be an
+ * operator-visible contradiction between two files meant to agree. `,` and `:` are excluded because they
+ * are `PI_WAIT_PROFILES`' own separators, so a name carrying either could not round-trip.
+ */
+const PROFILE_NAME = /^[A-Za-z0-9._-]+$/;
+
+/**
+ * The epoch ms of a legal `after` value, or `null` when the text is not one.
+ *
+ * ONE DEFINITION, READ BY BOTH SIDES. The loader calls this to refuse a malformed instant at load; the
+ * pickup gate calls it to decide how long to hold. A second spelling in either place is how a file that
+ * loads clean starts holding for an instant nobody wrote.
+ */
+export function parseAfterInstant(text) {
+	if (typeof text !== "string" || !AFTER_INSTANT.test(text)) return { error: "shape" };
+	const ms = Date.parse(text);
+	if (!Number.isFinite(ms)) return { error: "calendar" }; // an out-of-range month: "2026-13-01T00:00Z"
+	// An out-of-range DAY does not reach that check, and this is the surprise worth refusing: `Date.parse`
+	// rejects month 13 but silently ROLLS "2026-02-31T00:00:00Z" forward to March 3rd. An operator who
+	// mistyped a date would get a hold that ends on a day they did not write, with the file still reading
+	// as though they had. The calendar fields are checked against the date part alone, so an offset in the
+	// text cannot move the day being validated: the operator wrote that date, whatever zone it is in.
+	const [year, month, day] = text.slice(0, 10).split("-").map(Number);
+	const probe = new Date(Date.UTC(year, month - 1, day));
+	// `Date.UTC` maps a two-digit year onto 19xx, so a year below 100 would fail the comparison below and
+	// refuse for a reason that has nothing to do with the operator's date. Absurd as a wait, but a refusal
+	// the code did not mean to make is still a refusal nobody can act on.
+	if (year < 100) probe.setUTCFullYear(year);
+	if (probe.getUTCFullYear() !== year || probe.getUTCMonth() !== month - 1 || probe.getUTCDate() !== day) {
+		return { error: "calendar" };
+	}
+	return { ms };
+}
+
+/**
+ * The epoch ms of a legal `after` value, or `null`. The runtime spelling of `parseAfterInstant`, which the
+ * gate wants because at pickup there is nothing to say: the loader already refused anything unparseable, so
+ * by then the only question is what instant to hold until.
+ */
+export function afterInstantMs(text) {
+	const parsed = parseAfterInstant(text);
+	return parsed.error ? null : parsed.ms;
+}
+
+/**
+ * Parse `PI_WAIT_PROFILES`: `name:/abs/path,other:/abs/other`. Returns `{ [name]: path }`, empty when
+ * unset. Throws (config-tagged, so the worker refuses to boot) on anything malformed.
+ *
+ * The twin of `parseSecretProfiles`, deliberately duplicated rather than generalised into a shared helper.
+ * The two variables mean different things (one resolves a value, one answers a question), they will drift
+ * apart as this one grows bounds the other has no use for, and a shared parser would have to be told which
+ * variable name to put in every error message -- which is the only part an operator reads.
+ *
+ * SET-BUT-GARBLED FAILS LOUD: a silently dropped entry is a profile the operator believes is wired, and
+ * every trigger naming it would refuse at delivery time with the operator looking at the line that appears
+ * to declare it. Each entry splits on its FIRST colon, so `prod:C:\pi\wait.cmd` parses on Windows.
+ */
+export function parseWaitProfiles(raw) {
+	if (raw === undefined || raw === null || String(raw).trim() === "") return Object.create(null);
+	// PROTOTYPE-FREE, and that is a correctness requirement rather than hygiene: a profile named `toString`
+	// or `constructor` passes ID_CHARSET, so on a `{}` table `profiles[name]` answers with an inherited
+	// FUNCTION instead of `undefined` and the gate's "is this profile declared?" check silently says yes --
+	// for a name no operator declared, with the variable unset entirely. `Object.create(null)` also stops
+	// `name in profiles` from reporting a duplicate that was never written.
+	const profiles = Object.create(null);
+	for (const entry of String(raw).split(",")) {
+		const text = entry.trim();
+		if (text === "") continue; // a trailing comma is a typo, not a declaration
+		const cut = text.indexOf(":");
+		if (cut <= 0) {
+			throw configError(`PI_WAIT_PROFILES entries must be name:/absolute/path, got ${JSON.stringify(text)}`);
+		}
+		const name = text.slice(0, cut).trim();
+		const path = text.slice(cut + 1).trim();
+		if (!PROFILE_NAME.test(name)) {
+			throw configError(`PI_WAIT_PROFILES profile name ${JSON.stringify(name)} may use letters, digits, dot, dash and underscore only`);
+		}
+		if (name in profiles) {
+			throw configError(`PI_WAIT_PROFILES declares ${JSON.stringify(name)} twice -- one of the two is not the check you think is running`);
+		}
+		if (path === "" || !isAbsolutePath(path)) {
+			throw configError(`PI_WAIT_PROFILES profile ${JSON.stringify(name)} needs an ABSOLUTE path to its check -- a service manager's working directory is not your shell's, so a relative path is a different file on every host`);
+		}
+		profiles[name] = path;
+	}
+	return profiles;
+}
+
+/** Absolute on this platform, accepting a Windows drive letter or UNC root as `service.mjs` does. */
+function isAbsolutePath(path) {
+	return isAbsolute(path) || /^([A-Za-z]:[\\/]|\\\\)/.test(path);
+}
+
+/**
+ * True when this job carries any wait condition at all. The ONE guard every wait code path is behind, so
+ * an unflagged job takes zero new branches and its record, job data and key set stay byte-identical to a
+ * run prepared before this field existed.
+ */
+export function waitArmed(job) {
+	return Array.isArray(job?.waitFor) && job.waitFor.length > 0;
+}
+
+/**
+ * The `after` instant this job is holding for, or `null`. At most one `after` per array is legal (the
+ * loader refuses a second), so this is a lookup, not a fold.
+ */
+export function afterMs(job) {
+	if (!waitArmed(job)) return null;
+	for (const condition of job.waitFor) {
+		const ms = afterInstantMs(condition?.after);
+		if (ms !== null) return ms;
+	}
+	return null;
+}
+
+/**
+ * The conditions on this job that this worker cannot read, as written.
+ *
+ * The loader refuses all of these, and the loader is a DIFFERENT PROCESS: `job.data.waitFor` reaches the
+ * gate over Redis from the receiver, so a receiver newer than the worker can enqueue a condition shape this
+ * build has no branch for. Returning them rather than ignoring them is what lets the gate refuse instead of
+ * silently treating an unreadable condition as a satisfied one -- the forward half of the same version skew
+ * `makeCheckWaitSkew` closes backward.
+ *
+ * Deliberately structural rather than a re-run of the loader's validator: this asks only "can I act on
+ * this?", so a future worker that learns a new condition answers differently here without the two
+ * validators having to agree on every message.
+ */
+export function unreadableConditions(job) {
+	if (!waitArmed(job)) return [];
+	// A SECOND `after` is unreadable even though each one parses: the conjunction has no defined meaning
+	// with two instants, `afterMs` answers with whichever comes first, and the loader refuses the shape --
+	// so a job carrying one arrived from something newer or something wrong, which is this function's whole
+	// subject. Counted here rather than in the shape filter below, which sees one condition at a time.
+	let afters = 0;
+	for (const c of job.waitFor) if (c && typeof c === "object" && !Array.isArray(c) && Object.keys(c)[0] === "after") afters += 1;
+	if (afters > 1) return [...job.waitFor];
+	return job.waitFor.filter((condition) => {
+		if (condition === null || typeof condition !== "object" || Array.isArray(condition)) return true;
+		const keys = Object.keys(condition);
+		if (keys.length !== 1 || !WAIT_CONDITION_KEYS.includes(keys[0])) return true;
+		if (keys[0] === "after") return afterInstantMs(condition.after) === null;
+		return !isDeclarableName(condition.profile);
+	});
+}
+
+/** A profile name this worker will put in a log line and a public comment. See `waitLabel`. */
+function isDeclarableName(value) {
+	return typeof value === "string" && PROFILE_NAME.test(value);
+}
+
+/**
+ * The profile names this job waits on, in the operator's writing order. Order matters here even though the
+ * conditions are a CONJUNCTION and the tiers are evaluated cheapest-first: within tier 2 the checks run
+ * sequentially, and naming the first one that says "not yet" is what makes a held row readable.
+ */
+export function waitProfileNames(job) {
+	if (!waitArmed(job)) return [];
+	const names = [];
+	for (const condition of job.waitFor) {
+		// Charset-checked HERE, not merely upstream. The loader enforces the same set, but that runs in a
+		// different process and this value ends up in a PUBLIC forge comment and a log line: a name carrying
+		// a backtick breaks the code span it is rendered in, and one carrying a newline is a log injection.
+		// The module that makes the no-attacker-bytes claim is the one that has to enforce it -- the same
+		// reasoning as this file's prototype-free profile table.
+		if (isDeclarableName(condition?.profile)) names.push(condition.profile);
+	}
+	return names;
+}
+
+/**
+ * A short, PII-free label for what this job is waiting on, for the held row and the log line.
+ *
+ * Every byte of it is checked HERE against `PROFILE_NAME` or the instant grammar, rather than trusted to
+ * have been checked by the loader in another process. Nothing attacker-chosen can reach it,
+ * which is the property that lets it sit in a panel row and a log line at all (`INT-RUN-HISTORY-FILE-CONTRACT`'s
+ * PII-free-by-construction rule, applied to a surface that is not the record).
+ */
+export function waitLabel(job) {
+	if (!waitArmed(job)) return null;
+	const parts = [];
+	for (const condition of job.waitFor) {
+		// Both arms are re-validated for the reason waitProfileNames states: this string is rendered to
+		// operators, and "it was validated upstream" is a claim about another process.
+		if (afterInstantMs(condition?.after) !== null) parts.push(`after ${condition.after}`);
+		else if (isDeclarableName(condition?.profile)) parts.push(condition.profile);
+	}
+	return parts.length > 0 ? parts.join(" + ") : null;
+}

package/src/wait-state.mjs

@@ -0,0 +1,263 @@
+/**
+ * The `wait:` keyspace: what the worker remembers about a job it is holding (issue #230).
+ *
+ * Four things need remembering, and none of them can be derived from the delayed set alone:
+ *
+ *   1. WHEN THE HOLD STARTED. `job.timestamp` is the ENQUEUE instant, so it counts pause-window time,
+ *      scope-mutex time and retry backoff as "waited": a job enqueued into a 22:00-08:00 quiet window would
+ *      burn ten hours of its wait budget before the first check existed, then terminate having actually
+ *      waited fourteen. The panel would report the wrong number for the same reason.
+ *   2. WHICH JOB HOLDS A TARGET. A second delivery for the same repo/issue/flow while the first is held
+ *      would otherwise hold too, and both would run when the condition cleared -- two paid runs for one
+ *      intent, which the acceptance criteria refuse. The obvious fix, widening the queue's dedup window,
+ *      is worse: that key carries no trigger identity (so it would suppress an unflagged sibling on the
+ *      same target) and it OUTLIVES completion (so it would go on suppressing after this job finished).
+ *   3. HOW MANY TIMES A CHECK HAS FAILED TO ANSWER (the enforcement slice writes it; the field is
+ *      reserved here so both writers agree on the shape).
+ *   4. WHAT TO SHOW AN OPERATOR: an id-only target and an operator-authored condition label, chosen by
+ *      the worker rather than projected out of a delayed job's `.data`, which holds the issue title and
+ *      body.
+ *
+ * WHY THIS IS NOT THE REDIS STATE `OQ-008` AND #242 REFUSED. That refusal is about a claim that would
+ * survive a crash WRONGLY -- an in-flight count asserting a container the boot reaper had just killed.
+ * This describes DELAYED JOBS, which are themselves Redis-persisted, so it is the same source of truth
+ * rather than a second one. Every key carries a TTL sized to the hold it describes; a polled holder also
+ * refreshes its own lease on each wake, while an `after` holder has no wakes to refresh on, which is
+ * exactly why the supersede path VERIFIES the holder is still in the queue before it refuses anyone. A
+ * holder it cannot verify admits rather than refuses. So the worst a leak costs is a duplicate run or a
+ * stale panel row, never work that is dropped.
+ *
+ * The keys, all under one prefix so an operator can see the whole feature with one `KEYS wait:*`:
+ *   wait:held                 SET of held job ids -- an index the READER prunes, never a source of truth
+ *   wait:job:<jobId>          HASH { since, faults, throttles, target, label, dedupId }, TTL sized to the hold
+ *   wait:done:<dedupId>       STRING jobId -- this target's wait was satisfied, briefly remembered
+ *   wait:key:<dedupId>        STRING jobId -- the supersede lease, TTL sized to the expected hold
+ */
+
+/** Slack added to every lease so a hold that wakes exactly on time never races its own key's expiry. */
+const LEASE_SLACK_MS = 60 * 60 * 1000;
+
+/** The floor on a lease, so a short hold still leaves a trace long enough for a panel refresh to see it. */
+const LEASE_MIN_MS = 5 * 60 * 1000;
+
+// An INDEX SET, with the leak handled by the reader rather than by not having one.
+//
+// The first cut of this file had no index, on the reasoning that a SET cannot expire its members, so every
+// hold ending by any route except the clean one would leave a member nothing removes. That reasoning was
+// right about the leak and wrong about the cost of avoiding it: enumerating `wait:job:*` instead means a
+// SCAN of the whole keyspace, and the panel does it every second -- worst on deployments with NOTHING held,
+// because the exit-early condition never fires. Measured at 200k unrelated keys that is most of a refresh
+// interval, for a feature nobody is using.
+//
+// So the index exists and the READER prunes it: a member whose hash is gone is stale by definition, and the
+// reader removes it as it goes. Staleness is bounded by the next read, the hashes remain the source of
+// truth, and the cost of listing held jobs becomes O(held) instead of O(keyspace).
+export const HELD_SET = "wait:held";
+export const jobKey = (jobId) => `wait:job:${jobId}`;
+export const leaseKey = (dedupId) => `wait:key:${dedupId}`;
+export const satisfiedKey = (dedupId) => `wait:done:${dedupId}`;
+
+// How long "this target's wait has been satisfied" is remembered. The backoff ceiling, because that is the
+// longest two jobs that cleared together can wake apart.
+const SATISFIED_MS = 15 * 60 * 1000;
+
+/**
+ * Build the state accessor. `redis` is the same client the budget uses; nothing here is on the paid path,
+ * so every method is written to FAIL OPEN: a redis blip must never refuse a job or wedge a hold, it may
+ * only cost the panel a row. The one exception is `claim`, whose failure mode is spelled out on it.
+ */
+export function makeWaitState({ redis, now = () => Date.now(), counterTtlMs = 25 * 3600 * 1000 }) {
+	const leaseMs = (untilMs) => Math.max(LEASE_MIN_MS, (untilMs ?? 0) - now() + LEASE_SLACK_MS);
+
+	return {
+		/**
+		 * Record (or refresh) a hold. `since` is written ONCE with HSETNX so a re-pick does not restart the
+		 * clock -- that is the whole point of not using `job.timestamp`, and a plain HSET here would
+		 * reintroduce the bug from the other side by resetting the wait on every wake.
+		 */
+		async hold(jobId, { dedupId, target, label, untilMs }) {
+			try {
+				const key = jobKey(jobId);
+				const ttl = leaseMs(untilMs);
+				await redis.hsetnx(key, "since", String(now()));
+				await redis.hset(key, "target", target ?? "", "label", label ?? "", "dedupId", dedupId ?? "");
+				await redis.pexpire(key, ttl);
+				await redis.sadd(HELD_SET, jobId);
+				// Refresh OUR lease only, and re-read to find out whether it IS ours: `hold` is no longer
+				// reached solely after a successful claim (the throttle path holds to stamp a clock without
+				// ever winning a lease), so ownership is a question rather than an assumption.
+				// Extending a lease this job LOST would hand the winner a longer deafening window than its own
+				// hold asked for, which is the failure the three-way claim exists to avoid.
+				if (dedupId && (await redis.get(leaseKey(dedupId))) === jobId) await redis.pexpire(leaseKey(dedupId), ttl);
+			} catch {
+				// Fail open: an unrecorded hold is an invisible row, not a wrong decision.
+			}
+		},
+
+		/**
+		 * Take the supersede lease for a target, or say what to do instead. Returns `{ ok: true }` when this
+		 * job may hold, `{ heldBy }` when another job verifiably still does, or `{ retry: true }` when the
+		 * holder could not be checked.
+		 *
+		 * `SET NX PX` is the coalescing mechanism: atomic, so two deliveries arriving together cannot both win.
+		 *
+		 * THE THREE-WAY ANSWER IS THE POINT, and a two-way one is wrong in both directions. Admitting an
+		 * unverified holder means two jobs hold the same target and BOTH are paid when it clears -- the exact
+		 * accumulation `REQ-WAIT-FOR` promises will not happen. Refusing one means a holder that is gone
+		 * deafens the target for the rest of its lease, and a refused forge delivery is gone for good. So a
+		 * holder we cannot check produces neither: the caller re-defers and asks again, which costs one wake
+		 * and decides nothing until the answer is known.
+		 *
+		 * `isLive` returns true, false, or null/undefined for "cannot tell" (no probe wired, or it threw).
+		 */
+		async claim(jobId, { dedupId, untilMs, isLive }) {
+			if (!dedupId) return { ok: true }; // no semantic identity to coalesce on (a job id is unique already)
+			try {
+				const key = leaseKey(dedupId);
+				const won = await redis.set(key, jobId, "PX", leaseMs(untilMs), "NX");
+				if (won) return { ok: true };
+				const holder = await redis.get(key);
+				if (!holder || holder === jobId) return { ok: true }; // expired between the two calls, or it is us
+
+				let live = null;
+				if (typeof isLive === "function") {
+					try {
+						live = await isLive(holder);
+					} catch {
+						live = null; // a probe that threw has not told us the holder is alive
+					}
+				}
+				if (live === false) {
+					// The holder can no longer wake. Take the lease over rather than leaving a key that
+					// refuses every later delivery for this target until it expires.
+					await redis.set(key, jobId, "PX", leaseMs(untilMs));
+					return { ok: true, tookOverFrom: holder };
+				}
+				if (live !== true) return { retry: true, holder };
+				return { heldBy: holder };
+			} catch {
+				// A redis fault reached us before any decision was made. Ask again rather than guess: both
+				// guesses are wrong in a way an operator cannot see.
+				return { retry: true, holder: null };
+			}
+		},
+
+		/**
+		 * Mark a target as SATISFIED by this job, and read who satisfied it.
+		 *
+		 * The lease alone cannot close one window. Two deliveries can both be holding when a worker outage
+		 * outlives the lease TTL -- the delayed jobs survive, being Redis-persisted, while the lease does not
+		 * -- and if the condition cleared during that outage each one wakes, finds no holder, claims cleanly,
+		 * and runs. Two paid runs for one intent, which is the accumulation `REQ-WAIT-FOR` promises against.
+		 *
+		 * So the job that clears a target says so, and a sibling waking after it refuses. The marker lives for
+		 * `SATISFIED_MS` because that is the longest two jobs which cleared TOGETHER can wake apart: the
+		 * backoff ceiling. It is deliberately not longer -- a delivery arriving well after a wait completed is
+		 * a genuinely new intent, and coalescing it would be the dedup-window mistake this design already
+		 * refused once.
+		 */
+		async markSatisfied(jobId, { dedupId }) {
+			if (!dedupId) return;
+			try {
+				await redis.set(satisfiedKey(dedupId), jobId, "PX", SATISFIED_MS);
+			} catch {
+				// Fail open: the cost is the duplicate this marker exists to prevent, not a wrong refusal.
+			}
+		},
+
+		async satisfiedBy(dedupId) {
+			if (!dedupId) return null;
+			try {
+				return await redis.get(satisfiedKey(dedupId));
+			} catch {
+				return null;
+			}
+		},
+
+		/** Drop a hold: the job is running, refusing, or expiring. Only clears a lease this job actually owns. */
+		async release(jobId, { dedupId } = {}) {
+			try {
+				await redis.srem(HELD_SET, jobId);
+				await redis.del(jobKey(jobId));
+				if (dedupId) {
+					const holder = await redis.get(leaseKey(dedupId));
+					if (holder === jobId) await redis.del(leaseKey(dedupId));
+				}
+			} catch {
+				// Fail open: the TTLs are the backstop, and every field here is advisory.
+			}
+		},
+
+		/**
+		 * The two per-job counters the polled tier keeps: how many checks have run, and how many of them in a
+		 * ROW could not answer. Both live beside `since` so one hash carries the whole hold, and both come
+		 * back 0 when nothing was written -- an unrecorded hold must read as a fresh one rather than as a
+		 * job that has already exhausted its budget.
+		 */
+		async counters(jobId) {
+			try {
+				const h = await redis.hget(jobKey(jobId), "checks");
+				const f = await redis.hget(jobKey(jobId), "faults");
+				const t = await redis.hget(jobKey(jobId), "throttles");
+				return { checks: Number(h) || 0, faults: Number(f) || 0, throttles: Number(t) || 0 };
+			} catch {
+				return { checks: 0, faults: 0, throttles: 0 };
+			}
+		},
+
+		/**
+		 * Count a wake that was denied the check lease, or clear the count when one is granted.
+		 *
+		 * This is the only observable the capacity bound has. The lease caps how much wall-clock a worker
+		 * spends checking, which is what it is for -- but a cap that is being hit constantly means demand
+		 * exceeds it, and the plan's own economics say that arrives quietly: paid jobs starve behind checks
+		 * that spend nothing, and nothing in the product explains why. A consecutive-denial count is the
+		 * symptom an operator can actually see, so it is kept and its overflow is logged rather than absorbed.
+		 */
+		async noteThrottle(jobId, { denied }) {
+			try {
+				if (!denied) return void (await redis.hset(jobKey(jobId), "throttles", "0"));
+				const n = Number(await redis.hincrby(jobKey(jobId), "throttles", 1)) || 0;
+				await redis.pexpire(jobKey(jobId), counterTtlMs); // see noteCheck: a counter may CREATE this hash
+				return n;
+			} catch {
+				return 0;
+			}
+		},
+
+		/**
+		 * Record what a check answered. `checks` only ever grows; `faults` is a CONSECUTIVE count, so a check
+		 * that answered resets it -- a script that works after an outage has not accumulated a debt.
+		 */
+		async noteCheck(jobId, { fault }) {
+			try {
+				await redis.hincrby(jobKey(jobId), "checks", 1);
+				if (fault) await redis.hincrby(jobKey(jobId), "faults", 1);
+				else await redis.hset(jobKey(jobId), "faults", "0");
+				// A counter can CREATE this hash: on a job's first polled wake nothing has held yet, and a path
+				// that exits before `hold` (a lease denial, an unverified supersede) would otherwise leave a
+				// key with NO expiry at all. That breaks the invariant this whole keyspace rests on -- "every
+				// hash carries a TTL", which is why there is no index set to leak instead -- and the panel
+				// would render a held row nothing could ever remove.
+				await redis.pexpire(jobKey(jobId), counterTtlMs);
+			} catch {
+				// Fail open: a lost counter costs a longer wait, never a wrong verdict.
+			}
+		},
+
+		/** How long this job has been held, in ms, or null when nothing was recorded. */
+		async heldForMs(jobId) {
+			try {
+				const since = await redis.hget(jobKey(jobId), "since");
+				// ABSENT IS NOT ZERO, and conflating them is not a rounding error: `Number(null)` is 0, so a
+				// job with no recorded hold would read as held since the epoch and every bound measured from
+				// here -- the maximum hold above all -- would fire on its first check.
+				if (typeof since !== "string" || since.trim() === "") return null;
+				const ms = Number(since);
+				return Number.isFinite(ms) && ms > 0 ? Math.max(0, now() - ms) : null;
+			} catch {
+				return null;
+			}
+		},
+	};
+}