@edgehero/pi-dispatch 4.3.1 → 5.1.0

package/src/defer-reasons.mjs

@@ -0,0 +1,200 @@
+/**
+ * Why a job waits (issue #737): the closed vocabulary the worker writes into `defer:job:<id>` when it puts a job back
+ * on the delayed set, and the rules a reader holds each field to. A LEAF: it imports nothing but the BullMQ leaf below,
+ * so the admin console can import it (`@edgehero/pi-dispatch/defer-reasons`) without pulling the worker's dependencies
+ * into its bundle. The writer is `defer-write.mjs`; the reader is the admin's `readWaiting`.
+ *
+ * The record is the worker's own words for its own decision, written at the moment it defers, so a reader never has to
+ * guess a reason from a job's data (which it must not read: `INT-WAIT-PROFILES-CONTRACT`) or from its timing. A job
+ * with no record says so ("No reason was recorded"), and one whose record names a reason this console does not know
+ * says that; each is true, never a guess. Every value is judged here on the way in (`deferHsetArgs`) and on the way out
+ * (`parseDefer`), because the hash lives in a shared Valkey that anything with its URL can write.
+ */
+
+/** The hash's key prefix; the job's id follows it. */
+export const DEFER_KEY = "defer:job:";
+
+/** The defer record's key for a job id. */
+export const deferKey = (jobId) => `${DEFER_KEY}${jobId}`;
+
+/**
+ * The buckets a console groups waiting jobs by, in the order it shows them. The last four are the reader's own and no
+ * reason in `DEFER_WHYS` is in them: `paused` holds a job in a paused queue, `due` a delayed job whose due instant has
+ * passed with no worker taking it (judged from the delayed set and the read's clock, whatever its record says),
+ * `unknown` a job whose record names a reason this console does not know (it has the reason charset but is not in
+ * `DEFER_WHYS`: most often a newer worker's), and `none` a job with no record, or with one whose reason fails the
+ * charset (no reason at all). Each title is true of every job in its bucket, which is why a reason this console cannot
+ * explain has a bucket of its own rather than sharing `none`'s "No reason was recorded".
+ */
+export const DEFER_BUCKETS = Object.freeze(["slot", "run", "check", "retry", "pause", "ready", "paused", "due", "unknown", "none"]);
+
+/** Each bucket's words: a waitlist's title and its short form (a chip, a row's tag). */
+export const DEFER_BUCKET_WORDS = Object.freeze({
+	slot: Object.freeze({ title: "Waiting for a free slot", short: "Free slot" }),
+	run: Object.freeze({ title: "Waiting for another run to finish", short: "Another run" }),
+	check: Object.freeze({ title: "Waiting on a time or a check", short: "Time or check" }),
+	retry: Object.freeze({ title: "Retry backoff", short: "Retry backoff" }),
+	pause: Object.freeze({ title: "In a pause window", short: "Pause window" }),
+	ready: Object.freeze({ title: "Waiting for its host to get ready", short: "Host getting ready" }),
+	paused: Object.freeze({ title: "In a paused queue", short: "Queue paused" }),
+	due: Object.freeze({ title: "Due, and no worker has taken it", short: "Due, not taken" }),
+	unknown: Object.freeze({ title: "A reason this console does not know", short: "Unknown reason" }),
+	none: Object.freeze({ title: "No reason was recorded", short: "No reason recorded" }),
+});
+
+/**
+ * The `due` bucket's line, whose `{due}` is the job's due instant (the delayed set's, drawn as a `<time>`). Not a reason
+ * in `DEFER_WHYS`: no worker records it, since the job is overdue exactly because no worker has looked at it since.
+ */
+export const DEFER_DUE_SENTENCE = "Due since {due}; no worker has taken it";
+
+/**
+ * Every reason the worker can record: its bucket and the sentence a console shows, whose `{placeholders}` the
+ * console fills from the record's own fields (escaped; times drawn as `<time>`). A reason not in this table is
+ * written by nobody: `deferHsetArgs` throws on one (a programming error), and `parseDefer` reads one as unknown.
+ */
+export const DEFER_WHYS = Object.freeze({
+	canary: Object.freeze({ bucket: "run", sentence: "Waiting for the operator model canary to finish" }),
+	"pause-window": Object.freeze({ bucket: "pause", sentence: "In a pause window until {until}" }),
+	"size-never-fits": Object.freeze({ bucket: "slot", sentence: "Its size fits no slot on {host}; another host may take it" }),
+	"host-slot": Object.freeze({ bucket: "slot", sentence: "{host} has no free job slot" }),
+	"scope-folder": Object.freeze({ bucket: "slot", sentence: "Its folder is busy with another run" }),
+	"scope-repo": Object.freeze({ bucket: "slot", sentence: "Its repository's concurrent limit is full" }),
+	"scope-project": Object.freeze({ bucket: "slot", sentence: "Its project's concurrent limit is full" }),
+	endpoint: Object.freeze({ bucket: "slot", sentence: "Every slot of model endpoint {endpoint} is taken" }),
+	"subscription-quota": Object.freeze({ bucket: "check", sentence: "The subscription account is exhausted; waits for newer evidence, a known reset or an operator trial. Asks again at {until}" }),
+	"subscription-owner": Object.freeze({ bucket: "ready", sentence: "Waits for the host that owns its subscription connection" }),
+	"subscription-auth": Object.freeze({ bucket: "check", sentence: "Waits until the subscription credential can be renewed at {until}" }),
+	"model-cooldown": Object.freeze({ bucket: "check", sentence: "No reviewed model is available; a model is cooling down. Asks again at {until}" }),
+	"model-window": Object.freeze({ bucket: "check", sentence: "No reviewed model is available; a model's dollar window is full. Asks again at {until}" }),
+	"host-budget": Object.freeze({ bucket: "slot", sentence: "{host}'s budget has no room for it yet" }),
+	"bundle-profile": Object.freeze({ bucket: "slot", sentence: "Waits for a host that serves its secret bundle profile" }),
+	chained: Object.freeze({ bucket: "run", sentence: "Chained after {holder}; starts when its folder is free" }),
+	"recovery-wait": Object.freeze({ bucket: "run", sentence: "Recovers run {holder}, whose attempt is still running" }),
+	"wait-after": Object.freeze({ bucket: "check", sentence: "Held until {until}" }),
+	"wait-hold": Object.freeze({ bucket: "check", sentence: "Held on its wait check; asks again at {until}" }),
+	"wait-throttled": Object.freeze({ bucket: "check", sentence: "Its wait check waits for a free check slot" }),
+	"wait-supersede": Object.freeze({ bucket: "check", sentence: "Asks again whether the delivery holding its target still waits" }),
+	"wait-aborted": Object.freeze({ bucket: "check", sentence: "Its wait check was stopped by a restart or a cancel; asks again soon" }),
+	"wait-unclear": Object.freeze({ bucket: "check", sentence: "Its wait check answered in a way the worker cannot read; asks again at {until}" }),
+	"podman-restart": Object.freeze({ bucket: "ready", sentence: "Waits for rootful Podman's service on {host} to restart" }),
+	"netns-keeper": Object.freeze({ bucket: "ready", sentence: "Waits for the network keeper on {host} to settle" }),
+	"newer-release": Object.freeze({ bucket: "ready", sentence: "Queued or configured by a newer pi-dispatch: waits for this host's upgrade" }),
+	retry: Object.freeze({ bucket: "retry", sentence: "Attempt {attempt} of {attempts} at {until}" }),
+});
+
+/** The hash's fields, in the order a reader's HMGET asks for them. `n` is the writer's own nonce, never shown. */
+export const DEFER_FIELDS = Object.freeze(["why", "at", "until", "project", "host", "holder", "endpoint", "budgetWhy", "trigger", "attempt", "attempts", "n"]);
+
+/** A reason's charset: what `parseDefer` takes as a reason word at all (a word outside the table reads as unknown). */
+export const DEFER_WHY_RE = /^[a-z][a-z0-9-]{0,31}$/;
+/** The writer's nonce: 16 hex characters, so a compare-and-delete can tell its own record from another worker's. */
+export const DEFER_N_RE = /^[0-9a-f]{16}$/;
+/** A job id the record may name (`holder`). Pinned equal to the admin reader's `RUN_ID_RE` by a test. */
+export const DEFER_ID_RE = /^[A-Za-z0-9._:-]{1,200}$/;
+/** A project id. A copy of `project-id.mjs`'s `PROJECT_ID_RE`, pinned equal to it by a test (this module imports nothing). */
+export const DEFER_PROJECT_RE = /^[a-z0-9][a-z0-9-]{0,31}$/;
+/** A host's name. A copy of `worker-name.mjs`'s `WORKER_NAME_RE`, pinned equal to it by a test. */
+export const DEFER_HOST_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
+/** A model endpoint's id. A copy of `model-endpoints.mjs`'s `MODEL_ENDPOINT_ID_RE`, pinned equal to it by a test. */
+export const DEFER_ENDPOINT_RE = /^[a-z0-9-]{1,32}$/;
+
+/**
+ * The last instant a UTC date-time can be written with a four-digit year: 9999-12-31T23:59:59.999Z. Every instant this
+ * leaf passes on (`at`, `until`, a due time) is at most this, so a console can always draw it. NOT the latest instant a
+ * JavaScript Date holds (8.64e15 ms, in the year 275760): `toISOString` writes a year past 9999 as six digits with a
+ * sign (`+275760-09-13T00:00:00.000Z`), and a console that draws the first sixteen characters as `YYYY-MM-DD HH:MM`
+ * would show a date that is not the instant's.
+ */
+export const DEFER_TIME_MAX_MS = 253402300799999;
+
+/** How long a record outlives the job's due time, so a record is still there while the job is promoted and picked up. */
+export const DEFER_TTL_SLACK_MS = 60_000;
+/** A record's lifetime: until the job is due, plus the slack, and never less than the slack. */
+export const deferTtlMs = (untilMs, nowMs) => Math.max(DEFER_TTL_SLACK_MS, untilMs - nowMs + DEFER_TTL_SLACK_MS);
+
+/**
+ * The delayed set's score and a scheduler's next run are BullMQ's facts, so they live in `bullmq-internals.mjs` (issue
+ * #761) and are re-exported here, where they were first published: a reader of this subpath keeps every name.
+ */
+export { DELAYED_SCORE_SHIFT, dueOfScore, isSchedulerNext } from "./bullmq-internals.mjs";
+
+/** A decimal safe integer, from a number or its string form, or null. */
+function intOf(value) {
+	if (typeof value === "number") return Number.isSafeInteger(value) ? value : null;
+	if (typeof value !== "string" || !/^(?:0|[1-9][0-9]{0,15})$/.test(value)) return null;
+	const n = Number(value);
+	return Number.isSafeInteger(n) ? n : null;
+}
+
+/** An integer within [lo, hi], or null. */
+function intIn(value, lo, hi) {
+	const n = intOf(value);
+	return n !== null && n >= lo && n <= hi ? n : null;
+}
+
+/** A string that passes `re`, or null. */
+const textOf = (value, re) => (typeof value === "string" && re.test(value) ? value : null);
+
+/** Each field's rule: the value as the record holds it (a number or a string), or null when it fails. */
+const FIELD_RULES = Object.freeze({
+	why: (v) => (typeof v === "string" && Object.hasOwn(DEFER_WHYS, v) ? v : null),
+	at: (v) => intIn(v, 0, DEFER_TIME_MAX_MS),
+	until: (v) => intIn(v, 0, DEFER_TIME_MAX_MS),
+	project: (v) => textOf(v, DEFER_PROJECT_RE),
+	host: (v) => textOf(v, DEFER_HOST_RE),
+	holder: (v) => textOf(v, DEFER_ID_RE),
+	endpoint: (v) => textOf(v, DEFER_ENDPOINT_RE),
+	budgetWhy: (v) => textOf(v, DEFER_WHY_RE),
+	trigger: (v) => intIn(v, 0, 9999),
+	attempt: (v) => intIn(v, 1, 1000),
+	attempts: (v) => intIn(v, 1, 1000),
+	n: (v) => textOf(v, DEFER_N_RE),
+});
+
+/**
+ * The HSET arguments for a record, `[field, value, ...]`, each value judged by its field's rule and left out when it
+ * fails (a record with a missing field is still worth writing; one with a wrong field is not worth trusting). A
+ * reason not in `DEFER_WHYS` throws: every call site names its reason by a literal, so an unknown one is a bug.
+ */
+export function deferHsetArgs(rec) {
+	const why = rec?.why;
+	if (typeof why !== "string" || !Object.hasOwn(DEFER_WHYS, why)) throw new RangeError(`defer-reasons: unknown reason ${JSON.stringify(why)}`);
+	const args = [];
+	for (const field of DEFER_FIELDS) {
+		const value = FIELD_RULES[field](rec[field]);
+		if (value !== null) args.push(field, String(value));
+	}
+	return args;
+}
+
+/**
+ * A record from an HMGET of `DEFER_FIELDS`, in that order: null when there is no `why` (no record, or none worth
+ * reading); otherwise every field, each null when it is absent or fails its rule. A `why` with the reason charset but
+ * not in the table is read as unknown (`known` false, bucket `unknown`): a newer worker's reason an older console does
+ * not know. A `why` outside the charset is no reason at all (`why` null, bucket `none`). The nonce `n` is the writer's
+ * and is never returned.
+ */
+export function parseDefer(values) {
+	if (!Array.isArray(values)) return null;
+	const raw = values[0];
+	if (raw === null || raw === undefined || raw === "") return null;
+	const why = typeof raw === "string" && DEFER_WHY_RE.test(raw) ? raw : null;
+	const known = why !== null && Object.hasOwn(DEFER_WHYS, why);
+	const at = (i) => values[i];
+	return {
+		why,
+		known,
+		bucket: known ? DEFER_WHYS[why].bucket : why !== null ? "unknown" : "none",
+		at: FIELD_RULES.at(at(1)),
+		until: FIELD_RULES.until(at(2)),
+		project: FIELD_RULES.project(at(3)),
+		host: FIELD_RULES.host(at(4)),
+		holder: FIELD_RULES.holder(at(5)),
+		endpoint: FIELD_RULES.endpoint(at(6)),
+		budgetWhy: FIELD_RULES.budgetWhy(at(7)),
+		trigger: FIELD_RULES.trigger(at(8)),
+		attempt: FIELD_RULES.attempt(at(9)),
+		attempts: FIELD_RULES.attempts(at(10)),
+	};
+}

package/src/defer-write.mjs

@@ -0,0 +1,208 @@
+/**
+ * The worker's half of "why does this job wait" (issue #737): every gate that puts a job back on the delayed set
+ * writes its reason into `defer:job:<id>` (`defer-reasons.mjs` holds the vocabulary and the field rules) just before
+ * BullMQ's move, through ONE function, `deferTo`, so no deferral can move a job without saying why. A retry after an
+ * infrastructure failure, which BullMQ delays by itself after the processor throws, is noted by `noteRetry` before
+ * the throw. Worker-only, and free of bullmq itself: the processor hands it BullMQ's `DelayedError` class (index.mjs
+ * imports bullmq; `valkey-clients.test.mjs` keeps every other module from naming that library), so the throw is the
+ * very class the processor's own catch tells a deferral apart by. The admin console never loads it (it reads the leaf).
+ *
+ * THE RECORD IS ADVISORY, so it FAILS OPEN and is BOUNDED. A deferral never fails or changes for its reason, and waits
+ * for it at most `DEFER_WRITE_TIMEOUT_MS` (2 s): a write that fails or takes longer is logged once per episode and the
+ * job is moved all the same, and a logger that throws is swallowed rather than rejecting the deferral. The bound is not
+ * decoration: the worker's client queues commands during an outage rather than rejecting them
+ * (`maxRetriesPerRequest: null`, the host registry's reason), so an unbounded write would hold a gate's deferral for
+ * the whole outage. The move itself is BullMQ's, exactly as it was before this file.
+ *
+ * WRITE, THEN MOVE. A reader starts from the delayed set and looks up the records of the ids it finds there, so a
+ * record written before the move is in place the moment the job is delayed, and one for a job that is still active
+ * is never read. The write is one MULTI (`DEL`, `HSET`, `PEXPIRE`), so a reader never sees this deferral's fields
+ * mixed with an earlier one's, and every record carries a TTL (`deferTtlMs`: the due time plus a minute's slack,
+ * so the record outlives the promotion and the pickup that follows it).
+ *
+ * A MOVE THAT REJECTS (a lost lock, Valkey gone) leaves a record for a job that is not delayed, so its own record is
+ * removed by a compare-and-delete on the writer's nonce (`n`): a plain DEL could remove the record another worker
+ * wrote for a later deferral of the same job id. The rejection is rethrown unchanged, so it reaches BullMQ's
+ * failed-attempt handling exactly as a raw `moveToDelayed` rejection always has.
+ *
+ * A TIMED-OUT WRITE MAY STILL LAND. The bound stops the waiting, not the command: a queued MULTI runs when the
+ * connection comes back. That is why a timed-out write still returns its nonce: the compare-and-delete is sent on
+ * the same connection after it, so it runs after the MULTI if that lands at all. A record that lands for a job that
+ * then ran is not read while the job is not delayed, and expires with its TTL.
+ *
+ * WHAT A RECORD CANNOT PROMISE. A write that failed leaves the previous deferral's record, until it expires, beside a
+ * job now delayed for another reason; and a job BullMQ delays by itself (a move's rejection retried, a backoff
+ * `retryDelayOf` cannot know) gets no fresh record. Both show as a record whose `until` is not the job's due time in
+ * the delayed set, which is how a reader tells a record that describes the wait from one that does not.
+ */
+import { randomBytes } from "node:crypto";
+import { DEFER_N_RE, DEFER_WHY_RE, deferHsetArgs, deferKey, deferTtlMs } from "./defer-reasons.mjs";
+
+/** How long a gate waits for its reason's write, and a failed move for its compare-and-delete, before going on. */
+export const DEFER_WRITE_TIMEOUT_MS = 2_000;
+
+/** Delete the record only while it still carries this writer's nonce: never another worker's later record. */
+export const DEFER_CAD_LUA = "if redis.call('HGET', KEYS[1], 'n') == ARGV[1] then return redis.call('DEL', KEYS[1]) end return 0";
+
+/**
+ * The delay BullMQ will put on this job's retry after the current attempt throws, in ms, or null when it will not
+ * retry it on a delay this function can know. Mirrors the pinned BullMQ's `shouldRetryJob` and its builtin backoff
+ * strategies (job.js, backoffs.js at 5.80.4): it retries while `attemptsMade + 1 < opts.attempts` and the job is not
+ * discarded; a number is a `fixed` backoff; `fixed` waits `delay`, `exponential` waits `round(2^attemptsMade * delay)`
+ * (BullMQ calls the strategy with `attemptsMade + 1` and raises 2 to one less).
+ *
+ * Null for everything else, never a guess: a jittered backoff (its delay is random, so no instant could be told), a
+ * custom strategy (it is the queue's code), and a delay at or below zero (BullMQ then retries at once, through the
+ * wait list rather than the delayed set).
+ */
+export function retryDelayOf(job) {
+	const attempts = job?.opts?.attempts;
+	const made = job?.attemptsMade;
+	if (!Number.isInteger(attempts) || !Number.isInteger(made) || made < 0) return null;
+	if (!(made + 1 < attempts) || job.discarded === true) return null;
+	const raw = job.opts.backoff;
+	const backoff = Number.isFinite(raw) ? { type: "fixed", delay: raw } : raw;
+	if (backoff === null || typeof backoff !== "object") return null;
+	// BullMQ's own test (`jitter > 0`), so a value it ignores is ignored here too.
+	if (backoff.jitter > 0) return null;
+	const delay = backoff.delay;
+	if (!Number.isFinite(delay) || delay <= 0) return null;
+	if (backoff.type === "fixed") return delay;
+	if (backoff.type === "exponential") return Math.round(2 ** made * delay);
+	return null;
+}
+
+/** Settle `promise` or give up after `ms`, never rejecting: `{ value }`, `{ error }` or `{ timedOut: true }`. */
+function settleWithin(promise, ms) {
+	return new Promise((resolve) => {
+		const timer = setTimeout(() => resolve({ timedOut: true }), ms);
+		Promise.resolve(promise).then(
+			(value) => {
+				clearTimeout(timer);
+				resolve({ value });
+			},
+			(error) => {
+				clearTimeout(timer);
+				resolve({ error });
+			},
+		);
+	});
+}
+
+/** A failure's name for the log: a fixed word or an errno-like code, never an error's message (it can quote a URL). */
+function failureOf(outcome) {
+	if (outcome.timedOut) return "timeout";
+	if (outcome.threw) return "threw";
+	if (outcome.error) return typeof outcome.error.code === "string" && /^[A-Z][A-Z0-9_]{0,31}$/.test(outcome.error.code) ? outcome.error.code : "rejected";
+	return "error-reply";
+}
+
+/**
+ * Build the writer for one processor. `redis` is the processor's client; `DelayedError` is BullMQ's class (required:
+ * a stand-in would pass BullMQ's name test and fail the processor's `instanceof`, so a deferral would read as a
+ * failure there); `hostName` is this worker's declared name, recorded as the record's host when it passes the host
+ * rule; `now` is the processor's clock, the one every gate's due time is computed from.
+ *
+ * Returns:
+ *   - `writeReason(job, { why, untilMs, atMs, project, holder, endpoint, budgetWhy, attempt, attempts })`: the record's
+ *     write, bounded. `atMs` is the record's `at`, by default the clock when the write is built; a caller that measured
+ *     `untilMs` from its own reading passes that reading, so the two are exactly the delay apart. Resolves `{ n }` once
+ *     the MULTI was sent (it landed, or may still land: see the header) and null when nothing was sent: a client with
+ *     no `multi` (a bare test wiring), a job with no id, or a write that could not be built. Never rejects. `why` must
+ *     be a reason of `DEFER_WHYS`; any other field that fails its rule is left out of the record rather than failing it.
+ *   - `deferTo(job, token, untilMs, why, extra)`: the one deferral. Writes the reason, then `job.moveToDelayed(untilMs,
+ *     token)`, then throws BullMQ's `DelayedError`, the throw by which BullMQ learns the job was deferred. NEVER
+ *     returns. A rejected move removes this write's record and is rethrown unchanged.
+ *   - `noteRetry(job, { project })`: before an infrastructure retry's throw, the `retry` record, when BullMQ will
+ *     delay the retry by a known amount (`retryDelayOf`). Never throws.
+ *
+ * The log says `defer_reason_unwritten { why, reason }` on the first failed write and nothing more until a write
+ * succeeds again, which says `defer_reason_restored { why }` once: an outage is one episode, not a line per wake.
+ */
+export function makeDeferTo({ redis, DelayedError, now = () => Date.now(), hostName = "", log = () => {}, nonce = () => randomBytes(8).toString("hex"), timeoutMs = DEFER_WRITE_TIMEOUT_MS } = {}) {
+	if (typeof DelayedError !== "function") throw new TypeError("makeDeferTo needs BullMQ's DelayedError class, the one the processor's catch recognises a deferral by");
+	let failing = false;
+	const said = (why) => (typeof why === "string" && DEFER_WHY_RE.test(why) ? why : null);
+	// Every line goes through `say`, so a logger that throws is swallowed here: the log describes a deferral, and a line
+	// that could not be written must not turn the deferral into a rejection before its move (the job left active, BullMQ
+	// counting a failed attempt). The episode's state is set before the line, so it holds even when the line is lost.
+	const say = (event, fields) => {
+		try {
+			log(event, fields);
+		} catch {
+			// The line is lost, the deferral goes on.
+		}
+	};
+	const failed = (why, outcome) => {
+		if (failing) return;
+		failing = true;
+		say("defer_reason_unwritten", { why: said(why), reason: failureOf(outcome) });
+	};
+	const succeeded = (why) => {
+		if (!failing) return;
+		failing = false;
+		say("defer_reason_restored", { why: said(why) });
+	};
+
+	const writeReason = async (job, { why, untilMs, atMs, project = null, holder, endpoint, budgetWhy, attempt, attempts } = {}) => {
+		if (typeof redis?.multi !== "function") return null;
+		if (typeof job?.id !== "string" || job.id === "") return null;
+		let sent;
+		let n = null;
+		try {
+			const at = Number.isFinite(atMs) ? Math.floor(atMs) : now();
+			const drawn = nonce();
+			n = typeof drawn === "string" && DEFER_N_RE.test(drawn) ? drawn : null;
+			// Whole milliseconds: the record's rule takes integers, and a gate's arithmetic (a jittered delay, a keeper's
+			// wait) need not produce one.
+			const until = Number.isFinite(untilMs) ? Math.floor(untilMs) : null;
+			const args = deferHsetArgs({ why, at, until, project, host: hostName, holder, endpoint, budgetWhy, trigger: job.data?.trigger?.matched?.index, attempt, attempts, n });
+			const key = deferKey(job.id);
+			const ttl = Math.ceil(deferTtlMs(until ?? at, at));
+			sent = redis.multi().del(key).hset(key, ...args).pexpire(key, ttl).exec();
+		} catch (error) {
+			// Nothing was sent: a client whose transaction could not be built, or a reason outside the table (a defect at
+			// the call site, which `defer-write.test.mjs` pins by literal).
+			failed(why, { threw: true, error });
+			return null;
+		}
+		const outcome = await settleWithin(sent, timeoutMs);
+		const replies = outcome.value;
+		const errorReply = Array.isArray(replies) ? replies.some((reply) => Array.isArray(reply) && reply[0]) : replies !== undefined;
+		if (outcome.timedOut || outcome.error || errorReply) failed(why, outcome);
+		else succeeded(why);
+		return { n };
+	};
+
+	const deferTo = async (job, token, untilMs, why, extra = {}) => {
+		const written = await writeReason(job, { ...extra, why, untilMs });
+		try {
+			await job.moveToDelayed(untilMs, token);
+		} catch (error) {
+			if (written?.n && typeof redis?.eval === "function") {
+				let removal;
+				try {
+					removal = redis.eval(DEFER_CAD_LUA, 1, deferKey(job.id), written.n);
+				} catch {
+					removal = null;
+				}
+				await settleWithin(removal, timeoutMs);
+			}
+			throw error;
+		}
+		throw new DelayedError();
+	};
+
+	const noteRetry = async (job, { project = null } = {}) => {
+		try {
+			const delay = retryDelayOf(job);
+			if (delay === null) return;
+			const at = now();
+			await writeReason(job, { why: "retry", untilMs: at + delay, atMs: at, project, attempt: job.attemptsMade + 2, attempts: job.opts.attempts });
+		} catch {
+			// Never in the way of the throw it precedes: the retry happens whether or not its reason was written.
+		}
+	};
+
+	return { writeReason, deferTo, noteRetry };
+}

package/src/deployment-venue.mjs

@@ -4,11 +4,14 @@
  * different venue than the service they described. The deployment's `.env` is read with `readStackKeys`, exactly as
  * `service install` reads it, and a key this shell sets differently stops the command, since the service runs the file.
  */
-import { parseBackendList } from "./backends.mjs";
+import { parseBackendList, venuesOf } from "./backends.mjs";
 import { egressArmed, egressProxyName } from "./egress.mjs";
 import { envValueShown } from "./env-file.mjs";
 import { STACK_KEYS, VALKEY_SHARED_KEY, readStackKeys, readValkeyKeys } from "./podman-stack.mjs";
 
+/** The venue keys only the openshell venues read (issue #707). */
+const OPENSHELL_KEYS = Object.freeze(["PI_OPENSHELL_ENGINE", "PI_OPENSHELL_GATEWAY"]);
+
 /** How each command says what a shell/file disagreement would make it do, the clause before "while the service ...". */
 const COMMAND_SAYS = Object.freeze({
 	up: "up would drive the shell's venue",
@@ -39,10 +42,16 @@ export function deploymentVenueEnv({ env, fs, envPath, platform, command = "up",
 			text = fs.readFileSync(envPath);
 		} catch (err) {
 			// Said rather than silent (round 2 nit): the venue then comes from this shell alone.
-			notes.push(`${envPath} could not be read (${err?.message}), so PI_BACKENDS, PI_EGRESS and PI_EGRESS_PROXY come from this shell alone`);
+			notes.push(`${envPath} could not be read (${err?.message}), so the venue keys (${STACK_KEYS.join(", ")}) come from this shell alone`);
 		}
 		if (text !== null) {
-			const read = readStackKeys(text, { loader, path: envPath });
+			// The openshell keys are refused only where an openshell venue is blessed (review round 2 of #715): the file's
+			// PI_BACKENDS says so to the reader itself, this shell's is said here.
+			let shellOpenshell = false;
+			try {
+				shellOpenshell = typeof env.PI_BACKENDS === "string" && venuesOf({ PI_BACKENDS: env.PI_BACKENDS }).openshellUsed;
+			} catch {}
+			const read = readStackKeys(text, { loader, path: envPath, openshellBlessed: shellOpenshell });
 			if (read.error && linux) return { error: read.error };
 			if (read.error) notes.push(`${read.error}; off Linux the podman venue refuses this host anyway, so this pass reads the venue from this shell`);
 			else keys = read.keys;
@@ -57,7 +66,7 @@ export function deploymentVenueEnv({ env, fs, envPath, platform, command = "up",
 		if (typeof env[key] === "string") {
 			// Compared as the worker READS them (round 3, D4), not as strings: ` podman` and `podman,podman` are the list
 			// `podman`, and refusing them sent an operator to reconcile two values that already agree.
-			if (venueKeyMeaning(key, env[key]) !== venueKeyMeaning(key, keys[key])) conflicts.push(`${key} is ${quotedShown(env[key])} in this shell and ${quotedShown(keys[key])} in ${envPath}`);
+			if (venueKeyMeaning(key, env[key]) !== venueKeyMeaning(key, keys[key])) conflicts.push({ key, text: `${key} is ${quotedShown(env[key])} in this shell and ${quotedShown(keys[key])} in ${envPath}` });
 			continue;
 		}
 		merged[key] = keys[key];
@@ -66,8 +75,12 @@ export function deploymentVenueEnv({ env, fs, envPath, platform, command = "up",
 	// A KNOWN disagreement is a refusal (round 2, E5), not a warning: this pass would stand up one venue's stack and the
 	// service installed next would run the other. The file is what the service runs, so it is the one to change unless
 	// the shell's value was a one-off.
-	if (conflicts.length > 0) {
-		return { error: `${conflicts.join("; ")}. ${COMMAND_SAYS[command] ?? COMMAND_SAYS.up} while the service runs the file's. Make them agree: change ${envPath} (what the service reads), or unset the key in this shell` };
+	// The openshell venues' two keys disagree only where one of those venues is blessed (review round 1 of #715): elsewhere
+	// neither the worker nor any command reads them, so a stray value in one place is no reason to stop.
+	const openshellBlessed = venuesOf(merged).openshellUsed;
+	const said = conflicts.filter((c) => openshellBlessed || !OPENSHELL_KEYS.includes(c.key)).map((c) => c.text);
+	if (said.length > 0) {
+		return { error: `${said.join("; ")}. ${COMMAND_SAYS[command] ?? COMMAND_SAYS.up} while the service runs the file's. Make them agree: change ${envPath} (what the service reads), or unset the key in this shell` };
 	}
 	return { env: merged, fromFile, notes };
 }
@@ -146,6 +159,10 @@ function venueKeyMeaning(key, value) {
 	try {
 		if (key === "PI_BACKENDS") return `list:${parseBackendList(value).join(",")}`;
 		if (key === "PI_EGRESS") return `egress:${egressArmed({ PI_EGRESS: value }) ? "on" : "off"}`;
+		// Issue #707: the openshell venues' two keys, as `openshellFrom` reads them: an empty engine is docker, an empty
+		// gateway the CLI's active one. Not trimmed, as the worker trims neither.
+		if (key === "PI_OPENSHELL_ENGINE") return `engine:${value === "" ? "docker" : value}`;
+		if (key === "PI_OPENSHELL_GATEWAY") return `gateway:${value}`;
 		return `proxy:${egressProxyName({ PI_EGRESS_PROXY: value })}`;
 	} catch {
 		return `raw:${value}`;

package/src/docker-run.mjs

@@ -22,6 +22,7 @@
 export { containerSpec, CONTAINER_GLOBAL_PI_DIR, CONTAINER_SESSION_DIR, CONTAINER_SESSION_FILE } from "./container-spec.mjs";
 import { isAbsolute, relative } from "node:path";
 import { SIZE_LABEL_CPU, SIZE_LABEL_MEM, assertCidFile, assertJobUser, containerSpec } from "./container-spec.mjs";
+import { LAYOUTS } from "./job-layout.mjs";
 import { CGROUP_PARENT, CPU_SHARES_MAX, CPU_SHARES_MIN } from "./job-size.mjs";
 
 /** The fixed isolation flags. Not configurable -- these ARE the boundary. */
@@ -165,6 +166,10 @@ export const DOCKER_EXTRA_FORBIDDEN = [
 	// The job user is a spec field (issue #341). Here so a repeat cannot override it with uid 0.
 	"--user",
 	"-u",
+	// Issue #683: the builder pins the working directory (see `--workdir=` in `argsFromSpec`). A repeat would start the
+	// job outside its workspace, where a relative path the agent writes lands in the image's own tree.
+	"--workdir",
+	"-w",
 ];
 
 /**
@@ -331,6 +336,12 @@ function argsFromSpec(spec, { userns }) {
 		args.push("-v", `${m.host}:${m.container}${options.length > 0 ? `:${options.join(",")}` : ""}`);
 	}
 
+	// Issue #683: the job starts in its workspace mount because the ARGV says so, not because the image's WORKDIR
+	// happens to match. The shipped image's WORKDIR is now /sandbox (a copying venue's writable tree, image/Dockerfile),
+	// so without this a binding venue's job would start in an empty /sandbox and every relative path the agent writes
+	// would land outside the repository it was asked to change. A binding venue always uses the `mounts` layout. An
+	// older image, whose WORKDIR was /workspace, starts exactly where it did.
+	args.push(`--workdir=${LAYOUTS.mounts.workspace}`);
 	args.push(spec.image);
 	return args;
 }

package/src/doctor-bundles.mjs

@@ -0,0 +1,130 @@
+/**
+ * doctor's project secret bundle lines (issue #704, REQ-TRIGGER-SECRETS, DES-PROJECT-SECRET-BUNDLES).
+ *
+ * Its own module, the doctor-openshell.mjs precedent: doctor.mjs is a ten-thousand-line file, and these lines read
+ * three files and (with `--secrets`) spawn resolvers, so a seam per input keeps them testable without a host.
+ *
+ * WHAT IS CHECKED, and why each is a failure rather than a warning:
+ *   - a bundle naming a profile this host does not declare (or declares twice, or declares in the panel outside
+ *     PI_SECRET_RESOLVER_ROOTS): every job of that project refuses pre-spend on this host, deliberately;
+ *   - a bundle naming a project projects.json does not hold: the binding can never apply, and the operator reads it
+ *     as granting an environment. That is the silent no-op this project refuses;
+ *   - bundle profiles declared with no triggers file to read: no job on this host can get a bundle, while the host
+ *     reads as serving them.
+ * Two bundles naming one project is refused by the triggers file's own validator, so it surfaces as that file's
+ * parse failure, which doctor already prints, rather than as a second line here.
+ *
+ * And a WARNING per bundle whose project holds a folder: a local job's /workspace IS the operator's folder, so an
+ * agent that writes a credential to `.env` there writes the whole bundle's into a real repository. The run.secrets
+ * disclosure, raised to a warning because a bundle is a whole environment, not one field.
+ *
+ * `--secrets` (`secrets: true`) resolves each bundle ON THIS HOST and prints its variable NAMES and their count, never
+ * a value. It is opt-in because it spawns every bundle resolver, which reads the manager (and counts against its
+ * rate limit) once per bundle.
+ */
+
+import { FORGE_KINDS } from "./forges.mjs";
+import { mergeSecretProfiles, parseSecretBundleProfiles, servableSecretProfiles } from "./secret-profiles.mjs";
+
+/** Whether a projects.json member is a FOLDER (a local job's scope), not a forge-qualified repository. */
+function isFolderMember(member) {
+	return typeof member === "string" && !FORGE_KINDS.some((kind) => member.startsWith(`${kind}:`));
+}
+
+/** `PI_SECRET_BUNDLES`, never throwing: `{ bundles }` or `{ error }` (the worker refuses to boot on the latter). */
+export function parseSecretBundleProfilesSafe(raw) {
+	try {
+		return { bundles: parseSecretBundleProfiles(raw), error: null };
+	} catch (err) {
+		return { bundles: {}, error: err?.message ?? "unparseable" };
+	}
+}
+
+/**
+ * The checks. Inputs, every one already read by the caller:
+ *   - `bundles`: the triggers file's validated `secretBundles` (`{}` when none);
+ *   - `triggersPresent`: whether the worker has a triggers file to read them from;
+ *   - `projects` / `projectsError`: projects.json as the worker loads it, or its load failure;
+ *   - `envRaw`: `PI_SECRET_BUNDLES` as set; `overlayBundles`: the overlay's `secretBundles`; `roots`: PI_SECRET_RESOLVER_ROOTS;
+ *   - `realExecutablePath`: the resolver's own probe (secrets.mjs `realExecutable`), or null when pi did not load;
+ *   - `resolveBundle`: `(binding) => resolve result`, only with `--secrets`.
+ */
+export async function secretBundleChecks({ bundles = {}, triggersPresent = true, projects = [], projectsError = null, envRaw, overlayBundles = {}, roots = [], realExecutablePath = null, resolveBundle = null } = {}) {
+	const checks = [];
+	const { bundles: envBundles, error } = parseSecretBundleProfilesSafe(envRaw);
+	if (error) {
+		checks.push({ ok: false, label: "PI_SECRET_BUNDLES does not parse", fix: `${error} -- the worker refuses to boot until this is fixed` });
+		return checks;
+	}
+	const ids = Object.keys(bundles);
+	const declaredNames = [...new Set([...Object.keys(envBundles), ...Object.keys(overlayBundles)])].sort();
+	if (ids.length === 0 && declaredNames.length === 0) return checks;
+
+	// The overlay half counts only where the worker would use it: an executable inside the roots (servableSecretProfiles
+	// asks the resolver's own question). Without the probe (pi did not load) an overlay profile counts as written.
+	const overlayPaths = Object.fromEntries(Object.entries(overlayBundles).map(([name, entry]) => [name, entry?.path]));
+	const usableOverlay = realExecutablePath === null ? overlayPaths : servableSecretProfiles({ envProfiles: {}, overlayProfiles: overlayPaths, roots, realExecutablePath }).profiles;
+	const ambiguous = mergeSecretProfiles(envBundles, overlayBundles).ambiguous !== undefined;
+	const ambiguousNames = Object.keys(overlayBundles).filter((name) => Object.hasOwn(envBundles, name));
+
+	if (declaredNames.length > 0) {
+		checks.push({ ok: true, label: `Secret bundle profiles declared: ${declaredNames.map((n) => `${n} (${(Object.hasOwn(envBundles, n) ? envBundles[n] : overlayBundles[n])?.format ?? "?"})`).join(", ")}` });
+		// The rate figure, stated once: what a manager's rate limit counts is job starts.
+		checks.push({ ok: true, label: "A project with a secret bundle costs one resolver call per job start (never one per variable), so a secrets manager's rate limit counts job starts: size it for your busiest hour of jobs across every bundled project" });
+		if (!triggersPresent) {
+			checks.push({ ok: false, label: "secret bundle profiles are declared, but the worker has no triggers file to read secretBundles from", fix: "set PI_TRIGGERS_FILE (or put triggers.json in the worker's folder) on this host -- the worker reads which project holds a bundle from its own copy of that file, so without it no job on this host gets one" });
+		}
+	}
+	if (ambiguous) {
+		checks.push({ ok: false, label: `${ambiguousNames.length} secret bundle profile name(s) are declared in both PI_SECRET_BUNDLES and the settings overlay: ${ambiguousNames.join(", ")}`, fix: "remove each name from one of the two (neither source wins a collision) -- until then every bundled project's jobs refuse as secret-bundle-profile-ambiguous" });
+	}
+
+	const projectIds = new Set((projects ?? []).map((p) => p?.id));
+	for (const id of ids) {
+		const b = bundles[id];
+		const fromEnv = Object.hasOwn(envBundles, b.profile);
+		const fromOverlay = Object.hasOwn(overlayBundles, b.profile);
+		const format = fromEnv ? envBundles[b.profile].format : fromOverlay ? overlayBundles[b.profile]?.format : null;
+		checks.push({ ok: true, label: `secret bundle ${id}: profile ${b.profile}${format ? ` (${format})` : ""}, project ${b.project}${b.forkReviews === true ? ", fork reviews included" : ""}` });
+		if (!fromEnv && !fromOverlay) {
+			checks.push({ ok: false, label: `secret bundle ${id} names profile ${b.profile}, which this host does not declare`, fix: "declare it in PI_SECRET_BUNDLES as name:/absolute/path:format (format dotenv or json), on EVERY host that takes this project's jobs -- until then they refuse pre-spend as secret-bundle-profile-unknown" });
+		} else if (!fromEnv && !Object.hasOwn(usableOverlay, b.profile)) {
+			checks.push({ ok: false, label: `secret bundle ${id} names profile ${b.profile}, declared in the settings overlay but not usable`, fix: roots.length === 0 ? "PI_SECRET_RESOLVER_ROOTS is unset, so the worker uses no overlay-declared resolver: set it, or declare the profile in PI_SECRET_BUNDLES" : "the resolver must be an executable file whose real path sits inside PI_SECRET_RESOLVER_ROOTS" });
+		}
+		if (projectsError !== null) {
+			checks.push({ ok: false, label: `secret bundle ${id} names project ${b.project}, and projects.json does not load`, fix: "fix projects.json (its own line above says why) -- a bundle applies to a project's jobs only once the worker knows the project" });
+		} else if (!projectIds.has(b.project)) {
+			checks.push({ ok: false, label: `secret bundle ${id} names project ${b.project}, which projects.json does not hold`, fix: "add the project to PI_PROJECTS_FILE, or correct the bundle's project -- until then the bundle reaches no job, while the file reads as granting it" });
+		} else {
+			const project = projects.find((p) => p?.id === b.project);
+			const folders = (project?.members ?? []).filter(isFolderMember);
+			if (folders.length > 0) {
+				checks.push({ ok: true, warn: true, label: `secret bundle ${id}'s project ${b.project} holds ${folders.length} folder(s), whose local jobs run IN the operator's own folder with the whole bundle in their environment`, fix: "a value the agent writes to .env, .netrc or .git-credentials there lands in your real repository (and in a retained sandbox), and nothing scans for that: keep those folders out of anything you push, or set run.secretBundle: false on their triggers" });
+			}
+		}
+	}
+
+	if (typeof resolveBundle === "function") {
+		for (const id of ids) {
+			const b = bundles[id];
+			let read;
+			try {
+				read = await resolveBundle({ id, ...b });
+			} catch {
+				read = { unreachable: true, failure: "spawn" };
+			}
+			if (read?.ok === true) {
+				const names = Object.keys(read.vars).sort();
+				checks.push({ ok: true, label: `secret bundle ${id} resolved on this host: ${names.length} variable(s): ${names.join(", ")}` });
+			} else if (read?.reserved !== undefined) {
+				checks.push({ ok: false, label: `secret bundle ${id} carries ${read.reserved}, a name this deployment reserves`, fix: "rename it in the secrets manager -- every job of the project refuses pre-spend as secret-name-reserved until you do" });
+			} else if (read?.profileUnknown !== undefined || read?.ambiguous !== undefined) {
+				// Already its own line above.
+			} else {
+				const what = read?.unreachable ? "could not reach its manager (exit 1, a timeout or a spawn fault: retried at job time)" : `was refused (${read?.failure ?? "unknown"}${read?.name ? `, variable ${read.name}` : ""}${read?.line ? `, line ${read.line}` : ""})`;
+				checks.push({ ok: false, label: `secret bundle ${id}'s resolver ${what}`, fix: "run the profile's resolver by hand with the bundle's ref as its argument -- doctor prints no byte of what it printed" });
+			}
+		}
+	}
+	return checks;
+}