@edgehero/pi-dispatch 4.3.0 → 5.0.0

package/src/container-spec.mjs

@@ -48,6 +48,13 @@ export const CONTAINER_GLOBAL_PI_DIR = "/opt/pi-global";
 export const CONTAINER_SESSION_DIR = "/session";
 export const CONTAINER_SESSION_FILE = `${CONTAINER_SESSION_DIR}/current.jsonl`;
 
+/**
+ * Where a steer-armed job's inbox lands (issue #646, INT-STEER-INBOX-CONTRACT): the runner (image/runner/src/inbox.mjs
+ * `INBOX_DIR`) reads it, the worker's relay writes the host side. A constant path, like `/session`, so nothing about
+ * the host's layout is legible inside the job.
+ */
+export const CONTAINER_INBOX_DIR = "/inbox";
+
 /**
  * The job image's home, and the uid its `USER` directive runs as (issue #341, `DES-JOB-USER-INFERRED-READ-BACK-ON-
  * REQUEST`). A job run under `--user` needs `HOME=CONTAINER_HOME` beside it, because such a uid has no passwd entry in
@@ -132,6 +139,8 @@ export function assertUserns(userns) {
  * @param sessionDir host path to this job's OWN copy of its session transcript (REQ-RESUMABLE-SESSION);
  *                   mounted /session:rw. Per-job, like jobDir -- never the shared store.
  * @param globalPiDir host path to the operator's global pi overlay (REQ-GLOBAL-PI-OVERLAY); mounted /opt/pi-global:ro
+ * @param steerInboxDir host path to a steer-armed job's inbox (issue #646), `<jobDir>.inbox`; mounted /inbox:ro. Absent
+ *                   for every job that is not armed, so its argv is byte-identical to one built before the feature.
  * @param name       container name (for `docker stop` at the timeout)
  * @param size       the job's size, `{ memMiB, cpuCenti }` (issue #596, `job-size.mjs`); the built-in 4g and 2 when
  *                   absent. It becomes `memory`, `memorySwap` (equal: no swap beyond memory), `cpuShares` and `shmSize`
@@ -169,6 +178,7 @@ export function containerSpec({
 	outboxDir,
 	sessionDir,
 	globalPiDir,
+	steerInboxDir,
 	name,
 	size = DEFAULT_JOB_SIZE,
 	hostCpus = null,
@@ -181,6 +191,7 @@ export function containerSpec({
 	extraFlags = [],
 	relabel = false,
 	workspaceOwned = false,
+	workspaceReadOnly = false,
 }) {
 	if (!image) throw new Error("docker run: image is required");
 	assertJobUser(user);
@@ -194,6 +205,7 @@ export function containerSpec({
 	// Booleans, strictly: a truthy string from a caller that forwarded an option bag must not re-own host directories.
 	if (typeof relabel !== "boolean") throw new Error(`docker run: relabel must be a boolean; got ${typeof relabel}`);
 	if (typeof workspaceOwned !== "boolean") throw new Error(`docker run: workspaceOwned must be a boolean; got ${typeof workspaceOwned}`);
+	if (typeof workspaceReadOnly !== "boolean") throw new Error("docker run: workspaceReadOnly must be a boolean");
 
 	// Issue #355. A SPREAD rather than a `relabel: null` field, so a spec built without it has mount objects byte-identical
 	// to every one built before this feature existed. PRIVATE, never shared: each of these directories is made for this
@@ -208,7 +220,7 @@ export function containerSpec({
 	// second container mounting it afterwards is denied, and the label an operator chose is gone. So only a workspace the
 	// worker made is relabelled; an operator's folder needs the `semanage fcontext` fix doctor names, and the runner
 	// refuses one it cannot read before any spend.
-	mounts.push({ host: workspace, container: "/workspace", readOnly: false, ...(workspaceOwned ? own : {}) });
+	mounts.push({ host: workspace, container: "/workspace", readOnly: workspaceReadOnly, ...(workspaceOwned ? own : {}) });
 	// Local jobs get a writable /outbox host bind, the same host-bind mechanism as /workspace
 	// (DES-WORKER-ON-HOST). github jobs pass no outboxDir, so the request channel does not exist for
 	// them -- an untrusted issue author cannot chain (INT-OUTBOX-CONTRACT).
@@ -223,6 +235,13 @@ export function containerSpec({
 	// resolved, so an unarmed job's argv is byte-identical to one built before this feature existed.
 	if (sessionDir) mounts.push({ host: sessionDir, container: CONTAINER_SESSION_DIR, readOnly: false, ...own });
 
+	// A steer-armed job's inbox (issue #646, DES-STEER-VIA-READONLY-INBOX-MOUNT, INT-STEER-INBOX-CONTRACT). READ-ONLY: the
+	// worker writes the messages, the job only reads them, so the channel is one way and nothing the agent writes there
+	// can reach the worker (the delivery ack travels in the transcript, MACed, never through this directory). Per job and
+	// BESIDE the job directory, never inside it: `/job` is merge-gated input only. Relabelled like the other per-job
+	// directories (issue #355), since it is made for this one container. Absent unless armed.
+	if (steerInboxDir) mounts.push({ host: steerInboxDir, container: CONTAINER_INBOX_DIR, readOnly: true, ...own });
+
 	// The operator's global pi overlay (REQ-GLOBAL-PI-OVERLAY): custom models, global skills, a global
 	// persona, layered UNDER each repo's own .pi/. Read-only -- it is operator-authored deploy-time config,
 	// the same trust class as the baked floor, but the agent still must not rewrite it. Both job kinds.

package/src/cpu-reserve.mjs

@@ -179,13 +179,16 @@ export function cgroupParentFor(runtime) {
  *   `parent`  false when jobs run without the parent (`cgroupParentFor`): their shares are then capped instead
  *   `method`  `user-systemd` | `helper` | `system-systemd` | null (the quota is not kept here; `why` says which)
  *   `why`     a fixed token: `podman-cgroupfs`, `cgroup-v1`, `remote-daemon`, `rootless-docker`, `driver-unknown`,
- *             `podman-rootful-remote` or null
+ *             `podman-rootful-remote`, `venue-unmanaged` or null
  * `facts` is the venue's parsed info: for `local` the `docker info` facts (`parseDaemonFacts`, with `cgroupDriver` and
  * `cgroupVersion`), for `podman` the `podman info` read (`parsePodmanInfo`). `endpointLocal` is the docker endpoint's
  * observed locality, and `platform` this process's: a systemd read on THIS host says nothing about a remote daemon's.
  */
 export function reservePlan({ venue, facts, endpointLocal = false, platform = process.platform }) {
 	const plan = (parent, method, why) => ({ venue, parent, method, why });
+	// Issue #681: only the two venues whose facts this reads are planned. Another venue used to fall through to docker's
+	// rules and could be handed docker's helper; it keeps no quota here, said with a token rather than guessed.
+	if (venue !== "local" && venue !== "podman") return plan(false, null, "venue-unmanaged");
 	if (venue === "podman") {
 		if (cgroupParentFor({ podman: true, cgroupManager: facts?.cgroupManager }) === null) return plan(false, null, "podman-cgroupfs");
 		if (facts?.cgroupVersion && facts.cgroupVersion !== "v2") return plan(true, null, "cgroup-v1");

package/src/defer-reasons.mjs

@@ -0,0 +1,200 @@
+/**
+ * Why a job waits (issue #737): the closed vocabulary the worker writes into `defer:job:<id>` when it puts a job back
+ * on the delayed set, and the rules a reader holds each field to. A LEAF: it imports nothing but the BullMQ leaf below,
+ * so the admin console can import it (`@edgehero/pi-dispatch/defer-reasons`) without pulling the worker's dependencies
+ * into its bundle. The writer is `defer-write.mjs`; the reader is the admin's `readWaiting`.
+ *
+ * The record is the worker's own words for its own decision, written at the moment it defers, so a reader never has to
+ * guess a reason from a job's data (which it must not read: `INT-WAIT-PROFILES-CONTRACT`) or from its timing. A job
+ * with no record says so ("No reason was recorded"), and one whose record names a reason this console does not know
+ * says that; each is true, never a guess. Every value is judged here on the way in (`deferHsetArgs`) and on the way out
+ * (`parseDefer`), because the hash lives in a shared Valkey that anything with its URL can write.
+ */
+
+/** The hash's key prefix; the job's id follows it. */
+export const DEFER_KEY = "defer:job:";
+
+/** The defer record's key for a job id. */
+export const deferKey = (jobId) => `${DEFER_KEY}${jobId}`;
+
+/**
+ * The buckets a console groups waiting jobs by, in the order it shows them. The last four are the reader's own and no
+ * reason in `DEFER_WHYS` is in them: `paused` holds a job in a paused queue, `due` a delayed job whose due instant has
+ * passed with no worker taking it (judged from the delayed set and the read's clock, whatever its record says),
+ * `unknown` a job whose record names a reason this console does not know (it has the reason charset but is not in
+ * `DEFER_WHYS`: most often a newer worker's), and `none` a job with no record, or with one whose reason fails the
+ * charset (no reason at all). Each title is true of every job in its bucket, which is why a reason this console cannot
+ * explain has a bucket of its own rather than sharing `none`'s "No reason was recorded".
+ */
+export const DEFER_BUCKETS = Object.freeze(["slot", "run", "check", "retry", "pause", "ready", "paused", "due", "unknown", "none"]);
+
+/** Each bucket's words: a waitlist's title and its short form (a chip, a row's tag). */
+export const DEFER_BUCKET_WORDS = Object.freeze({
+	slot: Object.freeze({ title: "Waiting for a free slot", short: "Free slot" }),
+	run: Object.freeze({ title: "Waiting for another run to finish", short: "Another run" }),
+	check: Object.freeze({ title: "Waiting on a time or a check", short: "Time or check" }),
+	retry: Object.freeze({ title: "Retry backoff", short: "Retry backoff" }),
+	pause: Object.freeze({ title: "In a pause window", short: "Pause window" }),
+	ready: Object.freeze({ title: "Waiting for its host to get ready", short: "Host getting ready" }),
+	paused: Object.freeze({ title: "In a paused queue", short: "Queue paused" }),
+	due: Object.freeze({ title: "Due, and no worker has taken it", short: "Due, not taken" }),
+	unknown: Object.freeze({ title: "A reason this console does not know", short: "Unknown reason" }),
+	none: Object.freeze({ title: "No reason was recorded", short: "No reason recorded" }),
+});
+
+/**
+ * The `due` bucket's line, whose `{due}` is the job's due instant (the delayed set's, drawn as a `<time>`). Not a reason
+ * in `DEFER_WHYS`: no worker records it, since the job is overdue exactly because no worker has looked at it since.
+ */
+export const DEFER_DUE_SENTENCE = "Due since {due}; no worker has taken it";
+
+/**
+ * Every reason the worker can record: its bucket and the sentence a console shows, whose `{placeholders}` the
+ * console fills from the record's own fields (escaped; times drawn as `<time>`). A reason not in this table is
+ * written by nobody: `deferHsetArgs` throws on one (a programming error), and `parseDefer` reads one as unknown.
+ */
+export const DEFER_WHYS = Object.freeze({
+	canary: Object.freeze({ bucket: "run", sentence: "Waiting for the operator model canary to finish" }),
+	"pause-window": Object.freeze({ bucket: "pause", sentence: "In a pause window until {until}" }),
+	"size-never-fits": Object.freeze({ bucket: "slot", sentence: "Its size fits no slot on {host}; another host may take it" }),
+	"host-slot": Object.freeze({ bucket: "slot", sentence: "{host} has no free job slot" }),
+	"scope-folder": Object.freeze({ bucket: "slot", sentence: "Its folder is busy with another run" }),
+	"scope-repo": Object.freeze({ bucket: "slot", sentence: "Its repository's concurrent limit is full" }),
+	"scope-project": Object.freeze({ bucket: "slot", sentence: "Its project's concurrent limit is full" }),
+	endpoint: Object.freeze({ bucket: "slot", sentence: "Every slot of model endpoint {endpoint} is taken" }),
+	"subscription-quota": Object.freeze({ bucket: "check", sentence: "The subscription account is exhausted; waits for newer evidence, a known reset or an operator trial. Asks again at {until}" }),
+	"subscription-owner": Object.freeze({ bucket: "ready", sentence: "Waits for the host that owns its subscription connection" }),
+	"subscription-auth": Object.freeze({ bucket: "check", sentence: "Waits until the subscription credential can be renewed at {until}" }),
+	"model-cooldown": Object.freeze({ bucket: "check", sentence: "No reviewed model is available; a model is cooling down. Asks again at {until}" }),
+	"model-window": Object.freeze({ bucket: "check", sentence: "No reviewed model is available; a model's dollar window is full. Asks again at {until}" }),
+	"host-budget": Object.freeze({ bucket: "slot", sentence: "{host}'s budget has no room for it yet" }),
+	"bundle-profile": Object.freeze({ bucket: "slot", sentence: "Waits for a host that serves its secret bundle profile" }),
+	chained: Object.freeze({ bucket: "run", sentence: "Chained after {holder}; starts when its folder is free" }),
+	"recovery-wait": Object.freeze({ bucket: "run", sentence: "Recovers run {holder}, whose attempt is still running" }),
+	"wait-after": Object.freeze({ bucket: "check", sentence: "Held until {until}" }),
+	"wait-hold": Object.freeze({ bucket: "check", sentence: "Held on its wait check; asks again at {until}" }),
+	"wait-throttled": Object.freeze({ bucket: "check", sentence: "Its wait check waits for a free check slot" }),
+	"wait-supersede": Object.freeze({ bucket: "check", sentence: "Asks again whether the delivery holding its target still waits" }),
+	"wait-aborted": Object.freeze({ bucket: "check", sentence: "Its wait check was stopped by a restart or a cancel; asks again soon" }),
+	"wait-unclear": Object.freeze({ bucket: "check", sentence: "Its wait check answered in a way the worker cannot read; asks again at {until}" }),
+	"podman-restart": Object.freeze({ bucket: "ready", sentence: "Waits for rootful Podman's service on {host} to restart" }),
+	"netns-keeper": Object.freeze({ bucket: "ready", sentence: "Waits for the network keeper on {host} to settle" }),
+	"newer-release": Object.freeze({ bucket: "ready", sentence: "Queued or configured by a newer pi-dispatch: waits for this host's upgrade" }),
+	retry: Object.freeze({ bucket: "retry", sentence: "Attempt {attempt} of {attempts} at {until}" }),
+});
+
+/** The hash's fields, in the order a reader's HMGET asks for them. `n` is the writer's own nonce, never shown. */
+export const DEFER_FIELDS = Object.freeze(["why", "at", "until", "project", "host", "holder", "endpoint", "budgetWhy", "trigger", "attempt", "attempts", "n"]);
+
+/** A reason's charset: what `parseDefer` takes as a reason word at all (a word outside the table reads as unknown). */
+export const DEFER_WHY_RE = /^[a-z][a-z0-9-]{0,31}$/;
+/** The writer's nonce: 16 hex characters, so a compare-and-delete can tell its own record from another worker's. */
+export const DEFER_N_RE = /^[0-9a-f]{16}$/;
+/** A job id the record may name (`holder`). Pinned equal to the admin reader's `RUN_ID_RE` by a test. */
+export const DEFER_ID_RE = /^[A-Za-z0-9._:-]{1,200}$/;
+/** A project id. A copy of `project-id.mjs`'s `PROJECT_ID_RE`, pinned equal to it by a test (this module imports nothing). */
+export const DEFER_PROJECT_RE = /^[a-z0-9][a-z0-9-]{0,31}$/;
+/** A host's name. A copy of `worker-name.mjs`'s `WORKER_NAME_RE`, pinned equal to it by a test. */
+export const DEFER_HOST_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
+/** A model endpoint's id. A copy of `model-endpoints.mjs`'s `MODEL_ENDPOINT_ID_RE`, pinned equal to it by a test. */
+export const DEFER_ENDPOINT_RE = /^[a-z0-9-]{1,32}$/;
+
+/**
+ * The last instant a UTC date-time can be written with a four-digit year: 9999-12-31T23:59:59.999Z. Every instant this
+ * leaf passes on (`at`, `until`, a due time) is at most this, so a console can always draw it. NOT the latest instant a
+ * JavaScript Date holds (8.64e15 ms, in the year 275760): `toISOString` writes a year past 9999 as six digits with a
+ * sign (`+275760-09-13T00:00:00.000Z`), and a console that draws the first sixteen characters as `YYYY-MM-DD HH:MM`
+ * would show a date that is not the instant's.
+ */
+export const DEFER_TIME_MAX_MS = 253402300799999;
+
+/** How long a record outlives the job's due time, so a record is still there while the job is promoted and picked up. */
+export const DEFER_TTL_SLACK_MS = 60_000;
+/** A record's lifetime: until the job is due, plus the slack, and never less than the slack. */
+export const deferTtlMs = (untilMs, nowMs) => Math.max(DEFER_TTL_SLACK_MS, untilMs - nowMs + DEFER_TTL_SLACK_MS);
+
+/**
+ * The delayed set's score and a scheduler's next run are BullMQ's facts, so they live in `bullmq-internals.mjs` (issue
+ * #761) and are re-exported here, where they were first published: a reader of this subpath keeps every name.
+ */
+export { DELAYED_SCORE_SHIFT, dueOfScore, isSchedulerNext } from "./bullmq-internals.mjs";
+
+/** A decimal safe integer, from a number or its string form, or null. */
+function intOf(value) {
+	if (typeof value === "number") return Number.isSafeInteger(value) ? value : null;
+	if (typeof value !== "string" || !/^(?:0|[1-9][0-9]{0,15})$/.test(value)) return null;
+	const n = Number(value);
+	return Number.isSafeInteger(n) ? n : null;
+}
+
+/** An integer within [lo, hi], or null. */
+function intIn(value, lo, hi) {
+	const n = intOf(value);
+	return n !== null && n >= lo && n <= hi ? n : null;
+}
+
+/** A string that passes `re`, or null. */
+const textOf = (value, re) => (typeof value === "string" && re.test(value) ? value : null);
+
+/** Each field's rule: the value as the record holds it (a number or a string), or null when it fails. */
+const FIELD_RULES = Object.freeze({
+	why: (v) => (typeof v === "string" && Object.hasOwn(DEFER_WHYS, v) ? v : null),
+	at: (v) => intIn(v, 0, DEFER_TIME_MAX_MS),
+	until: (v) => intIn(v, 0, DEFER_TIME_MAX_MS),
+	project: (v) => textOf(v, DEFER_PROJECT_RE),
+	host: (v) => textOf(v, DEFER_HOST_RE),
+	holder: (v) => textOf(v, DEFER_ID_RE),
+	endpoint: (v) => textOf(v, DEFER_ENDPOINT_RE),
+	budgetWhy: (v) => textOf(v, DEFER_WHY_RE),
+	trigger: (v) => intIn(v, 0, 9999),
+	attempt: (v) => intIn(v, 1, 1000),
+	attempts: (v) => intIn(v, 1, 1000),
+	n: (v) => textOf(v, DEFER_N_RE),
+});
+
+/**
+ * The HSET arguments for a record, `[field, value, ...]`, each value judged by its field's rule and left out when it
+ * fails (a record with a missing field is still worth writing; one with a wrong field is not worth trusting). A
+ * reason not in `DEFER_WHYS` throws: every call site names its reason by a literal, so an unknown one is a bug.
+ */
+export function deferHsetArgs(rec) {
+	const why = rec?.why;
+	if (typeof why !== "string" || !Object.hasOwn(DEFER_WHYS, why)) throw new RangeError(`defer-reasons: unknown reason ${JSON.stringify(why)}`);
+	const args = [];
+	for (const field of DEFER_FIELDS) {
+		const value = FIELD_RULES[field](rec[field]);
+		if (value !== null) args.push(field, String(value));
+	}
+	return args;
+}
+
+/**
+ * A record from an HMGET of `DEFER_FIELDS`, in that order: null when there is no `why` (no record, or none worth
+ * reading); otherwise every field, each null when it is absent or fails its rule. A `why` with the reason charset but
+ * not in the table is read as unknown (`known` false, bucket `unknown`): a newer worker's reason an older console does
+ * not know. A `why` outside the charset is no reason at all (`why` null, bucket `none`). The nonce `n` is the writer's
+ * and is never returned.
+ */
+export function parseDefer(values) {
+	if (!Array.isArray(values)) return null;
+	const raw = values[0];
+	if (raw === null || raw === undefined || raw === "") return null;
+	const why = typeof raw === "string" && DEFER_WHY_RE.test(raw) ? raw : null;
+	const known = why !== null && Object.hasOwn(DEFER_WHYS, why);
+	const at = (i) => values[i];
+	return {
+		why,
+		known,
+		bucket: known ? DEFER_WHYS[why].bucket : why !== null ? "unknown" : "none",
+		at: FIELD_RULES.at(at(1)),
+		until: FIELD_RULES.until(at(2)),
+		project: FIELD_RULES.project(at(3)),
+		host: FIELD_RULES.host(at(4)),
+		holder: FIELD_RULES.holder(at(5)),
+		endpoint: FIELD_RULES.endpoint(at(6)),
+		budgetWhy: FIELD_RULES.budgetWhy(at(7)),
+		trigger: FIELD_RULES.trigger(at(8)),
+		attempt: FIELD_RULES.attempt(at(9)),
+		attempts: FIELD_RULES.attempts(at(10)),
+	};
+}

package/src/defer-write.mjs

@@ -0,0 +1,208 @@
+/**
+ * The worker's half of "why does this job wait" (issue #737): every gate that puts a job back on the delayed set
+ * writes its reason into `defer:job:<id>` (`defer-reasons.mjs` holds the vocabulary and the field rules) just before
+ * BullMQ's move, through ONE function, `deferTo`, so no deferral can move a job without saying why. A retry after an
+ * infrastructure failure, which BullMQ delays by itself after the processor throws, is noted by `noteRetry` before
+ * the throw. Worker-only, and free of bullmq itself: the processor hands it BullMQ's `DelayedError` class (index.mjs
+ * imports bullmq; `valkey-clients.test.mjs` keeps every other module from naming that library), so the throw is the
+ * very class the processor's own catch tells a deferral apart by. The admin console never loads it (it reads the leaf).
+ *
+ * THE RECORD IS ADVISORY, so it FAILS OPEN and is BOUNDED. A deferral never fails or changes for its reason, and waits
+ * for it at most `DEFER_WRITE_TIMEOUT_MS` (2 s): a write that fails or takes longer is logged once per episode and the
+ * job is moved all the same, and a logger that throws is swallowed rather than rejecting the deferral. The bound is not
+ * decoration: the worker's client queues commands during an outage rather than rejecting them
+ * (`maxRetriesPerRequest: null`, the host registry's reason), so an unbounded write would hold a gate's deferral for
+ * the whole outage. The move itself is BullMQ's, exactly as it was before this file.
+ *
+ * WRITE, THEN MOVE. A reader starts from the delayed set and looks up the records of the ids it finds there, so a
+ * record written before the move is in place the moment the job is delayed, and one for a job that is still active
+ * is never read. The write is one MULTI (`DEL`, `HSET`, `PEXPIRE`), so a reader never sees this deferral's fields
+ * mixed with an earlier one's, and every record carries a TTL (`deferTtlMs`: the due time plus a minute's slack,
+ * so the record outlives the promotion and the pickup that follows it).
+ *
+ * A MOVE THAT REJECTS (a lost lock, Valkey gone) leaves a record for a job that is not delayed, so its own record is
+ * removed by a compare-and-delete on the writer's nonce (`n`): a plain DEL could remove the record another worker
+ * wrote for a later deferral of the same job id. The rejection is rethrown unchanged, so it reaches BullMQ's
+ * failed-attempt handling exactly as a raw `moveToDelayed` rejection always has.
+ *
+ * A TIMED-OUT WRITE MAY STILL LAND. The bound stops the waiting, not the command: a queued MULTI runs when the
+ * connection comes back. That is why a timed-out write still returns its nonce: the compare-and-delete is sent on
+ * the same connection after it, so it runs after the MULTI if that lands at all. A record that lands for a job that
+ * then ran is not read while the job is not delayed, and expires with its TTL.
+ *
+ * WHAT A RECORD CANNOT PROMISE. A write that failed leaves the previous deferral's record, until it expires, beside a
+ * job now delayed for another reason; and a job BullMQ delays by itself (a move's rejection retried, a backoff
+ * `retryDelayOf` cannot know) gets no fresh record. Both show as a record whose `until` is not the job's due time in
+ * the delayed set, which is how a reader tells a record that describes the wait from one that does not.
+ */
+import { randomBytes } from "node:crypto";
+import { DEFER_N_RE, DEFER_WHY_RE, deferHsetArgs, deferKey, deferTtlMs } from "./defer-reasons.mjs";
+
+/** How long a gate waits for its reason's write, and a failed move for its compare-and-delete, before going on. */
+export const DEFER_WRITE_TIMEOUT_MS = 2_000;
+
+/** Delete the record only while it still carries this writer's nonce: never another worker's later record. */
+export const DEFER_CAD_LUA = "if redis.call('HGET', KEYS[1], 'n') == ARGV[1] then return redis.call('DEL', KEYS[1]) end return 0";
+
+/**
+ * The delay BullMQ will put on this job's retry after the current attempt throws, in ms, or null when it will not
+ * retry it on a delay this function can know. Mirrors the pinned BullMQ's `shouldRetryJob` and its builtin backoff
+ * strategies (job.js, backoffs.js at 5.80.4): it retries while `attemptsMade + 1 < opts.attempts` and the job is not
+ * discarded; a number is a `fixed` backoff; `fixed` waits `delay`, `exponential` waits `round(2^attemptsMade * delay)`
+ * (BullMQ calls the strategy with `attemptsMade + 1` and raises 2 to one less).
+ *
+ * Null for everything else, never a guess: a jittered backoff (its delay is random, so no instant could be told), a
+ * custom strategy (it is the queue's code), and a delay at or below zero (BullMQ then retries at once, through the
+ * wait list rather than the delayed set).
+ */
+export function retryDelayOf(job) {
+	const attempts = job?.opts?.attempts;
+	const made = job?.attemptsMade;
+	if (!Number.isInteger(attempts) || !Number.isInteger(made) || made < 0) return null;
+	if (!(made + 1 < attempts) || job.discarded === true) return null;
+	const raw = job.opts.backoff;
+	const backoff = Number.isFinite(raw) ? { type: "fixed", delay: raw } : raw;
+	if (backoff === null || typeof backoff !== "object") return null;
+	// BullMQ's own test (`jitter > 0`), so a value it ignores is ignored here too.
+	if (backoff.jitter > 0) return null;
+	const delay = backoff.delay;
+	if (!Number.isFinite(delay) || delay <= 0) return null;
+	if (backoff.type === "fixed") return delay;
+	if (backoff.type === "exponential") return Math.round(2 ** made * delay);
+	return null;
+}
+
+/** Settle `promise` or give up after `ms`, never rejecting: `{ value }`, `{ error }` or `{ timedOut: true }`. */
+function settleWithin(promise, ms) {
+	return new Promise((resolve) => {
+		const timer = setTimeout(() => resolve({ timedOut: true }), ms);
+		Promise.resolve(promise).then(
+			(value) => {
+				clearTimeout(timer);
+				resolve({ value });
+			},
+			(error) => {
+				clearTimeout(timer);
+				resolve({ error });
+			},
+		);
+	});
+}
+
+/** A failure's name for the log: a fixed word or an errno-like code, never an error's message (it can quote a URL). */
+function failureOf(outcome) {
+	if (outcome.timedOut) return "timeout";
+	if (outcome.threw) return "threw";
+	if (outcome.error) return typeof outcome.error.code === "string" && /^[A-Z][A-Z0-9_]{0,31}$/.test(outcome.error.code) ? outcome.error.code : "rejected";
+	return "error-reply";
+}
+
+/**
+ * Build the writer for one processor. `redis` is the processor's client; `DelayedError` is BullMQ's class (required:
+ * a stand-in would pass BullMQ's name test and fail the processor's `instanceof`, so a deferral would read as a
+ * failure there); `hostName` is this worker's declared name, recorded as the record's host when it passes the host
+ * rule; `now` is the processor's clock, the one every gate's due time is computed from.
+ *
+ * Returns:
+ *   - `writeReason(job, { why, untilMs, atMs, project, holder, endpoint, budgetWhy, attempt, attempts })`: the record's
+ *     write, bounded. `atMs` is the record's `at`, by default the clock when the write is built; a caller that measured
+ *     `untilMs` from its own reading passes that reading, so the two are exactly the delay apart. Resolves `{ n }` once
+ *     the MULTI was sent (it landed, or may still land: see the header) and null when nothing was sent: a client with
+ *     no `multi` (a bare test wiring), a job with no id, or a write that could not be built. Never rejects. `why` must
+ *     be a reason of `DEFER_WHYS`; any other field that fails its rule is left out of the record rather than failing it.
+ *   - `deferTo(job, token, untilMs, why, extra)`: the one deferral. Writes the reason, then `job.moveToDelayed(untilMs,
+ *     token)`, then throws BullMQ's `DelayedError`, the throw by which BullMQ learns the job was deferred. NEVER
+ *     returns. A rejected move removes this write's record and is rethrown unchanged.
+ *   - `noteRetry(job, { project })`: before an infrastructure retry's throw, the `retry` record, when BullMQ will
+ *     delay the retry by a known amount (`retryDelayOf`). Never throws.
+ *
+ * The log says `defer_reason_unwritten { why, reason }` on the first failed write and nothing more until a write
+ * succeeds again, which says `defer_reason_restored { why }` once: an outage is one episode, not a line per wake.
+ */
+export function makeDeferTo({ redis, DelayedError, now = () => Date.now(), hostName = "", log = () => {}, nonce = () => randomBytes(8).toString("hex"), timeoutMs = DEFER_WRITE_TIMEOUT_MS } = {}) {
+	if (typeof DelayedError !== "function") throw new TypeError("makeDeferTo needs BullMQ's DelayedError class, the one the processor's catch recognises a deferral by");
+	let failing = false;
+	const said = (why) => (typeof why === "string" && DEFER_WHY_RE.test(why) ? why : null);
+	// Every line goes through `say`, so a logger that throws is swallowed here: the log describes a deferral, and a line
+	// that could not be written must not turn the deferral into a rejection before its move (the job left active, BullMQ
+	// counting a failed attempt). The episode's state is set before the line, so it holds even when the line is lost.
+	const say = (event, fields) => {
+		try {
+			log(event, fields);
+		} catch {
+			// The line is lost, the deferral goes on.
+		}
+	};
+	const failed = (why, outcome) => {
+		if (failing) return;
+		failing = true;
+		say("defer_reason_unwritten", { why: said(why), reason: failureOf(outcome) });
+	};
+	const succeeded = (why) => {
+		if (!failing) return;
+		failing = false;
+		say("defer_reason_restored", { why: said(why) });
+	};
+
+	const writeReason = async (job, { why, untilMs, atMs, project = null, holder, endpoint, budgetWhy, attempt, attempts } = {}) => {
+		if (typeof redis?.multi !== "function") return null;
+		if (typeof job?.id !== "string" || job.id === "") return null;
+		let sent;
+		let n = null;
+		try {
+			const at = Number.isFinite(atMs) ? Math.floor(atMs) : now();
+			const drawn = nonce();
+			n = typeof drawn === "string" && DEFER_N_RE.test(drawn) ? drawn : null;
+			// Whole milliseconds: the record's rule takes integers, and a gate's arithmetic (a jittered delay, a keeper's
+			// wait) need not produce one.
+			const until = Number.isFinite(untilMs) ? Math.floor(untilMs) : null;
+			const args = deferHsetArgs({ why, at, until, project, host: hostName, holder, endpoint, budgetWhy, trigger: job.data?.trigger?.matched?.index, attempt, attempts, n });
+			const key = deferKey(job.id);
+			const ttl = Math.ceil(deferTtlMs(until ?? at, at));
+			sent = redis.multi().del(key).hset(key, ...args).pexpire(key, ttl).exec();
+		} catch (error) {
+			// Nothing was sent: a client whose transaction could not be built, or a reason outside the table (a defect at
+			// the call site, which `defer-write.test.mjs` pins by literal).
+			failed(why, { threw: true, error });
+			return null;
+		}
+		const outcome = await settleWithin(sent, timeoutMs);
+		const replies = outcome.value;
+		const errorReply = Array.isArray(replies) ? replies.some((reply) => Array.isArray(reply) && reply[0]) : replies !== undefined;
+		if (outcome.timedOut || outcome.error || errorReply) failed(why, outcome);
+		else succeeded(why);
+		return { n };
+	};
+
+	const deferTo = async (job, token, untilMs, why, extra = {}) => {
+		const written = await writeReason(job, { ...extra, why, untilMs });
+		try {
+			await job.moveToDelayed(untilMs, token);
+		} catch (error) {
+			if (written?.n && typeof redis?.eval === "function") {
+				let removal;
+				try {
+					removal = redis.eval(DEFER_CAD_LUA, 1, deferKey(job.id), written.n);
+				} catch {
+					removal = null;
+				}
+				await settleWithin(removal, timeoutMs);
+			}
+			throw error;
+		}
+		throw new DelayedError();
+	};
+
+	const noteRetry = async (job, { project = null } = {}) => {
+		try {
+			const delay = retryDelayOf(job);
+			if (delay === null) return;
+			const at = now();
+			await writeReason(job, { why: "retry", untilMs: at + delay, atMs: at, project, attempt: job.attemptsMade + 2, attempts: job.opts.attempts });
+		} catch {
+			// Never in the way of the throw it precedes: the retry happens whether or not its reason was written.
+		}
+	};
+
+	return { writeReason, deferTo, noteRetry };
+}

package/src/deployment-venue.mjs

@@ -4,11 +4,14 @@
  * different venue than the service they described. The deployment's `.env` is read with `readStackKeys`, exactly as
  * `service install` reads it, and a key this shell sets differently stops the command, since the service runs the file.
  */
-import { parseBackendList } from "./backends.mjs";
+import { parseBackendList, venuesOf } from "./backends.mjs";
 import { egressArmed, egressProxyName } from "./egress.mjs";
 import { envValueShown } from "./env-file.mjs";
 import { STACK_KEYS, VALKEY_SHARED_KEY, readStackKeys, readValkeyKeys } from "./podman-stack.mjs";
 
+/** The venue keys only the openshell venues read (issue #707). */
+const OPENSHELL_KEYS = Object.freeze(["PI_OPENSHELL_ENGINE", "PI_OPENSHELL_GATEWAY"]);
+
 /** How each command says what a shell/file disagreement would make it do, the clause before "while the service ...". */
 const COMMAND_SAYS = Object.freeze({
 	up: "up would drive the shell's venue",
@@ -39,10 +42,16 @@ export function deploymentVenueEnv({ env, fs, envPath, platform, command = "up",
 			text = fs.readFileSync(envPath);
 		} catch (err) {
 			// Said rather than silent (round 2 nit): the venue then comes from this shell alone.
-			notes.push(`${envPath} could not be read (${err?.message}), so PI_BACKENDS, PI_EGRESS and PI_EGRESS_PROXY come from this shell alone`);
+			notes.push(`${envPath} could not be read (${err?.message}), so the venue keys (${STACK_KEYS.join(", ")}) come from this shell alone`);
 		}
 		if (text !== null) {
-			const read = readStackKeys(text, { loader, path: envPath });
+			// The openshell keys are refused only where an openshell venue is blessed (review round 2 of #715): the file's
+			// PI_BACKENDS says so to the reader itself, this shell's is said here.
+			let shellOpenshell = false;
+			try {
+				shellOpenshell = typeof env.PI_BACKENDS === "string" && venuesOf({ PI_BACKENDS: env.PI_BACKENDS }).openshellUsed;
+			} catch {}
+			const read = readStackKeys(text, { loader, path: envPath, openshellBlessed: shellOpenshell });
 			if (read.error && linux) return { error: read.error };
 			if (read.error) notes.push(`${read.error}; off Linux the podman venue refuses this host anyway, so this pass reads the venue from this shell`);
 			else keys = read.keys;
@@ -57,7 +66,7 @@ export function deploymentVenueEnv({ env, fs, envPath, platform, command = "up",
 		if (typeof env[key] === "string") {
 			// Compared as the worker READS them (round 3, D4), not as strings: ` podman` and `podman,podman` are the list
 			// `podman`, and refusing them sent an operator to reconcile two values that already agree.
-			if (venueKeyMeaning(key, env[key]) !== venueKeyMeaning(key, keys[key])) conflicts.push(`${key} is ${quotedShown(env[key])} in this shell and ${quotedShown(keys[key])} in ${envPath}`);
+			if (venueKeyMeaning(key, env[key]) !== venueKeyMeaning(key, keys[key])) conflicts.push({ key, text: `${key} is ${quotedShown(env[key])} in this shell and ${quotedShown(keys[key])} in ${envPath}` });
 			continue;
 		}
 		merged[key] = keys[key];
@@ -66,8 +75,12 @@ export function deploymentVenueEnv({ env, fs, envPath, platform, command = "up",
 	// A KNOWN disagreement is a refusal (round 2, E5), not a warning: this pass would stand up one venue's stack and the
 	// service installed next would run the other. The file is what the service runs, so it is the one to change unless
 	// the shell's value was a one-off.
-	if (conflicts.length > 0) {
-		return { error: `${conflicts.join("; ")}. ${COMMAND_SAYS[command] ?? COMMAND_SAYS.up} while the service runs the file's. Make them agree: change ${envPath} (what the service reads), or unset the key in this shell` };
+	// The openshell venues' two keys disagree only where one of those venues is blessed (review round 1 of #715): elsewhere
+	// neither the worker nor any command reads them, so a stray value in one place is no reason to stop.
+	const openshellBlessed = venuesOf(merged).openshellUsed;
+	const said = conflicts.filter((c) => openshellBlessed || !OPENSHELL_KEYS.includes(c.key)).map((c) => c.text);
+	if (said.length > 0) {
+		return { error: `${said.join("; ")}. ${COMMAND_SAYS[command] ?? COMMAND_SAYS.up} while the service runs the file's. Make them agree: change ${envPath} (what the service reads), or unset the key in this shell` };
 	}
 	return { env: merged, fromFile, notes };
 }
@@ -146,6 +159,10 @@ function venueKeyMeaning(key, value) {
 	try {
 		if (key === "PI_BACKENDS") return `list:${parseBackendList(value).join(",")}`;
 		if (key === "PI_EGRESS") return `egress:${egressArmed({ PI_EGRESS: value }) ? "on" : "off"}`;
+		// Issue #707: the openshell venues' two keys, as `openshellFrom` reads them: an empty engine is docker, an empty
+		// gateway the CLI's active one. Not trimmed, as the worker trims neither.
+		if (key === "PI_OPENSHELL_ENGINE") return `engine:${value === "" ? "docker" : value}`;
+		if (key === "PI_OPENSHELL_GATEWAY") return `gateway:${value}`;
 		return `proxy:${egressProxyName({ PI_EGRESS_PROXY: value })}`;
 	} catch {
 		return `raw:${value}`;

package/src/docker-run.mjs

@@ -22,6 +22,7 @@
 export { containerSpec, CONTAINER_GLOBAL_PI_DIR, CONTAINER_SESSION_DIR, CONTAINER_SESSION_FILE } from "./container-spec.mjs";
 import { isAbsolute, relative } from "node:path";
 import { SIZE_LABEL_CPU, SIZE_LABEL_MEM, assertCidFile, assertJobUser, containerSpec } from "./container-spec.mjs";
+import { LAYOUTS } from "./job-layout.mjs";
 import { CGROUP_PARENT, CPU_SHARES_MAX, CPU_SHARES_MIN } from "./job-size.mjs";
 
 /** The fixed isolation flags. Not configurable -- these ARE the boundary. */
@@ -165,6 +166,10 @@ export const DOCKER_EXTRA_FORBIDDEN = [
 	// The job user is a spec field (issue #341). Here so a repeat cannot override it with uid 0.
 	"--user",
 	"-u",
+	// Issue #683: the builder pins the working directory (see `--workdir=` in `argsFromSpec`). A repeat would start the
+	// job outside its workspace, where a relative path the agent writes lands in the image's own tree.
+	"--workdir",
+	"-w",
 ];
 
 /**
@@ -331,6 +336,12 @@ function argsFromSpec(spec, { userns }) {
 		args.push("-v", `${m.host}:${m.container}${options.length > 0 ? `:${options.join(",")}` : ""}`);
 	}
 
+	// Issue #683: the job starts in its workspace mount because the ARGV says so, not because the image's WORKDIR
+	// happens to match. The shipped image's WORKDIR is now /sandbox (a copying venue's writable tree, image/Dockerfile),
+	// so without this a binding venue's job would start in an empty /sandbox and every relative path the agent writes
+	// would land outside the repository it was asked to change. A binding venue always uses the `mounts` layout. An
+	// older image, whose WORKDIR was /workspace, starts exactly where it did.
+	args.push(`--workdir=${LAYOUTS.mounts.workspace}`);
 	args.push(spec.image);
 	return args;
 }