@edgehero/pi-dispatch 1.10.3 → 2.0.0

package/src/connection.mjs

@@ -1,4 +1,129 @@
-import { Redis } from "ioredis";
+import { AbstractConnector, Command, Redis } from "ioredis";
+import { Socket, connect as netConnect } from "node:net";
+import { connect as tlsConnect } from "node:tls";
+import { valkeyAuthRefusal } from "./valkey-auth.mjs";
+import { valkeySchemeOf } from "./podman-stack.mjs";
+import { decodedUserinfo, judgeValkeyAtStart as judgeEndpointAtStart, judgedEndpoint, killSwitchValkeyUrls, defaultValkeyContext, urlShown, useValkeyContext, valkeyClientContext, valkeyContextFromKeys, VALKEY_CONTEXT_KEYS, valkeyPasswordFor, valkeyDbRangeSentence, valkeyRefusal, valkeyUrlFor, valkeyUrlProblem } from "./valkey-endpoint.mjs";
+
+// Re-exported for the receiver and the admin, which import this module from the package (issues #464 and #468).
+export { killSwitchValkeyUrls, defaultValkeyContext, urlShown, useValkeyContext, valkeyClientContext, valkeyContextFromKeys, VALKEY_CONTEXT_KEYS, valkeyPasswordFor, valkeyUrlFor, valkeyUrlProblem };
+
+/**
+ * The VALKEY_URL a CLI command uses from `cwd` (`valkeyUrlFor`), with a disagreement or an unreadable `.env` written as
+ * one warning line through `warn`. The one resolver every CLI verb and `service restart --drain` share.
+ */
+export function cliValkeyUrl(env, { cwd, warn = (line) => process.stderr.write(line) } = {}) {
+	const resolved = valkeyUrlFor(valkeyClientContext({ env, ...(cwd ? { cwd } : {}) }));
+	if (resolved.note) warn(`warning: ${resolved.note}\n`);
+	return resolved.url;
+}
+
+/**
+ * An error a Valkey client reports, without the command it failed on (issue #468, PR #475's review). ioredis hangs
+ * `command: { name, args }` on every reply error, and for a failed AUTH the args ARE the password: after a rotation,
+ * a client still sending the old one emitted `WRONGPASS` with it attached, and BullMQ, whose Queue and Worker had no
+ * `error` listener, fell back to `console.error(err)`, which prints the whole object, password included (measured
+ * against a real Redis with CONFIG SET requirepass and CLIENT KILL). The args also carry job data (task text) for any
+ * other failed command. So only the command's NAME is kept. Mutates and returns `err`; anything else passes through.
+ */
+export function scrubValkeyError(err) {
+	if (err && typeof err === "object" && err.command && typeof err.command === "object") {
+		try {
+			err.command = { name: err.command.name };
+		} catch {
+			// A frozen error: nothing this client made.
+		}
+	}
+	return err;
+}
+
+/**
+ * WHERE the scrub runs (PR #475's review, round 2 corrected the first version of this comment, which claimed the emit
+ * hook alone covered every path). ioredis hands an error to three kinds of consumer, and a failed AUTH reaches all
+ * three with the password in `command.args`:
+ *   - a listener, through `emit("error")`;
+ *   - nobody, through `silentEmit("error")`, which SKIPS `emit()` when the client has no `error` listener (the worker's
+ *     shared client, a CLI probe): ioredis then prints only `error.stack`, but the same object was already handed
+ *   - to every command waiting on the connection, through that command's `reject` (`flushQueue`, `abortError`, a
+ *     reply error): an awaited or unhandled rejection that anything may print whole (measured by the review: two
+ *     occurrences of the password from a listener-less client after a rotation).
+ * So the hook sits where each of them receives the object: `Redis.prototype.emit` and `silentEmit` for the events, and
+ * every `Command`'s `reject` (wrapped as `initPromise` makes it) for the rejections. BullMQ and this module share the
+ * one ioredis, so its clients are covered by the same hooks.
+ */
+const SCRUBS = Symbol.for("pi-dispatch.valkey-error-scrub");
+if (!Redis.prototype[SCRUBS]) {
+	const emit = Redis.prototype.emit;
+	Redis.prototype.emit = function emitScrubbed(event, ...args) {
+		if (event === "error") scrubValkeyError(args[0]);
+		return emit.call(this, event, ...args);
+	};
+	const silentEmit = Redis.prototype.silentEmit;
+	Redis.prototype.silentEmit = function silentEmitScrubbed(event, arg, ...rest) {
+		if (event === "error") scrubValkeyError(arg);
+		return silentEmit.call(this, event, arg, ...rest);
+	};
+	const initPromise = Command.prototype.initPromise;
+	Command.prototype.initPromise = function initPromiseScrubbed(...args) {
+		const out = initPromise.apply(this, args);
+		const reject = this.reject;
+		if (typeof reject === "function") this.reject = (err) => reject(scrubValkeyError(err));
+		return out;
+	};
+	Redis.prototype[SCRUBS] = true;
+}
+
+/**
+ * A database the server refuses is a REFUSAL, never database 0 (gate round 2 of PR #478). ioredis 5.11.1 applies `db`
+ * with a SELECT in its connect handler (redis/event_handler.js), before the ready check, and on a failure only
+ * `silentEmit`s the error and carries on: the client went ready on database 0, so `status`, `cancel`, the worker and
+ * doctor used database 0 of a Valkey whose `/16` does not exist, possibly another deployment's queue (measured against
+ * Valkey: "ERR DB index is out of range", then a write through the `/16` client read back from database 0).
+ *
+ * Here, where that error surfaces, the client is stopped before its ready check can finish: the refusal
+ * (`valkeyRefusal`, a configuration error naming the index) rejects every command queued so far, the client closes
+ * without reconnecting, `JudgedConnector.check` fails the ready check that was already in flight, and a later connect
+ * rejects with the same refusal. Every client of the project is built on that connector, so none goes ready on
+ * another database. The start judgements (`judgeValkeyAtStart`, the worker's `refuseValkeyAuth`) and doctor read the
+ * server's `databases` count for their sentence (`valkeyAuthState`).
+ */
+export const DB_REFUSED = Symbol.for("pi-dispatch.valkey-db-refused");
+const SELECT_HOOK = Symbol.for("pi-dispatch.valkey-select-refusal");
+/** Whether `err` is a server's refusal of a SELECT for a database it does not have. */
+function isSelectRangeError(err) {
+	return Boolean(err) && typeof err === "object" && String(err.command?.name ?? "").toLowerCase() === "select" && /DB index is out of range|invalid DB index/i.test(String(err.message ?? ""));
+}
+if (!Redis.prototype[SELECT_HOOK]) {
+	const silentEmit = Redis.prototype.silentEmit;
+	Redis.prototype.silentEmit = function silentEmitSelectRefused(event, arg, ...rest) {
+		if (event === "error" && isSelectRangeError(arg)) {
+			if (!this[DB_REFUSED]) {
+				const refusal = Object.assign(valkeyRefusal(valkeyDbRangeSentence(this.options?.[JUDGE]?.url ?? "", this.condition?.select ?? this.options?.db)), { valkeyDbRange: true, cause: scrubValkeyError(arg) });
+				this[DB_REFUSED] = refusal;
+				if (this.connector) this.connector[DB_REFUSED] = refusal;
+				try {
+					this.flushQueue(refusal);
+				} catch {}
+				this.disconnect();
+			}
+			// Said to a listener only: without one, every command of the client already rejects with the refusal, and
+			// ioredis' own "Unhandled error event" print would add a stack trace to that sentence.
+			return this.listeners("error").length > 0 ? silentEmit.call(this, event, this[DB_REFUSED], ...rest) : false;
+		}
+		return silentEmit.call(this, event, arg, ...rest);
+	};
+	Redis.prototype[SELECT_HOOK] = true;
+}
+
+/**
+ * The `error` listener every BullMQ Queue and Worker of the project carries (issue #468): without one, BullMQ prints
+ * the whole error object with `console.error`. This writes one line, the message only (scrubbed of any command), and
+ * `what` naming whose error it is.
+ */
+export function onValkeyError(emitter, what, write = (line) => process.stderr.write(line)) {
+	emitter.on?.("error", (err) => write(`[pi-dispatch] Valkey error (${what}): ${String(scrubValkeyError(err)?.message ?? err)}\n`));
+	return emitter;
+}
 
 /**
  * Connection helpers for BullMQ and the budget's raw Redis client, both from one VALKEY_URL.
@@ -15,14 +140,39 @@ import { Redis } from "ioredis";
  * error in a couple of seconds with a clear message, not hang forever. The long-running WORKER
  * uses the default (persistent) options -- it should ride out a Valkey restart, not give up.
  */
-export function parseConnection(url, { failFast = false } = {}) {
+export function parseConnection(url, { failFast = false, servername = null, context = null, judge = null, withoutPassword = false } = {}) {
 	const u = new URL(url);
+	// PR #478's gate: a path that is not a database number was `db: NaN`, and the SELECT ioredis then sent failed outside
+	// every caller's await (an unhandled rejection). Refused here, where every client of the project is made, as the
+	// configuration refusal it is; `judgeValkeyAtStart` says it first at each start.
+	const dbProblem = valkeyUrlProblem(url);
+	if (dbProblem) throw valkeyRefusal(dbProblem);
+	const where = context ?? defaultValkeyContext();
+	// Issue #468: the password this client sends, by `valkeyPasswordFor`'s rule (the URL's own, else VALKEY_PASSWORD from
+	// the environment, else from the deployment `.env` for a loopback host). Here and nowhere else, so every client of the
+	// project (the worker, the CLI, the receiver, the admin panel, doctor) authenticates the same way. The URL's userinfo
+	// is percent-decoded: it was handed over raw before, so a password with `@` written `%40` never matched.
+	const { password } = valkeyPasswordFor(url, where, { withoutPassword });
 	return {
-		host: u.hostname || "127.0.0.1",
+		// Issue #464 (gate round 2 follow-up): EVERY client made from these options connects through `JudgedConnector`,
+		// which judges and pins the address (valkey-endpoint.mjs) before each connect; `host` below is only what ioredis
+		// reports. `context` is where the client stands (`valkeyClientContext`), the process's own when not given.
+		Connector: JudgedConnector,
+		// `judge` is a test seam only: the function asked for the address (judgedEndpoint when not given).
+		[JUDGE]: { url, context: where, ...(judge ? { judge } : {}) },
+		// WHATWG URL keeps an IPv6 literal's brackets in `hostname` (`[::1]`), and net.connect then looks up the name
+		// "[::1]", which never resolves, so a `redis://[::1]:6379` VALKEY_URL never connected (issue #464, measured on
+		// Fedora 44: ETIMEDOUT). The address is the part inside them.
+		host: u.hostname.replace(/^\[(.*)\]$/, "$1") || "127.0.0.1",
 		port: Number(u.port || 6379),
-		...(u.password ? { password: u.password } : {}),
-		...(u.username ? { username: u.username } : {}),
+		...(password ? { password } : {}),
+		...(u.username && !withoutPassword ? { username: decodedUserinfo(u.username) } : {}),
 		...(u.pathname && u.pathname !== "/" ? { db: Number(u.pathname.slice(1)) } : {}),
+		// TLS for `rediss:` (issue #464, gate round 2). Host and port alone dropped it, so a `rediss:` VALKEY_URL reached
+		// BullMQ as plaintext. `servername` is the name a certificate is checked against when the worker connects to a
+		// pinned address rather than to that name (`pinnedValkeyUrl`).
+		// `valkeys:` is `rediss:`'s alias (`valkeySchemeOf`, gate round 3): TLS as well.
+		...(valkeySchemeOf(u.protocol) === "rediss:" ? { tls: servername ? { servername } : {} } : {}),
 		maxRetriesPerRequest: null, // required for BullMQ blocking connections
 		...(failFast
 			? {
@@ -34,7 +184,223 @@ export function parseConnection(url, { failFast = false } = {}) {
 	};
 }
 
-/** A raw ioredis client for the budget's INCR/EXPIRE. */
-export function makeRedisClient(url) {
-	return new Redis(url, { maxRetriesPerRequest: null });
+/**
+ * A raw ioredis client (the budget's INCR/EXPIRE, the registry and fleet reads, doctor's PING). Built from
+ * `parseConnection`'s options, so it connects through `JudgedConnector` like every other client. `lazyConnect` and
+ * `failFast` as ioredis and `parseConnection` read them.
+ */
+export function makeRedisClient(url, { servername = null, context = null, failFast = false, lazyConnect = false, judge = null, withoutPassword = false } = {}) {
+	return new Redis({ ...parseConnection(url, { failFast, servername, context, judge, withoutPassword }), ...(lazyConnect ? { lazyConnect: true } : {}) });
+}
+
+/**
+ * How the Valkey at `url` answers this client's credential (issue #468), through a fail-fast client built like every
+ * other: `{ state }`, one of
+ *   "ok"          PING answered PONG with what this client sends (a password or none);
+ *   "noauth"      it requires a password and this client sent none (NOAUTH);
+ *   "wrongpass"   it refused the password this client sent (WRONGPASS, or "invalid password");
+ *   "unreachable" anything else (nothing answers, a refused owner), with its `error` text.
+ * `withoutPassword` asks as a client that sends none would: "ok" then means this Valkey answers ANY local account.
+ * The password itself never leaves this function; the answer names only which state it is. Never throws.
+ */
+export async function valkeyAuthState(url, { context = null, servername = null, withoutPassword = false, timeoutMs = 5000, makeClient = makeRedisClient } = {}) {
+	const seen = [];
+	let client;
+	try {
+		client = makeClient(url, { failFast: true, lazyConnect: true, context, servername, withoutPassword });
+	} catch (err) {
+		return { state: "unreachable", error: err?.message ?? String(err) };
+	}
+	// ioredis reports an AUTH or ready-check refusal as an "error" event and then reconnects (bounded by failFast), while
+	// connect() itself may reject with only "Connection is closed.", so the events are kept and read with the reply.
+	client.on("error", (err) => seen.push(err?.message ?? String(err)));
+	let timer;
+	try {
+		const reply = await Promise.race([
+			client.connect().then(() => client.ping()),
+			new Promise((_, reject) => {
+				timer = setTimeout(() => reject(new Error(`no answer within ${timeoutMs} ms`)), timeoutMs);
+			}),
+		]);
+		return reply === "PONG" ? { state: "ok" } : { state: "unreachable", error: `PING answered ${String(reply).slice(0, 40)}` };
+	} catch (err) {
+		if (client[DB_REFUSED]) return { state: "dbrange", error: await dbRangeSentenceFor(url, client, { context, servername, makeClient, timeoutMs }) };
+		const all = [err?.message ?? String(err), ...seen].join("\n");
+		if (/\bWRONGPASS\b|invalid password/i.test(all)) return { state: "wrongpass" };
+		if (/\bNOAUTH\b/.test(all)) return { state: "noauth" };
+		return { state: "unreachable", error: err?.message ?? String(err) };
+	} finally {
+		clearTimeout(timer);
+		client.disconnect();
+	}
+}
+
+/**
+ * The refusal sentence for a client whose SELECT was refused, with the server's `databases` count read by a second
+ * client on database 0 (`CONFIG GET databases`; null where that is refused, a managed Valkey say). The second client
+ * only reads that setting and never touches a key.
+ */
+async function dbRangeSentenceFor(url, client, { context, servername, makeClient, timeoutMs }) {
+	let databases = null;
+	let probe;
+	let timer;
+	try {
+		const u = new URL(url);
+		u.pathname = "";
+		probe = makeClient(u.toString(), { failFast: true, lazyConnect: true, context, servername });
+		probe.on?.("error", () => {});
+		const reply = await Promise.race([
+			probe.connect().then(() => probe.config("GET", "databases")),
+			new Promise((_, reject) => {
+				timer = setTimeout(() => reject(new Error("timeout")), timeoutMs);
+			}),
+		]);
+		const n = Array.isArray(reply) ? Number(reply[1]) : NaN;
+		databases = Number.isInteger(n) && n > 0 ? n : null;
+	} catch {
+		// Unreadable: the sentence says so.
+	} finally {
+		clearTimeout(timer);
+		probe?.disconnect?.();
+	}
+	return valkeyDbRangeSentence(url, client.options?.db ?? client.condition?.select, databases);
+}
+
+/**
+ * `judgeValkeyAtStart` of valkey-endpoint.mjs (the owner judgement, retried while nothing answers), then this client's
+ * credential asked once (issue #468): a Valkey that refuses it is a refusal (a configError, exit 2, like a refused
+ * owner), since a restart sends the same credential, and names VALKEY_PASSWORD without its value. Nothing answering
+ * stays what it was: the command's own connect says it could not reach Valkey. `checkAuth` is a seam, the real check
+ * unless the caller injected its own `judge` (a test standing in for the host, which has no Valkey to ask).
+ */
+export async function judgeValkeyAtStart(url, context, opts = {}) {
+	const endpoint = await judgeEndpointAtStart(url, context, opts);
+	const checkAuth = opts.checkAuth !== undefined ? opts.checkAuth : opts.judge ? null : valkeyAuthState;
+	if (checkAuth) {
+		const where = context ?? defaultValkeyContext();
+		const { state, error } = await checkAuth(url, { context: where, servername: endpoint?.servername ?? null });
+		// Gate round 2 of PR #478: a database the server does not have is a refusal too, never database 0.
+		if (state === "dbrange") throw Object.assign(new Error(error), { piDispatchConfig: true, valkeyRefused: true, valkeyDbRange: true });
+		const refusal = state === "noauth" || state === "wrongpass" ? authRefusalFor(state, url, where) : null;
+		if (refusal) throw Object.assign(new Error(refusal), { piDispatchConfig: true, valkeyRefused: true, valkeyAuth: state });
+	}
+	return endpoint;
+}
+
+/** The refusal sentence for an auth state, naming where the password would come from (never its value). */
+export function authRefusalFor(state, url, context) {
+	const { password, from } = valkeyPasswordFor(url, context);
+	return valkeyAuthRefusal(new Error(state === "noauth" ? "NOAUTH" : "WRONGPASS"), { passwordSet: Boolean(password), from, envPath: context?.envPath ?? "the deployment's .env" });
+}
+
+/**
+ * The key under which a client's options carry what `JudgedConnector` judges: the URL as written and the context. A
+ * string, not a Symbol: ioredis merges its options with lodash `defaults`, which copies string keys only.
+ */
+const JUDGE = "piDispatchValkeyJudge";
+
+/**
+ * ioredis' connector for every client of this project (issue #464, gate round 2 follow-up): on each connect it asks
+ * `judgedEndpoint` for the address, then dials that literal (with a `rediss:` URL's name kept as the TLS servername),
+ * exactly as ioredis' own StandaloneConnector dials a host.
+ *
+ * A judgement that throws (nothing answers while the Valkey restarts, a refusal, a name that does not resolve) must NOT
+ * reject `connect()`: ioredis 5.11.1 then sets the client's status to "end" and never retries, so one Valkey restart
+ * killed every client for good (gate round 3, measured: "Connection is closed." on the next job, never retried). So
+ * `connect()` resolves a socket already destroyed, with the error as `firstError`: ioredis' own branch for a stream that
+ * failed before it was handed over, which reports that error and takes the close path, retrying by the client's
+ * strategy and judging again on the next connect. A client with a bounded strategy (the CLI's failFast) then ends with
+ * that error. Destroyed before it is returned, never on a later tick: a socket that is neither connecting nor destroyed
+ * when ioredis looks is taken as connected, and its ready check then fails on a dead stream (measured on the VMs:
+ * "Stream isn't writeable" printed beside every failed judgement).
+ */
+export class JudgedConnector extends AbstractConnector {
+	constructor(options) {
+		super(options.disconnectTimeout);
+		this.options = options;
+	}
+
+	/** ioredis' ready check: failed for a client whose SELECT was refused (`DB_REFUSED`), so it never goes ready. */
+	check() {
+		return !this[DB_REFUSED];
+	}
+
+	connect() {
+		const { options } = this;
+		// A database the server refused stays refused: no reconnect goes ready on another one.
+		if (this[DB_REFUSED]) return Promise.reject(this[DB_REFUSED]);
+		this.connecting = true;
+		const judge = options[JUDGE];
+		if (!judge) return Promise.reject(new Error("a Valkey client was built without parseConnection's judged options"));
+		return (judge.judge ?? judgedEndpoint)(judge.url, judge.context).then(
+			(endpoint) => {
+				if (!this.connecting) throw new Error("Connection is closed.");
+				const target = { host: endpoint.host, port: options.port, ...(options.family != null ? { family: options.family } : {}) };
+				if (options.tls) Object.assign(target, options.tls, endpoint.servername ? { servername: endpoint.servername } : {});
+				this.stream = options.tls ? JudgedConnector.dial.tls(target) : JudgedConnector.dial.net(target);
+				this.stream.once("error", (err) => {
+					this.firstError = err;
+				});
+				return this.stream;
+			},
+			(err) => {
+				if (!this.connecting) throw new Error("Connection is closed.");
+				const stream = new Socket();
+				this.stream = stream;
+				this.firstError = err;
+				// The destroyed socket's own "error" event comes on a later tick; ioredis reports firstError instead.
+				stream.on("error", () => {});
+				stream.destroy(err);
+				return stream;
+			},
+		);
+	}
+}
+
+/** How `JudgedConnector` dials, net and tls: a seam, so a test sees the options a pinned connect is made with. */
+JudgedConnector.dial = { net: netConnect, tls: tlsConnect };
+
+/**
+ * Whether `connection` (options for BullMQ, or an ioredis client) connects through `JudgedConnector`. `makeQueue` and the
+ * worker refuse any other, so no Queue or Worker of this project can be built on an unjudged connection.
+ */
+export function isJudgedConnection(connection) {
+	const options = connection?.options ?? connection;
+	return options?.Connector === JudgedConnector && Boolean(options?.[JUDGE]);
+}
+
+/** Throws unless `connection` is judged (`isJudgedConnection`). */
+export function assertJudgedConnection(connection) {
+	if (!isJudgedConnection(connection)) throw new TypeError("a Valkey connection must be built by parseConnection (connection.mjs), which judges and pins the address it dials (issue #464)");
+}
+
+/**
+ * Record, then read back, which deployment folder the queue in a Valkey belongs to (PR #475's review, the volume gap):
+ * `SET pi-dispatch:owner <folder> NX`, then `GET`. Docker volume labels cannot be added after creation, so a volume made
+ * before the label (adopted with consent) carries its owner here, inside the data it describes, and every later start
+ * on it checks this key. Retried for `waitMs` while a Valkey that was just started comes up. `{ owner, claimed }` where
+ * `owner` is what the key holds (this folder, or another one's), or `{ error }` when it could not be read.
+ */
+export async function claimValkeyOwner(url, folder, { context = null, waitMs = 15000, makeClient = makeRedisClient, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) } = {}) {
+	const { OWNER_MARKER_KEY } = await import("./valkey-auth.mjs");
+	const until = Date.now() + waitMs;
+	let last = "no attempt";
+	for (;;) {
+		let client;
+		try {
+			client = makeClient(url, { failFast: true, lazyConnect: true, context });
+			client.on("error", () => {});
+			await client.connect();
+			const set = await client.set(OWNER_MARKER_KEY, folder, "NX");
+			const owner = await client.get(OWNER_MARKER_KEY);
+			return { owner, claimed: set === "OK" };
+		} catch (err) {
+			last = err?.message ?? String(err);
+			// NOAUTH and WRONGPASS will not change by waiting.
+			if (/\bNOAUTH\b|\bWRONGPASS\b/.test(last) || Date.now() >= until) return { error: last };
+		} finally {
+			client?.disconnect();
+		}
+		await sleep(500);
+	}
 }

package/src/container-spec.mjs

@@ -45,6 +45,59 @@ export const CONTAINER_GLOBAL_PI_DIR = "/opt/pi-global";
 export const CONTAINER_SESSION_DIR = "/session";
 export const CONTAINER_SESSION_FILE = `${CONTAINER_SESSION_DIR}/current.jsonl`;
 
+/**
+ * The job image's home, and the uid its `USER` directive runs as (issue #341, `DES-JOB-USER-INFERRED-READ-BACK-ON-
+ * REQUEST`). A job run under `--user` needs `HOME=CONTAINER_HOME` beside it, because such a uid has no passwd entry in
+ * the image: Docker gives it `HOME=/`, Podman `HOME=/workspace` (both measured). The builder does not pair them; a job
+ * or sandbox path that passes a `user` must. `SHIPPED_IMAGE_UID` is the one uid that needs no `--user` at all, since
+ * the image already runs as it; a test pins it against `image/Dockerfile`'s `useradd` so the two cannot drift.
+ */
+export const CONTAINER_HOME = "/home/pi";
+export const SHIPPED_IMAGE_UID = 1001;
+
+// "<uid>:<gid>", both non-zero decimal. A bare uid is refused because docker then takes the group from the image's
+// passwd, which is gid 0 for a uid it has no entry for; a name is refused because the image decides what it means.
+const JOB_USER_RE = /^[1-9]\d{0,9}:[1-9]\d{0,9}$/;
+
+/**
+ * The host path `docker run` writes the container's ID to (issue #345), or `null`. Absolute (a POSIX path, or a Windows
+ * drive path for a worker on Windows), no `..` segment and no control byte, because it rides a single
+ * `--cidfile=<path>` token (the CLI splits on the first `=`) and names a file the worker later reads and removes. The worker puts it BESIDE the job
+ * directory, never inside the `/job:ro` mount.
+ */
+export function assertCidFile(cidFile) {
+	if (cidFile === null || cidFile === undefined) return;
+	const absolute = typeof cidFile === "string" && (cidFile.startsWith("/") || /^[A-Za-z]:[\\/]/.test(cidFile));
+	if (!absolute || cidFile.split(/[\\/]/).includes("..") || /[\u0000-\u001f\u007f]/.test(cidFile)) {
+		throw new Error(`docker run: refusing a cidfile that is not a plain absolute path: ${JSON.stringify(cidFile)}`);
+	}
+}
+
+/** Throws unless `user` is null/undefined or a non-root "<uid>:<gid>" (`nonRoot`, issue #341). */
+export function assertJobUser(user) {
+	if (user === null || user === undefined) return;
+	if (typeof user !== "string" || !JOB_USER_RE.test(user)) {
+		throw new Error(`docker run: refusing a job user that is not "<uid>:<gid>" with both parts non-zero: ${JSON.stringify(user)}`);
+	}
+}
+
+/**
+ * The user-namespace modes a spec may ask for (issue #354): `null`, the runtime's default, or `"keep-id"`, which maps the
+ * worker's own uid and gid to the SAME ids inside a rootless Podman container, so a job run as the worker's own
+ * "<uid>:<gid>" owns its bind mounts exactly as the worker does. A CLOSED set, and refused rather than passed through,
+ * because the value reaches an argv as one token and every other spelling Podman accepts (`host`, `auto`, `nomap`, a
+ * path to another process's namespace) changes which uids a job can reach on the host.
+ */
+export const USERNS_MODES = Object.freeze(["keep-id"]);
+
+/** Throws unless `userns` is null/undefined or a member of `USERNS_MODES` (issue #354). */
+export function assertUserns(userns) {
+	if (userns === null || userns === undefined) return;
+	if (typeof userns !== "string" || !USERNS_MODES.includes(userns)) {
+		throw new Error(`container spec: refusing a userns other than null or ${USERNS_MODES.map((m) => JSON.stringify(m)).join(", ")}: ${JSON.stringify(userns)}`);
+	}
+}
+
 /**
  * WHAT the box is, with no Docker vocabulary in it.
  *
@@ -57,8 +110,8 @@ export const CONTAINER_SESSION_FILE = `${CONTAINER_SESSION_DIR}/current.jsonl`;
  * strings, because the flattening IS the Docker part: a runtime that does not bind-mount has to be able to
  * see which host path becomes which container path, and what may be written.
  *
- * `dockerExtra` is named for what it is. It carries raw Docker flags (`-i -t --entrypoint bash`, a
- * Linux-only `--user`), so it is the one field a non-Docker consumer must refuse rather than translate.
+ * `dockerExtra` is named for what it is. It carries raw Docker flags (`-i -t --entrypoint bash`), so it is the one
+ * field a non-Docker consumer must refuse rather than translate. The job user is NOT in it: it is `user`, a field.
  * Calling it `extraFlags` at the boundary would have hidden that.
  *
  * @param image      pinned job image tag/digest
@@ -73,7 +126,18 @@ export const CONTAINER_SESSION_FILE = `${CONTAINER_SESSION_DIR}/current.jsonl`;
  * @param memory     e.g. "4g"; cpus e.g. "2"
  * @param network    the per-job egress network this container joins (REQ-EGRESS-ALLOWLIST); null = the
  *                   docker default bridge, which is what every job did before that requirement existed
- * @param extraFlags escape hatch for a Linux-only --user uid:gid on a bind-mounted local folder
+ * @param user       "<uid>:<gid>" the job runs as (issue #341), or null for the image's own USER. Portable: it says WHO
+ *                   runs the box, which a non-docker runtime must honour too.
+ * @param userns     null (the runtime's own default) or "keep-id" (issue #354): how the job user's ids map to the host's.
+ *                   Portable in meaning, but only one runtime spells it: `dockerArgsFromSpec` REFUSES a non-null one,
+ *                   because the docker CLI rejects `--userns=keep-id` client-side (exit 125, measured in issue #345), and
+ *                   dropping it instead would run the job under a uid mapping nobody asked for.
+ * @param extraFlags raw docker flags for the few callers that need them (the sandbox's -i -t --entrypoint bash)
+ * @param relabel    true where the daemon confines containers with SELinux (`relabelsPrivateMounts`, issue #355): the
+ *                   worker's own per-job mounts then carry `relabel: "private"`. Portable: it says WHICH host paths the
+ *                   runtime may re-own for this one container, which a non-docker runtime must honour or refuse.
+ * @param workspaceOwned true when `workspace` is the worker's own per-job directory (a forge job's clone, a sandbox's
+ *                   retained copy), false when it is the operator's folder. Read only when `relabel` is true.
  */
 export function containerSpec({
 	image,
@@ -87,21 +151,41 @@ export function containerSpec({
 	memory = "4g",
 	cpus = "2",
 	network = null,
+	user = null,
+	userns = null,
+	cidFile = null,
 	extraFlags = [],
+	relabel = false,
+	workspaceOwned = false,
 }) {
 	if (!image) throw new Error("docker run: image is required");
+	assertJobUser(user);
+	assertUserns(userns);
+	assertCidFile(cidFile);
 	if (!name) throw new Error("docker run: container name is required");
 	if (!workspace) throw new Error("docker run: workspace mount is required");
+	// Booleans, strictly: a truthy string from a caller that forwarded an option bag must not re-own host directories.
+	if (typeof relabel !== "boolean") throw new Error(`docker run: relabel must be a boolean; got ${typeof relabel}`);
+	if (typeof workspaceOwned !== "boolean") throw new Error(`docker run: workspaceOwned must be a boolean; got ${typeof workspaceOwned}`);
 
+	// Issue #355. A SPREAD rather than a `relabel: null` field, so a spec built without it has mount objects byte-identical
+	// to every one built before this feature existed. PRIVATE, never shared: each of these directories is made for this
+	// one container, and the shared form (`:z`) would leave it readable by every other container on the host, which is
+	// the job-to-job reach the per-job directory exists to deny.
+	const own = relabel ? { relabel: "private" } : {};
 	const mounts = [];
 	// The WHOLE /job dir is read-only (INT-CONTAINER-JOB-INPUTS): it holds prompt.md and pi/, and
 	// the agent cannot rewrite any of it. /workspace is the only writable mount.
-	if (jobDir) mounts.push({ host: jobDir, container: "/job", readOnly: true });
-	mounts.push({ host: workspace, container: "/workspace", readOnly: false });
+	if (jobDir) mounts.push({ host: jobDir, container: "/job", readOnly: true, ...own });
+	// A local job's /workspace IS the operator's folder, and `:Z` would re-own it for this one container: measured, a
+	// second container mounting it afterwards is denied, and the label an operator chose is gone. So only a workspace the
+	// worker made is relabelled; an operator's folder needs the `semanage fcontext` fix doctor names, and the runner
+	// refuses one it cannot read before any spend.
+	mounts.push({ host: workspace, container: "/workspace", readOnly: false, ...(workspaceOwned ? own : {}) });
 	// Local jobs get a writable /outbox host bind, the same host-bind mechanism as /workspace
 	// (DES-WORKER-ON-HOST). github jobs pass no outboxDir, so the request channel does not exist for
 	// them -- an untrusted issue author cannot chain (INT-OUTBOX-CONTRACT).
-	if (outboxDir) mounts.push({ host: outboxDir, container: "/outbox", readOnly: false });
+	if (outboxDir) mounts.push({ host: outboxDir, container: "/outbox", readOnly: false, ...own });
 
 	// This job's OWN copy of its session transcript (REQ-RESUMABLE-SESSION, INT-SESSION-STORE-CONTRACT).
 	// Writable, because pi appends to it as the agent works -- and per-job, exactly like jobDir, which is
@@ -110,11 +194,14 @@ export function containerSpec({
 	// read and rewrite every other branch's and every other repository's transcripts, which is not a
 	// weakening of that constraint but its inversion. Absent unless the trigger armed run.resume AND a key
 	// resolved, so an unarmed job's argv is byte-identical to one built before this feature existed.
-	if (sessionDir) mounts.push({ host: sessionDir, container: CONTAINER_SESSION_DIR, readOnly: false });
+	if (sessionDir) mounts.push({ host: sessionDir, container: CONTAINER_SESSION_DIR, readOnly: false, ...own });
 
 	// The operator's global pi overlay (REQ-GLOBAL-PI-OVERLAY): custom models, global skills, a global
 	// persona, layered UNDER each repo's own .pi/. Read-only -- it is operator-authored deploy-time config,
 	// the same trust class as the baked floor, but the agent still must not rewrite it. Both job kinds.
+	// NEVER relabelled (issue #355): every job mounts this one directory, so a private label would lock each job out of
+	// it in turn, and a shared one would re-own an operator's directory on every run. It needs the operator's
+	// `semanage fcontext` fix, which doctor names.
 	if (globalPiDir) mounts.push({ host: globalPiDir, container: CONTAINER_GLOBAL_PI_DIR, readOnly: true });
 
 	return {
@@ -123,6 +210,14 @@ export function containerSpec({
 		memory,
 		cpus,
 		network,
+		user,
+		// Issue #354. Always present and `null` by default (the parameter default turns an `undefined` into it, so a spec
+		// has ONE spelling of "the runtime's default"), beside `user` because it qualifies it: the same "<uid>:<gid>" names
+		// a different host identity under a different mapping.
+		userns,
+		// Issue #345: where the runtime writes the container ID once it creates one, so a CLI that exits as if nothing
+		// started (a lost API connection) can be told apart from a container that runs on without it. null = absent.
+		cidFile,
 		// UNCONDITIONALLY true, and there is deliberately no parameter that can unset it. The boundary is
 		// not a thing a caller opts into -- CONST-ISOLATION-CONTAINER-PER-JOB is why every other flag here
 		// exists -- so the spec is simply unable to describe an unisolated container, and the builder in