@edgehero/pi-dispatch 4.3.1 → 5.1.0

package/src/sandbox-sessions.mjs

@@ -0,0 +1,161 @@
+import { createHash, randomBytes } from "node:crypto";
+import { WORKER_NAME_RE } from "./worker-name.mjs";
+
+/** Operator-private observations, never authority over a job or a reusable container name (#771). */
+export const SANDBOX_SESSION_TTL_MS = 45_000;
+export const SANDBOX_SESSION_POLL_MS = 1_000;
+export const SANDBOX_SESSION_MAX = 256;
+export const SANDBOX_SESSION_INDEX = "sandbox:shells";
+export const SANDBOX_NOTICE_INDEX = "sandbox:notices";
+export const SANDBOX_NOTICE_TTL_MS = 86_400_000;
+export const SANDBOX_SESSION_LABEL = "org.pi-dispatch.sandbox-session";
+export const SANDBOX_SESSION_ID_RE = /^[A-Za-z0-9._:-]{1,200}$/;
+const nonceRe = /^[a-f0-9]{32}$/;
+const containerRe = /^[a-f0-9]{64}$/;
+const memberRe = /^[A-Za-z0-9._-]{1,64}:[a-f0-9]{64}$/;
+const instant = (v) => Number.isSafeInteger(v) && v >= 0 && v <= 8_640_000_000_000_000;
+const digest = (id) => createHash("sha256").update(id).digest("hex");
+export const sandboxSessionMember = (host, id) => `${host}:${digest(id)}`;
+export const sandboxSessionKey = (host, id) => `sandbox:shell:${sandboxSessionMember(host, id)}`;
+export const sandboxNoticeKey = (host, id) => `sandbox:notice:${sandboxSessionMember(host, id)}`;
+export const sandboxEndKey = (nonce) => `sandbox:end:${nonce}`;
+export const sandboxEndAckKey = (nonce, requestId) => `sandbox:ended:${nonce}:${requestId}`;
+
+export function parseSandboxSession(raw) {
+	let s;
+	try { s = typeof raw === "string" && raw.length <= 4096 ? JSON.parse(raw) : raw; } catch { return null; }
+	if (!s || ["host", "id", "nonce", "containerId", "venue", "by"].some((k) => typeof s[k] !== "string") || s.v !== 1 || !WORKER_NAME_RE.test(s.host) || !SANDBOX_SESSION_ID_RE.test(s.id) || !nonceRe.test(s.nonce) || !containerRe.test(s.containerId) || !["local", "podman"].includes(s.venue) || !/^[A-Za-z0-9._@-]{1,64}$/.test(s.by) || !instant(s.since) || !instant(s.at) || s.at < s.since) return null;
+	return { v: 1, host: s.host, id: s.id, nonce: s.nonce, containerId: s.containerId, venue: s.venue, by: s.by, since: s.since, at: s.at };
+}
+
+export function sandboxSessionAccount(name) {
+	return typeof name === "string" && /^[A-Za-z0-9._@-]{1,64}$/.test(name) ? name : "operator";
+}
+
+export function sandboxBounded(promise, ms = 2_000) {
+	return new Promise((resolve, reject) => {
+		const timer = setTimeout(() => reject(new Error("sandbox control timed out")), ms);
+		Promise.resolve(promise).then((v) => { clearTimeout(timer); resolve(v); }, (e) => { clearTimeout(timer); reject(e); });
+	});
+}
+
+// A delayed renew/delete must never replace or erase the next shell's lease.
+const RENEW = `local raw=redis.call('GET',KEYS[1]); if raw then local ok,s=pcall(cjson.decode,raw); if not ok or s.nonce~=ARGV[1] or s.containerId~=ARGV[2] then return 0 end end; redis.call('SET',KEYS[1],ARGV[3],'PX',ARGV[4]); local t=redis.call('TIME'); local at=t[1]*1000+math.floor(t[2]/1000); redis.call('ZADD',KEYS[2],at+tonumber(ARGV[4]),ARGV[5]); redis.call('ZREMRANGEBYSCORE',KEYS[2],'-inf',at-tonumber(ARGV[4])); return 1`;
+const RELEASE = `local raw=redis.call('GET',KEYS[1]); if not raw then return 0 end; local ok,s=pcall(cjson.decode,raw); if not ok or s.nonce~=ARGV[1] or s.containerId~=ARGV[2] then return 0 end; return redis.call('DEL',KEYS[1])`;
+const REMOVE_REQUEST = `if redis.call('GET',KEYS[1])==ARGV[1] then return redis.call('DEL',KEYS[1]) end; return 0`;
+
+export async function renewSandboxSession(redis, session, { now = Date.now, ttlMs = SANDBOX_SESSION_TTL_MS, timeoutMs = 2_000 } = {}) {
+	const value = parseSandboxSession({ ...session, v: 1, at: now() });
+	if (!value) throw new Error("invalid sandbox session");
+	const owned = await sandboxBounded(redis.eval(RENEW, 2, sandboxSessionKey(value.host, value.id), SANDBOX_SESSION_INDEX, value.nonce, value.containerId, JSON.stringify(value), String(ttlMs), sandboxSessionMember(value.host, value.id)), timeoutMs);
+	if (owned !== 1) return false;
+	return true;
+}
+
+async function redisNow(redis, timeoutMs = 2_000) {
+	const value = await sandboxBounded(redis.time(), timeoutMs);
+	if (!Array.isArray(value) || value.length !== 2 || !/^\d+$/.test(value[0]) || !/^\d+$/.test(value[1])) throw new Error("the sandbox lease clock is unreadable");
+	const at = Number(value[0]) * 1000 + Math.floor(Number(value[1]) / 1000);
+	if (!instant(at)) throw new Error("the sandbox lease clock is invalid");
+	return at;
+}
+
+export async function releaseSandboxSession(redis, session, { timeoutMs = 2_000 } = {}) {
+	return sandboxBounded(redis.eval(RELEASE, 1, sandboxSessionKey(session.host, session.id), session.nonce, session.containerId), timeoutMs);
+}
+
+export async function publishSandboxNotice(redis, { host, id, reason }, { now = Date.now } = {}) {
+	if (!WORKER_NAME_RE.test(host ?? "") || !SANDBOX_SESSION_ID_RE.test(id ?? "") || !["network-kept", "network-cleanup-failed", "runtime-unreadable"].includes(reason)) throw new Error("invalid sandbox notice");
+	const at = now();
+	const serverAt = await redisNow(redis);
+	await sandboxBounded(redis.set(sandboxNoticeKey(host, id), JSON.stringify({ host, id, at, reason }), "PX", SANDBOX_NOTICE_TTL_MS));
+	await sandboxBounded(redis.zadd(SANDBOX_NOTICE_INDEX, serverAt + SANDBOX_NOTICE_TTL_MS, sandboxSessionMember(host, id)));
+	await sandboxBounded(redis.zremrangebyscore(SANDBOX_NOTICE_INDEX, "-inf", serverAt));
+}
+
+async function readNotices(redis, { now, timeoutMs, hosts }) {
+	const members = await sandboxBounded(redis.zrangebyscore(SANDBOX_NOTICE_INDEX, now(), "+inf", "LIMIT", 0, SANDBOX_SESSION_MAX + 1), timeoutMs);
+	if (!Array.isArray(members)) throw new Error("unreadable notices");
+	const wanted = members.slice(0, SANDBOX_SESSION_MAX).filter((m) => memberRe.test(m) && (!hosts.length || hosts.some((h) => m.startsWith(`${h}:`))));
+	const values = wanted.length ? await sandboxBounded(redis.mget(...wanted.map((m) => `sandbox:notice:${m}`)), timeoutMs) : [];
+	if (!Array.isArray(values)) throw new Error("unreadable sandbox notice values");
+	let unreadable = values.length !== wanted.length;
+	const notices = values.flatMap((raw, i) => {
+		try {
+			if (raw === null) return [];
+			if (typeof raw !== "string" || raw.length > 1024) { unreadable = true; return []; }
+			const n = JSON.parse(raw);
+			if (typeof n?.host !== "string" || !WORKER_NAME_RE.test(n.host) || typeof n.id !== "string" || !SANDBOX_SESSION_ID_RE.test(n.id) || !instant(n.at) || !["network-kept", "network-cleanup-failed", "runtime-unreadable"].includes(n.reason) || sandboxSessionMember(n.host, n.id) !== wanted[i]) { unreadable = true; return []; }
+			return [{ host: n.host, id: n.id, at: n.at, reason: n.reason }];
+		} catch { unreadable = true; return []; }
+	});
+	return { notices, partial: members.length > SANDBOX_SESSION_MAX || unreadable };
+}
+
+/** The console reads only these bounded keys; an unreachable read is never an invented empty list. */
+export async function readSandboxSessions(redis, { hosts = [], now = Date.now, timeoutMs = 2_000 } = {}) {
+	try {
+		const serverAt = await redisNow(redis, timeoutMs);
+		const raw = await sandboxBounded(redis.zrangebyscore(SANDBOX_SESSION_INDEX, serverAt, "+inf", "LIMIT", 0, SANDBOX_SESSION_MAX + 1), timeoutMs);
+		if (!Array.isArray(raw)) throw new Error("unreadable index");
+		const members = [...new Set(raw.filter((m) => typeof m === "string" && memberRe.test(m) && (!hosts.length || hosts.some((h) => m.startsWith(`${h}:`)))))].sort();
+		const wanted = members.slice(0, SANDBOX_SESSION_MAX);
+		const values = wanted.length ? await sandboxBounded(redis.mget(...wanted.map((m) => `sandbox:shell:${m}`)), timeoutMs) : [];
+		if (!Array.isArray(values) || values.length !== wanted.length) throw new Error("unreadable leases");
+		const sessions = [];
+		let unreadable = 0;
+		for (const [i, value] of values.entries()) {
+			if (value === null) continue;
+			const s = parseSandboxSession(value);
+			if (!s || sandboxSessionMember(s.host, s.id) !== wanted[i]) { unreadable++; continue; }
+			// Key existence/TTL is the lease. Host and console clocks can disagree by more than its lifetime.
+			sessions.push(s);
+		}
+		let notices = [], noticesUnavailable = false;
+		try { const read = await readNotices(redis, { now: () => serverAt, timeoutMs, hosts }); notices = read.notices; noticesUnavailable = read.partial; } catch { noticesUnavailable = true; }
+		return { sessions: sessions.sort((a, b) => a.since - b.since), more: Math.max(0, raw.length - SANDBOX_SESSION_MAX, members.length - wanted.length), unreadable, notices, ...(noticesUnavailable ? { noticesUnavailable: true } : {}) };
+	} catch { return { sessions: [], unreachable: true }; }
+}
+
+/** End is an attended request for ONE immutable container, never a name that can be reused. */
+export async function requestSandboxEnd({ redis, host, id, nonce, containerId, now = Date.now, ackTimeoutMs = 10_000, pollMs = 250, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), timeoutMs = 2_000, random = () => randomBytes(16).toString("hex") }) {
+	if ([host, id, nonce, containerId].some((v) => typeof v !== "string") || !WORKER_NAME_RE.test(host) || !SANDBOX_SESSION_ID_RE.test(id) || !nonceRe.test(nonce) || !containerRe.test(containerId)) return { refused: "invalid-session", message: "The sandbox session is invalid; reload its current state." };
+	const current = parseSandboxSession(await sandboxBounded(redis.get(sandboxSessionKey(host, id)), timeoutMs));
+	if (!current || current.nonce !== nonce || current.containerId !== containerId) return { refused: "session-changed", message: "The sandbox session ended or changed; reload before asking to end it." };
+	const requestId = random();
+	if (!nonceRe.test(requestId)) throw new Error("invalid sandbox End request identity");
+	const value = JSON.stringify({ host, id, nonce, containerId, requestId });
+	await sandboxBounded(redis.set(sandboxEndKey(nonce), value, "PX", Math.max(ackTimeoutMs + 5_000, 15_000)), timeoutMs);
+	for (let elapsed = 0; ; elapsed += pollMs) {
+		const ack = await sandboxBounded(redis.get(sandboxEndAckKey(nonce, requestId)), timeoutMs);
+		if (ack === "ended") return { ended: true };
+		if (ack === "stop-failed") return { refused: "stop-failed", message: "The runtime did not confirm this sandbox stopped. Check its terminal and reload." };
+		if (elapsed + pollMs > ackTimeoutMs) break;
+		await sleep(pollMs);
+	}
+	await sandboxBounded(redis.eval(REMOVE_REQUEST, 1, sandboxEndKey(nonce), value), timeoutMs).catch(() => {});
+	return { timeout: true };
+}
+
+/** One monitor tick, injected so identity races, outages and stop failures are exercised without a daemon. */
+export async function sandboxSessionTick({ redis, session, inspect, stop, now = Date.now }) {
+	const state = await inspect(session.containerId);
+	if (state.unknown) return { unknown: true };
+	if (!state.present) {
+		await releaseSandboxSession(redis, session);
+		return { gone: true };
+	}
+	if (state.containerId !== session.containerId || state.nonce !== session.nonce) return { refused: "container-changed" };
+	if (!await renewSandboxSession(redis, session, { now })) return { refused: "lease-changed" };
+	const raw = await sandboxBounded(redis.get(sandboxEndKey(session.nonce)));
+	let request;
+	try { request = JSON.parse(raw); } catch { return { live: true }; }
+	if (!request || !nonceRe.test(request.requestId ?? "") || ["host", "id", "nonce", "containerId"].some((k) => request[k] !== session[k])) return { live: true };
+	const stopped = await stop(session.containerId);
+	const after = await inspect(session.containerId);
+	const ended = stopped && !after.unknown && !after.present;
+	await sandboxBounded(redis.set(sandboxEndAckKey(session.nonce, request.requestId), ended ? "ended" : "stop-failed", "PX", 60_000));
+	await sandboxBounded(redis.eval(REMOVE_REQUEST, 1, sandboxEndKey(session.nonce), raw));
+	if (ended) { await releaseSandboxSession(redis, session); return { gone: true, ended: true }; }
+	return { live: true, stopFailed: true };
+}

package/src/sandbox-store.mjs

@@ -3,6 +3,7 @@ import { isAbsolute, join, relative } from "node:path";
 import { sanitizeJobId } from "./run-history.mjs";
 import { scrubCredentials } from "./redact.mjs";
 import { TRANSIENT_READ_ERRORS } from "./runtime-observations.mjs";
+import { neededVersion } from "./store-version.mjs";
 
 /**
  * sandbox-store.mjs -- the host side of a resurrectable sandbox (REQ-RESURRECTABLE-SANDBOX,
@@ -37,6 +38,53 @@ import { TRANSIENT_READ_ERRORS } from "./runtime-observations.mjs";
 /** The manifest filename inside a retained directory. */
 export const SANDBOX_MANIFEST = "manifest.json";
 
+/**
+ * The highest manifest version this build reads (issue #761, `DES-STORE-VERSION-WHEN-NEEDED`): a top-level `version`,
+ * absent meaning 1, written only when the manifest uses a field that needs more, so every manifest of today's fields is
+ * the shape it always was. Defined here, beside the writers (`retainJobDir`, `pinSandbox`, the preparer), and read by
+ * `sandbox-policy.mjs`'s refusal, which already imports this module (the reverse would be a cycle).
+ *
+ * THE DEADLINE FIELDS NEVER CHANGE MEANING: `createdAt`, `retainUntil` and `keepUntil` keep theirs in every version, so
+ * an older worker's retention sweep (after a rollback, or on a host a newer CLI pinned) still deletes a newer run on
+ * time. That is why the sweep never consults the version, while opening and pinning refuse a newer manifest.
+ */
+export const SANDBOX_MANIFEST_VERSION = 1;
+
+/** Each manifest field whose absence an older build would misread, with the version that introduced it. Empty until one does. */
+export const SANDBOX_MANIFEST_FIELD_VERSIONS = Object.freeze({});
+
+/** The lowest version `manifest` needs from its fields (`version` and the reader-added `dir` aside): 1 while none needs more. */
+export function sandboxManifestVersionFor(manifest, { table = SANDBOX_MANIFEST_FIELD_VERSIONS } = {}) {
+	if (manifest === null || typeof manifest !== "object") return 1;
+	return neededVersion(Object.keys(manifest).filter((key) => key !== "version" && key !== "dir"), table);
+}
+
+/**
+ * `manifest` with `version` set to what it needs: the higher of the version it carries (a manifest a newer build
+ * wrote, rewritten here by a pin) and the one its fields need, in its place when it had one and first when added, and
+ * no key at all while that is 1. So a manifest of today's fields is returned unchanged, byte for byte once serialized.
+ */
+export function withManifestVersion(manifest, { table = SANDBOX_MANIFEST_FIELD_VERSIONS } = {}) {
+	const declared = Number.isSafeInteger(manifest?.version) && manifest.version >= 1 ? manifest.version : 1;
+	const needed = Math.max(declared, sandboxManifestVersionFor(manifest, { table }));
+	const had = Object.hasOwn(manifest ?? {}, "version");
+	if (needed <= 1 && !had) return manifest;
+	const { version: _owned, ...rest } = manifest;
+	if (needed <= 1) return rest;
+	return had ? { ...manifest, version: needed } : { version: needed, ...rest };
+}
+
+/**
+ * Whether this build cannot read `manifest`'s version: `{ version }` when `version` is present and is not an integer
+ * from 1 to `supported` (a newer build's manifest, version N; or a value nothing here wrote, version null), else null.
+ */
+export function sandboxManifestNewer(manifest, { supported = SANDBOX_MANIFEST_VERSION } = {}) {
+	if (manifest === null || typeof manifest !== "object" || !Object.hasOwn(manifest, "version")) return null;
+	const declared = manifest.version;
+	if (Number.isSafeInteger(declared) && declared >= 1 && declared <= supported) return null;
+	return { version: Number.isSafeInteger(declared) && declared >= 1 ? declared : null };
+}
+
 const HOUR_MS = 3600000;
 const DAY_MS = 86400000;
 
@@ -152,6 +200,9 @@ export function retainJobDir(prepared, { sandboxDir, retentionHours = null, fs =
 		// with a different, operator-extendable lifetime would silently extend that TTL, which is not a
 		// weakening of the session policy so much as an end-run around it.
 		fs.rmSync(join(jobDir, "session"), { recursive: true, force: true });
+		// Issue #796: a canary's candidate is third-party code that was on trial for that one job. Reopening the retained directory loads neither it nor the overlay, so it would only
+		// sit there looking as if it were part of the run's inputs.
+		fs.rmSync(join(jobDir, "canary"), { recursive: true, force: true });
 
 		fs.mkdirSync(sandboxDir, { recursive: true, mode: 0o700 });
 		// Issue #464 (gate round 1): this account's, asked again here and not only at boot. A sandbox dir absent at boot is
@@ -167,8 +218,10 @@ export function retainJobDir(prepared, { sandboxDir, retentionHours = null, fs =
 		fs.rmSync(dest, { recursive: true, force: true });
 		fs.renameSync(jobDir, dest);
 
-		const manifest = {
+		// Issue #761: `withManifestVersion` stamps a version only when a field needs one, so today's manifest is unchanged.
+		const manifest = withManifestVersion({
 			jobId: meta.jobId,
+			...(typeof prepared.sha === "string" && /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(prepared.sha) ? { baseSha: prepared.sha } : {}),
 			kind: meta.kind ?? null,
 			image: meta.image ?? null,
 			// The venue the job resolved to (#277), which `resolveSandbox` refuses by when it is not this
@@ -196,7 +249,7 @@ export function retainJobDir(prepared, { sandboxDir, retentionHours = null, fs =
 			// reader's window alone, as every manifest from before the key does.
 			retainUntil: Number.isFinite(retentionHours) && retentionHours > 0 ? new Date(created + retentionHours * HOUR_MS).toISOString() : null,
 			keepUntil: null,
-		};
+		});
 		fs.writeFileSync(join(dest, SANDBOX_MANIFEST), `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 });
 		log("sandbox_retained", { jobId: meta.jobId, kind: manifest.kind });
 		return manifest;
@@ -267,12 +320,13 @@ export function readManifest({ sandboxDir, jobId, fs = defaultFs }) {
  * An entry with no readable manifest is skipped rather than surfaced: it cannot be resurrected, and the
  * reaper removes it on the next sweep.
  */
-export function listSandboxes({ sandboxDir, fs = defaultFs }) {
+export function listSandboxes({ sandboxDir, fs = defaultFs, throwOnReadFailure = false }) {
 	if (!sandboxDir) return [];
 	let names;
 	try {
 		names = fs.readdirSync(sandboxDir);
-	} catch {
+	} catch (error) {
+		if (throwOnReadFailure) throw error;
 		return [];
 	}
 	const out = [];
@@ -297,9 +351,12 @@ export function listSandboxes({ sandboxDir, fs = defaultFs }) {
  * is how a directory holding a full repository clone per run becomes unbounded, and the acceptance this
  * feature was written against says retention stays swept.
  */
-export function pinSandbox({ sandboxDir, jobId, pinDays, fs = defaultFs, now = () => Date.now(), euid = process.geteuid?.() }) {
+export function pinSandbox({ sandboxDir, jobId, pinDays, fs = defaultFs, now = () => Date.now(), euid = process.geteuid?.(), supported = SANDBOX_MANIFEST_VERSION }) {
 	const manifest = readManifest({ sandboxDir, jobId, fs });
 	if (!manifest) return { pinned: false, reason: "absent" };
+	// Issue #761: refused BEFORE the rewrite. This build would write back a manifest it cannot read, under a stamp it
+	// cannot vouch for; the release that kept the run pins it. (The sweep still honours the deadline either way.)
+	if (sandboxManifestNewer(manifest, { supported })) return { pinned: false, reason: "manifest-newer" };
 	const keepUntil = new Date(now() + pinDays * DAY_MS).toISOString();
 	const { dir, ...body } = manifest;
 	// ATOMIC (issue #429 review): a temp file beside it, then a rename over it. `writeFileSync` on the manifest itself
@@ -319,7 +376,7 @@ export function pinSandbox({ sandboxDir, jobId, pinDays, fs = defaultFs, now = (
 	try {
 		const original = fs.lstatSync(target);
 		const mode = Number.isInteger(original?.mode) ? original.mode & 0o777 : 0o600;
-		fs.writeFileSync(tmp, `${JSON.stringify({ ...body, keepUntil }, null, 2)}\n`, { mode, flag: "wx" });
+		fs.writeFileSync(tmp, `${JSON.stringify(withManifestVersion({ ...body, keepUntil }), null, 2)}\n`, { mode, flag: "wx" });
 		created = true;
 		if (Number.isInteger(original?.uid) && Number.isInteger(original?.gid)) {
 			try {

package/src/sandbox-workspace-guard.mjs

@@ -0,0 +1,39 @@
+/** A retained workspace is human-only, including when an operator later points a local trigger at it (#771). */
+import * as nodeFs from "node:fs";
+import { dirname, resolve, sep } from "node:path";
+
+export const LOCAL_FOLDER_IS_SANDBOX = "local-folder-is-sandbox";
+const policy = () => ({ outcome: "policy", reason: LOCAL_FOLDER_IS_SANDBOX });
+const within = (path, root) => path === root || path.startsWith(root.endsWith(sep) ? root : `${root}${sep}`);
+const absent = (error) => error?.code === "ENOENT" || error?.code === "ENOTDIR";
+const identity = (stat) => stat && ((typeof stat.dev === "bigint" && typeof stat.ino === "bigint") || (Number.isSafeInteger(stat.dev) && Number.isSafeInteger(stat.ino))) ? `${stat.dev}:${stat.ino}` : null;
+
+/** Policy is returned; an unanswered filesystem read is separate so each caller can preserve infrastructure retry. */
+export function checkSandboxWorkspace(job, { sandboxDir, fs = nodeFs, resolvedFolder = undefined } = {}) {
+	if (job?.kind !== "local" || typeof job.folder !== "string" || typeof sandboxDir !== "string" || !sandboxDir) return null;
+	const named = resolve(job.folder), root = resolve(sandboxDir);
+	if (within(named, root)) return policy();
+	const realpath = fs.realpathSync?.native ?? fs.realpathSync;
+	if (typeof realpath !== "function" || typeof fs.statSync !== "function") return { unavailable: true };
+	let actualRoot, folder;
+	try { actualRoot = realpath(sandboxDir); }
+	catch (error) { return absent(error) ? null : { unavailable: true }; }
+	if (within(named, actualRoot)) return policy();
+	try { folder = resolvedFolder ?? realpath(job.folder); }
+	catch (error) { return absent(error) ? null : { unavailable: true }; }
+	if (within(folder, root) || within(folder, actualRoot)) return policy();
+	let rootId;
+	try { rootId = identity(fs.statSync(actualRoot, { bigint: true })); }
+	catch (error) { return absent(error) ? null : { unavailable: true }; }
+	if (rootId === null) return { unavailable: true };
+	// Case variants and macOS firmlinks can name the same directory without matching either spelling above.
+	// Compare the root's identity to every resolved ancestor before trusting a text-only "outside" verdict.
+	for (let path = folder;; path = dirname(path)) {
+		let here;
+		try { here = identity(fs.statSync(path, { bigint: true })); }
+		catch (error) { return absent(error) ? null : { unavailable: true }; }
+		if (here === null) return { unavailable: true };
+		if (here === rootId) return policy();
+		if (dirname(path) === path) return null;
+	}
+}

package/src/sandbox.mjs

@@ -5,10 +5,10 @@ import { release as osRelease } from "node:os";
 import { promisify } from "node:util";
 import { execDockerBounded, makeDockerEndpointResolver } from "./backend-local.mjs";
 import { PODMAN_CONF_WIDENS_JOB, PODMAN_JOB_USER_FIX, decidePodmanJobUser, judgePodmanVenue, keeperPreflight, makePodmanInfoReader, resolvePodmanImageUser } from "./backend-podman.mjs";
-import { DEFAULT_BACKEND, PODMAN_BACKEND, UNATTRIBUTED_BACKEND, parseBackendFloor, parseBackendList } from "./backends.mjs";
+import { DEFAULT_BACKEND, PODMAN_BACKEND } from "./backends.mjs";
 import { configError } from "./config.mjs";
 import { assertJobUser, CONTAINER_HOME, SHIPPED_IMAGE_UID } from "./container-spec.mjs";
-import { buildDockerRunArgs, buildPodmanRunArgs, insideDir } from "./docker-run.mjs";
+import { insideDir } from "./docker-run.mjs";
 import { CGROUP_PARENT, cgroupParentFor } from "./cpu-reserve.mjs";
 import { DEFAULT_JOB_SIZE, recordedJobSize } from "./job-size.mjs";
 import { makeImagePreflight } from "./image-preflight.mjs";
@@ -17,7 +17,12 @@ import { DEFAULT_EGRESS_PROXY, NETWORK_SUFFIX, createJobNetwork, egressArmed, eg
 import { NETNS_KEEPER, makeDetachGate, runtimeFromFacts } from "./netns-keeper.mjs";
 import { NETNS_KEEPER_START } from "./podman-stack.mjs";
 import { makePodmanServiceReader, observeRootfulConf, rootfulConfRefusal } from "./runtime-observations.mjs";
-import { isSandboxTombstone, readManifest, readRetained, sandboxDeadline, sandboxEntryName } from "./sandbox-store.mjs";
+import { isSandboxTombstone, readManifest, readRetained, sandboxEntryName } from "./sandbox-store.mjs";
+import { SANDBOX_LAUNCHERS, SANDBOX_OPEN_GRACE_MS, sandboxLauncher, sandboxSyncRefusal, sandboxVenueOf, sandboxVenuePolicy, sandboxVenueRefusal, sandboxVenues, sandboxWindowRefusal } from "./sandbox-policy.mjs";
+
+// The manifest-only policy lives in sandbox-policy.mjs (issue #710), a leaf the operator console imports without this
+// file's graph; re-exported here under the same names, so every existing importer reads the one definition.
+export { SANDBOX_LAUNCHERS, SANDBOX_OPEN_GRACE_MS, sandboxLauncher, sandboxSyncRefusal, sandboxVenueOf, sandboxVenuePolicy, sandboxVenueRefusal, sandboxVenues, sandboxWindowRefusal };
 
 /**
  * sandbox.mjs -- the operator session's container shape (INT-SANDBOX-CONTRACT).
@@ -47,52 +52,6 @@ import { isSandboxTombstone, readManifest, readRetained, sandboxDeadline, sandbo
 
 const exec = promisify(execFile);
 
-/**
- * The venues a sandbox opens on, and how each one spells its container (issue #429): the CLI every spawn for that
- * venue goes through, and the job argv builder the session's argv is built by. Keyed by the venue name the manifest
- * records, so a run is reopened ONLY in the runtime that ran it: a podman run under docker would reproduce none of it
- * (another store, no keep-id, another uid map), and the reverse is as wrong.
- *
- * A TABLE rather than a `bin` read off the backend bundles, and that alternative was the obvious one: the bundles are
- * built by the worker at boot, and the CLI and the admin panel never build them (they would construct a
- * `runContainer`, a reaper and a `podman info` reader to open a shell). What the sandbox needs from a venue is two
- * facts, and both are the same values the bundles are built with (`bin: "podman"` and `buildPodmanRunArgs` in
- * `makePodmanBackend`); `sandbox.test.mjs` pins the pairing. A venue not in this table has no sandbox, by
- * construction: nothing here can reopen a run under a runtime it did not run in.
- *
- * Not "any venue declaring `remote: false`": a future non-remote venue on a third runtime would pass that and be
- * reopened under one of these two. Such a venue must add its own row, deliberately.
- */
-export const SANDBOX_LAUNCHERS = Object.freeze({
-	[DEFAULT_BACKEND]: Object.freeze({ bin: "docker", build: buildDockerRunArgs }),
-	[PODMAN_BACKEND]: Object.freeze({ bin: "podman", build: buildPodmanRunArgs }),
-});
-
-/** The launcher for `venue`, or null. `Object.hasOwn`, so `"toString"` is not a venue. */
-export function sandboxLauncher(venue) {
-	return typeof venue === "string" && Object.hasOwn(SANDBOX_LAUNCHERS, venue) ? SANDBOX_LAUNCHERS[venue] : null;
-}
-
-/**
- * The blessed venues a sandbox can open on, in `PI_BACKENDS` order: the venues this host both runs and has a launcher
- * for. The retention reaper asks each one which sandboxes are open, and `--list` draws its RUNNING column from them.
- */
-export function sandboxVenues(blessed) {
-	return (Array.isArray(blessed) ? blessed : []).filter((venue) => sandboxLauncher(venue) !== null);
-}
-
-/**
- * `{ blessed, backendFloor }` as a sandbox opened from `env` sees them, through the worker's own two parsers. For the
- * admin panel, which deliberately never calls `loadConfig`; the CLI has the parsed config already. THROWS on a
- * malformed `PI_BACKENDS` or `PI_BACKEND_FLOOR`, exactly as the worker refuses to boot on one: a typo must never read
- * as "podman is not blessed" (a refusal an operator then chases in the wrong place) or as "no floor" (a podman sandbox
- * opened without the observations the deployment asks for). Read from THIS process's environment and never a
- * deployment's `.env`, which `OQ-038` records for the panel.
- */
-export function sandboxVenuePolicy(env) {
-	return { blessed: parseBackendList(env?.PI_BACKENDS), backendFloor: parseBackendFloor(env?.PI_BACKEND_FLOOR) };
-}
-
 /**
  * The name namespace, and it is load-bearing. The boot reaper filters `name=pi-job-`
  * (`makeReaper` in backend-local.mjs) and docker matches that as a SUBSTRING, so a sandbox must not contain it --
@@ -407,11 +366,13 @@ export function makeSandboxRuntimeWatch({ sandboxDir, blessed = [], fs = { lstat
 }
 
 /** The store this account's Podman uses right now, from one bounded `podman info`, or null when it cannot say. */
-async function defaultPodmanStore() {
-	const read = await makePodmanInfoReader()();
-	return read?.answered === true ? (read.info?.graphRoot ?? null) : null;
+export async function readPodmanStore({ read = makePodmanInfoReader() } = {}) {
+	const result = await read();
+	return result?.answered === true ? (result.info?.graphRoot ?? null) : null;
 }
 
+const defaultPodmanStore = readPodmanStore;
+
 /**
  * The sweep's runner for one runtime: bounded, both streams, never throws. `execDockerBounded` already settles on its
  * own timer and kills with SIGKILL, which matters here for the reason `retention-sweep.mjs` records -- this
@@ -745,153 +706,6 @@ export async function sandboxKeeperCheck({ proxy, info = makePodmanInfoReader(),
 	};
 }
 
-/**
- * Whether a retained run can be re-opened HERE, judged by the venue it ran in (issue #277). `null` when it
- * can, else `{ refused, message }`. Exported for the admin panel, which asks before advertising the key.
- *
- * WHY THIS IS PER JOB. A sandbox opens a shell on THIS host, in the runtime the job ran in, against the job's retained
- * directory, so it can reproduce only a run whose container was built here. The check used to be deployment-wide
- * (refuse every sandbox when any blessed venue was remote) because the command takes a job id and could not learn its
- * venue; the manifest now records it, and the answer belongs to the job.
- *
- * HELD MEANS A VENUE THIS FILE HAS A LAUNCHER FOR (`SANDBOX_LAUNCHERS`: `local` through the docker CLI, `podman`
- * through this account's rootless podman), AND ONE `blessed` NAMES (issue #429). `blessed` is `PI_BACKENDS` as the
- * CALLER's process reads it: the CLI's `loadConfig`, the panel's own environment (`sandboxVenuePolicy`). A shell whose
- * `PI_BACKENDS` does not bless a venue has said it does not run that runtime, so reopening a run there would spawn a CLI
- * that environment took out of service. It is NOT what keeps an open shell's directory from the retention reaper, and
- * an earlier version of this comment said it was: the opener's `PI_BACKENDS` and the worker's routinely differ
- * (`OQ-038`), so the reaper asks the runtime each retained run records instead, whatever either blesses
- * (`makeSandboxRuntimeWatch`). It defaults to `local` alone, which is what `PI_BACKENDS` unset means, so a caller from
- * before issue #429 refuses and admits exactly as it did.
- *
- * Not "any venue declaring `remote: false`": a future non-remote venue on a third runtime would pass that and be
- * reopened under one of these two, reproducing a run from a runtime it never ran in. Such a venue must add its own
- * launcher row, deliberately.
- *
- * A MANIFEST WITH NO `backend` KEY predates venue attribution and ran on `local` (`UNATTRIBUTED_BACKEND`).
- * A key that is PRESENT but not a name is refused: `typeof` first, because the table's `backendFor(null)`
- * returns `local`, and a null stamp means the venue was never known, not that it was local.
- */
-export function sandboxVenueRefusal({ jobId, manifest, blessed = [DEFAULT_BACKEND] }) {
-	const venue = sandboxVenueOf(manifest);
-	if (typeof venue !== "string" || venue === "") {
-		return { refused: "venue-unreachable", message: `the manifest for ${jobId} names no backend, so this host cannot tell whether the run happened here` };
-	}
-	const launcher = sandboxLauncher(venue);
-	const held = Array.isArray(blessed) && blessed.includes(venue);
-	if (launcher && held) return null;
-	if (launcher) {
-		// A venue this file CAN open, on a host (as this process reads it) that does not bless it. The fix is an
-		// environment, and it is named with where it is read: the CLI and the panel read `PI_BACKENDS` from the process
-		// they run in, never from a deployment's `.env`, and "set it where you run this" is the whole remedy.
-		return {
-			refused: "venue-unreachable",
-			message: `${jobId} ran on the ${JSON.stringify(venue)} backend, which PI_BACKENDS in this process's environment does not bless (it reads ${JSON.stringify((Array.isArray(blessed) ? blessed : []).join(","))}), so no sandbox opens on it here: a sandbox opens only on a venue this host blesses, through that venue's own CLI (${launcher.bin}). Set PI_BACKENDS where you run this as the worker's is set; it is read from this environment, not from the deployment's .env.`,
-		};
-	}
-	return {
-		refused: "venue-unreachable",
-		// Names BOTH sides (issue #354): the venue that ran it, and the venues a sandbox opens through. "Not on this host's
-		// docker daemon" was true only while every venue but `local` was elsewhere.
-		message: `${jobId} ran on the ${JSON.stringify(venue)} backend, and a sandbox opens a shell only through a venue it has a launcher for on this host (${Object.entries(SANDBOX_LAUNCHERS).map(([name, l]) => `${JSON.stringify(name)} through the ${l.bin} CLI`).join(", ")}) against the retained directory, so it cannot reproduce that run. Open it on the venue that ran it.`,
-	};
-}
-
-/** The venue a retained manifest records: its `backend` key when present (whatever it holds), else `UNATTRIBUTED_BACKEND`. */
-export function sandboxVenueOf(manifest) {
-	return Object.hasOwn(manifest ?? {}, "backend") ? manifest.backend : UNATTRIBUTED_BACKEND;
-}
-
-/**
- * The refusals `resolveSandbox` decides from the MANIFEST ALONE, in the order an operator should read
- * them. Not every manifest-only refusal on the `b` path: `decideSandboxJobUser` can refuse a run from
- * `manifest.jobUser` too, and it stays where it is for two MEASURED reasons rather than the vaguer "it
- * would change what the CLI refuses and when" this comment used to give (issue #367, item 5).
- *
- * IT IS NOT MANIFEST-ONLY. It returns `{ user: null, home: null }` on `darwin` and `win32` before it ever
- * reads the stamp, so the SAME malformed `jobUser` refuses everywhere else and does not refuse on macOS or
- * Windows. Measured across twelve platform strings: the split is darwin and win32 against every other
- * value, linux, the BSDs, sunos, aix and the empty string alike, which is wider than "linux" and is why
- * this says it that way. Folding it in would make this function's answer, and therefore whether the panel
- * advertises `b` at all, depend on the operator's own OS for an identical run. Every other refusal here is
- * a property of the run, and that is the whole of the argument.
- *
- * NOT because it is async, which is true of the function and is NOT a reason about this refusal: measured,
- * `job-user-stamp-invalid` is reached with zero calls to the endpoint resolver, the daemon-facts reader
- * and the image preflight, on every platform and every malformed shape. It is decided from the manifest
- * before any await that does work. A well-formed stamp does reach the daemon, which is why this function
- * stays async and out of a predicate called on every left and right between runs, but that is a cost of
- * moving the WHOLE function, not of the refusal #367 item 5 is about.
- *
- * Extracted (issue #337) because the admin panel needs the same answer before it advertises `b`, and the
- * alternative is the shape this file's own `openSandbox` docblock warns about: "Two callers assembling
- * the same session from parts is how one of them drops a part." The panel had exactly that, checking the
- * venue and not the other two, so a run whose manifest names no image was offered the key, given two
- * lines of detail about the session it would get, and refused the moment the key was pressed.
- *
- * SYNCHRONOUS AND MANIFEST-ONLY is the boundary, not an accident of what fitted. The panel reads this
- * inside a key handler, once per record; anything needing docker (a proxy that is not running, a sandbox
- * already up) stays in `openSandbox` where it belongs and is `OQ-038`'s residual for the panel.
- *
- * The VENUE comes first, and that ordering is #277's: for a run from another venue the image and the
- * workspace are symptoms, and the first refusal an operator reads should be the cause. A workspace that
- * happens to exist at the same path on this host would otherwise pass and silently reproduce the wrong
- * run.
- */
-export function sandboxSyncRefusal({ jobId, manifest, fileExists = existsSync, blessed }) {
-	const venue = sandboxVenueRefusal({ jobId, manifest, blessed });
-	if (venue) return venue;
-	if (!manifest?.image) {
-		return { refused: "no-image", message: `the manifest for ${jobId} names no image, so the sandbox cannot reproduce the run` };
-	}
-	if (!manifest?.workspace || !fileExists(manifest.workspace)) {
-		// The common cause for a local run: the operator's folder moved or was deleted. Naming the path is
-		// the whole diagnosis, so name it.
-		return { refused: "workspace-gone", message: `the workspace for ${jobId} is no longer at ${manifest.workspace} — a local folder that moved cannot be re-opened` };
-	}
-	return null;
-}
-
-/**
- * How close to its deadline a run may be and still open without `--pin` (issue #446): five minutes.
- *
- * WHY A MARGIN AND NOT THE DEADLINE ITSELF. The sweep holds a run whose sandbox a runtime reports open, but an open is
- * not in any `ps` until its container starts, and the opener spends that stretch on its own runtime asks (the running
- * check, the job user, the image, the network), each bounded in seconds. A sweep whose `ps` came before the container
- * and whose clock (`at`, read after its asks) is past the deadline would still delete the directory under the new
- * shell: window 1 of #446, reproduced. A run whose deadline is more than this far away cannot be past it by the time a
- * pass that missed the container decides, so the refusal closes that window with room to spare; inside it, `--pin`
- * writes a new deadline FIRST, which the sweep's own re-reads honour.
- */
-export const SANDBOX_OPEN_GRACE_MS = 5 * 60 * 1000;
-
-/**
- * The refusal of a run past its deadline, or within `SANDBOX_OPEN_GRACE_MS` of it, unless the open pins it first
- * (issue #446). `null` when the run may open.
- *
- * The deadline is `sandboxDeadline`'s, the sweep's own rule (the pin; else the earlier of the `retainUntil` the worker
- * wrote and `createdAt` plus THIS opener's window), so an opener whose own PI_SANDBOX_RETENTION_HOURS is larger than
- * the worker's is refused by the worker's deadline, not admitted by its own. The one case it cannot see, a worker
- * whose window was lowered after the run was retained, is the post-launch look's (`openSandbox`).
- * Shared by `resolveSandbox` and the admin panel's `readSandboxInfo`, which must not advertise `b` for a run the key
- * press would refuse; the panel has no pin, so its refusal is this one and it names the CLI command.
- */
-export function sandboxWindowRefusal({ jobId, manifest, retentionHours, at, pin = false }) {
-	if (pin) return null;
-	const { until } = sandboxDeadline(manifest, retentionHours);
-	const fix = `open it with \`pi-dispatch sandbox ${jobId} --pin\`, which extends its retention before anything starts`;
-	if (until === null) {
-		return { refused: "past-window", message: `the retained workspace for ${jobId} records no creation time, so the retention sweep deletes it on its next pass; ${fix}` };
-	}
-	if (until - at > SANDBOX_OPEN_GRACE_MS) return null;
-	const when = new Date(until).toISOString();
-	const where = until <= at ? `is past its retention window (it closed at ${when})` : `is within ${Math.round(SANDBOX_OPEN_GRACE_MS / 60000)} minutes of the end of its retention window (${when})`;
-	return {
-		refused: "past-window",
-		message: `the retained workspace for ${jobId} ${where}, so the retention sweep may delete it under the shell; ${fix}`,
-	};
-}
-
 /**
  * Resolve one retained run into a launchable argv, or a NAMED refusal.
  *
@@ -1028,6 +842,8 @@ export async function openSandbox({
 	detachGate = null,
 	// Issue #452, gate round 4: `(network, reason) => void`, told of a teardown the gate refused, which leaves the network.
 	onNetworkKept = () => {},
+	// The CLI's detached monitor registers the shell and owns eventual network cleanup (#771).
+	monitorSession = null,
 }) {
 	// The posture is REQUIRED, and a boolean. Every other part of this function defaults safely; this one would
 	// default to the open bridge, which is the dropped part this function exists to stop a caller dropping.
@@ -1037,6 +853,11 @@ export async function openSandbox({
 	const pinning = typeof pin === "function";
 	const resolved = resolveSandbox({ jobId, sandboxDir, retentionHours, publish, blessed, now, pin: pinning, ...(fs ? { fs } : {}), ...(fileExists ? { fileExists } : {}) });
 	if (resolved.refused) return resolved;
+	if (resolved.manifest.origin === "prepared") {
+		const saved = resolved.manifest.sandboxEgress;
+		if (typeof saved?.armed !== "boolean" || typeof saved.proxy !== "string" || saved.proxy !== egress.proxy || (egress.armed && !saved.armed)) return { refused: "sandbox-egress-changed", message: "The prepared sandbox's egress settings changed. Prepare it again with this deployment's current settings." };
+		egress = { armed: saved.armed, proxy: saved.proxy };
+	}
 	// Admitted by `resolveSandbox`, so never null here.
 	const { bin } = sandboxLauncher(resolved.venue);
 	// Issue #596: the run's size, decided before anything is asked or created. A recorded size that is malformed refuses.
@@ -1196,6 +1017,7 @@ export async function openSandbox({
 		};
 	}
 	let detached = false;
+	let sessionWatch = null;
 	// THE POST-LAUNCH LOOK (issue #446). The refusal and the pin decide from what was on disk before the launch, and
 	// two things can still take the run from under the new shell: the accepted residual (a worker whose window was
 	// lowered below this opener's, whose pass missed the container), and the sweep's own rename and restore. So once
@@ -1274,6 +1096,10 @@ export async function openSandbox({
 		// the network was created, so a leftover network printed the banner and then refused. Inside the `try`, so a hook
 		// that throws still has this session's network removed by the `finally`.
 		await beforeLaunch({ resolved, args, network, runtime: bin });
+		if (monitorSession) {
+			try { sessionWatch = await monitorSession({ resolved, args, network, runtime: bin, admittedRuntime: jobUser?.runtime ?? null }); }
+			catch { return { refused: "sandbox-tracking-unavailable", message: "Sandbox tracking is unavailable. Check Valkey before opening a shell." }; }
+		}
 		const watching = look().catch(() => {});
 		let launched;
 		try {
@@ -1327,7 +1153,8 @@ export async function openSandbox({
 			: {};
 		return { code: code ?? null, error: error ?? null, ...(detached ? { detached: true } : {}), ...during };
 	} finally {
-		if (network && !detached) {
+		sessionWatch?.finish?.();
+		if (network && !detached && !sessionWatch?.ownsNetwork) {
 			// The runtime this session was ADMITTED on (issue #452, gate round 4), never a fresh read at the teardown; a
 			// refused teardown is said with its token.
 			await removeJobNetwork(spawnNetwork, {