@edgehero/pi-dispatch 1.10.3 → 2.1.0

package/src/runtime-settings.mjs

@@ -1,6 +1,6 @@
 import * as nodeFs from "node:fs";
 import { dirname } from "node:path";
-import { defaultSettingsFile } from "./config.mjs";
+import { ensureUnderAccountRoot } from "./config.mjs";
 
 /**
  * Runtime-settings overlay: the shared, durable truth between the admin extension and the worker
@@ -56,13 +56,15 @@ function isIntInRange(value, min, max) {
 }
 
 /**
- * The absolute path of the settings overlay. Mirrors config.mjs: `PI_SETTINGS_FILE` wins, an unset or
- * empty value falls back to the shared default so the admin extension and the worker resolve the same
- * path without either coupling to the other.
+ * The absolute path of the settings overlay: `PI_SETTINGS_FILE` wins, an unset or empty value falls back
+ * to the durable default.
+ *
+ * RE-EXPORTED, not re-derived. It used to be a second copy of `config.mjs`'s expression, and two copies
+ * of a defaulting rule are how a panel and a worker come to read different files. This module keeps the
+ * name because `@edgehero/pi-dispatch/runtime-settings` is the import path the admin and the export-map
+ * probe already use; config.mjs owns the rule because it owns the default the rule falls back to.
  */
-export function settingsFilePath(env = process.env) {
-	return env.PI_SETTINGS_FILE || defaultSettingsFile();
-}
+export { settingsFilePath } from "./config.mjs";
 
 /**
  * Validate a parsed overlay against the key contract, returning the sanitized overlay (known keys only)
@@ -201,9 +203,12 @@ export function writeOverlay(path, candidate, { fs = nodeFs, log = () => {} } =
 	if (result.invalid) return { invalid: result.invalid };
 
 	try {
+		// Issue #464: on an account with no home the overlay's default lives in the per-account temp root, which is made
+		// this account's (0700) or refused, as the worker's boot does, before a file that outranks .env is written there.
+		ensureUnderAccountRoot(dirname(path), { fs: { mkdirSync: fs.mkdirSync, lstatSync: fs.lstatSync ?? nodeFs.lstatSync, statSync: fs.statSync ?? nodeFs.statSync, chmodSync: fs.chmodSync ?? nodeFs.chmodSync } });
 		fs.mkdirSync(dirname(path), { recursive: true });
 	} catch (err) {
-		return { invalid: `settings dir unwritable (${err?.code ?? "mkdir-error"})` };
+		return { invalid: err?.piDispatchConfig ? `settings dir refused: ${err.message}` : `settings dir unwritable (${err?.code ?? "mkdir-error"})` };
 	}
 
 	const tmp = `${path}.tmp`;

package/src/sandbox-cli.mjs

@@ -1,29 +1,9 @@
 import { spawn } from "node:child_process";
 import { parseArgs } from "node:util";
-import { DEFAULT_BACKEND, backendFor } from "./backends.mjs";
 import { loadConfig } from "./config.mjs";
-import { sanitizeJobId } from "./run-history.mjs";
-import { createJobNetwork, egressEnv, networkNameFor, removeJobNetwork } from "./egress.mjs";
-import { buildSandboxRunArgs, launchSandbox, listRunningSandboxes, parsePublish, resolveSandbox, sandboxContainerName } from "./sandbox.mjs";
-import { listSandboxes, pinSandbox } from "./sandbox-store.mjs";
-
-/**
- * Would this deployment's venues put a job's retained directory out of this command's reach? Exported ONLY
- * so it can be driven.
- *
- * The refusal it answers is unreachable today: the table holds one backend and it is local, so
- * `PI_BACKENDS` cannot name a remote venue. An unreachable rule with no test is a rule a mutation pass
- * deletes in silence -- which is precisely what happened to the trigger-side remote refusal one slice ago,
- * so this one is a pure predicate over an explicit config from the start.
- *
- * TRUE when any blessed venue is remote, not merely when the DEFAULT is: a job that ran there has a
- * retained directory this host never had, and `pi-dispatch sandbox <jobId>` takes a job id rather than a
- * venue, so it cannot know which one it is being asked about until the directory is already missing.
- */
-export function sandboxUnreachableFrom(config) {
-	const names = config?.backends ?? [config?.defaultBackend ?? DEFAULT_BACKEND];
-	return names.some((n) => backendFor(n)?.remote !== false);
-}
+import { basename } from "node:path";
+import { SANDBOX_OPEN_GRACE_MS, launchSandbox, listRunningSandboxes, openSandbox, parsePublish, sandboxContainerName, sandboxLauncher, sandboxVenueOf, sandboxVenueRefusal, sandboxVenues } from "./sandbox.mjs";
+import { listSandboxes, pinSandbox, sandboxDeadline } from "./sandbox-store.mjs";
 
 /**
  * `pi-dispatch sandbox` -- re-open a finished run's sandbox as an interactive shell
@@ -46,6 +26,10 @@ export async function runSandbox(argv = [], { env = process.env, deps = {} } = {
 		// The docker spawn used for this session's egress network, seamed like `launch` so the tests never
 		// touch a daemon. Not used when PI_EGRESS=0.
 		spawnNetwork = spawn,
+		// Issue #341: which uid the shell runs as. Seamed so the tests never ask a daemon.
+		resolveJobUser,
+		// Issue #452, gate round 2: the keeper check an egress-armed podman open asks first. Seamed like `resolveJobUser`.
+		keeperCheck,
 		now = () => Date.now(),
 	} = deps;
 
@@ -69,7 +53,21 @@ export async function runSandbox(argv = [], { env = process.env, deps = {} } = {
 	// A docker that cannot be reached costs a column, never the command: `listRunningSandboxes` throws so
 	// the REAPER can tell "none" from "could not ask", and these callers only draw a marker. Asked only
 	// where it is used, and never before the arguments are known good -- a typo should not shell out.
-	const liveSandboxes = async () => new Set(await running().catch(() => []));
+	//
+	// EVERY runtime a sandbox can open on here (issue #429), each asked on its own and each degrading on its own: a
+	// podman that cannot be asked must not blank docker's column, and the reverse. `running({ bin })` is the same seam
+	// `openSandbox` asks with, so one fake answers both.
+	const liveSandboxes = async () => {
+		const live = new Set();
+		for (const venue of sandboxVenues(config.backends)) {
+			for (const id of await Promise.resolve()
+				.then(() => running({ bin: sandboxLauncher(venue).bin }))
+				.catch(() => [])) {
+				live.add(id);
+			}
+		}
+		return live;
+	};
 
 	if (values.list) {
 		return renderList({ config, live: await liveSandboxes(), out, now });
@@ -85,11 +83,6 @@ export async function runSandbox(argv = [], { env = process.env, deps = {} } = {
 		return fail(err, "`pi-dispatch sandbox` needs a terminal — it opens an interactive shell, so it cannot run from a pipe, a script without a TTY, or CI");
 	}
 
-	// `listRunningSandboxes` yields ids, already sanitized, so this compares like with like.
-	if ((await liveSandboxes()).has(sanitizeJobId(jobId))) {
-		return fail(err, `a sandbox for ${jobId} is already running — attach to it with \`docker attach ${sandboxContainerName(jobId)}\`, or exit it first`);
-	}
-
 	let publish;
 	try {
 		publish = parsePublish(values.publish ?? []);
@@ -97,74 +90,63 @@ export async function runSandbox(argv = [], { env = process.env, deps = {} } = {
 		return fail(err, error.message);
 	}
 
-	// #227. THE SANDBOX IS LOCAL-ONLY, and this refusal is what keeps that true rather than accidental.
-	//
-	// `buildSandboxRunArgs` is a SECOND container producer, outside the `runContainer` seam and hard-wired to
-	// this host's docker CLI. It reopens a retained job directory: `manifest.workspace` is a path on THIS
-	// machine, which for a job that ran in another venue either does not exist or exists and reproduces a
-	// run from the wrong host, silently. `INT-SANDBOX-CONTRACT` is already "a SIBLING ... never an
-	// amendment" of the container contract, and this is the clause that says which sibling.
-	//
-	// Refused BEFORE `resolveSandbox` reads anything, so a deployment that blesses a remote venue is told
-	// what is wrong rather than handed a confusing missing-directory message.
-	if (sandboxUnreachableFrom(config)) {
-		return fail(
-			err,
-			`pi-dispatch sandbox opens a shell on THIS host's docker daemon against the job's retained directory, so it cannot reach a job that ran in a remote venue (PI_BACKENDS: ${config.backends.join(", ")}). Run it on the host that ran the job.`,
-		);
-	}
-
-	const resolved = resolveSandbox({
+	// #227, #277. Everything from here to the shell is `openSandbox`, shared with the admin panel: every
+	// refusal (the per-job venue refusal included, which replaced a deployment-wide one this command used to
+	// apply on its own), the already-running refusal, this session's egress network and its teardown. The
+	// panel assembled the same session from parts and dropped the network; one function is what stops that.
+	// The run's venue's CLI, learned when the session is known to be openable, so the lines after the shell name the
+	// runtime that ran it (`podman attach`, not `docker attach`). Stays "docker" only for a failure before that point,
+	// which never reaches those lines.
+	let runtime = "docker";
+	// The container's name as `resolveSandbox` built it, off the run's directory (a run retained before #446's escape
+	// keeps its old one); the id's own spelling only before that point.
+	let containerName = sandboxContainerName(jobId);
+	const result = await openSandbox({
 		jobId,
 		sandboxDir: config.sandboxDir,
 		retentionHours: config.sandboxRetentionHours,
 		publish,
-	});
-	if (resolved.refused) return fail(err, resolved.message);
-
-	// Pin BEFORE the shell, not after: the operator asked to keep this one, and a session that ends in a
-	// crashed terminal or a closed laptop lid must not be the reason the pin never landed.
-	if (values.pin) {
-		const pinned = pinSandbox({ sandboxDir: config.sandboxDir, jobId, pinDays: config.sandboxPinDays, now });
-		if (pinned.pinned) out(`pinned ${jobId} until ${pinned.keepUntil} (${config.sandboxPinDays}d)\n`);
-		else err(`warning: could not pin ${jobId}: ${pinned.reason}\n`);
-	}
-
-	// REQ-EGRESS-ALLOWLIST: this session's own network, exactly like a job's, named off its own container
-	// so the reaper's `pi-job-` filter never touches it -- a worker restart must not tear the network out
-	// from under a shell an operator is sitting in.
-	const network = config.egress ? networkNameFor(resolved.name) : null;
-	const args = buildSandboxRunArgs({
-		image: resolved.manifest.image,
-		name: resolved.name,
-		workspace: resolved.manifest.workspace,
-		jobDir: resolved.manifest.dir,
-		publish,
 		// env-internal TERM: the operator's own terminal type, forwarded so the sandbox shell renders the
 		// way their terminal does. Nothing a deployment declares.
 		term: env.TERM,
 		idleSeconds: config.sandboxIdleMinutes * 60,
-		network,
-		egressEnv: egressEnv({ proxy: config.egressProxy, armed: config.egress }),
+		egress: { armed: config.egress, proxy: config.egressProxy },
+		running,
+		launch,
+		spawnNetwork,
+		...(resolveJobUser ? { resolveJobUser } : {}),
+		...(keeperCheck ? { keeperCheck } : {}),
+		// Issue #452, gate round 4: a teardown the detach gate refused leaves the session's network, said here with its reason.
+		onNetworkKept: (network, reason) => err(`warning: the egress network ${network} was kept (${reason}): detaching the running proxy from it now could cut the proxy's route out on a rootless Podman 4.x whose rootless network keeper does not hold (issue #458); start the keeper, and the worker's retention sweep removes it\n`),
+		// Issue #429: which venues open here, and what a podman sandbox's observations must show, as the worker reads them.
+		blessed: config.backends,
+		backendFloor: config.backendFloor,
+		now,
+		// Pin BEFORE the shell, not after: the operator asked to keep this one, and a session that ends in a crashed
+		// terminal or a closed laptop lid must not be the reason the pin never landed. And since issue #446 before
+		// ANY runtime call, inside `openSandbox` straight after `resolveSandbox`, where a pin that fails refuses the open
+		// instead of printing a warning above a shell whose workspace the sweep may take.
+		pin: values.pin
+			? () => {
+					const pinned = pinSandbox({ sandboxDir: config.sandboxDir, jobId, pinDays: config.sandboxPinDays, now });
+					if (pinned.pinned) out(`pinned ${jobId} until ${pinned.keepUntil} (${config.sandboxPinDays}d)\n`);
+					return pinned;
+				}
+			: null,
+		beforeLaunch: ({ resolved, runtime: cli }) => {
+			runtime = cli;
+			containerName = resolved.name;
+			out(`opening ${resolved.name} — image ${resolved.manifest.image}, workspace ${resolved.manifest.workspace}\n`);
+			out("no credentials are set in this container. exit the shell to dispose of it.\n");
+			if (publish.length > 0) out(`published: ${publish.filter((f) => f !== "-p").join(", ")}\n`);
+		},
 	});
-
-	out(`opening ${resolved.name} — image ${resolved.manifest.image}, workspace ${resolved.manifest.workspace}\n`);
-	out("no credentials are set in this container. exit the shell to dispose of it.\n");
-	if (publish.length > 0) out(`published: ${publish.filter((f) => f !== "-p").join(", ")}\n`);
-
-	// No pre-spend gate here, deliberately: that is a MONEY gate and a sandbox spends nothing. A missing
-	// proxy fails at `docker run` with docker's own message, in front of an operator at a terminal, which
-	// is the one place a late failure is cheap.
-	if (network && !(await createJobNetwork(spawnNetwork, { network, proxy: config.egressProxy }))) {
-		return fail(err, `could not create the egress network ${network} -- is the proxy running? \`docker compose -f deploy/docker-compose.yml --profile egress up -d\``);
-	}
-	try {
-		const { code, error } = await launch({ args });
-		if (error) return fail(err, `could not start docker: ${error.message}`);
-		return code ?? 0;
-	} finally {
-		if (network) await removeJobNetwork(spawnNetwork, { network, proxy: config.egressProxy });
-	}
+	if (result.refused) return fail(err, result.message);
+	if (result.error) return fail(err, `could not start ${runtime}: ${result.error.message}`);
+	// Issue #446: a run lost from under a session after it started is said when the shell exits, never acted on then.
+	if (result.lost) err(`note: ${result.message}\n`);
+	if (result.detached) out(`detached: ${containerName} is still running with its egress network, which is left in place after it exits -- \`${runtime} attach ${containerName}\` to return\n`);
+	return result.code ?? 0;
 }
 
 /**
@@ -187,19 +169,42 @@ function renderList({ config, live, out, now }) {
 	for (const row of rows) {
 		const id = String(row.jobId ?? "?").padEnd(width);
 		const kind = String(row.kind ?? "?").padEnd(8);
-		const state = live.has(sanitizeJobId(row.jobId)) ? "RUNNING" : remaining(row, config.sandboxRetentionHours, now());
+		// A run this host cannot re-open is still listed (it is still retained and still swept), but not as
+		// time left on something re-openable (#277): the list answers "what can I open", and the venue says why not.
+		// A venue this command CAN open that this environment's PI_BACKENDS leaves out (issue #429) says which variable,
+		// because it is the one cause here an operator fixes in their own shell.
+		// Judged on the venue the refusal itself read (`sandboxVenueOf`): a manifest with no `backend` key is a `local`
+		// run from before attribution, so in a shell whose PI_BACKENDS leaves out `local` its cause is that variable,
+		// not a missing record.
+		const venue = sandboxVenueOf(row);
+		const state = sandboxVenueRefusal({ jobId: row.jobId, manifest: row, blessed: config.backends })
+			? typeof venue === "string" && venue !== ""
+				? sandboxLauncher(venue)
+					? `not here (PI_BACKENDS lacks ${venue})`
+					: `not here (ran on ${venue})`
+				: "not openable (no venue recorded)"
+			: // The directory's own name, which is what the runtime reports (a run retained before the escape keeps its old one).
+				live.has(basename(row.dir))
+				? "RUNNING"
+				: remaining(row, config.sandboxRetentionHours, now());
 		out(`${id}  ${kind}  ${state}\n`);
 	}
 	return 0;
 }
 
-/** How long this one has left, from the manifest's own timestamps -- never from mtime, which a live sandbox moves. */
+/**
+ * How long this one has left, from the manifest's own timestamps -- never from mtime, which a live sandbox moves.
+ * Through `sandboxDeadline` (issue #446), the sweep's own order, so a run the worker retained with a shorter window
+ * than this shell's PI_SANDBOX_RETENTION_HOURS shows the worker's deadline, which is the one that deletes it.
+ */
 function remaining(row, retentionHours, at) {
-	const keepUntil = Date.parse(row.keepUntil ?? "");
-	if (Number.isFinite(keepUntil)) return `pinned, ${humanise(keepUntil - at)} left`;
-	const createdAt = Date.parse(row.createdAt ?? "");
-	if (!Number.isFinite(createdAt)) return "expired";
-	return `${humanise(createdAt + retentionHours * 3600000 - at)} left`;
+	const { until, source } = sandboxDeadline(row, retentionHours);
+	if (until === null) return "expired";
+	// Said, not counted down to "0h" (gate round 2): past the deadline, or inside the opener's grace, a plain open is
+	// refused, and the list is where an operator decides what to type.
+	if (until <= at) return "past its window (open with --pin)";
+	if (until - at <= SANDBOX_OPEN_GRACE_MS) return "within the grace (open with --pin)";
+	return source === "pin" ? `pinned, ${humanise(until - at)} left` : `${humanise(until - at)} left`;
 }
 
 function humanise(ms) {