@edgehero/pi-dispatch 2.1.0 → 3.0.0

package/src/egress-cli.mjs

@@ -0,0 +1,316 @@
+/**
+ * `pi-dispatch egress render` (issue #503, INT-MODEL-ENDPOINTS-FILE-CONTRACT, INT-EGRESS-POLICY-CONTRACT): writes the
+ * egress proxy's include, `model-endpoints.conf` in the deployment folder, from the declared model endpoints, then
+ * prints the command that reloads the proxy. It never reloads the proxy itself: the worker does not manage the proxy's
+ * lifecycle, and a reload is the operator's step.
+ *
+ * IN PLACE, and that is the measured part (issue #503, Docker 29.1.3 and rootless and rootful Podman 4.9.3 and 5.8.1 on
+ * Linux, 2026-09-30). The include is a single-file bind mount, and such a mount holds the file's inode. A temp file
+ * renamed over the path is a new inode the running container never sees, and `squid -k reconfigure` then reloads the
+ * OLD rules and says nothing. So the existing file is opened without O_CREAT, truncated and written, and the inode the
+ * proxy holds is the one that changes. The render is built and checked in memory first, so a refused declaration never
+ * truncates anything.
+ *
+ * Refused at the path: a symlink (a write through it would land wherever it points, and the mount holds the target,
+ * not the link), a directory (Docker creates one for a missing bind source, and squid then reads it as empty with no
+ * warning, measured on Docker Desktop), anything else that is not a regular file, and nothing at all. A missing file
+ * is not created: `pi-dispatch init` scaffolds it, and a render run from the wrong folder must say so rather than
+ * write a file no proxy mounts and then print a reload that changes nothing.
+ *
+ * The reload is `squid -k reconfigure` through `docker exec` or `podman exec`, on every venue (measured the same day):
+ * it re-reads the include and the allowlist in well under a second, keeps the same squid, its networks and their
+ * addresses, and keeps every open tunnel, so a running job is not cut off. A restart kills every tunnel, takes about
+ * 11 s with this image, and changes the proxy's addresses on Podman.
+ */
+import { closeSync, constants as fsConstants, fstatSync, ftruncateSync, lstatSync, openSync, readFileSync, fsyncSync, writeSync, existsSync } from "node:fs";
+import { homedir } from "node:os";
+import { join, resolve } from "node:path";
+import { venuesOf } from "./backends.mjs";
+import { DEFAULT_VALKEY_URL } from "./config.mjs";
+import { deploymentVenueEnv } from "./deployment-venue.mjs";
+import { egressArmed, egressProxyName } from "./egress.mjs";
+import { rulesIncludeEndpoints } from "./egress-proxy-state.mjs";
+import { proxyConfCopyPath } from "./podman-stack.mjs";
+import { readEnvAssignments } from "./env-file.mjs";
+import { MODEL_ENDPOINTS_INCLUDE_NAME, loadModelEndpoints, renderEndpointsInclude } from "./model-endpoints.mjs";
+import { valkeyClientContext } from "./valkey-endpoint.mjs";
+
+const USAGE = `pi-dispatch egress render
+    write model-endpoints.conf in this deployment folder from model-endpoints.json (PI_MODEL_ENDPOINTS_FILE
+    overrides), in place, then print the command that reloads the egress proxy. It never reloads the proxy itself.
+`;
+
+/** The reload, for one runtime and one proxy name: a reconfigure, never a restart (see the header). */
+export function reloadCommand(bin, proxy) {
+	return `${bin} exec ${proxy} squid -k reconfigure`;
+}
+
+/**
+ * PI_MODEL_ENDPOINTS_FILE as the service reads it: this shell's value where it sets one, else the deployment `.env`'s.
+ * Both set differently is refused, since the service runs the file's and a render of the other file would write rules
+ * the worker does not believe in. A `.env` line this reader cannot vouch for is refused too. Returns `{ value }`
+ * (null for unset) or `{ error }`.
+ */
+export function endpointsFileSetting({ env, cwd, platform = process.platform, readEnv = (p) => readFileSync(p), exists = existsSync }) {
+	const envPath = join(cwd, ".env");
+	const shell = typeof env.PI_MODEL_ENDPOINTS_FILE === "string" ? env.PI_MODEL_ENDPOINTS_FILE : null;
+	let file = null;
+	if (exists(envPath)) {
+		let text;
+		try {
+			text = String(readEnv(envPath));
+		} catch (err) {
+			return { error: `${envPath} could not be read (${err?.code ?? err?.message}), so where the declared model endpoints are is not known` };
+		}
+		const loader = platform === "linux" ? "systemd" : platform === "win32" ? "cmd" : "shell";
+		const found = readEnvAssignments(text, ["PI_MODEL_ENDPOINTS_FILE"], { loader }).PI_MODEL_ENDPOINTS_FILE;
+		if (found) {
+			if (!found.plain) return { error: `${envPath} line ${found.line} sets PI_MODEL_ENDPOINTS_FILE in a way this command cannot read the same as the service does: write it as a plain KEY=value line` };
+			file = found.value;
+		}
+	}
+	// Compared as the paths they name, resolved against the deployment folder as the loader resolves them, so
+	// `./x.json` and `x.json` agree.
+	if (shell !== null && file !== null && (shell === "" || file === "" ? shell !== file : resolve(cwd, shell) !== resolve(cwd, file))) {
+		return { error: `PI_MODEL_ENDPOINTS_FILE is ${JSON.stringify(shell)} in this shell and ${JSON.stringify(file)} in ${envPath}, and the service reads the file's. Make them agree, or unset it in this shell` };
+	}
+	// Set in this shell alone: the service does not see it, so its worker reads another file than this render did.
+	const note = shell !== null && file === null ? `PI_MODEL_ENDPOINTS_FILE comes from this shell only, and the service reads ${envPath}, which does not set it: put it in ${envPath}, or the worker reads another file than this render did` : null;
+	return { value: shell ?? file, note };
+}
+
+/** VALKEY_URL as the service reads it (the `.env`'s, else this shell's, else the default), for the port refusal. */
+function serviceValkeyUrl({ env, cwd }) {
+	const context = valkeyClientContext({ env, cwd });
+	return context?.url?.file ?? context?.url?.environment ?? DEFAULT_VALKEY_URL;
+}
+
+/**
+ * What is at the include's path, by lstat (a symlink is judged as itself): `{ ok: true }` for a regular file, else
+ * `{ error }` naming what is there and what to do.
+ */
+export function includePathProblem(path, { lstat = lstatSync } = {}) {
+	let st;
+	try {
+		st = lstat(path);
+	} catch (err) {
+		if (err?.code === "ENOENT") return `${path} does not exist: \`pi-dispatch init\` in the deployment folder writes it (create-only), and the egress proxy mounts it from there. Run this command from that folder`;
+		return `${path} could not be read (${err?.code ?? err?.message})`;
+	}
+	if (st.isSymbolicLink()) return `${path} is a symlink: the proxy mounts the file itself, so a render through a link could write a file the proxy does not read. Replace the link with a regular file (\`pi-dispatch init\` writes one when nothing is there)`;
+	if (st.isDirectory()) return `${path} is a directory, not a file (a runtime creates one when it mounts a path that does not exist), and squid reads it as no rules at all. Remove the directory, then \`pi-dispatch init\` writes the file`;
+	if (!st.isFile()) return `${path} is not a regular file`;
+	return null;
+}
+
+/**
+ * Write `text` over the regular file at `path` IN PLACE: no O_CREAT, so nothing new is ever made here; O_NOFOLLOW where
+ * the platform has it, so a link swapped in after the check is refused by the kernel; O_TRUNC, so the inode the
+ * proxy's mount holds is the one that changes. Then fsync. Throws on any failure.
+ */
+export function writeInPlace(path, text, fs = { openSync, fstatSync, ftruncateSync, writeSync, fsyncSync, closeSync }) {
+	// Opened WITHOUT O_TRUNC, so a refusal below leaves the file as it was; truncated only once it is judged.
+	const flags = fsConstants.O_WRONLY | (fsConstants.O_NOFOLLOW ?? 0);
+	const fd = fs.openSync(path, flags);
+	try {
+		const st = fs.fstatSync(fd);
+		if (!st.isFile()) throw Object.assign(new Error(`${path} is not a regular file`), { untouched: true });
+		// A second name for this inode (a hard link) is a file the render would also rewrite, wherever it is.
+		if (st.nlink > 1) throw Object.assign(new Error(`${path} has ${st.nlink} hard links: a write in place would change every one of them. Replace it with a file of its own`), { untouched: true });
+		fs.ftruncateSync(fd, 0);
+		try {
+			const bytes = Buffer.from(text, "utf8");
+			let off = 0;
+			while (off < bytes.length) off += fs.writeSync(fd, bytes, off, bytes.length - off);
+			fs.fsyncSync(fd);
+		} catch (e) {
+			// A PARTIAL include is not safe: a prefix ending in the middle of an endpoint's lines can be an allow without
+			// its port ACL, which opens every port of that host (measured). An EMPTY include is valid and closes every
+			// endpoint, so a failed write leaves it empty, never half written.
+			try {
+				fs.ftruncateSync(fd, 0);
+				fs.fsyncSync(fd);
+				e.emptied = true;
+			} catch {
+				e.emptied = false;
+			}
+			throw e;
+		}
+	} finally {
+		fs.closeSync(fd);
+	}
+}
+
+/**
+ * The reload lines for this deployment's venues: docker's for the docker venue (Docker Engine, Docker Desktop, and
+ * rootful Podman through its Docker API, all reached as `docker`), podman's for the native rootless one. Both when
+ * the deployment runs both. The proxy's name is PI_EGRESS_PROXY's when it names the operator's own proxy, which then
+ * has to mount the include itself.
+ */
+export function reloadLines(env) {
+	const venues = venuesOf(env);
+	const proxy = egressProxyName(env);
+	const lines = [];
+	if (venues.localUsed) lines.push(reloadCommand("docker", proxy));
+	if (venues.podmanUsed) lines.push(reloadCommand("podman", proxy));
+	return { lines, proxy };
+}
+
+/**
+ * The verb. `deps` are seams for tests: `cwd`, `env`, `platform`, `out`, `err`, `fsRead` (the include's current text),
+ * `lstat`, `write` (the in-place writer), `load` (the endpoints loader), `venueEnv` (the deployment venue resolver).
+ */
+export async function runEgress(argv = [], deps = {}) {
+	const {
+		env = process.env,
+		cwd = process.cwd(),
+		platform = process.platform,
+		out = (s) => process.stdout.write(s),
+		err = (s) => process.stderr.write(s),
+		lstat = lstatSync,
+		readInclude = (p) => readFileSync(p, "utf8"),
+		write = writeInPlace,
+		load = loadModelEndpoints,
+		readEnv = (p) => readFileSync(p),
+		exists = existsSync,
+		venueEnv = (args) => deploymentVenueEnv(args),
+		valkeyUrl = serviceValkeyUrl,
+		home = homedir(),
+		readRules = (p, enc) => readFileSync(p, enc),
+	} = deps;
+	const sub = argv[0];
+	if (sub === undefined || sub === "--help" || sub === "-h" || sub === "help") {
+		out(USAGE);
+		return sub === undefined ? 1 : 0;
+	}
+	if (sub !== "render") {
+		err(`pi-dispatch egress: unknown subcommand ${JSON.stringify(sub)}\n${USAGE}`);
+		return 1;
+	}
+	if (argv.length > 1) {
+		err(`pi-dispatch egress render takes no argument (got ${argv.slice(1).map((a) => JSON.stringify(a)).join(" ")})\n`);
+		return 1;
+	}
+	const setting = endpointsFileSetting({ env, cwd, platform, readEnv, exists });
+	if (setting.error) {
+		err(`✗ ${setting.error}\n`);
+		return 1;
+	}
+	if (setting.note) out(`⚠ ${setting.note}\n`);
+	const path = join(cwd, MODEL_ENDPOINTS_INCLUDE_NAME);
+	// Validated and rendered in memory BEFORE the include is touched: a refused declaration leaves the rules in force.
+	let text;
+	let endpoints;
+	try {
+		endpoints = load({ modelEndpointsFile: setting.value, valkeyUrl: valkeyUrl({ env, cwd }) }, { cwd });
+		text = renderEndpointsInclude(endpoints);
+	} catch (e) {
+		err(`✗ ${e?.message ?? e}\n${exists(path) ? `${MODEL_ENDPOINTS_INCLUDE_NAME} was not changed.` : `Nothing was written.`}\n`);
+		return 1;
+	}
+	const problem = includePathProblem(path, { lstat });
+	if (problem) {
+		err(`✗ ${problem}\n`);
+		return 1;
+	}
+	let current = null;
+	try {
+		current = String(readInclude(path));
+	} catch {
+		// Unreadable reads as changed: the write below then says what is wrong.
+	}
+	const venue = venueEnv({ env, fs: { existsSync: exists, readFileSync: readEnv }, envPath: join(cwd, ".env"), platform, command: "egress" });
+	const unchanged = current === text;
+	if (unchanged) {
+		out(`✓ ${path} already matches the declared model endpoints: nothing written.\n`);
+	} else {
+		try {
+			write(path, text);
+		} catch (e) {
+			const state = e?.untouched ? "It was not changed" : e?.emptied ? "It was EMPTIED rather than left half written, which closes every declared endpoint until a render succeeds" : "It may be partly written";
+			err(`✗ could not write ${path} in place (${e?.message ?? e?.code ?? e}). ${state}: fix the cause and run this command again\n`);
+			return 1;
+		}
+		out(`✓ wrote ${path} (in place)\n`);
+	}
+	if (venue.error) {
+		out(`The reload depends on the venue, which could not be read: ${venue.error}\nOn Docker: ${reloadCommand("docker", "pi-dispatch-egress-proxy")}\nOn rootless Podman: ${reloadCommand("podman", "pi-dispatch-egress-proxy")}\n`);
+		return 0;
+	}
+	let armed = true;
+	try {
+		armed = egressArmed(venue.env);
+	} catch {
+		// A malformed PI_EGRESS is the worker's boot failure, and doctor's to report; it reads as armed here.
+	}
+	if (!armed) {
+		out("The egress policy is off (PI_EGRESS=0), so no proxy reads this file now. It takes effect once the policy is on and the proxy starts.\n");
+		return 0;
+	}
+	const { lines, proxy } = reloadLines(venue.env);
+	out(`${unchanged ? "If the proxy started before it last changed, reload it" : "Reload the egress proxy so it reads the new rules"} (no restart; open tunnels and running jobs are kept):\n${lines.map((l) => `  ${l}`).join("\n")}\n`);
+	if (proxy !== "pi-dispatch-egress-proxy") out(`${proxy} is your own proxy (PI_EGRESS_PROXY): it must mount ${path} at /etc/pi-dispatch/model-endpoints.conf and include it, as the shipped rules do.\n`);
+	// The one state the reload cannot fix (issue #503): endpoints declared, and the rules the proxy runs predate the include.
+	if (proxy === "pi-dispatch-egress-proxy" && endpoints.length > 0) {
+		const venues = venuesOf(venue.env);
+		for (const v of [...(venues.localUsed ? ["docker"] : []), ...(venues.podmanUsed ? ["podman"] : [])]) {
+			const rules = runningRulesPath(v, { cwd, home });
+			// No rules file at all is not that state: init or `service install` writes the current rules, include and all.
+			if (exists(rules) && !rulesFileIncludes(rules, { readFileSync: readRules })) out(`⚠ ${rulesPredateEndpointsLine(v)}\n`);
+		}
+	}
+	return 0;
+}
+
+/**
+ * The rules file the shipped proxy runs, per venue (issue #503): docker's mounts the deployment folder's
+ * `deploy/egress-proxy.conf`; the rootless podman venue's mounts the account-owned copy `service install` writes.
+ */
+export function runningRulesPath(venue, { cwd, home }) {
+	return venue === "podman" ? proxyConfCopyPath(home) : join(cwd, "deploy/egress-proxy.conf");
+}
+
+/** Whether the rules file at `path` includes the model endpoints' file. Unreadable or absent reads as no. */
+export function rulesFileIncludes(path, fs) {
+	try {
+		return rulesIncludeEndpoints(String(fs.readFileSync(path, "utf8")));
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * THE ONE LINE for endpoints declared under rules that predate #503 (issue #503's governing rule): `up`, `doctor` and
+ * `egress render` print it alike. Replacing or reloading the proxy fixes nothing here, since its rules do not read the
+ * include; the rules refresh does, and it brings the proxy's third mount with it.
+ */
+export function rulesPredateEndpointsLine(venue) {
+	return venue === "podman"
+		? "model endpoints are declared, but the podman proxy's rules (~/.config/pi-dispatch/egress-proxy.conf) predate #503 and do not include model-endpoints.conf, so the endpoints stay unreachable until the rules are refreshed: `pi-dispatch service install --force`"
+		: "model endpoints are declared, but deploy/egress-proxy.conf predates #503 and does not include model-endpoints.conf, so the endpoints stay unreachable until the rules are refreshed: `pi-dispatch up`, and accept the refresh";
+}
+
+/**
+ * Whether model endpoints are declared for this deployment, read as the service reads them. A declaration that does
+ * not load counts as none here: doctor names why it does not load, and the worker refuses it.
+ */
+export function endpointsDeclaredIn({ env, cwd, fs, platform = process.platform }) {
+	return declaredEndpointsIn({ env, cwd, fs, platform }).length > 0;
+}
+
+/**
+ * The declared model endpoints for this deployment, read as the service reads them (the same rule as
+ * `endpointsDeclaredIn`, which counts them): `[]` for none, and for a declaration that does not load, which doctor's
+ * boot-file line names. Doctor's endpoint rows (issue #503) are built from this list and from nothing else.
+ */
+export function declaredEndpointsIn({ env, cwd, fs, platform = process.platform, valkeyUrl = null }) {
+	try {
+		const setting = endpointsFileSetting({ env, cwd, platform, readEnv: (p) => fs.readFileSync(p), exists: (p) => fs.existsSync(p) });
+		if (setting.error) return [];
+		// `valkeyUrl` null skips the queue-port refusal; doctor's keyless line passes the service's (issue #503), so an
+		// endpoint on the queue's port, which refuses the worker's boot, is not called keyless.
+		return loadModelEndpoints({ modelEndpointsFile: setting.value, valkeyUrl }, { cwd, readFileSync: (p, enc) => fs.readFileSync(p, enc), existsSync: (p) => fs.existsSync(p) });
+	} catch {
+		return [];
+	}
+}

package/src/egress-proxy-state.mjs

@@ -12,7 +12,14 @@
  *     of it, or a "present" that leaves it running, keeps that policy. Compared: the image's digest, the entrypoint and
  *     command against the pinned image's own (read from its registry config, identical on every platform in the
  *     manifest list), and the mounts: the two bind sources against this folder's `deploy/egress-proxy.conf` and
- *     `egress-allowlist.conf`, the image's two anonymous volumes allowed, anything else stale.
+ *     `egress-allowlist.conf`, the image's two anonymous volumes allowed, anything else stale. A third bind, of
+ *     `model-endpoints.conf` (issue #503), must be this folder's file when it is there. When it is not there, the proxy
+ *     is still current (one made before #503 mounts two files) unless this folder's rules `include` the file: squid
+ *     refuses to start without it, so that proxy's next restart would crash-loop. That is the one rule, and it is the
+ *     only thing that makes a missing third mount drift. Endpoints declared under rules without the include are NOT
+ *     drift: replacing the proxy would cut every tunnel and change nothing, since its rules still lack the include.
+ *     That state is the rules refresh's to fix, and `up`, `doctor` and `egress render` name it alike
+ *     (`rulesPredateEndpointsLine` in egress-cli.mjs).
  *
  * MEASURED (Docker Engine 29.8.1, compose 5.5.1, rootful, gate round 2; pinned in egress-proxy-state.test.mjs): a proxy
  * made by `docker compose --profile egress up -d` and one made by `up`'s argv carry the pinned reference exactly as
@@ -52,9 +59,19 @@ const VM_PREFIXES = Object.freeze(["/host_mnt/", "/run/desktop/mnt/host/"]);
 export const PROXY_STATE_FORMAT =
 	'--format={"status":{{json .State.Status}},"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}"none"{{end}},"image":{{json .Config.Image}},"entrypoint":{{json .Config.Entrypoint}},"cmd":{{json .Config.Cmd}},"mounts":{{json .Mounts}},"networks":{{json .NetworkSettings.Networks}}}';
 
-/** Where the proxy's two files are mounted inside it (compose's and `up`'s mounts alike). */
+/** Where the proxy's files are mounted inside it (compose's and `up`'s mounts alike). */
 const SQUID_CONF = "/etc/squid/squid.conf";
 const ALLOWLIST = "/etc/pi-dispatch/allowlist.conf";
+/** The declared model endpoints' rules (issue #503), the path the shipped rules `include`. */
+export const MODEL_ENDPOINTS_TARGET = "/etc/pi-dispatch/model-endpoints.conf";
+
+/**
+ * Whether a copy of the proxy's rules `include`s the model endpoints' file (issue #503): an uncommented include line
+ * naming exactly that path. A copy from before #503 does not, and a proxy started on it needs no third mount.
+ */
+export function rulesIncludeEndpoints(text) {
+	return new RegExp(`^[ \\t]*include[ \\t]+${MODEL_ENDPOINTS_TARGET.replace(/[.]/g, "\\.")}[ \\t]*$`, "m").test(String(text ?? ""));
+}
 
 /**
  * `{ status, health, image, entrypoint, cmd, mounts, networks }` from `PROXY_STATE_FORMAT`'s stdout, or null when it
@@ -97,9 +114,12 @@ const sameList = (a, b) => Array.isArray(a) && a.length === b.length && a.every(
  * sentences that make it stale, `unknown` why the mounts could not be compared here (null when they were).
  * `realpath` throws for a path that does not resolve on this host. `compareMounts: false` judges the image, entrypoint and
  * command alone, for a caller that does not know which folder the service uses. `platform` decides whether a source
- * that does not resolve can be a path of the runtime's own VM (below).
+ * that does not resolve can be a path of the runtime's own VM (below). `rulesInclude` (this folder's rules include the
+ * model endpoints' file, `rulesIncludeEndpoints`) makes an absent third mount drift (issue #503); without it, a proxy
+ * mounting the two files is current, as every proxy made before #503 does. `outOfDate` is true when the missing third
+ * mount is the ONLY drift: the proxy is this folder's, made before #503.
  */
-export function shippedProxyDrift(state, { cwd, realpath = (p) => p, compareMounts = true, platform = "linux" }) {
+export function shippedProxyDrift(state, { cwd, realpath = (p) => p, compareMounts = true, platform = "linux", rulesInclude = false }) {
 	const drift = [];
 	if (digestOf(state.image) !== digestOf(EGRESS_PROXY_IMAGE)) drift.push(`it was created from ${state.image || "an unnamed image"}, not the pinned ${EGRESS_PROXY_IMAGE}`);
 	if (!sameList(state.entrypoint, EGRESS_PROXY_ENTRYPOINT)) drift.push(`its entrypoint is ${JSON.stringify(state.entrypoint)}, not the image's ${JSON.stringify(EGRESS_PROXY_ENTRYPOINT)}`);
@@ -128,6 +148,15 @@ export function shippedProxyDrift(state, { cwd, realpath = (p) => p, compareMoun
 			return false;
 		}
 	};
+	// The third bind (issue #503): compared like the two when it is there; its absence is drift only when it is needed.
+	const endpointsWant = resolve(cwd, "model-endpoints.conf");
+	const endpointsBind = binds.find((m) => m.destination === MODEL_ENDPOINTS_TARGET);
+	if (endpointsBind) expected[MODEL_ENDPOINTS_TARGET] = endpointsWant;
+	let includeMissing = false;
+	if (!endpointsBind && rulesInclude) {
+		includeMissing = true;
+		drift.push(`nothing is mounted at ${MODEL_ENDPOINTS_TARGET}, where ${endpointsWant} belongs: this folder's rules include it, and squid will not start again without it`);
+	}
 	const unknownSources = [];
 	for (const [destination, want] of Object.entries(expected)) {
 		const bind = binds.find((m) => m.destination === destination);
@@ -142,7 +171,8 @@ export function shippedProxyDrift(state, { cwd, realpath = (p) => p, compareMoun
 		drift.push(`it has a ${m.type || "mount"} at ${m.destination} (from ${m.source || "nowhere named"}) that the shipped proxy does not`);
 	}
 	const unknown = unknownSources.length > 0 ? `${unknownSources.join(", ")} ${unknownSources.length === 1 ? "is a path" : "are paths"} this host cannot resolve (the runtime's own VM's, as Docker Desktop reports its sources), so ${unknownSources.length === 1 ? "that mount cannot" : "those mounts cannot"} be compared from here` : null;
-	return { drift, unknown };
+	// `outOfDate` only when true, so a current proxy's judgement keeps its two-key shape.
+	return includeMissing && drift.length === 1 && unknown === null ? { drift, unknown, outOfDate: true } : { drift, unknown };
 }
 
 /** The job and sandbox networks attached to the proxy, which a removal would cut off (issue #453, gate round 2). */

package/src/egress.mjs

@@ -137,6 +137,18 @@ export function egressCanaryProbe(slug, pid) {
 	return `${EGRESS_CANARY_PROBE_PREFIX}${slug}-${pid}`;
 }
 
+/**
+ * The probe containers doctor runs per declared model endpoint (issue #503), on the same canary network. UNDER the
+ * canary's probe prefix on purpose: every line and page that tells an operator what a leftover probe looks like
+ * (`pi-dispatch-egress-probe-...`) stays true, and the dead-pid sweep matches these by an anchored pattern of its own.
+ */
+export const EGRESS_ENDPOINT_PROBE_PREFIX = `${EGRESS_CANARY_PROBE_PREFIX}endpoint-`;
+
+/** One endpoint probe container: per probe, per endpoint id (`[a-z0-9-]{1,32}`, the parser's rule) and per doctor process. */
+export function egressEndpointProbe(slug, id, pid) {
+	return `${EGRESS_ENDPOINT_PROBE_PREFIX}${slug}-${id}-${pid}`;
+}
+
 /**
  * How a container reaches the proxy: by NAME, resolved by docker's embedded DNS on the user-defined
  * network. `docs/sandbox.md`'s recipe had to write a bare gateway IP because the DEFAULT bridge has no

package/src/env-allowlist.mjs

@@ -33,6 +33,8 @@ import { egressEnv } from "./egress.mjs";
 import { forgeSpec } from "./forges.mjs";
 import { apiKeyVariable } from "./provider-key.mjs";
 import { isDeterminateFsCode } from "./transient.mjs";
+import { KEYLESS_HOW, keylessVerdict } from "./model-endpoints.mjs";
+import { KEYLESS_ENV_NAME } from "./reserved-env.mjs";
 
 function configError(message) {
 	const error = new Error(message);
@@ -111,8 +113,15 @@ export function piProviders() {
  * API-key credentials only; an OAuth/subscription login is refused (it expires, the container cannot refresh
  * it, and it is not the credential for an unattended service). Throws a config-tagged error (pre-spend
  * refusal) when neither source yields a credential.
+ *
+ * Third way in (issue #503), only for a provider pi does not know: a custom provider in the overlay `models.json`
+ * whose every model is served by a declared `keyless` model endpoint passes with no credential, and the answer is
+ * `{ PI_DISPATCH_KEYLESS: "keyless" }`, the fixed value its `"apiKey": "$PI_DISPATCH_KEYLESS"` resolves to in the job.
+ * `modelEndpoints` is the pickup's snapshot `{ endpoints, models }` (never a read of its own), so the free gate and
+ * the container env decide on the same declaration. A builtin provider whose baseUrl an operator pointed at a local
+ * server is NOT keyless (a named residual): pi knows it, so it still needs its key.
  */
-export function resolveProviderCredential({ provider, hostEnv, authFromPi = false, agentDir, readFile = readFileSync, forwardEnv = [] }) {
+export function resolveProviderCredential({ provider, hostEnv, authFromPi = false, agentDir, readFile = readFileSync, forwardEnv = [], modelEndpoints = null }) {
 	// The NAMES come from pi, the VALUES from `hostEnv`, and this asks each question of the thing that can
 	// answer it (issue #311). It used to be one `providerKeyVars(provider, hostEnv)` call, which conflates
 	// them: that is `findEnvKeys`, whose presence test falls back to the REAL `process.env` for any name the
@@ -129,7 +138,32 @@ export function resolveProviderCredential({ provider, hostEnv, authFromPi = fals
 	// is the DI seam that was dishonest, which is worth fixing where the seam is the whole test surface.
 	// ONE read per name, checked and returned, for the same reason: two reads could decide on one value and
 	// ship another.
-	const held = providerKeyCandidates(provider)
+	const candidates = providerKeyCandidates(provider);
+	// Issue #503: a provider pi does not know at all. The same two questions, in the same order, as the auth.json
+	// refusal below and doctor's `noKeyVariableCheck`: no key variable, and not in pi's catalog. Such a provider can
+	// never take a key through this gate (there is no variable to write it under), so it is answered HERE, before
+	// auth.json is read: either it is keyless, or it is refused with both ways in named. That used to be decided
+	// after an auth.json read, which could only end in a refusal too, and whose wording ("set the key in .env, or run
+	// `pi login`") was advice no custom provider can follow.
+	if (candidates.length === 0 && !piProviders().includes(provider)) {
+		const ids = keylessEndpointsFor(provider, modelEndpoints);
+		// The one variable this branch writes, fixed and non-secret. It is the credential's slot in the closed map,
+		// so buildContainerEnv carries it exactly when this gate passes keyless, and on no other job.
+		if (ids !== null) return { [KEYLESS_ENV_NAME]: KEYLESS_VALUE };
+		// The overlay could not be read at this pickup for a TRANSIENT reason (index.mjs, `modelsUnreadable`): the provider
+		// may well be keyless, so this is no verdict at all. UNTAGGED and marked transient, never `piDispatchConfig`: a
+		// config refusal is permanent, refunded and commented publicly, and the same job may pass on the next attempt. The
+		// processor turns this into an infra retry (CONST-RETRY-INFRA-ONLY); absent or invalid JSON stays determinate.
+		const unreadable = modelEndpoints?.modelsUnreadable;
+		if (unreadable && Array.isArray(modelEndpoints?.endpoints) && modelEndpoints.endpoints.length > 0) {
+			const error = new Error(`the overlay models.json could not be read at this pickup (${unreadable.code}), so whether provider "${provider}" is keyless is not known yet`);
+			error.piDispatchTransient = true;
+			error.code = unreadable.code;
+			throw error;
+		}
+		throw configError(unknownProviderMessage(provider));
+	}
+	const held = candidates
 		.map((name) => [name, hostEnv[name]])
 		.filter(([, value]) => value);
 	// EVERY held candidate is forwarded, in pi's order, including the two that are not API keys: a host
@@ -157,7 +191,26 @@ export function resolveProviderCredential({ provider, hostEnv, authFromPi = fals
 		const { name, value } = credentialFromPiAuth(provider, agentDir ?? defaultAgentDir(hostEnv), readFile, { hostEnv, forwardEnv });
 		return { [name]: value };
 	}
-	throw configError(`provider ${provider} has no configured credential in the worker environment`);
+	throw configError(`provider ${provider} has no configured credential in the worker environment. Set its key there, or, ${KEYLESS_HOW}.`);
+}
+
+/** The value of `PI_DISPATCH_KEYLESS`: fixed and non-secret, since the model server takes no key. */
+export const KEYLESS_VALUE = "keyless";
+
+/**
+ * The keyless endpoint ids serving this provider, or null (issue #503). Asked only for a provider pi does not know (the
+ * caller's predicate), against the pickup's snapshot `{ endpoints, models }` (index.mjs, one read per pickup). No
+ * snapshot, or no endpoint declared, is null: with nothing declared the overlay is never read, and nothing is keyless.
+ */
+export function keylessEndpointsFor(provider, modelEndpoints) {
+	const endpoints = modelEndpoints?.endpoints;
+	if (!Array.isArray(endpoints) || endpoints.length === 0) return null;
+	const verdict = keylessVerdict({ models: modelEndpoints.models, provider, endpoints });
+	return verdict.keyless ? verdict.endpoints : null;
+}
+
+function unknownProviderMessage(provider) {
+	return `pi has no provider "${provider}", so there is no key variable to give it a key. Use one of pi's provider ids with its key in the worker environment or in pi's auth.json, or, ${KEYLESS_HOW}.`;
 }
 
 function defaultAgentDir(hostEnv) {
@@ -245,7 +298,9 @@ function credentialFromPiAuth(provider, agentDir, readFile, { hostEnv = {}, forw
 				`pi authenticates "${provider}" without an API-key environment variable (an AWS profile or an OAuth login), and the container env is a closed set of variables, so it has no way in. Configure a provider whose credential is a single environment variable.`,
 			);
 		}
-		throw configError(`could not determine the environment variable pi expects for provider "${provider}" — pi has no such provider, so check PI_PROVIDER against pi's own ids`);
+		// Unreachable through resolveProviderCredential, which answers an unknown provider before auth.json is read. Kept
+		// as the backstop with the same words, so a later caller cannot reopen the old advice.
+		throw configError(unknownProviderMessage(provider));
 	}
 	return { name, value: cred.key };
 }
@@ -296,12 +351,14 @@ function resolveEnvName(provider) {
  * `allowGlobalExtensions` defaults to TRUE here, matching loadConfig's default (REQ-GLOBAL-PI-OVERLAY): a
  * caller that says nothing gets the operator's staged setup, and only an explicit `false` withholds it.
  */
-export function buildContainerEnv({ provider, model, maxTurns, maxTokens, jobId, githubToken, forgeKind, forgeHosts = {}, hostEnv, allowGlobalExtensions = true, packagePaths = [], forwardEnv = [], secrets = {}, sessionFile = null, flow = null, command = null, excludeTools = [], authFromPi = false, egress = false, egressProxy, agentDir, home = null, readFile = readFileSync }) {
+export function buildContainerEnv({ provider, model, maxTurns, maxTokens, maxCostMicros = null, jobId, githubToken, forgeKind, forgeHosts = {}, hostEnv, allowGlobalExtensions = true, packagePaths = [], forwardEnv = [], secrets = {}, sessionFile = null, flow = null, command = null, excludeTools = [], allowedModels = null, authFromPi = false, egress = false, egressProxy, agentDir, home = null, readFile = readFileSync, modelEndpoints = null, exitAuth = false }) {
 	// The provider credential(s), by pi's expected variable name(s) -- from the worker env, or (when
 	// PI_AUTH_FROM_PI is set and the env has none) host-side from pi's auth.json. Throws (config) if
 	// neither source yields one, which the processor turns into a policy refusal that refunds any reserve
 	// (issue #310); the same call is made by its free credential gate, before anything is reserved at all.
-	const credEnv = resolveProviderCredential({ provider, hostEnv, authFromPi, agentDir, readFile, forwardEnv });
+	// `modelEndpoints` is the pickup's snapshot (issue #503): the keyless branch answers `PI_DISPATCH_KEYLESS` in this
+	// map, so the variable is set exactly when the gate passed keyless and is absent on every other job.
+	const credEnv = resolveProviderCredential({ provider, hostEnv, authFromPi, agentDir, readFile, forwardEnv, modelEndpoints });
 
 	const env = {
 		PI_PROVIDER: provider,
@@ -356,6 +413,13 @@ export function buildContainerEnv({ provider, model, maxTurns, maxTokens, jobId,
 		// second-validator rule directly above). Absent means the full pinned default set, never an empty
 		// string, for PI_PACKAGES' reason.
 		PI_EXCLUDE_TOOLS: excludeTools.length > 0 ? excludeTools.join(",") : undefined,
+		// The job's EFFECTIVE allowed-model list (issue #502): its trigger's `run.models`, else the deployment's
+		// PI_ALLOWED_MODELS. Comma-joined `provider/model` entries, which the runner splits at each entry's first `/`
+		// (image/runner/src/config.mjs). Comma is safe because both list parsers refuse an entry carrying one. NULL is
+		// unrestricted and emits NO variable, never an empty string: the runner refuses an empty value as a config
+		// error, because an empty allow list read as "unset" would fail open. An image too old to enforce a list is
+		// refused before this is ever built (`modelPolicy`, CAPABILITY_GATES).
+		PI_ALLOWED_MODELS: Array.isArray(allowedModels) && allowedModels.length > 0 ? allowedModels.join(",") : undefined,
 		// Kill switch for job-time package installation, UNCONDITIONAL for every job. pi's resolver shells out
 		// to a REAL `npm install` for any npm:/git: source unless offline mode is on, and `~/.pi/agent` IS
 		// writable in the container. We emit only local paths, so nothing should reach that branch -- this
@@ -431,6 +495,43 @@ export function buildContainerEnv({ provider, model, maxTurns, maxTokens, jobId,
 	// env-internal HOME: written into the job's closed env map here, never read from the worker's environment.
 	if (typeof home === "string" && home !== "") env.HOME = home;
 
+	// Issue #503: PI_DISPATCH_KEYLESS is the credential gate's answer and nothing else's, so it is settled AFTER the
+	// PI_FORWARD_ENV and secrets loops, as HOME and the egress variables are: both lists refuse the name upstream, and
+	// this is the backstop that keeps a value from either one from replacing it, or from appearing on a keyed job.
+	if (credEnv[KEYLESS_ENV_NAME] === KEYLESS_VALUE) env[KEYLESS_ENV_NAME] = KEYLESS_VALUE;
+	else delete env[KEYLESS_ENV_NAME];
+
+	// Issue #501: the per-job dollar cap is the worker's answer and nothing else's, so it is settled again AFTER the
+	// PI_FORWARD_ENV and secrets loops, PI_DISPATCH_KEYLESS's backstop. Both lists refuse the name upstream (config.mjs
+	// refuses every CONTAINER_ENV_NAMES member in PI_FORWARD_ENV at boot, #502; the loader refuses it in run.secrets);
+	// this is the line that holds if either refusal is ever bypassed. A forwarded or secret PI_MAX_COST_MICROS would
+	// otherwise replace the computed cap (measured: a host value of 1e12 beat a computed 500000), and on a job with
+	// no cap it would send one past the `costCap` gate, which only runs for capped jobs, to an image that may ignore it.
+	// It is set ONLY here, once, after both loops. `!== null`, never truthiness: 0 is a real cap (no priced call at
+	// all, which a fail-closed job value reads as, `effectiveCostCapMicros`), and a truthy test would send NO cap for
+	// it, the widest one there is. Absent (null or undefined) leaves no variable, so a job with no cap carries the env
+	// it always did, and never an empty string (the runner refuses one).
+	// env-internal PI_MAX_COST_MICROS: written into the job's closed env map here, never read from the worker's environment.
+	if (maxCostMicros === null || maxCostMicros === undefined) delete env.PI_MAX_COST_MICROS;
+	else env.PI_MAX_COST_MICROS = String(maxCostMicros);
+
+	// Issue #545: the exit line's key waits on the container's stdin, and this variable tells the runner to read it.
+	// The value names the channel and is never the key: the environment is readable from /proc/1/environ by every
+	// process in the container (measured), which is why the key does not travel here. Settled after both loops like
+	// the cap above, and ONLY for a run the worker actually hands a key, so a runner never blocks on a stdin no one
+	// writes. `=== true`, so only run-container's own boolean asks for it.
+	// env-internal PI_EXIT_AUTH: written into the job's closed env map here, never read from the worker's environment.
+	if (exitAuth === true) env.PI_EXIT_AUTH = "stdin";
+	else delete env.PI_EXIT_AUTH;
+
+	// Issue #500: the runner sets these two in its own environment, for its child processes, and no value from outside
+	// may arrive first. Both lists refuse them upstream (config.mjs for PI_FORWARD_ENV, the triggers loader for
+	// run.secrets, through RUNNER_ENV_NAMES); this is the line that holds if either refusal is ever bypassed.
+	// env-internal PI_DISPATCH_CHILD_LEDGER: set by the runner inside the container, never by the worker, so removed here.
+	delete env.PI_DISPATCH_CHILD_LEDGER;
+	// env-internal PI_DISPATCH_RUNNER_PID: set by the runner inside the container, never by the worker, so removed here.
+	delete env.PI_DISPATCH_RUNNER_PID;
+
 	// Forge-backed jobs, and local cron jobs that opted in via run.github. Other local-folder jobs have
 	// no token (CONST-TOKEN-SCOPED-PER-JOB). The mint goes into BOTH of its forge's variables because
 	// each CLI has its own preference -- gh prefers GH_TOKEN over GITHUB_TOKEN, glab prefers GITLAB_TOKEN