@edgehero/pi-dispatch 1.10.3 → 2.0.0

package/src/egress-conf-copy.mjs

@@ -0,0 +1,166 @@
+/**
+ * A copy of the egress proxy's RULES that has gone stale (issue #484). Two copies of the package's
+ * `deploy/egress-proxy.conf` live outside the package: the deployment folder's `deploy/egress-proxy.conf`, which `init`
+ * writes create-only (issue #480) and the docker proxy mounts, and the podman venue's account-owned copy,
+ * `~/.config/pi-dispatch/egress-proxy.conf`, which `service install` writes (podman-stack.mjs, `proxyConfCopyPath`).
+ * Neither is rewritten by an upgrade, so a release that changes the shipped rules leaves the proxy on the old ones, and
+ * the mount-path comparison `up` and `doctor` make (egress-proxy-state.mjs) cannot see it: the path is right, only the
+ * bytes are old.
+ *
+ * So the bytes are compared with the INSTALLED package's copy, the one `init` copies from, and a difference is said.
+ * Said, never repaired on its own: a difference reads the same whether an upgrade left the file behind or the operator
+ * edited it on purpose, and nothing here can tell the two apart. `doctor` warns (⚠: a differing copy still runs a
+ * proxy, and may be exactly what the operator meant); `up` offers the refresh, shown and asked, and keeps the old file.
+ */
+import { randomBytes } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+/**
+ * The proxy's RULES (INT-EGRESS-POLICY-CONTRACT), the package's own copy: `../deploy/` from this module, the layout
+ * service.mjs resolves its templates by, which is worker/deploy in a checkout (byte-identical to the root deploy/,
+ * pinned by worker/test/publish.test.mjs) and the shipped deploy/ under npm. Issue #480: the docker proxy mounts
+ * `./deploy/egress-proxy.conf` from the deployment folder, and a folder made by `npx @edgehero/pi-dispatch up` had
+ * none, so `up` declined to start the proxy the policy is on by default for and every job was refused before it spent.
+ */
+export const PACKAGED_EGRESS_PROXY_CONF = fileURLToPath(new URL("../deploy/egress-proxy.conf", import.meta.url));
+
+/** The installed package's version, for the words "the package's copy (<version>)". Null when it cannot be read. */
+export const PACKAGE_VERSION = (() => {
+	try {
+		const version = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
+		return typeof version === "string" ? version : null;
+	} catch {
+		return null;
+	}
+})();
+
+/** The installed package's rules, as text: the file `init` copies. */
+export function readPackagedProxyConf() {
+	return String(readFileSync(PACKAGED_EGRESS_PROXY_CONF, "utf8"));
+}
+
+/** "the package's copy (2.0.0)", or without the version where it could not be read. */
+export function packageCopyName(version = PACKAGE_VERSION) {
+	return version ? `the package's copy (${version})` : "the package's copy";
+}
+
+/**
+ * A short account of how `current` differs from `packaged`, by lines counted as a multiset (a line's order is not
+ * compared, only whether it is there, which is what an operator reading the summary asks first). `diff` is named for
+ * the rest. Never the lines themselves: a rules file is short, but a summary that quoted it would print an operator's
+ * own edit into every doctor run.
+ */
+export function describeConfDifference(current, packaged) {
+	// Line endings alone (PR #491's review): an editor or a checkout that wrote CRLF changes every line, and "66 lines of
+	// it not in the package's copy" then reads as a rewrite. Said as what it is, naming which side has which.
+	const lf = (text) => String(text).replace(/\r\n/g, "\n");
+	if (lf(current) === lf(packaged)) {
+		const crlf = (text) => String(text).includes("\r\n");
+		return `only in its line endings, ${crlf(current) ? "CRLF" : "LF"} here and ${crlf(packaged) ? "CRLF" : "LF"} in the package's`;
+	}
+	const count = (text) => {
+		const m = new Map();
+		for (const line of String(text).split("\n")) m.set(line, (m.get(line) ?? 0) + 1);
+		return m;
+	};
+	const a = count(current);
+	const b = count(packaged);
+	let onlyHere = 0;
+	let onlyThere = 0;
+	for (const [line, n] of a) onlyHere += Math.max(0, n - (b.get(line) ?? 0));
+	for (const [line, n] of b) onlyThere += Math.max(0, n - (a.get(line) ?? 0));
+	if (onlyHere === 0 && onlyThere === 0) return "the same lines, in another order";
+	const lines = (n) => `${n} line${n === 1 ? "" : "s"}`;
+	return `${lines(onlyHere)} of it not in the package's copy, ${lines(onlyThere)} of the package's not in it`;
+}
+
+/**
+ * Judges one copy against the package's: `{ state }` is "absent" (nothing to compare; whoever mounts it says so),
+ * "same", "differs" (with `summary`), "unreadable" (with `error`, the copy could not be read), or "no-package" (with
+ * `error`, the package's own copy could not be read, which is a broken install and said as one). `read(path)` reads the
+ * copy; `readPackaged()` the package's.
+ */
+export function judgeProxyConfCopy({ path, read, readPackaged = readPackagedProxyConf }) {
+	let current;
+	try {
+		current = String(read(path));
+	} catch (err) {
+		if (err?.code === "ENOENT" || err?.code === "ENOTDIR") return { state: "absent" };
+		return { state: "unreadable", error: err?.code ?? err?.message ?? String(err) };
+	}
+	let packaged;
+	try {
+		packaged = String(readPackaged());
+	} catch (err) {
+		return { state: "no-package", error: err?.code ?? err?.message ?? String(err) };
+	}
+	if (current === packaged) return { state: "same" };
+	return { state: "differs", summary: describeConfDifference(current, packaged), packaged };
+}
+
+/** A timestamp for a backup's name, from the injected clock: 20260929T101500Z. No colon, which Windows refuses. */
+export function backupStamp(now) {
+	return new Date(now).toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[-:]/g, "");
+}
+
+/**
+ * Replaces the copy at `path` with `text`, keeping the old bytes as `<path>.bak-<stamp>` (issue #484). Returns
+ * `{ ok: true, backup }` or `{ ok: false, reason }`, where `reason` finishes the sentence "not replaced: ...".
+ *
+ * NO SYMLINK IS FOLLOWED, and both ends are asked with `lstat`: a symlinked parent (`deploy/` a link) would put the new
+ * file and the backup wherever it points, and a symlinked file would have the backup read through it; both are refused
+ * with nothing written, as `init` refuses a symlinked `deploy/` (PR #488's review). Anything that is not a regular file
+ * is refused too.
+ *
+ * ATOMIC: the new text is written to a temp file in the SAME directory (`wx`, so nothing that exists is written through)
+ * and renamed over the copy, so a reader (a proxy starting this second) sees the old file or the new one, never half of
+ * either, and a failed write leaves the old file exactly as it was. `rename` replaces the directory ENTRY, so a link
+ * that appeared at `path` after the check is replaced, never written through. The backup is written `wx` before the
+ * rename, so a backup that could not be written replaces nothing. A second refresh in the same second finds the backup
+ * name taken and refuses rather than overwrite the first backup.
+ */
+export function replaceProxyConfCopy({ path, text, fs, now = Date.now, random = () => randomBytes(6).toString("hex") }) {
+	if (typeof fs.lstatSync !== "function") return { ok: false, reason: "this build cannot check the path for a symlink, so it is left as it is" };
+	const dir = dirname(path);
+	const lst = (p) => {
+		try {
+			return fs.lstatSync(p);
+		} catch {
+			return null;
+		}
+	};
+	const parent = lst(dir);
+	if (!parent || parent.isSymbolicLink() || !parent.isDirectory()) return { ok: false, reason: `${dir} is ${!parent ? "not there" : parent.isSymbolicLink() ? "a symlink" : "not a directory"}, and the new file would be written wherever it leads` };
+	const own = lst(path);
+	if (!own) return { ok: false, reason: `${path} is not there` };
+	if (own.isSymbolicLink()) return { ok: false, reason: `${path} is a symlink, and the file it points to is not this folder's to replace` };
+	if (!own.isFile()) return { ok: false, reason: `${path} is not a regular file` };
+	const mode = typeof own.mode === "number" ? own.mode & 0o777 : 0o644;
+	let previous;
+	try {
+		previous = fs.readFileSync(path);
+	} catch (err) {
+		return { ok: false, reason: `${path} could not be read (${err?.code ?? err?.message})` };
+	}
+	const backup = `${path}.bak-${backupStamp(now())}`;
+	try {
+		fs.writeFileSync(backup, previous, { flag: "wx", mode });
+	} catch (err) {
+		return { ok: false, reason: `the backup ${backup} could not be written (${err?.code ?? err?.message}), so the copy was not touched` };
+	}
+	const tmp = join(dir, `.${basename(path)}.tmp-${random()}`);
+	try {
+		fs.writeFileSync(tmp, text, { flag: "wx", mode });
+		fs.renameSync(tmp, path);
+	} catch (err) {
+		try {
+			fs.unlinkSync(tmp);
+		} catch {
+			// Never written, or already renamed: nothing to take back.
+		}
+		return { ok: false, reason: `the new file could not be written beside it (${err?.code ?? err?.message}); the copy is as it was, and ${backup} holds the same bytes` };
+	}
+	return { ok: true, backup };
+}

package/src/egress-proxy-state.mjs

@@ -0,0 +1,151 @@
+/**
+ * THE SHIPPED EGRESS PROXY'S STATE, read one way by `up` and `doctor` on the docker venue (issue #453, gate rounds 1
+ * and 2).
+ *
+ * Two questions, both answered from one `docker inspect`:
+ *   - RUNNING is `{{.State.Status}}` === "running". Measured on Docker Engine 29.8.1: a paused container reads
+ *     `paused` with `.State.Running` TRUE, and one a restart policy is cycling reads `restarting` with `.State.Running`
+ *     TRUE (39 of 40 samples); Podman reports a paused one `paused` with Running false and one between restarts
+ *     `stopped` (4.9.3 and 5.8.1). Only the status word tells them from a proxy that carries traffic.
+ *   - CURRENT: a container of the shipped name made from another image (an older squid digest), with another
+ *     entrypoint or command, or with other files mounted, is not the proxy this deployment ships, and `docker start`
+ *     of it, or a "present" that leaves it running, keeps that policy. Compared: the image's digest, the entrypoint and
+ *     command against the pinned image's own (read from its registry config, identical on every platform in the
+ *     manifest list), and the mounts: the two bind sources against this folder's `deploy/egress-proxy.conf` and
+ *     `egress-allowlist.conf`, the image's two anonymous volumes allowed, anything else stale.
+ *
+ * MEASURED (Docker Engine 29.8.1, compose 5.5.1, rootful, gate round 2; pinned in egress-proxy-state.test.mjs): a proxy
+ * made by `docker compose --profile egress up -d` and one made by `up`'s argv carry the pinned reference exactly as
+ * written, the image's own entrypoint and command, the two binds and the image's two anonymous volumes; a folder
+ * reached through a symlink is reported by the symlink's path, which the realpath below makes the same folder.
+ *
+ * THE MOUNTS ARE COMPARED WHERE THEY CAN BE. A runtime that reports a source as a path of its own VM (Docker Desktop's
+ * `/host_mnt/...` and `/run/desktop/mnt/host/...`, not measured here) gives sources nothing here can resolve, and
+ * calling that proxy stale would have doctor fail forever and `up --yes` remove a healthy proxy on every run. So a
+ * source under one of those prefixes, or any unresolvable source on a macOS or Windows host, makes the mount
+ * comparison UNKNOWN, said as such and never the ground for a removal. On a Linux host an unresolvable source is gone,
+ * and that proxy is stale. The image, entrypoint and command are compared either way.
+ *
+ * Only the SHIPPED name is judged current or stale: a container `PI_EGRESS_PROXY` names is the operator's own.
+ */
+import { resolve } from "node:path";
+
+/** The pinned squid, as `deploy/docker-compose.yml` and `up`'s run argv name it (a test binds the three). */
+export const EGRESS_PROXY_IMAGE = "ubuntu/squid@sha256:6a097f68bae708cedbabd6188d68c7e2e7a38cedd05a176e1cc0ba29e3bbe029";
+
+/**
+ * The pinned image's own entrypoint and command, and its anonymous volumes, from its registry config (read on
+ * 2026-09-27 for every linux platform in the manifest list: amd64, arm64, ppc64le, s390x, all identical). Neither
+ * compose nor `up` sets either, so a container made by them carries these.
+ */
+export const EGRESS_PROXY_ENTRYPOINT = Object.freeze(["entrypoint.sh"]);
+export const EGRESS_PROXY_CMD = Object.freeze(["-f", "/etc/squid/squid.conf", "-NYC"]);
+const IMAGE_VOLUMES = Object.freeze(["/var/log/squid", "/var/spool/squid"]);
+
+/** Where Docker Desktop reports a bind source from its own VM (not measured here; the documented shapes). */
+const VM_PREFIXES = Object.freeze(["/host_mnt/", "/run/desktop/mnt/host/"]);
+
+/**
+ * The inspect format both callers use: one JSON object, so no field can be split on a separator a path might hold.
+ * `health` is doctor's advisory line; the rest is this module's.
+ */
+export const PROXY_STATE_FORMAT =
+	'--format={"status":{{json .State.Status}},"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}"none"{{end}},"image":{{json .Config.Image}},"entrypoint":{{json .Config.Entrypoint}},"cmd":{{json .Config.Cmd}},"mounts":{{json .Mounts}},"networks":{{json .NetworkSettings.Networks}}}';
+
+/** Where the proxy's two files are mounted inside it (compose's and `up`'s mounts alike). */
+const SQUID_CONF = "/etc/squid/squid.conf";
+const ALLOWLIST = "/etc/pi-dispatch/allowlist.conf";
+
+/**
+ * `{ status, health, image, entrypoint, cmd, mounts, networks }` from `PROXY_STATE_FORMAT`'s stdout, or null when it
+ * is not that shape. `mounts` is the list as `[{ type, source, destination }]`, `networks` the attached names.
+ */
+export function parseProxyState(stdout) {
+	let body;
+	try {
+		body = JSON.parse(String(stdout ?? "").trim());
+	} catch {
+		return null;
+	}
+	if (!body || typeof body !== "object" || typeof body.status !== "string" || !/^[a-z]{1,20}$/.test(body.status)) return null;
+	// An argv as a list. Podman 4.9.3's native inspect renders `.Config.Entrypoint` as a STRING (`"entrypoint.sh"`, and
+	// `""` for none; measured, gate round 3), where Docker Engine 29.8.1 and Podman 5.8.1 (native and through its Docker
+	// API) render a list; docker renders none as `null`. Each is normalised, so a correctly shipped proxy never reads
+	// stale on the runtime's spelling.
+	const strings = (v) => (Array.isArray(v) && v.every((x) => typeof x === "string") ? v : typeof v === "string" ? (v === "" ? [] : [v]) : v === null || v === undefined ? [] : null);
+	return {
+		status: body.status,
+		// Podman's Docker API answers "" where docker's template says "none" (measured, 5.8.1): both are "no healthcheck".
+		health: typeof body.health === "string" && body.health !== "" ? body.health : "none",
+		image: typeof body.image === "string" ? body.image : "",
+		entrypoint: strings(body.entrypoint),
+		cmd: strings(body.cmd),
+		mounts: (Array.isArray(body.mounts) ? body.mounts : []).filter((m) => m && typeof m === "object").map((m) => ({ type: String(m.Type ?? ""), source: String(m.Source ?? ""), destination: String(m.Destination ?? "") })),
+		networks: body.networks && typeof body.networks === "object" ? Object.keys(body.networks) : [],
+	};
+}
+
+/** The digest an image reference pins, or null. */
+function digestOf(image) {
+	return /@(sha256:[0-9a-f]{64})$/.exec(String(image ?? ""))?.[1] ?? null;
+}
+
+const sameList = (a, b) => Array.isArray(a) && a.length === b.length && a.every((x, i) => x === b[i]);
+
+/**
+ * What makes a container of the shipped name not this deployment's proxy, as `{ drift, unknown }`: `drift` the
+ * sentences that make it stale, `unknown` why the mounts could not be compared here (null when they were).
+ * `realpath` throws for a path that does not resolve on this host. `compareMounts: false` judges the image, entrypoint and
+ * command alone, for a caller that does not know which folder the service uses. `platform` decides whether a source
+ * that does not resolve can be a path of the runtime's own VM (below).
+ */
+export function shippedProxyDrift(state, { cwd, realpath = (p) => p, compareMounts = true, platform = "linux" }) {
+	const drift = [];
+	if (digestOf(state.image) !== digestOf(EGRESS_PROXY_IMAGE)) drift.push(`it was created from ${state.image || "an unnamed image"}, not the pinned ${EGRESS_PROXY_IMAGE}`);
+	if (!sameList(state.entrypoint, EGRESS_PROXY_ENTRYPOINT)) drift.push(`its entrypoint is ${JSON.stringify(state.entrypoint)}, not the image's ${JSON.stringify(EGRESS_PROXY_ENTRYPOINT)}`);
+	if (!sameList(state.cmd, EGRESS_PROXY_CMD)) drift.push(`its command is ${JSON.stringify(state.cmd)}, not the image's ${JSON.stringify(EGRESS_PROXY_CMD)}`);
+	if (!compareMounts) return { drift, unknown: null };
+	const binds = state.mounts.filter((m) => m.type === "bind");
+	const resolves = (source) => {
+		try {
+			realpath(source);
+			return true;
+		} catch {
+			return false;
+		}
+	};
+	// UNKNOWN only where a source can be a path of the runtime's own VM: under a VM prefix Docker Desktop is known to use,
+	// or on a macOS or Windows host, where every Linux daemon runs in a VM (gate round 3's simple rule). On a Linux host a
+	// source that does not resolve is simply gone (its folder deleted, say), and a proxy mounting nothing real is stale.
+	// EACH BIND ON ITS OWN (round-cap re-review): an unknown one is unknown for itself, and the other expected bind and
+	// every extra mount are still compared; a caller refuses a removal only while an EXPECTED bind is unknown.
+	const vmPath = (source) => VM_PREFIXES.some((prefix) => source.startsWith(prefix)) || platform === "darwin" || platform === "win32";
+	const expected = { [SQUID_CONF]: resolve(cwd, "deploy/egress-proxy.conf"), [ALLOWLIST]: resolve(cwd, "egress-allowlist.conf") };
+	const same = (source, want) => {
+		try {
+			return source === want || realpath(source) === realpath(want);
+		} catch {
+			return false;
+		}
+	};
+	const unknownSources = [];
+	for (const [destination, want] of Object.entries(expected)) {
+		const bind = binds.find((m) => m.destination === destination);
+		if (!bind) drift.push(`nothing is mounted at ${destination}, where ${want} belongs`);
+		else if (!resolves(bind.source) && vmPath(bind.source)) unknownSources.push(bind.source);
+		else if (!resolves(bind.source)) drift.push(`its ${destination} is ${bind.source}, which does not exist on this host`);
+		else if (!same(bind.source, want)) drift.push(`its ${destination} is ${bind.source}, not ${want}`);
+	}
+	for (const m of state.mounts) {
+		if (m.type === "bind" && Object.hasOwn(expected, m.destination)) continue;
+		if (m.type === "volume" && IMAGE_VOLUMES.includes(m.destination)) continue;
+		drift.push(`it has a ${m.type || "mount"} at ${m.destination} (from ${m.source || "nowhere named"}) that the shipped proxy does not`);
+	}
+	const unknown = unknownSources.length > 0 ? `${unknownSources.join(", ")} ${unknownSources.length === 1 ? "is a path" : "are paths"} this host cannot resolve (the runtime's own VM's, as Docker Desktop reports its sources), so ${unknownSources.length === 1 ? "that mount cannot" : "those mounts cannot"} be compared from here` : null;
+	return { drift, unknown };
+}
+
+/** The job and sandbox networks attached to the proxy, which a removal would cut off (issue #453, gate round 2). */
+export function jobNetworksOf(state) {
+	return state.networks.filter((n) => n.startsWith("pi-job-") || n.startsWith("pi-sandbox-"));
+}