cursedops 0.4.0 → 0.5.0

package/src/smoke.ts

@@ -31,9 +31,19 @@
  * |---|---|---|
  * | `0` | every check passed | keep the release |
  * | `1` | the APP is wrong | roll back |
- * | `2` | the EDGE is wrong — DNS, tunnel, hostname, policy | do **not** roll back |
+ * | `2` | NOT the app — the edge (DNS, tunnel, hostname, policy), the NETWORK the smoke ran on, or the smoke's OWN environment | do **not** roll back |
  *
  * Two is the load-bearing one and it is why {@link Smoke.edgeFault} exists.
+ *
+ * 🔴 **0.5.0 widened what a `2` can mean and did NOT add a code**, deliberately. Every deploy
+ * script in the fleet reads this contract as `0 → keep`, `2 → stop without reverting`, anything
+ * else → `git revert` (`apps/collections/scripts/deploy.ts:413-424` and its four siblings). A new
+ * `3` for "the smoke's credential is missing" would fall into the revert branch of every one of
+ * them — the exact failure it exists to prevent — so the distinction lives in {@link Smoke.verdict}
+ * and in the sentence {@link Smoke.report} prints, and the CODE stays one every reader already
+ * handles. The one direction that is never excused: a dead ORIGIN ({@link probeOrigin}) is `1`
+ * whatever else the run saw, because a live edge over a dead origin is precisely the green lie
+ * (incident 5 above) this module exists to refuse.
  * Reverting good application code cannot restore a DNS record, a tunnel route or
  * an access policy: a rollback there discards work while leaving the real fault
  * exactly where it is. {@link Smoke.finish} is the single place that code is
@@ -71,12 +81,29 @@
  */
 
 import { spawnSync } from "node:child_process";
+import { lookup } from "node:dns/promises";
+import { isIP } from "node:net";
 
 export interface Failure {
 	check: string;
 	detail: string;
 }
 
+/**
+ * What a finished smoke concluded, in the order {@link Smoke.report} decides it:
+ *
+ *   · `origin` — the process behind the edge did not answer on loopback. Exit 1, ALWAYS: a cache
+ *     in front of a dead origin is the one thing no other verdict may excuse.
+ *   · `environment` — the smoke could not judge the app, because something IT needs (a
+ *     credential, a file) is missing here. Exit 2: nothing about the app was measured.
+ *   · `network` — the public checks failed and this machine's resolver is substituting answers
+ *     for a Cloudflare-proxied hostname. Exit 2: the app is not at fault.
+ *   · `edge` — something in front of the app could not be reached or disagrees. Exit 2.
+ *   · `app` — the app answered wrongly. Exit 1, roll back.
+ *   · `pass` — exit 0.
+ */
+export type SmokeVerdict = "pass" | "app" | "origin" | "edge" | "network" | "environment";
+
 export interface SmokeOptions {
 	/** The origin every relative path is asked against. A trailing slash is trimmed. */
 	base: string;
@@ -91,6 +118,8 @@ export interface SmokeOptions {
 	headers?: () => Record<string, string>;
 	/** Injected for tests. Resolves a hostname against a public resolver. */
 	resolve?: (host: string) => string;
+	/** Injected for tests. How {@link Smoke.checkNetwork} asks the resolvers — see {@link diagnoseNetwork}. */
+	network?: NetworkProbe;
 	log?: (line: string) => void;
 	error?: (line: string) => void;
 }
@@ -109,12 +138,47 @@ export interface Smoke {
 	anonymous(path: string, init?: RequestInit): Promise<Response>;
 	/** Mark the run as an edge fault: reached nothing, so code 2 rather than 1. */
 	markEdgeFault(): void;
+	/**
+	 * Record that the ORIGIN is dead — code 1 whatever else happened. {@link probeOrigin} is the
+	 * one caller that should need this; it is public so an app with its own origin check can say so.
+	 */
+	markOriginFault(check: string, detail: string): void;
+	/** True once {@link markOriginFault} has been called. */
+	readonly originFault: boolean;
+	/**
+	 * Assert the SMOKE's own environment before asserting the app: every name in `needs` whose
+	 * value is empty is refused with `how` (what to do about it), and the run can no longer pass
+	 * or fail the app — it exits 2 with the sentence. Returns true when nothing was missing.
+	 *
+	 * 🔴 Measured 2026-09-17 on `desk`: `✗ FAILED — 5 of 6` about a deploy that was serving
+	 * correctly, because only the deploy script loaded the Access token and the smoke, run by a
+	 * person, hit the Access wall and charged it to the app.
+	 */
+	requireEnvironment(needs: Record<string, string | null | undefined>, how: string): boolean;
+	/** One free-form environment refusal — for a need that is not a variable (a file, a socket). */
+	refuseEnvironment(what: string, how: string): void;
+	/** Every environment refusal so far. */
+	readonly refusals: readonly Failure[];
+	/**
+	 * Ask whether this machine's NETWORK is lying about {@link base}'s hostname — see
+	 * {@link diagnoseNetwork}. When it is, the run is marked a network fault (exit 2) and the
+	 * lines say where the fault lives. Returns the diagnosis.
+	 */
+	checkNetwork(): Promise<NetworkDiagnosis>;
+	/** The conclusion {@link report} will print, without printing it. */
+	verdict(): SmokeVerdict;
 	/** A sentence to append to a fetch failure, or `""`. See {@link createSmoke}. */
 	dnsHint(error: Error): string;
 	/** Print the ledger and return the exit code. Does not exit. */
 	report(): number;
 	/** {@link report}, then `process.exit` with it. */
 	finish(): never;
+	/**
+	 * {@link checkNetwork} when the run failed for a reason other than a dead origin, then
+	 * {@link report}. Returns the exit code; does not exit. The recommended ending of a smoke
+	 * that asks a public hostname: `process.exit(await smoke.settle())`.
+	 */
+	settle(): Promise<number>;
 }
 
 /**
@@ -147,7 +211,10 @@ export function createSmoke(options: SmokeOptions): Smoke {
 
 	const passed: string[] = [];
 	const failures: Failure[] = [];
+	const refusals: Failure[] = [];
 	let edgeFault = false;
+	let originFault = false;
+	let network: NetworkDiagnosis | null = null;
 
 	const smoke: Smoke = {
 		base,
@@ -157,9 +224,15 @@ export function createSmoke(options: SmokeOptions): Smoke {
 		get failures() {
 			return failures;
 		},
+		get refusals() {
+			return refusals;
+		},
 		get edgeFault() {
 			return edgeFault;
 		},
+		get originFault() {
+			return originFault;
+		},
 
 		record(check, ok, detail) {
 			if (ok) passed.push(`${check} — ${detail}`);
@@ -187,6 +260,44 @@ export function createSmoke(options: SmokeOptions): Smoke {
 			edgeFault = true;
 		},
 
+		markOriginFault(check, detail) {
+			originFault = true;
+			failures.push({ check, detail });
+		},
+
+		requireEnvironment(needs, how) {
+			const missing = Object.entries(needs)
+				.filter(([, value]) => !value?.trim())
+				.map(([name]) => name);
+			if (missing.length === 0) return true;
+			smoke.refuseEnvironment(`${missing.join(", ")} ${missing.length === 1 ? "is" : "are"} not set for this smoke`, how);
+			return false;
+		},
+
+		refuseEnvironment(what, how) {
+			refusals.push({ check: what, detail: how });
+		},
+
+		async checkNetwork() {
+			let host: string;
+			try {
+				host = new URL(base).hostname;
+			} catch {
+				return { verdict: "not-applicable", lines: [`${base} is not a URL`] };
+			}
+			network = await diagnoseNetwork(host, options.network);
+			return network;
+		},
+
+		verdict() {
+			if (originFault) return "origin";
+			if (refusals.length > 0) return "environment";
+			if (failures.length === 0) return "pass";
+			if (network?.verdict === "intercepting") return "network";
+			if (edgeFault) return "edge";
+			return "app";
+		},
+
 		dnsHint(failed: Error) {
 			if (!LOOKS_LIKE_DNS.test(failed.message)) return "";
 			const host = new URL(base).hostname;
@@ -204,19 +315,50 @@ export function createSmoke(options: SmokeOptions): Smoke {
 		report() {
 			for (const line of passed) log(`  ✓ ${line}`);
 			for (const failure of failures) error(`  ✗ ${failure.check} — ${failure.detail}`);
+			for (const refusal of refusals) error(`  ⚠ ${refusal.check} — ${refusal.detail}`);
 			const total = passed.length + failures.length;
-			if (failures.length === 0) {
+			const verdict = smoke.verdict();
+			if (verdict === "pass") {
 				log(`\n✓ deploy smoke passed — ${total} checks against ${base}`);
 				return 0;
 			}
+			if (verdict === "environment") {
+				error(`\n⚠ this smoke could not judge the app at ${base} — its OWN environment is missing something (above).`);
+				error("  (exit 2 — not an app failure: nothing about the app was measured. Rolling back would not change it.)");
+				return 2;
+			}
 			error(`\n✗ deploy smoke FAILED — ${failures.length} of ${total} checks against ${base}`);
-			if (edgeFault) error("  (exit 2 — an EDGE fault. Rolling the app back would not change it.)");
-			return edgeFault ? 2 : 1;
+			if (verdict === "origin") {
+				error("  (exit 1 — the ORIGIN is dead. Whatever the public hostname said came from something in front of it.)");
+				return 1;
+			}
+			if (verdict === "network") {
+				error("\n🛜 this network is intercepting DNS — the app is not at fault:");
+				for (const line of network?.lines ?? []) error(`  ${line}`);
+				error("  (exit 2 — the NETWORK this ran on. Re-run from another network, or over encrypted DNS.)");
+				return 2;
+			}
+			if (verdict === "edge") {
+				error("  (exit 2 — an EDGE fault. Rolling the app back would not change it.)");
+				return 2;
+			}
+			return 1;
 		},
 
 		finish() {
 			process.exit(smoke.report());
 		},
+
+		async settle() {
+			if (failures.length > 0 && !originFault && refusals.length === 0) {
+				try {
+					await smoke.checkNetwork();
+				} catch {
+					// A diagnosis that could not run is not a verdict; the ledger stands as it is.
+				}
+			}
+			return smoke.report();
+		},
 	};
 
 	return smoke;
@@ -421,3 +563,333 @@ export async function sameDeployedShell(smoke: Smoke, addresses: string[], optio
 	});
 	return agreed;
 }
+
+// ── The ORIGIN, asked directly ────────────────────────────────────────────────────────────
+
+/** The origin probe's name in the ledger. */
+export const ORIGIN_CHECK = "the origin answers on loopback";
+
+export interface OriginOptions {
+	/** What the port was read from, for the sentence when there is none — usually the launchd label. */
+	label?: string;
+	/** The path to ask. `/healthz` unless the app's liveness answer lives elsewhere. */
+	path?: string;
+	/** What counts as alive. Default: a `200`. */
+	accept?: (response: Response) => boolean | Promise<boolean>;
+	timeoutMs?: number;
+	/** Injected for tests. Defaults to the global `fetch`. */
+	fetch?: (url: string, init: RequestInit) => Promise<Response>;
+}
+
+/**
+ * Ask the process itself, on `127.0.0.1:<port>`, and record a dead one as an ORIGIN fault —
+ * exit 1 whatever the public hostname said.
+ *
+ * 🔴 **A deployed smoke that only asks the public hostname is asking Cloudflare, not the app.**
+ * Measured 2026-09-15/17: `patterns` and `collections` each served a cached `200` over a `503`
+ * origin with every check green, and four apps' smokes (`patterns`, `roms`, `family`, `desk`)
+ * made no loopback request at all. Loopback cannot be cached by anything, so this is the one
+ * question whose answer the edge cannot forge.
+ *
+ * Pass the port from `livePort(LABEL)` in `cursedops/launchd` — the LIVE declaration, never
+ * `package.json`, which advertised the wrong port in eight of eight apps. A `null` port is
+ * recorded as the fault it is: no job, no origin, and a public `200` can only be a cache.
+ */
+export async function probeOrigin(smoke: Smoke, port: number | null, options: OriginOptions = {}): Promise<boolean> {
+	const path = options.path ?? "/healthz";
+	const accept = options.accept ?? ((response: Response) => response.status === 200);
+	const send = options.fetch ?? ((url: string, init: RequestInit) => fetch(url, init));
+	const where = options.label ? ` (${options.label})` : "";
+	if (port === null) {
+		smoke.markOriginFault(
+			ORIGIN_CHECK,
+			`there is no origin to ask — no live port${where}. The job is not loaded or declares no PORT, so any 200 the public hostname gave came from a cache.`,
+		);
+		return false;
+	}
+	const url = `http://127.0.0.1:${port}${path}${path.includes("?") ? "&" : "?"}cb=${Date.now().toString(36)}`;
+	try {
+		const response = await send(url, { redirect: "manual", signal: AbortSignal.timeout(options.timeoutMs ?? 10_000) });
+		if (await accept(response)) return smoke.record(ORIGIN_CHECK, true, `GET 127.0.0.1:${port}${path} → ${response.status}`);
+		smoke.markOriginFault(ORIGIN_CHECK, `GET 127.0.0.1:${port}${path} → ${response.status}${where} — the process behind the edge is not healthy`);
+	} catch (thrown) {
+		smoke.markOriginFault(ORIGIN_CHECK, `GET 127.0.0.1:${port}${path}${where} — ${(thrown as Error).message}. Nothing is serving behind the edge.`);
+	}
+	return false;
+}
+
+// ── The NETWORK the smoke ran on ──────────────────────────────────────────────────────────
+
+/**
+ * Cloudflare's published proxy ranges (`https://www.cloudflare.com/ips-v4` and `/ips-v6`), as of
+ * 2026-09-19. The fallback, never the first answer: {@link diagnoseNetwork} asks for the live
+ * list first, and uses this only when it cannot be fetched — which is most likely on the very
+ * network this exists to describe.
+ */
+export const CLOUDFLARE_RANGES_SNAPSHOT: readonly string[] = [
+	"173.245.48.0/20",
+	"103.21.244.0/22",
+	"103.22.200.0/22",
+	"103.31.4.0/22",
+	"141.101.64.0/18",
+	"108.162.192.0/18",
+	"190.93.240.0/20",
+	"188.114.96.0/20",
+	"197.234.240.0/22",
+	"198.41.128.0/17",
+	"162.158.0.0/15",
+	"104.16.0.0/13",
+	"104.24.0.0/14",
+	"172.64.0.0/13",
+	"131.0.72.0/22",
+	"2400:cb00::/32",
+	"2606:4700::/32",
+	"2803:f800::/32",
+	"2405:b500::/32",
+	"2405:8100::/32",
+	"2a06:98c0::/29",
+	"2c0f:f248::/32",
+];
+
+function ipToBig(ip: string): bigint | null {
+	if (isIP(ip) === 4) return ip.split(".").reduce((a, o) => (a << 8n) + BigInt(Number(o)), 0n);
+	if (isIP(ip) !== 6) return null;
+	const bare = ip.split("%")[0] as string;
+	// `::ffff:1.2.3.4` — the tail is dotted and is two groups, not one.
+	const dotted = /(\d+\.\d+\.\d+\.\d+)$/.exec(bare);
+	const text = dotted
+		? bare.replace(dotted[1] as string, (() => {
+				const n = ipToBig(dotted[1] as string) as bigint;
+				return `${(n >> 16n).toString(16)}:${(n & 0xffffn).toString(16)}`;
+			})())
+		: bare;
+	const [head, tail] = text.split("::");
+	const h = head ? head.split(":") : [];
+	const t = tail !== undefined ? (tail ? tail.split(":") : []) : [];
+	const mid = tail !== undefined ? Array(8 - h.length - t.length).fill("0") : [];
+	return [...h, ...mid, ...t].reduce((a, g) => (a << 16n) + BigInt(Number.parseInt(g || "0", 16)), 0n);
+}
+
+/** Is `ip` inside one of `ranges` (CIDR strings)? Lifted from `tools/check-public-dns.ts`. */
+export function inRanges(ip: string, ranges: readonly string[]): boolean {
+	const addr = ipToBig(ip);
+	if (addr === null) return false;
+	const v6 = isIP(ip) === 6;
+	for (const cidr of ranges) {
+		const [net, bitsRaw] = cidr.split("/");
+		if (!net || !bitsRaw || (isIP(net) === 6) !== v6) continue;
+		const base = ipToBig(net);
+		if (base === null) continue;
+		const width = v6 ? 128n : 32n;
+		const bits = BigInt(bitsRaw);
+		const mask = ((1n << bits) - 1n) << (width - bits);
+		if ((addr & mask) === (base & mask)) return true;
+	}
+	return false;
+}
+
+/** One independent resolver's answer; `null` when it could not be reached for this host. */
+export interface IndependentAnswer {
+	name: string;
+	answers: string[] | null;
+}
+
+/** How {@link diagnoseNetwork} asks. Every member is injectable, so a test never touches a network. */
+export interface NetworkProbe {
+	/** What THIS machine's resolver answers — the path a `fetch` really takes. Default: `dns.lookup`, all addresses. */
+	system?: (host: string) => Promise<string[]>;
+	/** What resolvers that cannot be silently substituted answer. Default: DoH to Cloudflare and Google. */
+	independent?: (host: string) => Promise<IndependentAnswer[]>;
+	/** Cloudflare's proxy ranges. Default: the published list, else {@link CLOUDFLARE_RANGES_SNAPSHOT}. */
+	ranges?: () => Promise<readonly string[]>;
+}
+
+export interface NetworkDiagnosis {
+	/**
+	 * `intercepting` — this machine's answer for a Cloudflare-proxied hostname is outside every
+	 * published range, or is one no independent resolver gives: the NETWORK is at fault.
+	 * `clean` — the network is honest; whatever failed, it was not this. `not-proxied` — the
+	 * hostname is not behind Cloudflare's proxy, so the range rule says nothing. `offline` —
+	 * nothing resolved it at all. `not-applicable` — an IP literal or a loopback name.
+	 */
+	verdict: "intercepting" | "clean" | "not-proxied" | "offline" | "not-applicable";
+	lines: string[];
+}
+
+async function systemAnswers(host: string): Promise<string[]> {
+	try {
+		return (await lookup(host, { all: true })).map((entry) => entry.address);
+	} catch {
+		return [];
+	}
+}
+
+const DOH_ENDPOINTS = [
+	["cloudflare-dns.com", "https://cloudflare-dns.com/dns-query"],
+	["dns.google", "https://dns.google/resolve"],
+] as const;
+
+async function dohAnswers(host: string): Promise<IndependentAnswer[]> {
+	const ask = async (endpoint: string, type: "A" | "AAAA"): Promise<string[] | null> => {
+		try {
+			const response = await fetch(`${endpoint}?name=${encodeURIComponent(host)}&type=${type}`, {
+				headers: { accept: "application/dns-json" },
+				signal: AbortSignal.timeout(8_000),
+			});
+			if (!response.ok) return null;
+			const body = (await response.json()) as { Answer?: Array<{ type: number; data: string }> };
+			return (body.Answer ?? []).filter((a) => a.type === 1 || a.type === 28).map((a) => a.data);
+		} catch {
+			return null;
+		}
+	};
+	return await Promise.all(
+		DOH_ENDPOINTS.map(async ([name, endpoint]) => {
+			const both = await Promise.all([ask(endpoint, "A"), ask(endpoint, "AAAA")]);
+			// 🔴 Either family failing means this endpoint did not answer — a partial answer read as
+			// complete is how a check reports "everything resolved" while blind to v6.
+			return { name, answers: both.some((b) => b === null) ? null : both.flatMap((b) => b ?? []) };
+		}),
+	);
+}
+
+async function publishedRanges(): Promise<readonly string[]> {
+	try {
+		const out: string[] = [];
+		for (const url of ["https://www.cloudflare.com/ips-v4", "https://www.cloudflare.com/ips-v6"]) {
+			const response = await fetch(url, { signal: AbortSignal.timeout(8_000) });
+			if (!response.ok) return CLOUDFLARE_RANGES_SNAPSHOT;
+			out.push(...(await response.text()).split("\n").map((l) => l.trim()).filter((l) => l.includes("/")));
+		}
+		return out.length > 0 ? out : CLOUDFLARE_RANGES_SNAPSHOT;
+	} catch {
+		return CLOUDFLARE_RANGES_SNAPSHOT;
+	}
+}
+
+/**
+ * Is the network this smoke is running on LYING about `host`?
+ *
+ * 🔴 Measured 2026-09-19 over an airline's wifi: the aircraft's resolver answered
+ * `desk.cursedalchemy.com` with `1.1.1.1` — Cloudflare's public RESOLVER, not an edge that serves
+ * customer zones — and Cloudflare refused the SNI with `403 · Error 1034`. Nothing in the fleet
+ * was wrong and nothing in the fleet could say so; every deployed smoke run there charges that
+ * 403 to the app, because a smoke's verdict is a function of whatever resolver its machine has.
+ *
+ * The mechanism is `tools/check-public-dns.ts`'s, lifted: a Cloudflare-proxied hostname MUST
+ * resolve inside the ranges Cloudflare publishes, so an answer outside them, or one no
+ * independent resolver gives, is the network substituting answers — true everywhere, and
+ * checkable from the bad network itself. Whether the hostname is proxied at all is read off the
+ * independent answers, so a caller never has to say.
+ *
+ * 🔴 **Diagnosis only. It never retries and never pins an address** — the fault is that a
+ * network-caused red was indistinguishable from an app-caused one, and the deliverable is the
+ * distinction, not a way around the network.
+ */
+export async function diagnoseNetwork(host: string, probe: NetworkProbe = {}): Promise<NetworkDiagnosis> {
+	if (isIP(host) !== 0 || host === "localhost" || host.endsWith(".localhost")) {
+		return { verdict: "not-applicable", lines: [`${host} is not a public hostname`] };
+	}
+	const [system, independent, ranges] = await Promise.all([
+		(probe.system ?? systemAnswers)(host),
+		(probe.independent ?? dohAnswers)(host),
+		(probe.ranges ?? publishedRanges)(),
+	]);
+	const reached = independent.filter((entry) => entry.answers !== null);
+	const world = [...new Set(reached.flatMap((entry) => entry.answers ?? []))];
+
+	if (reached.length === 0) {
+		if (system.length === 0) return { verdict: "offline", lines: [`nothing resolved ${host} — not this machine, not any independent resolver`] };
+		const outside = system.filter((ip) => !inRanges(ip, ranges));
+		return {
+			verdict: "intercepting",
+			lines: [
+				`no independent resolver (${DOH_ENDPOINTS.map(([name]) => name).join(", ")}) could be reached, while this machine's resolver answers ${system.join(", ")}.`,
+				...(outside.length > 0 ? [`${outside.join(", ")} is OUTSIDE every range Cloudflare publishes.`] : []),
+				"A network that blocks both encrypted resolvers is already answering the question.",
+			],
+		};
+	}
+	if (world.length > 0 && world.every((ip) => !inRanges(ip, ranges))) {
+		return { verdict: "not-proxied", lines: [`${host} is not served through Cloudflare's proxy (${world.join(", ")}), so the range rule says nothing`] };
+	}
+
+	const lines: string[] = [];
+	for (const ip of system) {
+		if (inRanges(ip, ranges)) continue;
+		lines.push(
+			`${host} resolved to ${ip} via this machine's resolver — OUTSIDE every range Cloudflare publishes. Connecting there with this SNI is what returns "403 · Error 1034 · Edge IP Restricted".`,
+		);
+	}
+	const strangers = system.filter((ip) => inRanges(ip, ranges) && !world.includes(ip));
+	if (strangers.length > 0) {
+		lines.push(`this machine's resolver answers ${strangers.join(", ")}, which NO independent resolver returned (they say ${world.join(", ")}).`);
+	}
+	if (system.length === 0 && world.length > 0) {
+		lines.push(
+			`this machine cannot resolve ${host}, which the independent resolvers answer (${world.join(", ")}) — a resolver in the way, or this Mac's negative DNS cache.`,
+		);
+	}
+	return lines.length > 0 ? { verdict: "intercepting", lines } : { verdict: "clean", lines: [`${host}: this machine and ${reached.length} independent resolver(s) agree`] };
+}
+
+// ── A version that has SETTLED ────────────────────────────────────────────────────────────
+
+export interface SteadyOptions {
+	/** Consecutive good reads before the answer is trusted. Default 3. */
+	reads?: number;
+	/** Reads before giving up and judging whatever answered. Default 40. */
+	tries?: number;
+	/** Pause after a GOOD read, before the next. Default 2 s. */
+	pauseMs?: number;
+	/** Pause after a BAD read. Default 5 s. */
+	retryMs?: number;
+	/** Injected for tests. */
+	sleep?: (ms: number) => Promise<void>;
+}
+
+export interface SteadyRead<T> {
+	/** The LAST response read, body unconsumed — judge this, never the wait. */
+	response: Response;
+	/** Its body as JSON, or `{}` when it was not JSON. */
+	body: T;
+	/** Consecutive good reads at the end. `>= reads` means it settled. */
+	steady: number;
+	/** How many times it asked. */
+	asked: number;
+}
+
+/**
+ * Ask until `ready` holds for `reads` reads IN A ROW, then hand back the last answer.
+ *
+ * 🔴 **Several reads, not one.** A deploy and each secret upload are separate Worker versions,
+ * and for some seconds consecutive requests land on different ones. Measured on `music`'s first
+ * deploy (2026-09-23, `a54ef71`): ONE good `/healthz` read was followed by thirteen gated routes
+ * answering `503` (the pre-secrets version) and `404`, because one isolate had the new version
+ * and the next did not. The copies in `vault` and `collections` still judged after one read.
+ *
+ * It never decides pass or fail: it returns what answered, and the caller judges that — a wait
+ * that swallowed its own timeout would be a check that cannot go red.
+ */
+export async function steadyHealth<T = Record<string, unknown>>(
+	ask: () => Promise<Response>,
+	ready: (body: T, response: Response) => boolean,
+	options: SteadyOptions = {},
+): Promise<SteadyRead<T>> {
+	const reads = options.reads ?? 3;
+	const tries = options.tries ?? 40;
+	const sleep = options.sleep ?? ((ms: number) => new Promise<void>((done) => setTimeout(done, ms)));
+	let steady = 0;
+	let asked = 0;
+	for (;;) {
+		const response = await ask();
+		asked++;
+		const body = (await response
+			.clone()
+			.json()
+			.catch(() => ({}))) as T;
+		steady = ready(body, response) ? steady + 1 : 0;
+		if (steady >= reads || asked >= tries) return { response, body, steady, asked };
+		await sleep(steady > 0 ? (options.pauseMs ?? 2_000) : (options.retryMs ?? 5_000));
+	}
+}