cursedops 0.21.0 → 0.22.0

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cursedops",
-	"version": "0.21.0",
+	"version": "0.22.0",
 	"description": "The build-and-ops answers this generation's apps wrote independently and identically: finding a generation's roots — and printing a command that runs when pasted — without knowing a path, the generation's whole-tree laws run over one repo from a checkout or a worktree, macOS launchd agent install/replace/remove and the live port a job serves, the check that a launchd job imports only what the LOCKED versions of its packages export (a lib-preview overlay's extra subpath or name is a job that dies at the next forced reinstall), the scaffolding and verdicts of a deployed smoke (origin probe, the smoke's own environment, a network that lies about DNS, a settled version), the static-serving helpers eight apps copied — the path-traversal guard among them — the API floor that keeps an unmatched /api/... from ever being answered with the app shell, the commit and dirty flag a checkout-served process reports, the Cloudflare Worker deploy toolkit four apps copied (the deploy sequence, exact-set secrets over a pipe, origin-first rollback, the curl edge fetch, the row-for-row D1 import proof, the billed-CPU tail check around a deploy's walk and smoke, and each app's worker:secrets and worker:smoke main as one function of its data), the relay a Worker fronts a Mac-bound app with (the Durable Object, the frames, the Mac's dialer and key rotation — lifted from station for roms — and the signed-in stage walk's skeleton and the relay app's whole worker:deploy), the Worker import-graph and await-port checks every Worker app's suite runs over its own source, and the public-surface ratchet three published libraries each carried a forked copy of. Mechanism only — no app knows its name from here. Bun, zero runtime dependencies (typescript is an optional peer, for public-surface only), ships source.",
 	"type": "module",
 	"scripts": {
@@ -158,6 +158,12 @@
 			"source": "./src/relayStage.ts",
 			"import": "./src/relayStage.ts"
 		},
+		"./stage-look": {
+			"types": "./src/stageLook.ts",
+			"bun": "./src/stageLook.ts",
+			"source": "./src/stageLook.ts",
+			"import": "./src/stageLook.ts"
+		},
 		"./container-deploy": {
 			"types": "./src/containerDeploy.ts",
 			"bun": "./src/containerDeploy.ts",
@@ -181,7 +187,8 @@
 		"public-surface": "./src/publicSurface.ts",
 		"forge-paths": "./src/paths.ts",
 		"forge-client": "./src/stagedClient.ts",
-		"forge-d1-schema": "./src/d1Schema.ts"
+		"forge-d1-schema": "./src/d1Schema.ts",
+		"stage-look": "./src/stageLook.ts"
 	},
 	"files": [
 		"src",

package/src/smoke.ts

@@ -932,6 +932,54 @@ export const HEALTH_READ_MAX_TIME_SEC = 10;
 /** The init a smoke's `/healthz` read carries — `cursedops/edge-fetch` honors it; a plain `fetch` ignores it. */
 const healthRead: EdgeRequestInit = { maxTimeSec: HEALTH_READ_MAX_TIME_SEC };
 
+/** How long one address's between-versions answers are asked again before they are judged. */
+export const BETWEEN_VERSIONS_MS = 60_000;
+
+/**
+ * One address's allowance for answers of the edge BETWEEN VERSIONS — a 5xx, or no answer at all —
+ * after `/healthz` has settled. Shared by every question asked of that address, so a Worker that
+ * really 500s costs one window, not one per route.
+ *
+ * 🔴 Measured 2026-10-28 (forge task 3882): auth's production-first smoke settled `/healthz` on
+ * bea42b2f three reads in a row, and in the next seconds — six `secret put`s, six versions —
+ * `/account` answered 500 on the preview while the public host gave 200, and five census routes
+ * answered 500 on the public host while the preview gave 401. Judged once each, that rolled back a
+ * deploy whose stage, on the same bytes, passed all 28 checks minutes later. `/healthz` reads no
+ * session; the routes behind the D1 snapshot were still on the turning edge.
+ *
+ * Only the window's shapes are asked again. A REAL wrong answer — a private route that answers 200,
+ * a shell that is JSON — is judged on its first read, and a 5xx that outlasts the window is red.
+ */
+interface VersionWindow {
+	/** Ask until the answer is not between versions, or the window is spent; the LAST answer is judged. */
+	settle<T>(ask: () => Promise<T>, between: (answer: T) => boolean): Promise<{ answer: T; first: T | null; waitedMs: number }>;
+}
+
+function versionWindow(options: SteadyOptions = {}): VersionWindow {
+	const sleep = options.sleep ?? ((ms: number) => new Promise<void>((done) => setTimeout(done, ms)));
+	const now = options.now ?? Date.now;
+	let opened: number | null = null;
+	return {
+		async settle(ask, between) {
+			const asked = now();
+			let first: Awaited<ReturnType<typeof ask>> | null = null;
+			for (;;) {
+				const answer = await ask();
+				if (!between(answer)) return { answer, first, waitedMs: now() - asked };
+				first ??= answer;
+				opened ??= now();
+				if (now() - opened >= BETWEEN_VERSIONS_MS) return { answer, first, waitedMs: now() - asked };
+				await sleep(options.retryMs ?? 3_000);
+			}
+		},
+	};
+}
+
+/** `, after <first> for Ns` — the row's note that an answer was asked again, so a settle is never silent. */
+function settledNote(first: unknown, waitedMs: number): string {
+	return first === null ? "" : ` (${first} first, between versions — asked again for ${Math.round(waitedMs / 1000)}s)`;
+}
+
 /** One question a deploy must answer the NEW way — `null` when it does, else why not. */
 export interface ConvergenceProbe {
 	name: string;
@@ -1154,14 +1202,21 @@ export async function runWorkerSmoke(spec: WorkerSmokeSpec): Promise<number> {
 
 		// Each probe is anonymous and refused at the gate, so it is replay-safe whatever its method,
 		// and one that gets no answer even then is this check's red — never a crash of the whole smoke.
+		const versions = versionWindow(spec.steady);
 		const open: string[] = [];
+		const settled: string[] = [];
 		for (const route of spec.census) {
 			const probe: EdgeRequestInit = { method: route.method, replaySafe: true };
-			const answered = await ask(route.path, probe).then(
-				(res) => res.status,
-				(error: Error) => `no answer (${error.message})`,
+			const { answer, first, waitedMs } = await versions.settle(
+				() =>
+					ask(route.path, probe).then(
+						(res): number | string => res.status,
+						(error: Error) => `no answer (${error.message})`,
+					),
+				(a) => typeof a === "string" || a >= 500,
 			);
-			if (answered !== 401) open.push(`${route.method} ${route.path} → ${answered}`);
+			if (answer !== 401) open.push(`${route.method} ${route.path} → ${answer}${settledNote(first, waitedMs)}`);
+			else if (first !== null) settled.push(`${route.method} ${route.path}${settledNote(first, waitedMs)}`);
 		}
 		smoke.record(
 			`${tag} origin-gate`,
@@ -1169,7 +1224,7 @@ export async function runWorkerSmoke(spec: WorkerSmokeSpec): Promise<number> {
 			spec.census.length === 0
 				? "🔴 the route census is EMPTY, so this proves nothing"
 				: open.length === 0
-					? `all ${spec.census.length} private routes refuse an anonymous caller`
+					? `all ${spec.census.length} private routes refuse an anonymous caller${settled.length ? ` — ${settled.join(", ")}` : ""}`
 					: `🔴 ${open.join(", ")}`,
 		);
 
@@ -1177,15 +1232,25 @@ export async function runWorkerSmoke(spec: WorkerSmokeSpec): Promise<number> {
 		await spec.extraChecks?.({ tag, address, ask, health, record: (check, ok, detail) => smoke.record(check, ok, detail) });
 
 		for (const path of spec.shellPaths) {
-			const res = await ask(path, { headers: { accept: "text/html" } });
+			const { answer: res, first, waitedMs } = await versions.settle(
+				() => ask(path, { headers: { accept: "text/html" } }),
+				(r) => r.status >= 500,
+			);
 			const type = res.headers.get("content-type") ?? "";
-			smoke.record(`${tag} shell ${path}`, res.status === 200 && type.includes("text/html"), `${res.status} ${type}`);
+			smoke.record(
+				`${tag} shell ${path}`,
+				res.status === 200 && type.includes("text/html"),
+				`${res.status} ${type}${settledNote(first?.status ?? null, waitedMs)}`,
+			);
 		}
-		const miss = await ask("/assets/index-no-such-bundle.js");
+		const { answer: miss, first, waitedMs } = await versions.settle(
+			() => ask("/assets/index-no-such-bundle.js"),
+			(r) => r.status >= 500,
+		);
 		smoke.record(
 			`${tag} missing-bundle`,
 			miss.status === 404 && !(miss.headers.get("content-type") ?? "").includes("text/html"),
-			`a missing bundle is a miss — ${miss.status}`,
+			`a missing bundle is a miss — ${miss.status}${settledNote(first?.status ?? null, waitedMs)}`,
 		);
 	}
 	return await smoke.settle();

package/src/stageLook.ts

@@ -0,0 +1,213 @@
+#!/usr/bin/env bun
+/**
+ * `cursedops/stage-look` — open `<app>-stage` in a real browser as its signed-in owner (forge task 3881).
+ *
+ * Rule 9 says every batch SEES its change on the stage, signed in, at 390 and 1440. Only collections
+ * had a tool for it (`apps/collections/scripts/stage-look.ts`), and it needed a walk's cookie jar
+ * first; batch 2026-10-28T0812 hand-built the same look twice more, for photos and family, and lost
+ * ~40 minutes to it. This is the seam those three shared:
+ *
+ *  - **The host is pinned to what 1.1.1.1 says.** This Mac's resolver can hold a NEGATIVE answer for
+ *    a stage hostname (`awaitLocalResolve`'s header in `relay-stage`), so Chromium is told the address.
+ *  - 🔴 **Cloudflare Access headers go to the stage's own origin ONLY** ({@link isStageRequest}). Sent
+ *    to binary-server too they turn every media fetch into a CORS preflight it refuses.
+ *  - **Sign-in is the stage's own door**: `grantStageOwner` (auth-stage's emailed second factor) and
+ *    the app's `/api/auth/sso/callback?grant=…`. Never production's accounts service.
+ *  - **A reaped binary-server sandbox is SAID, not screenshotted.** Its images answer 404, or 503 with
+ *    `x-bs-edge-cache: STAGE-ORIGIN-DOWN`; {@link brokenMediaOf} names them, and the CLI exits 1
+ *    with the recovery (`apps/binary-server` → `bun run sandbox:up`, then upload again).
+ *
+ * cursedops has no runtime dependencies, so playwright is the CALLER's: pass `chromium`, or let the
+ * look resolve `playwright` from the working directory (the app it is run in).
+ *
+ * CLI, from an app's directory (or `bun $FORGE/tools/stage-look.ts <app> …` from anywhere):
+ *
+ *     stage-look <app> [path] [--out <dir>]      → <dir>/stage-<app>-390.png, …-1440.png
+ */
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { grantStageOwner } from "cursedops/relay-stage";
+import { forgeCodeRoot, requireForgeState } from "cursedops/roots";
+
+/** `https://<app>-stage.cursedalchemy.com` — every stage's public origin. */
+export const stageOrigin = (app: string): string => `https://${app}-stage.cursedalchemy.com`;
+
+/** The route matcher the Access headers ride on: the stage's own origin, nothing else. */
+export const isStageRequest =
+	(origin: string) =>
+	(url: URL): boolean =>
+		url.origin === origin;
+
+/** `CF-Access-Client-Id` / `-Secret` from `$FORGE_STATE/secrets/cloudflare-access.env`. Throws when either is missing. */
+export function accessHeaders(state: string): Record<string, string> {
+	const file = join(state, "secrets", "cloudflare-access.env");
+	const text = existsSync(file) ? readFileSync(file, "utf8") : "";
+	const read = (key: string): string =>
+		(text.split("\n").find((l) => l.startsWith(`${key}=`)) ?? "")
+			.slice(key.length + 1)
+			.trim()
+			.replace(/^['"]|['"]$/g, "");
+	const id = read("CF_ACCESS_CLIENT_ID");
+	const secret = read("CF_ACCESS_CLIENT_SECRET");
+	if (!id || !secret) throw new Error(`${file} must declare CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET — the stage sits behind Cloudflare Access.`);
+	return { "CF-Access-Client-Id": id, "CF-Access-Client-Secret": secret };
+}
+
+/** `dig +short <host> @1.1.1.1`'s output. */
+export type Dig = (host: string) => string;
+const realDig: Dig = (host) => spawnSync("dig", ["+short", host, "@1.1.1.1"], { encoding: "utf8", timeout: 15_000 }).stdout ?? "";
+
+/** Chromium's `--host-resolver-rules` pinning `host` to 1.1.1.1's first IPv4 answer; none when it gave none. */
+export function pinnedResolverArgs(host: string, dig: Dig = realDig): string[] {
+	const ip = dig(host)
+		.split("\n")
+		.map((l) => l.trim())
+		.find((l) => /^\d+\.\d+\.\d+\.\d+$/.test(l));
+	return ip ? [`--host-resolver-rules=MAP ${host} ${ip}`] : [];
+}
+
+/** A response as the look reads it — the slice of playwright's `Response` it needs. */
+export interface LookResponse {
+	url(): string;
+	status(): number;
+	headers(): Record<string, string>;
+	request(): { resourceType(): string };
+}
+
+/** An image or media answer that says the stage's bytes are gone: a 404, or the edge's `STAGE-ORIGIN-DOWN`. */
+export function brokenMediaOf(r: LookResponse): string | null {
+	const kind = r.request().resourceType();
+	if (kind !== "image" && kind !== "media") return null;
+	const edge = r.headers()["x-bs-edge-cache"] ?? "";
+	// The URL without its query: a media URL carries a signed token, and this line lands in logs.
+	const url = r.url().split("?")[0];
+	if (edge === "STAGE-ORIGIN-DOWN" || r.status() === 404 || r.status() === 503) return `${r.status()}${edge ? ` ${edge}` : ""} ${url}`;
+	return null;
+}
+
+/** The slices of playwright this drives. Structural, so cursedops needs no dependency on it. */
+interface LookRoute {
+	request(): { headers(): Record<string, string> };
+	continue(options: { headers: Record<string, string> }): Promise<unknown>;
+}
+interface LookPage {
+	goto(url: string, options?: { waitUntil?: "load" | "networkidle" }): Promise<{ status(): number } | null>;
+	on(event: "response", listener: (r: LookResponse) => void): unknown;
+}
+interface LookContext {
+	route(match: (url: URL) => boolean, handler: (route: LookRoute) => Promise<unknown>): Promise<unknown>;
+	newPage(): Promise<LookPage>;
+	close(): Promise<unknown>;
+}
+interface LookBrowser {
+	newContext(options: Record<string, unknown>): Promise<LookContext>;
+	close(): Promise<unknown>;
+}
+export interface LookChromium {
+	launch(options: { args: string[] }): Promise<LookBrowser>;
+}
+
+export interface StageLookSpec {
+	app: string;
+	/** 390 is a phone (touch, DPR 3, 844 tall); anything else a desktop 900 tall. */
+	width: number;
+	/** Defaults: the roots above the working directory. */
+	forge?: string;
+	state?: string;
+	/** playwright's `chromium`; default: `playwright` resolved from the working directory. */
+	chromium?: LookChromium;
+	/** Extra Chromium flags — collections' audio daemon, say. */
+	args?: string[];
+	/** Seams for the suite. */
+	grant?: () => { ok: boolean; grant: string };
+	dig?: Dig;
+	headers?: Record<string, string>;
+}
+
+export interface StageLook<Page> {
+	page: Page;
+	/** The stage's origin, for `page.goto(`${origin}${path}`)`. */
+	origin: string;
+	/** Every broken image/media answer so far ({@link brokenMediaOf}). */
+	brokenMedia: () => string[];
+	close: () => Promise<unknown>;
+}
+
+async function defaultChromium(): Promise<LookChromium> {
+	const at = Bun.resolveSync("playwright", process.cwd());
+	return ((await import(at)) as { chromium: LookChromium }).chromium;
+}
+
+/**
+ * A page on `<app>-stage`, signed in as its owner, with Access headers on the stage's requests only.
+ * `Page` is the caller's playwright `Page` — the look builds it, so the cast is the caller's word.
+ */
+export async function openStagePage<Page = LookPage>(spec: StageLookSpec): Promise<StageLook<Page>> {
+	const cwd = process.cwd();
+	const forge = spec.forge ?? forgeCodeRoot(cwd);
+	if (!forge) throw new Error(`no forge.env above ${cwd} — run the look from inside the generation`);
+	const origin = stageOrigin(spec.app);
+	const access = spec.headers ?? accessHeaders(spec.state ?? requireForgeState(cwd));
+	const chromium = spec.chromium ?? (await defaultChromium());
+	const browser = await chromium.launch({ args: [...(spec.args ?? []), ...pinnedResolverArgs(new URL(origin).hostname, spec.dig)] });
+	try {
+		const phone = spec.width < 768;
+		const context = await browser.newContext({
+			viewport: { width: spec.width, height: phone ? 844 : 900 },
+			...(phone ? { isMobile: true, hasTouch: true, deviceScaleFactor: 3 } : {}),
+		});
+		await context.route(isStageRequest(origin), (route) => route.continue({ headers: { ...route.request().headers(), ...access } }));
+		const page = await context.newPage();
+		const broken: string[] = [];
+		page.on("response", (r) => {
+			const b = brokenMediaOf(r);
+			if (b) broken.push(b);
+		});
+		const granted = (spec.grant ?? (() => grantStageOwner({ forge, app: spec.app })))();
+		if (!granted.ok) throw new Error(`auth-stage gave no grant for ${spec.app} — its output above says why`);
+		const back = await page.goto(`${origin}/api/auth/sso/callback?grant=${encodeURIComponent(granted.grant)}`, { waitUntil: "load" });
+		if (back && back.status() >= 400) throw new Error(`${origin}'s sign-in callback answered ${back.status()}`);
+		return { page: page as Page, origin, brokenMedia: () => [...broken], close: () => browser.close() };
+	} catch (e) {
+		await browser.close();
+		throw e;
+	}
+}
+
+/** Shoot `path` at 390 and 1440. Exit 1 when the page or its media is broken, so a look never passes on broken tiles. */
+async function main(argv: string[]): Promise<number> {
+	const outAt = argv.indexOf("--out");
+	const out = outAt >= 0 ? argv[outAt + 1] : undefined;
+	const [app, path = "/"] = outAt >= 0 ? argv.filter((_, i) => i !== outAt && i !== outAt + 1) : argv;
+	if (!app || (outAt >= 0 && !out)) {
+		console.error("usage: stage-look <app> [path] [--out <dir>]   (run in the app's directory, or `bun $FORGE/tools/stage-look.ts <app> …`)");
+		return 2;
+	}
+	const dir = out ?? join(process.env.FORGE_BATCH_TMP ?? requireForgeState(process.cwd()), "stage-look");
+	mkdirSync(dir, { recursive: true });
+	let bad = false;
+	for (const width of [390, 1440]) {
+		type ShotPage = LookPage & { screenshot(o: { path: string; fullPage?: boolean }): Promise<unknown>; url(): string };
+		const look = await openStagePage<ShotPage>({ app, width });
+		try {
+			const res = await look.page.goto(`${look.origin}${path}`, { waitUntil: "networkidle" });
+			const shot = join(dir, `stage-${app}-${width}.png`);
+			await look.page.screenshot({ path: shot });
+			console.log(`${width}: ${res?.status() ?? "?"} ${look.page.url()} → ${shot}`);
+			if (!res || res.status() >= 400) bad = true;
+			const broken = look.brokenMedia();
+			if (broken.length) {
+				bad = true;
+				console.error(`  🔴 ${broken.length} image/media answer(s) broken — the stage's binary-server sandbox was likely reaped:`);
+				for (const b of broken.slice(0, 5)) console.error(`     ${b}`);
+				console.error("     recover: cd \"$FORGE/apps/binary-server\" && bun run sandbox:up — then upload the fixture again");
+			}
+		} finally {
+			await look.close();
+		}
+	}
+	return bad ? 1 : 0;
+}
+
+if (import.meta.main) process.exit(await main(process.argv.slice(2)));