cursedops 0.20.1 → 0.21.1

package/README.md

@@ -17,7 +17,7 @@ import shape, worked example, and the incident each one is pinned to.
 | `cursedops/launchd` | installing, replacing and removing a macOS launchd user agent, and the port a LIVE job serves on (`livePort`) |
 | `cursedops/launchd-imports` | the check that every launchd job — run straight out of an app's main checkout — imports only what the LOCKED versions of its packages export, so a lib-preview overlay's extra subpath or name cannot kill it at the next `bun install --force` (`launchdImportFaults`) |
 | `cursedops/smoke` | the scaffolding of a deployed smoke (ledger, fetch, DNS hint, exit code) — the one check no app owns — plus its VERDICTS: the origin asked on loopback, the smoke's own environment, a network that lies about DNS, and a version that has settled |
-| `cursedops/worker-deploy` | the sequence that ships a Cloudflare Worker — clean tree, stage first, schema, `--var` stamp, secrets, smoke — plus a billed-CPU tail wrapped around the walk and the smoke, and a ledger line per run |
+| `cursedops/worker-deploy` | the sequence that ships a Cloudflare Worker — clean tree, stage first (or `--prod-first`: production alone, rolled back on a red smoke), schema, `--var` stamp, secrets, smoke — plus a billed-CPU tail wrapped around the walk and the smoke, and a ledger line per run |
 | `cursedops/worker-secrets` | a Worker holding EXACTLY its deployment's secrets, uploaded over a pipe, read back up to five times before an inexact set is believed |
 | `cursedops/worker-rollback` | a hostname back on its Mac origin — origin first, route second, the route found rather than typed |
 | `cursedops/edge-fetch` | a request to a deployed Worker the way a script must make it — curl pinned past the Mac's negative DNS cache |

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cursedops",
-	"version": "0.20.1",
+	"version": "0.21.1",
 	"description": "The build-and-ops answers this generation's apps wrote independently and identically: finding a generation's roots — and printing a command that runs when pasted — without knowing a path, the generation's whole-tree laws run over one repo from a checkout or a worktree, macOS launchd agent install/replace/remove and the live port a job serves, the check that a launchd job imports only what the LOCKED versions of its packages export (a lib-preview overlay's extra subpath or name is a job that dies at the next forced reinstall), the scaffolding and verdicts of a deployed smoke (origin probe, the smoke's own environment, a network that lies about DNS, a settled version), the static-serving helpers eight apps copied — the path-traversal guard among them — the API floor that keeps an unmatched /api/... from ever being answered with the app shell, the commit and dirty flag a checkout-served process reports, the Cloudflare Worker deploy toolkit four apps copied (the deploy sequence, exact-set secrets over a pipe, origin-first rollback, the curl edge fetch, the row-for-row D1 import proof, the billed-CPU tail check around a deploy's walk and smoke, and each app's worker:secrets and worker:smoke main as one function of its data), the relay a Worker fronts a Mac-bound app with (the Durable Object, the frames, the Mac's dialer and key rotation — lifted from station for roms — and the signed-in stage walk's skeleton and the relay app's whole worker:deploy), the Worker import-graph and await-port checks every Worker app's suite runs over its own source, and the public-surface ratchet three published libraries each carried a forked copy of. Mechanism only — no app knows its name from here. Bun, zero runtime dependencies (typescript is an optional peer, for public-surface only), ships source.",
 	"type": "module",
 	"scripts": {

package/src/smoke.ts

@@ -932,6 +932,54 @@ export const HEALTH_READ_MAX_TIME_SEC = 10;
 /** The init a smoke's `/healthz` read carries — `cursedops/edge-fetch` honors it; a plain `fetch` ignores it. */
 const healthRead: EdgeRequestInit = { maxTimeSec: HEALTH_READ_MAX_TIME_SEC };
 
+/** How long one address's between-versions answers are asked again before they are judged. */
+export const BETWEEN_VERSIONS_MS = 60_000;
+
+/**
+ * One address's allowance for answers of the edge BETWEEN VERSIONS — a 5xx, or no answer at all —
+ * after `/healthz` has settled. Shared by every question asked of that address, so a Worker that
+ * really 500s costs one window, not one per route.
+ *
+ * 🔴 Measured 2026-10-28 (forge task 3882): auth's production-first smoke settled `/healthz` on
+ * bea42b2f three reads in a row, and in the next seconds — six `secret put`s, six versions —
+ * `/account` answered 500 on the preview while the public host gave 200, and five census routes
+ * answered 500 on the public host while the preview gave 401. Judged once each, that rolled back a
+ * deploy whose stage, on the same bytes, passed all 28 checks minutes later. `/healthz` reads no
+ * session; the routes behind the D1 snapshot were still on the turning edge.
+ *
+ * Only the window's shapes are asked again. A REAL wrong answer — a private route that answers 200,
+ * a shell that is JSON — is judged on its first read, and a 5xx that outlasts the window is red.
+ */
+interface VersionWindow {
+	/** Ask until the answer is not between versions, or the window is spent; the LAST answer is judged. */
+	settle<T>(ask: () => Promise<T>, between: (answer: T) => boolean): Promise<{ answer: T; first: T | null; waitedMs: number }>;
+}
+
+function versionWindow(options: SteadyOptions = {}): VersionWindow {
+	const sleep = options.sleep ?? ((ms: number) => new Promise<void>((done) => setTimeout(done, ms)));
+	const now = options.now ?? Date.now;
+	let opened: number | null = null;
+	return {
+		async settle(ask, between) {
+			const asked = now();
+			let first: Awaited<ReturnType<typeof ask>> | null = null;
+			for (;;) {
+				const answer = await ask();
+				if (!between(answer)) return { answer, first, waitedMs: now() - asked };
+				first ??= answer;
+				opened ??= now();
+				if (now() - opened >= BETWEEN_VERSIONS_MS) return { answer, first, waitedMs: now() - asked };
+				await sleep(options.retryMs ?? 3_000);
+			}
+		},
+	};
+}
+
+/** `, after <first> for Ns` — the row's note that an answer was asked again, so a settle is never silent. */
+function settledNote(first: unknown, waitedMs: number): string {
+	return first === null ? "" : ` (${first} first, between versions — asked again for ${Math.round(waitedMs / 1000)}s)`;
+}
+
 /** One question a deploy must answer the NEW way — `null` when it does, else why not. */
 export interface ConvergenceProbe {
 	name: string;
@@ -1154,14 +1202,21 @@ export async function runWorkerSmoke(spec: WorkerSmokeSpec): Promise<number> {
 
 		// Each probe is anonymous and refused at the gate, so it is replay-safe whatever its method,
 		// and one that gets no answer even then is this check's red — never a crash of the whole smoke.
+		const versions = versionWindow(spec.steady);
 		const open: string[] = [];
+		const settled: string[] = [];
 		for (const route of spec.census) {
 			const probe: EdgeRequestInit = { method: route.method, replaySafe: true };
-			const answered = await ask(route.path, probe).then(
-				(res) => res.status,
-				(error: Error) => `no answer (${error.message})`,
+			const { answer, first, waitedMs } = await versions.settle(
+				() =>
+					ask(route.path, probe).then(
+						(res): number | string => res.status,
+						(error: Error) => `no answer (${error.message})`,
+					),
+				(a) => typeof a === "string" || a >= 500,
 			);
-			if (answered !== 401) open.push(`${route.method} ${route.path} → ${answered}`);
+			if (answer !== 401) open.push(`${route.method} ${route.path} → ${answer}${settledNote(first, waitedMs)}`);
+			else if (first !== null) settled.push(`${route.method} ${route.path}${settledNote(first, waitedMs)}`);
 		}
 		smoke.record(
 			`${tag} origin-gate`,
@@ -1169,7 +1224,7 @@ export async function runWorkerSmoke(spec: WorkerSmokeSpec): Promise<number> {
 			spec.census.length === 0
 				? "🔴 the route census is EMPTY, so this proves nothing"
 				: open.length === 0
-					? `all ${spec.census.length} private routes refuse an anonymous caller`
+					? `all ${spec.census.length} private routes refuse an anonymous caller${settled.length ? ` — ${settled.join(", ")}` : ""}`
 					: `🔴 ${open.join(", ")}`,
 		);
 
@@ -1177,15 +1232,25 @@ export async function runWorkerSmoke(spec: WorkerSmokeSpec): Promise<number> {
 		await spec.extraChecks?.({ tag, address, ask, health, record: (check, ok, detail) => smoke.record(check, ok, detail) });
 
 		for (const path of spec.shellPaths) {
-			const res = await ask(path, { headers: { accept: "text/html" } });
+			const { answer: res, first, waitedMs } = await versions.settle(
+				() => ask(path, { headers: { accept: "text/html" } }),
+				(r) => r.status >= 500,
+			);
 			const type = res.headers.get("content-type") ?? "";
-			smoke.record(`${tag} shell ${path}`, res.status === 200 && type.includes("text/html"), `${res.status} ${type}`);
+			smoke.record(
+				`${tag} shell ${path}`,
+				res.status === 200 && type.includes("text/html"),
+				`${res.status} ${type}${settledNote(first?.status ?? null, waitedMs)}`,
+			);
 		}
-		const miss = await ask("/assets/index-no-such-bundle.js");
+		const { answer: miss, first, waitedMs } = await versions.settle(
+			() => ask("/assets/index-no-such-bundle.js"),
+			(r) => r.status >= 500,
+		);
 		smoke.record(
 			`${tag} missing-bundle`,
 			miss.status === 404 && !(miss.headers.get("content-type") ?? "").includes("text/html"),
-			`a missing bundle is a miss — ${miss.status}`,
+			`a missing bundle is a miss — ${miss.status}${settledNote(first?.status ?? null, waitedMs)}`,
 		);
 	}
 	return await smoke.settle();

package/src/workerDeploy.ts

@@ -50,6 +50,14 @@
  * rollback` puts the previous one back in one command — the message says so. The Mac-side
  * rollback, which is a different thing, is `cursedops/worker-rollback`.
  *
+ * ⏩ **Production first** ({@link PROD_FIRST_FLAG}, 0.21.0, forge task 3845) leaves out step 4 — the
+ * stage deploy and its signed-in walk — for the owner's Deploy now, which runs them BEHIND production
+ * instead. Station's Deploy now measured 210–252 s in `worker:deploy`, ~150–180 s of it the stage leg;
+ * production alone is install, build, upload and smoke, about 40 s. Everything else above still runs
+ * first. With no stage in front of it the smoke is production's only check, so under this flag a red
+ * smoke DOES roll back — to the version {@link liveVersionOf} read just before the upload, never the
+ * blind "previous" one (a secrets step makes a version of its own).
+ *
  * Everything that shells out is behind {@link DeployDeps}, so the suite proves the order and every
  * refusal without wrangler, git or a network.
  */
@@ -276,6 +284,11 @@ export interface WorkerDeploySpec {
 	build: readonly string[] | null;
 	/** Run in order BEFORE a production deploy; any red and production is not touched. Ignored for the stage. */
 	stageFirst?: readonly (readonly string[])[];
+	/**
+	 * ⏩ Production without {@link stageFirst} and {@link stageWalk} — the caller runs them after
+	 * (see {@link PROD_FIRST_FLAG}). Absent, {@link DeployDeps.argv} decides: `--prod-first` there.
+	 */
+	prodFirst?: boolean;
 	/**
 	 * The signed-in walk of the stage, run after `stageFirst` and before production is touched — and,
 	 * unlike `stageFirst`, under {@link WorkerDeploySpec.cpuTail} when that is set. Ignored for the stage.
@@ -343,7 +356,8 @@ export interface DeployDeps {
 	recordShip?: (line: ShipEventLine) => void;
 	/**
 	 * Take the app's hosted stage for the whole deploy (`./stageLease.ts`) — every deploy touches the
-	 * stage, a production one deploys and walks it first. {@link workerDeployDeps} takes the
+	 * stage, a production one deploys and walks it first — all but a production-first one, which
+	 * never touches it and takes none. {@link workerDeployDeps} takes the
 	 * generation's lease; absent (or `null` with no state root), nothing is leased.
 	 */
 	lease?: (app: string, what: string) => StageLeaseResult | null;
@@ -353,6 +367,47 @@ export interface DeployDeps {
 	 * {@link workerDeployDeps} makes one under the OS temp dir; absent, a failed deploy is not retried.
 	 */
 	wranglerOutput?: () => WranglerOutputFile;
+	/**
+	 * The deploy's own command line — where {@link PROD_FIRST_FLAG} is read when the spec does not say.
+	 * {@link workerDeployDeps} hands `process.argv.slice(2)`, so every app's `worker:deploy` takes the
+	 * flag with no edit of its own; absent, nothing is read.
+	 */
+	argv?: readonly string[];
+}
+
+/**
+ * ⏩ `bun run worker:deploy -- --prod-first`: production without the stage deploy and the signed-in
+ * stage walk in front of it ({@link WorkerDeploySpec.prodFirst}). The frozen install, every refusal,
+ * the build, the schema, the stamped upload, the secrets and the smoke still run, in order — and a
+ * red smoke rolls production back ({@link liveVersionOf}). Whoever passes it owes the stage after:
+ * autopilot's Deploy now runs `worker:deploy -- --env stage` and `worker:stage-walk` in its follow-up.
+ * An older cursedops ignores the word (no app's argv parsing refuses an unknown one), so it is safe
+ * to pass to any version — it only goes faster from 0.21.0.
+ */
+export const PROD_FIRST_FLAG = "--prod-first";
+
+/**
+ * The Worker version serving production right now, from `wrangler deployments status --json` — the
+ * version a production-first deploy rolls back TO. The one taking the most traffic; `null` when the
+ * answer is not a deployment (a first deploy, an API error), and then nothing is rolled back.
+ */
+export function liveVersionOf(stdout: string): string | null {
+	let doc: unknown;
+	try {
+		doc = JSON.parse(stdout);
+	} catch {
+		return null;
+	}
+	const deployment = Array.isArray(doc) ? doc[0] : doc;
+	const versions = (deployment as { versions?: unknown } | null)?.versions;
+	if (!Array.isArray(versions)) return null;
+	let best: { id: string; pct: number } | null = null;
+	for (const v of versions as Array<{ version_id?: unknown; percentage?: unknown }>) {
+		if (typeof v?.version_id !== "string" || !v.version_id) continue;
+		const pct = typeof v.percentage === "number" ? v.percentage : 0;
+		if (!best || pct > best.pct) best = { id: v.version_id, pct };
+	}
+	return best?.id ?? null;
 }
 
 /** One `wrangler` run's output file: the env that points wrangler at it, its text, and its removal. */
@@ -388,9 +443,11 @@ export interface DeployEventLine {
 	/**
 	 * Milliseconds per leg, in the order they ran: `prepare` (git and the refusals), `install`,
 	 * `stage` (the stage deploy a production deploy runs first), `walk`, `build`, `schema`,
-	 * `upload` (`wrangler deploy`), `secrets`, `smoke`. A leg that did not run is absent.
+	 * `upload` (`wrangler deploy`), `secrets`, `smoke`, `rollback`. A leg that did not run is absent.
 	 */
 	legs: Record<string, number>;
+	/** ⏩ Production first ({@link PROD_FIRST_FLAG}): no stage leg ran, and the stage is owed after. */
+	prodFirst?: true;
 }
 
 /** Append one line to the ledger under a state root. Never throws; `false` when it did not write. */
@@ -456,6 +513,7 @@ export function workerDeployDeps(cwd: string): DeployDeps {
 			const state = forgeState(cwd);
 			return state ? takeStageLease({ state, app, what }) : null;
 		},
+		argv: process.argv.slice(2),
 	};
 }
 
@@ -771,6 +829,8 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 		legFrom = at;
 	};
 	const tag = `[${spec.app}]`;
+	// ⏩ Production first — the stage leg runs behind it, the caller's to run (see PROD_FIRST_FLAG).
+	const prodFirst = spec.env === "production" && (spec.prodFirst ?? (deps.argv ?? []).includes(PROD_FIRST_FLAG));
 	const commit = deps.git(["rev-parse", "HEAD"]);
 	const porcelain = deps.git(["status", "--porcelain"]);
 	const dirty = porcelain.length > 0;
@@ -791,6 +851,7 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 				commit: commit.slice(0, 8),
 				dirty,
 				legs,
+				...(prodFirst ? { prodFirst: true as const } : {}),
 			});
 		} catch {
 			// never the deploy's verdict — see DeployDeps.record
@@ -843,7 +904,9 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 	}
 	// 🔴 One holder per hosted stage (task 3048): a stage deploy under the ship's held suite swapped
 	// the Worker and its link key under it. Taken before anything builds, held until `finish`.
-	const lease = deps.lease?.(spec.app, `${spec.app} worker:deploy${spec.env === "stage" ? " --env stage" : ""} @ ${commit.slice(0, 8)}`);
+	// ⏩ Not by a production-first deploy: it never touches the stage, so a stranger walking the stage
+	// is no reason for production to wait.
+	const lease = prodFirst ? null : deps.lease?.(spec.app,`${spec.app} worker:deploy${spec.env === "stage" ? " --env stage" : ""} @ ${commit.slice(0, 8)}`);
 	if (lease && !lease.ok) return stop("stage-lease", lease.refusal);
 	if (lease?.ok) release = lease.release;
 	for (const refusal of spec.beforeStage ?? []) {
@@ -881,7 +944,11 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 	if (previewed) return stop("preview", previewed);
 	if (previews.length > 0) log(`\n🟡 ${tag} this stage deploy ships an UNPUBLISHED ${describePreviews(previews)} — stamped dirty; production will refuse it until it is published`);
 	const shipsDirty = dirty || previews.length > 0;
-	if (spec.env === "production") {
+	const staged = spec.env === "production" && !prodFirst && Boolean(spec.stageFirst?.length || spec.stageWalk?.length);
+	if (prodFirst && (spec.stageFirst?.length || spec.stageWalk?.length)) {
+		log(`\n⏩ ${tag} production first (${PROD_FIRST_FLAG}) — the stage deploy and its signed-in walk are NOT run here; they are owed after, by whoever asked for this`);
+	}
+	if (spec.env === "production" && !prodFirst) {
 		for (const command of spec.stageFirst ?? []) {
 			step(`the stage first — production waits on it: ${command.join(" ")}`);
 			enter("stage");
@@ -896,7 +963,7 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 			else if (walked !== 0) return stop("stage", `\`${command.join(" ")}\` failed — production was NOT touched.`);
 		}
 	}
-	const beforeBuild = moved(spec.stageFirst?.length || spec.stageWalk?.length ? "while the stage deployed" : "before the build");
+	const beforeBuild = moved(staged ? "while the stage deployed" : "before the build");
 	if (beforeBuild) return beforeBuild;
 	if (spec.build) {
 		step("build");
@@ -971,6 +1038,40 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 	const stamp = spec.stampVars
 		? ["--var", `${spec.stampVars.commit}:${commit}`, "--var", `${spec.stampVars.dirty}:${shipsDirty ? "true" : "false"}`]
 		: [];
+	// ⏩ With no stage in front of it, a production-first deploy reads what production serves BEFORE it
+	// uploads, so a red smoke can put exactly that back. Unreadable is a line, never a refusal: the
+	// owner asked for production now, and the smoke still says whether it is broken.
+	let rollbackTo: string | null = null;
+	if (prodFirst) {
+		const status = deps.capture(["bunx", "wrangler", "deployments", "status", ...envArgs, "--name", spec.workerName, "--json"], spec.credential);
+		rollbackTo = status.code === 0 ? liveVersionOf(status.stdout) : null;
+		log(
+			rollbackTo
+				? `  production serves version ${rollbackTo} — a red smoke or secrets step rolls back to it`
+				: `  🟡 ${tag} what production serves could not be read (wrangler exited ${status.code}) — a red smoke will NOT be rolled back automatically`,
+		);
+	}
+	/** ⏩ A production-first deploy that went red AFTER the upload puts back what production served before it. */
+	const rollBack = (at: string, what: string): DeployResult => {
+		if (!rollbackTo) {
+			return stop(
+				at,
+				`the Worker is deployed production-first and ${what}. Nothing was rolled back — what production served before could not be read; \`bunx wrangler rollback\` puts the previous version back in one command.`,
+			);
+		}
+		step(`roll production back to ${rollbackTo} — ${what}, and no stage went first`);
+		enter("rollback");
+		const rolled = deps.run(
+			["bunx", "wrangler", "rollback", rollbackTo, ...envArgs, "--name", spec.workerName, "--yes", "--message", `${spec.app} ${commit.slice(0, 8)}: production-first ${at} red`],
+			spec.credential,
+		);
+		return stop(
+			at,
+			rolled === 0
+				? `the Worker was deployed production-first and ${what} — production is ROLLED BACK to ${rollbackTo}, the version it served before. Fix it, then deploy again.`
+				: `the Worker was deployed production-first and ${what}, and the rollback to ${rollbackTo} FAILED too (wrangler exited ${rolled}) — production serves the red version. \`bunx wrangler rollback ${rollbackTo}\` now.`,
+		);
+	};
 	enter("upload");
 	const deployArgv = ["bunx", "wrangler", "deploy", ...envArgs, ...stamp];
 	const deployOnce = (): { code: number; transient: string | null } => {
@@ -992,6 +1093,7 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 		step("attach the secrets — exactly this deployment's set, over a pipe");
 		enter("secrets");
 		if (deps.run(spec.secrets, spec.credential) !== 0) {
+			if (prodFirst) return rollBack("secrets", "its secrets are NOT attached, so it serves nothing");
 			return stop("secrets", "the Worker is deployed and its secrets are NOT attached — it serves nothing until they are.");
 		}
 	}
@@ -1003,6 +1105,7 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 		const smoked = deps.run(smoke, { [DEPLOYED_COMMIT_ENV]: commit });
 		if (spec.cpuTail && smoked === TAIL_UNATTACHED_EXIT) unmeasured.push("the smoke");
 		else if (smoked !== 0) {
+			if (prodFirst) return rollBack("smoke", "its smoke FAILED");
 			return stop(
 				"smoke",
 				"the Worker is deployed and its smoke FAILED. Nothing was rolled back: `bunx wrangler rollback` puts the previous version back in one command.",
@@ -1035,6 +1138,7 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
  *     bun run worker:deploy                   # the stage, walked signed in; then build, ship, smoke production
  *     bun run worker:deploy -- --env stage    # the stage alone: build, ship, smoke
  *     bun run worker:deploy -- --no-build     # ship what is already in dist/
+ *     bun run worker:deploy -- --prod-first   # ⏩ production alone — the stage and its walk are owed after (PROD_FIRST_FLAG)
  *
  * No secrets step: the one secret, the link key's digest, is the app's `link:key` to set, because
  * minting it is also rotating it. Returns the exit code — 2 for a missing credential or a bad
@@ -1105,7 +1209,7 @@ export function relayWorkerDeployCli(
 		error(`[${spec.app}] ${why instanceof Error ? why.message : why}`);
 		return 2;
 	}
-	const result = runWorkerDeploy(relayWorkerDeploySpec(spec, env, argv.includes("--no-build"), credential), deps);
+	const result = runWorkerDeploy({ ...relayWorkerDeploySpec(spec, env, argv.includes("--no-build"), credential), prodFirst: argv.includes(PROD_FIRST_FLAG) }, deps);
 	if (result.code === 0) log(`   ${env === "stage" ? spec.stageUrl : spec.previewUrl}`);
 	return result.code;
 }