cursedops 0.16.0 → 0.16.3

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cursedops",
-	"version": "0.16.0",
+	"version": "0.16.3",
 	"description": "The build-and-ops answers this generation's apps wrote independently and identically: finding a generation's roots — and printing a command that runs when pasted — without knowing a path, the generation's whole-tree laws run over one repo from a checkout or a worktree, macOS launchd agent install/replace/remove and the live port a job serves, the scaffolding and verdicts of a deployed smoke (origin probe, the smoke's own environment, a network that lies about DNS, a settled version), the static-serving helpers eight apps copied — the path-traversal guard among them — the API floor that keeps an unmatched /api/... from ever being answered with the app shell, the commit and dirty flag a checkout-served process reports, the Cloudflare Worker deploy toolkit four apps copied (the deploy sequence, exact-set secrets over a pipe, origin-first rollback, the curl edge fetch, the row-for-row D1 import proof, the billed-CPU tail check around a deploy's walk and smoke, and each app's worker:secrets and worker:smoke main as one function of its data), the relay a Worker fronts a Mac-bound app with (the Durable Object, the frames, the Mac's dialer and key rotation — lifted from station for roms — and the signed-in stage walk's skeleton and the relay app's whole worker:deploy), the Worker import-graph and await-port checks every Worker app's suite runs over its own source, and the public-surface ratchet three published libraries each carried a forked copy of. Mechanism only — no app knows its name from here. Bun, zero runtime dependencies (typescript is an optional peer, for public-surface only), ships source.",
 	"type": "module",
 	"scripts": {

package/src/edgeFetch.ts

@@ -59,10 +59,18 @@ export interface EdgeFetchOptions {
 }
 
 /**
- * A `RequestInit` this fetch reads one more field from: `maxTimeSec`, curl's `--max-time` for
- * THIS request, over the instance's {@link EdgeFetchOptions.maxTimeSec}. A plain `fetch` ignores it.
+ * A `RequestInit` this fetch reads two more fields from. A plain `fetch` ignores both.
+ *
+ *  · `maxTimeSec` — curl's `--max-time` for THIS request, over the instance's
+ *    {@link EdgeFetchOptions.maxTimeSec};
+ *  · `replaySafe` — the caller's word that sending this request twice does nothing the first did
+ *    not, WHATEVER its method: a smoke's anonymous census probe, which the origin gate refuses
+ *    before any handler runs. Such a request is replayed after a timeout or a dropped reply like a
+ *    GET. Measured 2026-10-13: family's stage smoke died on census route 55 of 101, an anonymous
+ *    `POST /api/media` that never reached the Worker (no trace of it), because a POST's timeout
+ *    was never replayed — a green Worker, a red ship lane (forge task 3115).
  */
-export type EdgeRequestInit = RequestInit & { maxTimeSec?: number };
+export type EdgeRequestInit = RequestInit & { maxTimeSec?: number; replaySafe?: boolean };
 
 /**
  * 🔴 How many MORE times a request is sent after a connection-level curl failure that carried
@@ -85,12 +93,13 @@ const EDGE_RETRIES = 2;
  *    three of them. Measured 2026-09-26: flix's prod smoke went red on one `curl exit 28` for
  *    `GET /healthz` while the same URL answered 200 in 0.2 s seconds later (task 2190).
  *
+ * A request its caller marked `replaySafe` ({@link EdgeRequestInit}) counts as safe to run twice.
  * Every other exit, and every HTTP status, whatever it is, is returned exactly as before.
  */
-function retryableCurlFailure(exit: number | null, stdout: Buffer, method = "GET", retry = 1): boolean {
+function retryableCurlFailure(exit: number | null, stdout: Buffer, method = "GET", retry = 1, replaySafe = false): boolean {
 	if (stdout.length > 0 || exit === null) return false;
 	if (exit === 6 || exit === 7 || exit === 35) return true;
-	const replayable = ["GET", "HEAD", "OPTIONS", "PUT", "DELETE"].includes(method.toUpperCase());
+	const replayable = replaySafe || ["GET", "HEAD", "OPTIONS", "PUT", "DELETE"].includes(method.toUpperCase());
 	if (exit === 28) return replayable && retry === 1;
 	return (exit === 52 || exit === 56) && replayable;
 }
@@ -181,7 +190,7 @@ export function createEdgeFetch(options: EdgeFetchOptions = {}): (url: string, i
 		}
 		try {
 			let out = curl([...head(), ...args, target.toString()]);
-			for (let retry = 1; retry <= EDGE_RETRIES && retryableCurlFailure(out.status, out.stdout, init.method, retry); retry++) {
+			for (let retry = 1; retry <= EDGE_RETRIES && retryableCurlFailure(out.status, out.stdout, init.method, retry, init.replaySafe); retry++) {
 				await new Promise((r) => setTimeout(r, options.retryDelayMs?.(retry) ?? 500 * retry));
 				// `head()` again: a host with no pin yet is asked again, so a retry can find the address.
 				out = curl([...head(), ...args, target.toString()]);

package/src/smoke.ts

@@ -1127,9 +1127,15 @@ export async function runWorkerSmoke(spec: WorkerSmokeSpec): Promise<number> {
 			`credentials go back to ${JSON.stringify(health.publicUrl)} — this deployment's ${deployment.publicUrl}`,
 		);
 
+		// Each probe is anonymous and refused at the gate, so it is replay-safe whatever its method,
+		// and one that gets no answer even then is this check's red — never a crash of the whole smoke.
 		const open: string[] = [];
 		for (const route of spec.census) {
-			const answered = (await ask(route.path, { method: route.method })).status;
+			const probe: EdgeRequestInit = { method: route.method, replaySafe: true };
+			const answered = await ask(route.path, probe).then(
+				(res) => res.status,
+				(error: Error) => `no answer (${error.message})`,
+			);
 			if (answered !== 401) open.push(`${route.method} ${route.path} → ${answered}`);
 		}
 		smoke.record(
@@ -1214,7 +1220,12 @@ async function accessDoorSmoke(
 		} else if (address === "preview") {
 			const open: string[] = [];
 			for (const route of spec.census) {
-				const answered = (await ask(route.path, { method: route.method })).status;
+				// Refused by Access before the Worker sees it: replay-safe, as the origin-gate census.
+				const probe: EdgeRequestInit = { method: route.method, replaySafe: true };
+				const answered = await ask(route.path, probe).then(
+					(res) => res.status,
+					(error: Error) => `no answer (${error.message})`,
+				);
 				if (answered !== 403) open.push(`${route.method} ${route.path} → ${answered}`);
 			}
 			for (const path of spec.shellPaths) {

package/src/workerDeploy.ts

@@ -54,7 +54,8 @@
  * refusal without wrangler, git or a network.
  */
 import { spawn, spawnSync } from "node:child_process";
-import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { liveColumns, liveColumnsQueries, missingColumnStatements, schemaTables } from "cursedops/d1-columns";
 import { wranglerRows } from "cursedops/d1-import";
@@ -345,6 +346,19 @@ export interface DeployDeps {
 	 * generation's lease; absent (or `null` with no state root), nothing is leased.
 	 */
 	lease?: (app: string, what: string) => StageLeaseResult | null;
+	/**
+	 * A fresh file for wrangler's ND-JSON output (`WRANGLER_OUTPUT_FILE_PATH`), which is where a failed
+	 * `wrangler deploy` says WHY in a form a program can read ({@link transientDeployFailure}).
+	 * {@link workerDeployDeps} makes one under the OS temp dir; absent, a failed deploy is not retried.
+	 */
+	wranglerOutput?: () => WranglerOutputFile;
+}
+
+/** One `wrangler` run's output file: the env that points wrangler at it, its text, and its removal. */
+export interface WranglerOutputFile {
+	env: Record<string, string>;
+	read: () => string;
+	dispose: () => void;
 }
 
 /**
@@ -420,6 +434,15 @@ export function workerDeployDeps(cwd: string): DeployDeps {
 		},
 		git: (args) => (spawnSync("git", [...args], { cwd, encoding: "utf8" }).stdout ?? "").trim(),
 		read: (path) => readFileSync(join(cwd, path), "utf8"),
+		wranglerOutput: () => {
+			const dir = mkdtempSync(join(tmpdir(), "cursedops-wrangler-"));
+			const path = join(dir, "output.ndjson");
+			return {
+				env: { WRANGLER_OUTPUT_FILE_PATH: path },
+				read: () => (existsSync(path) ? readFileSync(path, "utf8") : ""),
+				dispose: () => rmSync(dir, { recursive: true, force: true }),
+			};
+		},
 		record: (line) => {
 			recordDeployEvent(forgeState(cwd), line);
 		},
@@ -572,6 +595,38 @@ export function wranglerError(stdout: string): { reason: string; api: boolean }
 	return { reason, api: name === "APIError" || (kind === "error" && /cloudflare api/i.test(reason)) };
 }
 
+/**
+ * Why a failed `wrangler deploy` is worth ONE more try — or `null` when it would fail the same way.
+ *
+ * Measured 2026-10-11 (forge task 3053): roms' production deploy uploaded the Worker and its routes,
+ * then Cloudflare answered the cron-schedules PUT with `Internal authentication error: internal
+ * server error [code: 10500]`. wrangler exited 1, the ship's roms lane went red, and the ship batch
+ * re-ran a 50-minute `ship run` to redo a step that a second request would have done. A deploy is
+ * idempotent — the same script, the same triggers — so a Cloudflare-side failure is retried once.
+ * A bundling error, a refused binding or a script that throws at startup (10021) is not: wrangler's
+ * own `command-failed` line ({@link WranglerOutputFile}) must name the server's fault or a retry-after.
+ */
+export function transientDeployFailure(ndjson: string): string | null {
+	type Failed = { code?: unknown; message?: unknown; retry_after_ms?: unknown };
+	let failed: Failed | null = null;
+	for (const line of ndjson.split("\n")) {
+		if (!line.trim()) continue;
+		try {
+			const entry = JSON.parse(line) as (Failed & { type?: unknown }) | null;
+			if (entry?.type === "command-failed") failed = entry;
+		} catch {
+			// a torn line from a killed run — the ones around it still count
+		}
+	}
+	if (!failed) return null;
+	const message = typeof failed.message === "string" ? failed.message : "";
+	const first = message.split("\n").find((l) => l.trim()) ?? "";
+	const reason = (detail: string) => `${first.trim() || "wrangler deploy failed"} — ${detail}`;
+	if (typeof failed.retry_after_ms === "number") return reason(`Cloudflare asked for a retry after ${failed.retry_after_ms} ms`);
+	const server = message.match(/internal server error|\[code: (?:10500|10013|7403)\]|\b50[234]\b|ETIMEDOUT|ECONNRESET|socket hang up|fetch failed/i);
+	return server ? reason(`"${server[0]}" is Cloudflare's side`) : null;
+}
+
 /**
  * Did one schema chunk APPLY? Its exit code alone does not say.
  *
@@ -895,7 +950,22 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 		? ["--var", `${spec.stampVars.commit}:${commit}`, "--var", `${spec.stampVars.dirty}:${shipsDirty ? "true" : "false"}`]
 		: [];
 	enter("upload");
-	if (deps.run(["bunx", "wrangler", "deploy", ...envArgs, ...stamp], spec.credential) !== 0) return stop("deploy", "wrangler deploy failed.");
+	const deployArgv = ["bunx", "wrangler", "deploy", ...envArgs, ...stamp];
+	const deployOnce = (): { code: number; transient: string | null } => {
+		const output = deps.wranglerOutput?.();
+		try {
+			const code = deps.run(deployArgv, { ...spec.credential, ...output?.env });
+			return { code, transient: code !== 0 && output ? transientDeployFailure(output.read()) : null };
+		} finally {
+			output?.dispose();
+		}
+	};
+	let deployed = deployOnce();
+	if (deployed.code !== 0 && deployed.transient) {
+		error(`  🟡 ${deployed.transient} — retrying the deploy once`);
+		deployed = deployOnce();
+	}
+	if (deployed.code !== 0) return stop("deploy", "wrangler deploy failed.");
 	if (spec.secrets) {
 		step("attach the secrets — exactly this deployment's set, over a pipe");
 		enter("secrets");
@@ -919,7 +989,7 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 	const detail = `${spec.workerName} deployed${spec.secrets ? ", attached" : ""}${spec.smoke ? " and smoked" : ""} @ ${commit.slice(0, 8)}.`;
 	if (unmeasured.length > 0) {
 		const why =
-			`${detail} Worker CPU UNMEASURED — the tail never attached around ${unmeasured.join(" and ")}, which PASSED. ` +
+			`${detail} Worker CPU UNMEASURED — the tail heard nothing of ${unmeasured.join(" and ")}, which PASSED. ` +
 			"Nothing is wrong with the deployment and nothing needs rolling back; rerun the smoke under the app's CPU wrapper to measure it.";
 		error(`\n🟡 ${tag} ${why}`);
 		return finish({ code: TAIL_UNATTACHED_EXIT, step: "cpu-unmeasured", detail: why, commit, dirty });
@@ -1268,6 +1338,29 @@ export function workerTailEvents(traces: readonly unknown[], scriptName: string)
 	return out;
 }
 
+/**
+ * How many of this Worker's invocations a capture holds AFTER the trace carrying `cpu-tail=<marker>`,
+ * not counting any other marker. Pure. 0 when the start marker is not in the capture at all.
+ *
+ * 🔴 Why it exists (task 3110): music's stage walk PASSED under a tail that carried the start marker
+ * and then nothing — ~20 requests and the end marker, none heard in 45 s. A tail that went deaf after
+ * attaching measured nothing, exactly like one that never attached (decision 6), so it is unmeasured,
+ * not red. A PARTIAL capture (any of the command's requests heard) stays red: that one may really be
+ * missing the last requests.
+ */
+export function heardAfterMarker(traces: readonly unknown[], scriptName: string, marker: string): number {
+	let started = false;
+	let heard = 0;
+	for (const raw of traces as TailTraceLike[]) {
+		if (!raw || raw.scriptName !== scriptName) continue;
+		const url = raw.event?.request?.url;
+		const sent = url ? URL.parse(url)?.searchParams.get("cpu-tail") : null;
+		if (!started) started = sent === marker;
+		else if (sent == null) heard++;
+	}
+	return heard;
+}
+
 /** The verdict in words, for the deploy log. */
 export function formatTailVerdict(verdict: TailVerdict, bench: Pick<TailBench, "formatCpuBudgetReport">, scriptName: string): string {
 	const r = verdict.recorded;
@@ -1405,7 +1498,8 @@ export interface WorkerCpuTailSpec extends Omit<JudgeOpts, "scriptName"> {
 }
 
 /**
- * The exit code of a CPU tail that never attached, twice, around a command that PASSED (`EX_UNAVAILABLE`):
+ * The exit code of a CPU tail that never attached, twice — or went deaf once it had (task 3110) — around a
+ * command that PASSED (`EX_UNAVAILABLE`):
  * nothing was measured, and nothing was wrong with the deployment. Distinct from the command's own
  * codes so {@link runWorkerDeploy} can say which of the two happened — decision 6.
  */
@@ -1488,6 +1582,13 @@ export async function runWorkerCpuTail(spec: WorkerCpuTailSpec, deps: WorkerCpuD
 		);
 		return { code: ran, step: "command", verdict };
 	}
-	if (!drained) return { code: 1, step: "drain", verdict };
+	if (!drained) {
+		// Decision 6, again: a tail that heard none of a PASSED command went deaf — nothing was measured.
+		if (heardAfterMarker(spec.bench.parseTailLines(tail.captured()), spec.workerName, marker) === 0) {
+			deps.error(`🟡 ${tag} wrangler tail went deaf after it attached — it heard none of \`${spec.command.join(" ")}\`, which PASSED. CPU UNMEASURED.`);
+			return { code: TAIL_UNATTACHED_EXIT, step: "unmeasured", verdict };
+		}
+		return { code: 1, step: "drain", verdict };
+	}
 	return verdict.ok ? { code: 0, step: "done", verdict } : { code: 1, step: "budget", verdict };
 }