cursedops 0.16.1 → 0.17.0

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cursedops",
-	"version": "0.16.1",
+	"version": "0.17.0",
 	"description": "The build-and-ops answers this generation's apps wrote independently and identically: finding a generation's roots — and printing a command that runs when pasted — without knowing a path, the generation's whole-tree laws run over one repo from a checkout or a worktree, macOS launchd agent install/replace/remove and the live port a job serves, the scaffolding and verdicts of a deployed smoke (origin probe, the smoke's own environment, a network that lies about DNS, a settled version), the static-serving helpers eight apps copied — the path-traversal guard among them — the API floor that keeps an unmatched /api/... from ever being answered with the app shell, the commit and dirty flag a checkout-served process reports, the Cloudflare Worker deploy toolkit four apps copied (the deploy sequence, exact-set secrets over a pipe, origin-first rollback, the curl edge fetch, the row-for-row D1 import proof, the billed-CPU tail check around a deploy's walk and smoke, and each app's worker:secrets and worker:smoke main as one function of its data), the relay a Worker fronts a Mac-bound app with (the Durable Object, the frames, the Mac's dialer and key rotation — lifted from station for roms — and the signed-in stage walk's skeleton and the relay app's whole worker:deploy), the Worker import-graph and await-port checks every Worker app's suite runs over its own source, and the public-surface ratchet three published libraries each carried a forked copy of. Mechanism only — no app knows its name from here. Bun, zero runtime dependencies (typescript is an optional peer, for public-surface only), ships source.",
 	"type": "module",
 	"scripts": {

package/src/edgeFetch.ts

@@ -59,10 +59,18 @@ export interface EdgeFetchOptions {
 }
 
 /**
- * A `RequestInit` this fetch reads one more field from: `maxTimeSec`, curl's `--max-time` for
- * THIS request, over the instance's {@link EdgeFetchOptions.maxTimeSec}. A plain `fetch` ignores it.
+ * A `RequestInit` this fetch reads two more fields from. A plain `fetch` ignores both.
+ *
+ *  · `maxTimeSec` — curl's `--max-time` for THIS request, over the instance's
+ *    {@link EdgeFetchOptions.maxTimeSec};
+ *  · `replaySafe` — the caller's word that sending this request twice does nothing the first did
+ *    not, WHATEVER its method: a smoke's anonymous census probe, which the origin gate refuses
+ *    before any handler runs. Such a request is replayed after a timeout or a dropped reply like a
+ *    GET. Measured 2026-10-13: family's stage smoke died on census route 55 of 101, an anonymous
+ *    `POST /api/media` that never reached the Worker (no trace of it), because a POST's timeout
+ *    was never replayed — a green Worker, a red ship lane (forge task 3115).
  */
-export type EdgeRequestInit = RequestInit & { maxTimeSec?: number };
+export type EdgeRequestInit = RequestInit & { maxTimeSec?: number; replaySafe?: boolean };
 
 /**
  * 🔴 How many MORE times a request is sent after a connection-level curl failure that carried
@@ -85,12 +93,13 @@ const EDGE_RETRIES = 2;
  *    three of them. Measured 2026-09-26: flix's prod smoke went red on one `curl exit 28` for
  *    `GET /healthz` while the same URL answered 200 in 0.2 s seconds later (task 2190).
  *
+ * A request its caller marked `replaySafe` ({@link EdgeRequestInit}) counts as safe to run twice.
  * Every other exit, and every HTTP status, whatever it is, is returned exactly as before.
  */
-function retryableCurlFailure(exit: number | null, stdout: Buffer, method = "GET", retry = 1): boolean {
+function retryableCurlFailure(exit: number | null, stdout: Buffer, method = "GET", retry = 1, replaySafe = false): boolean {
 	if (stdout.length > 0 || exit === null) return false;
 	if (exit === 6 || exit === 7 || exit === 35) return true;
-	const replayable = ["GET", "HEAD", "OPTIONS", "PUT", "DELETE"].includes(method.toUpperCase());
+	const replayable = replaySafe || ["GET", "HEAD", "OPTIONS", "PUT", "DELETE"].includes(method.toUpperCase());
 	if (exit === 28) return replayable && retry === 1;
 	return (exit === 52 || exit === 56) && replayable;
 }
@@ -181,7 +190,7 @@ export function createEdgeFetch(options: EdgeFetchOptions = {}): (url: string, i
 		}
 		try {
 			let out = curl([...head(), ...args, target.toString()]);
-			for (let retry = 1; retry <= EDGE_RETRIES && retryableCurlFailure(out.status, out.stdout, init.method, retry); retry++) {
+			for (let retry = 1; retry <= EDGE_RETRIES && retryableCurlFailure(out.status, out.stdout, init.method, retry, init.replaySafe); retry++) {
 				await new Promise((r) => setTimeout(r, options.retryDelayMs?.(retry) ?? 500 * retry));
 				// `head()` again: a host with no pin yet is asked again, so a retry can find the address.
 				out = curl([...head(), ...args, target.toString()]);

package/src/relayStage.ts

@@ -20,6 +20,7 @@ import { lookup as dnsLookup } from "node:dns/promises";
 import { readdirSync, readFileSync, statSync } from "node:fs";
 import { join, relative } from "node:path";
 import { sha256Hex } from "cursedops/relay";
+import { awaitConverged, type ConvergenceOptions, type ConvergenceProbe } from "cursedops/smoke";
 
 /** The accounts door every stage signs in at — `apps/auth` deployed a second time. */
 export const STAGE_AUTH_URL = "https://auth-stage.cursedalchemy.com";
@@ -159,6 +160,30 @@ export const realRun: Run = (argv, options) => {
 	return { status: done.status, stdout: done.stdout ?? "", stderr: done.stderr ?? "" };
 };
 
+/**
+ * 🔴 How many rounds in a row every `/healthz` must name a just-put link digest before a walk uses it.
+ * A digest put is a new Worker version, and the edge takes it up instance by instance: the link
+ * staying up for 15 s proves only that the instance the link dialled has it. The Worker signs the
+ * Mac's identity with that digest, so a request answered by an instance still on the old one is the
+ * Mac's own 401 (and station's kept read a 503). Measured 2026-10-15 (forge task 3235): station's
+ * fast-lane walk at baf8495, the link steady for 15 s, then four writes 401 at the Mac; on the real
+ * stage the put needed 8 rounds to agree 5 in a row.
+ */
+export const LINK_KEY_ROUNDS = 5;
+
+/** The question {@link putStageLinkDigest} asks after a put: does this `/healthz` answer's `worker.linkKeyId` equal `expected`? */
+export function linkKeyProbe(health: () => Promise<Response>, expected: string): ConvergenceProbe {
+	return {
+		name: "/healthz linkKeyId",
+		check: async () => {
+			const res = await health();
+			if (!res.ok) return `status ${res.status}`;
+			const id = ((await res.json()) as { worker?: { linkKeyId?: string } }).worker?.linkKeyId ?? "";
+			return id === expected ? null : `answers ${id || "no key"}, not ${expected}`;
+		},
+	};
+}
+
 export interface DigestPut {
 	/** The app's checkout, where `wrangler.jsonc` is. */
 	cwd: string;
@@ -167,22 +192,32 @@ export interface DigestPut {
 	key: string;
 	/** `cloudflareCredential(...)`. */
 	credential: Record<string, string>;
+	/** One `/healthz` read of the stage Worker — asked until every answer names the new digest. */
+	health: () => Promise<Response>;
+	/** The fingerprint `/healthz` publishes for a digest — the app's `cursedauth/relay-door` `linkKeyId(app, digest)`. */
+	keyId: (digest: string) => Promise<string>;
 	run?: Run;
+	/** Injected for tests; `rounds` defaults to {@link LINK_KEY_ROUNDS}. */
+	convergence?: ConvergenceOptions;
 }
 
 /**
  * Put `key`'s digest on the STAGE Worker (`--env stage`, always — this module has no production
- * path). A per-run key for the walk, and again for the rotation leg. `ok: false` carries wrangler's
- * stderr. A secret put is a new version that reaches the edge in its own time: the caller waits for
- * a link that stays up ({@link linkedSteadily}), never a fixed sleep.
+ * path), then return only once every `/healthz` answer has named it {@link LINK_KEY_ROUNDS} rounds in
+ * a row — so no walk can put a digest and use it before the edge has (forge tasks 3235, 3237). A
+ * per-run key for the walk, and again for the rotation leg. `ok: false` is a refused put (wrangler's
+ * stderr) or an edge that never agreed; `detail` says which, for the walk's row.
  */
-export async function putStageLinkDigest(put: DigestPut): Promise<{ ok: boolean; stderr: string }> {
+export async function putStageLinkDigest(put: DigestPut): Promise<{ ok: boolean; stderr: string; detail: string }> {
+	const digest = await sha256Hex(put.key);
 	const done = (put.run ?? realRun)(["bunx", "wrangler", "secret", "put", put.secretName, "--env", "stage"], {
 		cwd: put.cwd,
-		input: await sha256Hex(put.key),
+		input: digest,
 		env: put.credential,
 	});
-	return { ok: done.status === 0, stderr: done.stderr };
+	if (done.status !== 0) return { ok: false, stderr: done.stderr, detail: `wrangler refused the put (exit ${done.status})` };
+	const held = await awaitConverged([linkKeyProbe(put.health, await put.keyId(digest))], { rounds: LINK_KEY_ROUNDS, ...put.convergence });
+	return { ok: held.converged, stderr: done.stderr, detail: `every answer holds it — ${held.detail}` };
 }
 
 /**

package/src/smoke.ts

@@ -1127,9 +1127,15 @@ export async function runWorkerSmoke(spec: WorkerSmokeSpec): Promise<number> {
 			`credentials go back to ${JSON.stringify(health.publicUrl)} — this deployment's ${deployment.publicUrl}`,
 		);
 
+		// Each probe is anonymous and refused at the gate, so it is replay-safe whatever its method,
+		// and one that gets no answer even then is this check's red — never a crash of the whole smoke.
 		const open: string[] = [];
 		for (const route of spec.census) {
-			const answered = (await ask(route.path, { method: route.method })).status;
+			const probe: EdgeRequestInit = { method: route.method, replaySafe: true };
+			const answered = await ask(route.path, probe).then(
+				(res) => res.status,
+				(error: Error) => `no answer (${error.message})`,
+			);
 			if (answered !== 401) open.push(`${route.method} ${route.path} → ${answered}`);
 		}
 		smoke.record(
@@ -1214,7 +1220,12 @@ async function accessDoorSmoke(
 		} else if (address === "preview") {
 			const open: string[] = [];
 			for (const route of spec.census) {
-				const answered = (await ask(route.path, { method: route.method })).status;
+				// Refused by Access before the Worker sees it: replay-safe, as the origin-gate census.
+				const probe: EdgeRequestInit = { method: route.method, replaySafe: true };
+				const answered = await ask(route.path, probe).then(
+					(res) => res.status,
+					(error: Error) => `no answer (${error.message})`,
+				);
 				if (answered !== 403) open.push(`${route.method} ${route.path} → ${answered}`);
 			}
 			for (const path of spec.shellPaths) {

package/src/workerDeploy.ts

@@ -989,7 +989,7 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 	const detail = `${spec.workerName} deployed${spec.secrets ? ", attached" : ""}${spec.smoke ? " and smoked" : ""} @ ${commit.slice(0, 8)}.`;
 	if (unmeasured.length > 0) {
 		const why =
-			`${detail} Worker CPU UNMEASURED — the tail never attached around ${unmeasured.join(" and ")}, which PASSED. ` +
+			`${detail} Worker CPU UNMEASURED — the tail heard nothing of ${unmeasured.join(" and ")}, which PASSED. ` +
 			"Nothing is wrong with the deployment and nothing needs rolling back; rerun the smoke under the app's CPU wrapper to measure it.";
 		error(`\n🟡 ${tag} ${why}`);
 		return finish({ code: TAIL_UNATTACHED_EXIT, step: "cpu-unmeasured", detail: why, commit, dirty });
@@ -1338,6 +1338,29 @@ export function workerTailEvents(traces: readonly unknown[], scriptName: string)
 	return out;
 }
 
+/**
+ * How many of this Worker's invocations a capture holds AFTER the trace carrying `cpu-tail=<marker>`,
+ * not counting any other marker. Pure. 0 when the start marker is not in the capture at all.
+ *
+ * 🔴 Why it exists (task 3110): music's stage walk PASSED under a tail that carried the start marker
+ * and then nothing — ~20 requests and the end marker, none heard in 45 s. A tail that went deaf after
+ * attaching measured nothing, exactly like one that never attached (decision 6), so it is unmeasured,
+ * not red. A PARTIAL capture (any of the command's requests heard) stays red: that one may really be
+ * missing the last requests.
+ */
+export function heardAfterMarker(traces: readonly unknown[], scriptName: string, marker: string): number {
+	let started = false;
+	let heard = 0;
+	for (const raw of traces as TailTraceLike[]) {
+		if (!raw || raw.scriptName !== scriptName) continue;
+		const url = raw.event?.request?.url;
+		const sent = url ? URL.parse(url)?.searchParams.get("cpu-tail") : null;
+		if (!started) started = sent === marker;
+		else if (sent == null) heard++;
+	}
+	return heard;
+}
+
 /** The verdict in words, for the deploy log. */
 export function formatTailVerdict(verdict: TailVerdict, bench: Pick<TailBench, "formatCpuBudgetReport">, scriptName: string): string {
 	const r = verdict.recorded;
@@ -1475,7 +1498,8 @@ export interface WorkerCpuTailSpec extends Omit<JudgeOpts, "scriptName"> {
 }
 
 /**
- * The exit code of a CPU tail that never attached, twice, around a command that PASSED (`EX_UNAVAILABLE`):
+ * The exit code of a CPU tail that never attached, twice — or went deaf once it had (task 3110) — around a
+ * command that PASSED (`EX_UNAVAILABLE`):
  * nothing was measured, and nothing was wrong with the deployment. Distinct from the command's own
  * codes so {@link runWorkerDeploy} can say which of the two happened — decision 6.
  */
@@ -1558,6 +1582,13 @@ export async function runWorkerCpuTail(spec: WorkerCpuTailSpec, deps: WorkerCpuD
 		);
 		return { code: ran, step: "command", verdict };
 	}
-	if (!drained) return { code: 1, step: "drain", verdict };
+	if (!drained) {
+		// Decision 6, again: a tail that heard none of a PASSED command went deaf — nothing was measured.
+		if (heardAfterMarker(spec.bench.parseTailLines(tail.captured()), spec.workerName, marker) === 0) {
+			deps.error(`🟡 ${tag} wrangler tail went deaf after it attached — it heard none of \`${spec.command.join(" ")}\`, which PASSED. CPU UNMEASURED.`);
+			return { code: TAIL_UNATTACHED_EXIT, step: "unmeasured", verdict };
+		}
+		return { code: 1, step: "drain", verdict };
+	}
 	return verdict.ok ? { code: 0, step: "done", verdict } : { code: 1, step: "budget", verdict };
 }