cursedops 0.15.0 → 0.16.1

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cursedops",
-	"version": "0.15.0",
+	"version": "0.16.1",
 	"description": "The build-and-ops answers this generation's apps wrote independently and identically: finding a generation's roots — and printing a command that runs when pasted — without knowing a path, the generation's whole-tree laws run over one repo from a checkout or a worktree, macOS launchd agent install/replace/remove and the live port a job serves, the scaffolding and verdicts of a deployed smoke (origin probe, the smoke's own environment, a network that lies about DNS, a settled version), the static-serving helpers eight apps copied — the path-traversal guard among them — the API floor that keeps an unmatched /api/... from ever being answered with the app shell, the commit and dirty flag a checkout-served process reports, the Cloudflare Worker deploy toolkit four apps copied (the deploy sequence, exact-set secrets over a pipe, origin-first rollback, the curl edge fetch, the row-for-row D1 import proof, the billed-CPU tail check around a deploy's walk and smoke, and each app's worker:secrets and worker:smoke main as one function of its data), the relay a Worker fronts a Mac-bound app with (the Durable Object, the frames, the Mac's dialer and key rotation — lifted from station for roms — and the signed-in stage walk's skeleton and the relay app's whole worker:deploy), the Worker import-graph and await-port checks every Worker app's suite runs over its own source, and the public-surface ratchet three published libraries each carried a forked copy of. Mechanism only — no app knows its name from here. Bun, zero runtime dependencies (typescript is an optional peer, for public-surface only), ships source.",
 	"type": "module",
 	"scripts": {
@@ -163,6 +163,12 @@
 			"bun": "./src/publicClient.ts",
 			"source": "./src/publicClient.ts",
 			"import": "./src/publicClient.ts"
+		},
+		"./stage-lease": {
+			"types": "./src/stageLease.ts",
+			"bun": "./src/stageLease.ts",
+			"source": "./src/stageLease.ts",
+			"import": "./src/stageLease.ts"
 		}
 	},
 	"bin": {
@@ -207,5 +213,9 @@
 				"script": "test"
 			}
 		]
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/wizardmordamir/cursedforge-cursedops.git"
 	}
 }

package/src/stageLease.ts

@@ -0,0 +1,176 @@
+/**
+ * `cursedops/stage-lease` — an app's hosted stage has ONE holder at a time.
+ *
+ * 🔴 Why (forge task 3048, 2026-10-11 02:26–02:40Z): the ship was in roms' `test:browser`, which
+ * HOLDS `roms-stage` (a key minted for that run, its digest on the Worker, a throwaway Mac end
+ * linked). A batch in the same minutes ran `worker:deploy -- --env stage` twice and its own
+ * `test:browser` twice — each a new link digest on the Worker, which disconnected the ship's Mac
+ * end. The ship's suite went red on 12 specs that pass alone, its alone-retry refused because the
+ * stage now served the batch's commit, and roms did not ship. Nothing refused either side: the
+ * ship's lane record guards PRODUCTION deploys, not the stage.
+ *
+ * The lease is a file, `$FORGE_STATE/locks/stage/<app>.lock`, created exclusively. Whoever holds the
+ * stage takes it — {@link runWorkerDeploy} for every deploy (a production deploy deploys and walks
+ * the stage first), and each app script that holds or walks its stage on its own — and anyone else
+ * is REFUSED with who holds it and since when.
+ *
+ *   · The holder id rides `FORGE_STAGE_LEASE` into every child, so a `test:browser` that spawns its
+ *     own `worker-stage.ts --hold` is let through by the lease it already holds.
+ *   · A process that runs several holders in a row (`autopilot ship run`'s lane: `worker:deploy
+ *     -- --env stage`, then `test:browser`, then `worker:deploy`) sets `FORGE_STAGE_LEASE` AND
+ *     `FORGE_STAGE_LEASE_PID` (its own pid) for each step: the first step's lease is recorded
+ *     against the LANE's pid and outlives the step, so no stranger slips in between steps. The lane
+ *     removes it when it ends ({@link dropStageLease}); a lane that dies leaves a dead pid.
+ *   · A lease whose pid is dead, or older than {@link STAGE_LEASE_MAX_MS}, is stale and taken over.
+ */
+import { randomUUID } from "node:crypto";
+import { mkdirSync, readFileSync, statSync, unlinkSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+/** The holder id, inherited by every child of the process that took the lease. */
+export const STAGE_LEASE_VAR = "FORGE_STAGE_LEASE";
+/** The pid that OWNS a lease taken under {@link STAGE_LEASE_VAR} — a lane running several holders. */
+export const STAGE_LEASE_PID_VAR = "FORGE_STAGE_LEASE_PID";
+/**
+ * Past this, a lease is stale even with a live pid — a recycled pid must not hold a stage forever.
+ * The longest honest holder is a ship lane: roms' suite is ~8 min, a ship run ~25 min.
+ */
+export const STAGE_LEASE_MAX_MS = 2 * 60 * 60 * 1000;
+/** A lease file that does not parse is a writer mid-write — stale only after this long. */
+const UNREADABLE_GRACE_MS = 10_000;
+
+export interface StageLease {
+	holder: string;
+	pid: number;
+	/** ISO time it was taken. */
+	started: string;
+	/** What took it — `roms worker:deploy --env stage`, `roms test:browser`. */
+	what: string;
+}
+
+export interface StageLeaseSpec {
+	/** The generation's state root (`forgeState`). */
+	state: string;
+	/** The app whose stage this is — `roms`. */
+	app: string;
+	what: string;
+}
+
+export interface StageLeaseDeps {
+	/** Read for the holder id, and WRITTEN with it when a fresh lease is taken. Default `process.env`. */
+	env?: NodeJS.ProcessEnv;
+	pid?: number;
+	now?: () => number;
+	alive?: (pid: number) => boolean;
+}
+
+export type StageLeaseResult =
+	| { ok: true; holder: string; inherited: boolean; release: () => void }
+	| { ok: false; refusal: string; held: StageLease | null };
+
+export function stageLeasePath(state: string, app: string): string {
+	return join(state, "locks", "stage", `${app}.lock`);
+}
+
+/** `kill -0`: ESRCH is dead; EPERM is a live process that is not ours. */
+export function pidAlive(pid: number): boolean {
+	if (!Number.isInteger(pid) || pid <= 0) return false;
+	try {
+		process.kill(pid, 0);
+		return true;
+	} catch (e) {
+		return (e as NodeJS.ErrnoException).code === "EPERM";
+	}
+}
+
+function readLease(path: string): StageLease | null {
+	try {
+		const parsed = JSON.parse(readFileSync(path, "utf8")) as Partial<StageLease>;
+		if (typeof parsed.holder !== "string" || typeof parsed.pid !== "number" || typeof parsed.started !== "string") return null;
+		return { holder: parsed.holder, pid: parsed.pid, started: parsed.started, what: String(parsed.what ?? "") };
+	} catch {
+		return null;
+	}
+}
+
+function minutes(ms: number): string {
+	return ms < 60_000 ? `${Math.max(0, Math.round(ms / 1000))}s` : `${Math.round(ms / 60_000)}m`;
+}
+
+/**
+ * Take `<app>`'s stage, or be refused. A fresh lease puts its holder id in `deps.env` so children
+ * inherit it; `release` removes the file only when this process created it for itself (a lease
+ * recorded against a lane's pid is the lane's to drop).
+ */
+export function takeStageLease(spec: StageLeaseSpec, deps: StageLeaseDeps = {}): StageLeaseResult {
+	const env = deps.env ?? process.env;
+	const now = deps.now ?? Date.now;
+	const alive = deps.alive ?? pidAlive;
+	const self = deps.pid ?? process.pid;
+	const named = env[STAGE_LEASE_VAR]?.trim() || "";
+	const lanePid = Number(env[STAGE_LEASE_PID_VAR]);
+	const owner = named && Number.isInteger(lanePid) && lanePid > 0 ? lanePid : self;
+	const path = stageLeasePath(spec.state, spec.app);
+	mkdirSync(dirname(path), { recursive: true });
+
+	for (let attempt = 0; attempt < 3; attempt++) {
+		const holder = named || randomUUID();
+		const lease: StageLease = { holder, pid: owner, started: new Date(now()).toISOString(), what: spec.what };
+		try {
+			writeFileSync(path, `${JSON.stringify(lease)}\n`, { flag: "wx" });
+			env[STAGE_LEASE_VAR] = holder;
+			const ours = owner === self;
+			return {
+				ok: true,
+				holder,
+				inherited: false,
+				release: () => {
+					if (ours) dropStageLease(spec.state, spec.app, holder);
+				},
+			};
+		} catch (e) {
+			if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e;
+		}
+		const held = readLease(path);
+		if (!held) {
+			let age = 0;
+			try {
+				age = now() - statSync(path).mtimeMs;
+			} catch {
+				continue; // gone between the two calls — try again
+			}
+			if (age < UNREADABLE_GRACE_MS) {
+				return { ok: false, held: null, refusal: `${spec.app}'s stage lease (${path}) is being written right now — try again in a few seconds.` };
+			}
+			dropStageLease(spec.state, spec.app, null);
+			continue;
+		}
+		if (named && held.holder === named) return { ok: true, holder: named, inherited: true, release: () => {} };
+		const age = now() - Date.parse(held.started);
+		if (!alive(held.pid) || !(age < STAGE_LEASE_MAX_MS)) {
+			dropStageLease(spec.state, spec.app, held.holder);
+			continue;
+		}
+		return {
+			ok: false,
+			held,
+			refusal:
+				`${spec.app}'s hosted stage is held by \`${held.what || "an unnamed holder"}\` (pid ${held.pid}, since ${held.started}, ${minutes(age)} ago). ` +
+				`A second holder would replace its Worker or its link key under it and turn its run red for nothing. ` +
+				`Wait for it to finish, then run this again. (The lease: ${path} — it is taken over by itself once pid ${held.pid} has exited.)`,
+		};
+	}
+	return { ok: false, held: readLease(path), refusal: `${spec.app}'s stage lease (${path}) changed hands three times while this run tried to take it — try again.` };
+}
+
+/** Remove `<app>`'s lease — only if `holder` still holds it (`null`: whatever is there). */
+export function dropStageLease(state: string, app: string, holder: string | null): boolean {
+	const path = stageLeasePath(state, app);
+	if (holder !== null && readLease(path)?.holder !== holder) return false;
+	try {
+		unlinkSync(path);
+		return true;
+	} catch {
+		return false;
+	}
+}

package/src/workerDeploy.ts

@@ -54,12 +54,14 @@
  * refusal without wrangler, git or a network.
  */
 import { spawn, spawnSync } from "node:child_process";
-import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { liveColumns, liveColumnsQueries, missingColumnStatements, schemaTables } from "cursedops/d1-columns";
 import { wranglerRows } from "cursedops/d1-import";
 import { forgeCodeRoot, forgeState } from "cursedops/roots";
 import { appendLedger, deferral, readShipSetting, type ShipEventLine, type ShipNowWhere, type ShipSetting } from "cursedops/ship";
+import { type StageLeaseResult, takeStageLease } from "cursedops/stage-lease";
 
 /** The two deployments every Worker app here has. */
 export type WorkerEnv = "production" | "stage";
@@ -338,6 +340,25 @@ export interface DeployDeps {
 	shipWhere?: ShipNowWhere;
 	/** Where a deferred production deploy's `ship` line goes. {@link workerDeployDeps} appends it to the ledger. */
 	recordShip?: (line: ShipEventLine) => void;
+	/**
+	 * Take the app's hosted stage for the whole deploy (`./stageLease.ts`) — every deploy touches the
+	 * stage, a production one deploys and walks it first. {@link workerDeployDeps} takes the
+	 * generation's lease; absent (or `null` with no state root), nothing is leased.
+	 */
+	lease?: (app: string, what: string) => StageLeaseResult | null;
+	/**
+	 * A fresh file for wrangler's ND-JSON output (`WRANGLER_OUTPUT_FILE_PATH`), which is where a failed
+	 * `wrangler deploy` says WHY in a form a program can read ({@link transientDeployFailure}).
+	 * {@link workerDeployDeps} makes one under the OS temp dir; absent, a failed deploy is not retried.
+	 */
+	wranglerOutput?: () => WranglerOutputFile;
+}
+
+/** One `wrangler` run's output file: the env that points wrangler at it, its text, and its removal. */
+export interface WranglerOutputFile {
+	env: Record<string, string>;
+	read: () => string;
+	dispose: () => void;
 }
 
 /**
@@ -413,6 +434,15 @@ export function workerDeployDeps(cwd: string): DeployDeps {
 		},
 		git: (args) => (spawnSync("git", [...args], { cwd, encoding: "utf8" }).stdout ?? "").trim(),
 		read: (path) => readFileSync(join(cwd, path), "utf8"),
+		wranglerOutput: () => {
+			const dir = mkdtempSync(join(tmpdir(), "cursedops-wrangler-"));
+			const path = join(dir, "output.ndjson");
+			return {
+				env: { WRANGLER_OUTPUT_FILE_PATH: path },
+				read: () => (existsSync(path) ? readFileSync(path, "utf8") : ""),
+				dispose: () => rmSync(dir, { recursive: true, force: true }),
+			};
+		},
 		record: (line) => {
 			recordDeployEvent(forgeState(cwd), line);
 		},
@@ -421,6 +451,10 @@ export function workerDeployDeps(cwd: string): DeployDeps {
 		recordShip: (line) => {
 			appendLedger(forgeState(cwd), line);
 		},
+		lease: (app, what) => {
+			const state = forgeState(cwd);
+			return state ? takeStageLease({ state, app, what }) : null;
+		},
 	};
 }
 
@@ -561,6 +595,38 @@ export function wranglerError(stdout: string): { reason: string; api: boolean }
 	return { reason, api: name === "APIError" || (kind === "error" && /cloudflare api/i.test(reason)) };
 }
 
+/**
+ * Why a failed `wrangler deploy` is worth ONE more try — or `null` when it would fail the same way.
+ *
+ * Measured 2026-10-11 (forge task 3053): roms' production deploy uploaded the Worker and its routes,
+ * then Cloudflare answered the cron-schedules PUT with `Internal authentication error: internal
+ * server error [code: 10500]`. wrangler exited 1, the ship's roms lane went red, and the ship batch
+ * re-ran a 50-minute `ship run` to redo a step that a second request would have done. A deploy is
+ * idempotent — the same script, the same triggers — so a Cloudflare-side failure is retried once.
+ * A bundling error, a refused binding or a script that throws at startup (10021) is not: wrangler's
+ * own `command-failed` line ({@link WranglerOutputFile}) must name the server's fault or a retry-after.
+ */
+export function transientDeployFailure(ndjson: string): string | null {
+	type Failed = { code?: unknown; message?: unknown; retry_after_ms?: unknown };
+	let failed: Failed | null = null;
+	for (const line of ndjson.split("\n")) {
+		if (!line.trim()) continue;
+		try {
+			const entry = JSON.parse(line) as (Failed & { type?: unknown }) | null;
+			if (entry?.type === "command-failed") failed = entry;
+		} catch {
+			// a torn line from a killed run — the ones around it still count
+		}
+	}
+	if (!failed) return null;
+	const message = typeof failed.message === "string" ? failed.message : "";
+	const first = message.split("\n").find((l) => l.trim()) ?? "";
+	const reason = (detail: string) => `${first.trim() || "wrangler deploy failed"} — ${detail}`;
+	if (typeof failed.retry_after_ms === "number") return reason(`Cloudflare asked for a retry after ${failed.retry_after_ms} ms`);
+	const server = message.match(/internal server error|\[code: (?:10500|10013|7403)\]|\b50[234]\b|ETIMEDOUT|ECONNRESET|socket hang up|fetch failed/i);
+	return server ? reason(`"${server[0]}" is Cloudflare's side`) : null;
+}
+
 /**
  * Did one schema chunk APPLY? Its exit code alone does not say.
  *
@@ -707,7 +773,9 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 	const commit = deps.git(["rev-parse", "HEAD"]);
 	const porcelain = deps.git(["status", "--porcelain"]);
 	const dirty = porcelain.length > 0;
+	let release = (): void => {};
 	const finish = (result: DeployResult): DeployResult => {
+		release();
 		enter("");
 		try {
 			deps.record?.({
@@ -755,6 +823,11 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 	if (spec.env === "production" && dirty) {
 		return stop("clean-tree", `the tree is dirty — production would report ${commit.slice(0, 8)} and run something else:\n${porcelain}`);
 	}
+	// 🔴 One holder per hosted stage (task 3048): a stage deploy under the ship's held suite swapped
+	// the Worker and its link key under it. Taken before anything builds, held until `finish`.
+	const lease = deps.lease?.(spec.app, `${spec.app} worker:deploy${spec.env === "stage" ? " --env stage" : ""} @ ${commit.slice(0, 8)}`);
+	if (lease && !lease.ok) return stop("stage-lease", lease.refusal);
+	if (lease?.ok) release = lease.release;
 	for (const refusal of spec.beforeStage ?? []) {
 		const why = refusal();
 		if (why) return stop("refusal", why);
@@ -877,7 +950,22 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 		? ["--var", `${spec.stampVars.commit}:${commit}`, "--var", `${spec.stampVars.dirty}:${shipsDirty ? "true" : "false"}`]
 		: [];
 	enter("upload");
-	if (deps.run(["bunx", "wrangler", "deploy", ...envArgs, ...stamp], spec.credential) !== 0) return stop("deploy", "wrangler deploy failed.");
+	const deployArgv = ["bunx", "wrangler", "deploy", ...envArgs, ...stamp];
+	const deployOnce = (): { code: number; transient: string | null } => {
+		const output = deps.wranglerOutput?.();
+		try {
+			const code = deps.run(deployArgv, { ...spec.credential, ...output?.env });
+			return { code, transient: code !== 0 && output ? transientDeployFailure(output.read()) : null };
+		} finally {
+			output?.dispose();
+		}
+	};
+	let deployed = deployOnce();
+	if (deployed.code !== 0 && deployed.transient) {
+		error(`  🟡 ${deployed.transient} — retrying the deploy once`);
+		deployed = deployOnce();
+	}
+	if (deployed.code !== 0) return stop("deploy", "wrangler deploy failed.");
 	if (spec.secrets) {
 		step("attach the secrets — exactly this deployment's set, over a pipe");
 		enter("secrets");