cursedops 0.14.0 → 0.16.0

package/README.md

@@ -35,6 +35,7 @@ import shape, worked example, and the incident each one is pinned to.
 | `cursedops/relay-link` | the MAC half — `dialLink`, `readLinkSettings`, `rotateLinkKey` |
 | `cursedops/relay-stage` | the signed-in STAGE WALK's skeleton both relay apps shared |
 | `cursedops/public-client` | does a relay app's PUBLIC hostname serve the client this release built? — `comparePublicClient`, `recordPublicClient`, `shipClientIfBehind` |
+| `cursedops/container-deploy` | a Cloudflare CONTAINER Worker deployed and proven — the env-file secret lines, Docker on PATH and up, `/healthz`, and `awaitRollout`: the container on the new image, not just the Worker answering |
 | `cursedops/public-surface` | the ratchet on a LIBRARY's public surface — a symbol count per export subpath against a committed baseline that may only fall |
 
 Bun, zero runtime dependencies, ships TypeScript source. Nothing here knows an app's

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cursedops",
-	"version": "0.14.0",
+	"version": "0.16.0",
 	"description": "The build-and-ops answers this generation's apps wrote independently and identically: finding a generation's roots — and printing a command that runs when pasted — without knowing a path, the generation's whole-tree laws run over one repo from a checkout or a worktree, macOS launchd agent install/replace/remove and the live port a job serves, the scaffolding and verdicts of a deployed smoke (origin probe, the smoke's own environment, a network that lies about DNS, a settled version), the static-serving helpers eight apps copied — the path-traversal guard among them — the API floor that keeps an unmatched /api/... from ever being answered with the app shell, the commit and dirty flag a checkout-served process reports, the Cloudflare Worker deploy toolkit four apps copied (the deploy sequence, exact-set secrets over a pipe, origin-first rollback, the curl edge fetch, the row-for-row D1 import proof, the billed-CPU tail check around a deploy's walk and smoke, and each app's worker:secrets and worker:smoke main as one function of its data), the relay a Worker fronts a Mac-bound app with (the Durable Object, the frames, the Mac's dialer and key rotation — lifted from station for roms — and the signed-in stage walk's skeleton and the relay app's whole worker:deploy), the Worker import-graph and await-port checks every Worker app's suite runs over its own source, and the public-surface ratchet three published libraries each carried a forked copy of. Mechanism only — no app knows its name from here. Bun, zero runtime dependencies (typescript is an optional peer, for public-surface only), ships source.",
 	"type": "module",
 	"scripts": {
@@ -152,11 +152,23 @@
 			"source": "./src/relayStage.ts",
 			"import": "./src/relayStage.ts"
 		},
+		"./container-deploy": {
+			"types": "./src/containerDeploy.ts",
+			"bun": "./src/containerDeploy.ts",
+			"source": "./src/containerDeploy.ts",
+			"import": "./src/containerDeploy.ts"
+		},
 		"./public-client": {
 			"types": "./src/publicClient.ts",
 			"bun": "./src/publicClient.ts",
 			"source": "./src/publicClient.ts",
 			"import": "./src/publicClient.ts"
+		},
+		"./stage-lease": {
+			"types": "./src/stageLease.ts",
+			"bun": "./src/stageLease.ts",
+			"source": "./src/stageLease.ts",
+			"import": "./src/stageLease.ts"
 		}
 	},
 	"bin": {
@@ -201,5 +213,9 @@
 				"script": "test"
 			}
 		]
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/wizardmordamir/cursedforge-cursedops.git"
 	}
 }

package/src/containerDeploy.ts

@@ -0,0 +1,174 @@
+/**
+ * `cursedops/container-deploy` — the pieces of deploying a Cloudflare CONTAINER Worker (a Worker
+ * fronting a Docker image) and proving it landed, which binary-server's `runner:deploy` (task 2723)
+ * wrote and music's `agent:deploy` (task 2816) copied body for body.
+ *
+ * ```ts
+ * import { awaitRollout, dockerUp, healthz, newestRollout, readEnvLine, withDockerBin, workersDevUrl, writeEnvLine } from "cursedops/container-deploy";
+ *
+ * const env = { ...process.env, PATH: withDockerBin(process.env.PATH) };
+ * const url = await workersDevUrl(account, apiToken, "thing-runner");
+ * const app = { account, token: apiToken, application: "thing-runner-encoder" };
+ * const before = await newestRollout(app);
+ * await dockerUp(env, "the runner");
+ * // … wrangler deploy, then healthz(url, token) until it answers at a new version …
+ * const { verdict } = await awaitRollout({ ...app, before, poke: () => startAnInstance(url), label: "encoder" });
+ * ```
+ *
+ * 🔴 **A Worker answering is not the container being on the new image.** Cloudflare rolls the
+ * container out separately and gradually, and a rollout only moves while an instance starts — so
+ * `awaitRollout` pokes one every time it looks (measured 2026-10-05: the Worker's version was live
+ * while the instances still ran the previous image).
+ *
+ * 🔴 **An application the deploy CREATED has no rollout at all** — it is born at version 1 with the
+ * image (`/rollouts` answered `[]` on music's first deploy), so waiting for one would wait the whole
+ * twenty minutes and then call a good deploy failed. `rolloutVerdict` says `created`.
+ *
+ * 🔴 **A network blip while looking is a look that failed, not a deploy that failed** — one
+ * "unknown certificate verification error" threw a landed deploy red. `awaitRollout` looks again.
+ */
+import { spawnSync } from "node:child_process";
+import { chmodSync, existsSync, readFileSync, writeFileSync } from "node:fs";
+
+/** The value of `key` in an env file's text, unquoted; `""` when it is not there. Pure. */
+export function readEnvLine(text: string, key: string): string {
+	return new RegExp(`^${key}=(.*)$`, "m").exec(text)?.[1]?.trim().replace(/^["']|["']$/g, "") ?? "";
+}
+
+/** Set `key=value` in an env file's text, replacing a line already there. Pure. */
+export function withEnvLine(text: string, key: string, value: string): string {
+	const line = `${key}=${value}`;
+	const re = new RegExp(`^${key}=.*$`, "m");
+	if (re.test(text)) return text.replace(re, line);
+	return `${text}${text.endsWith("\n") || text === "" ? "" : "\n"}${line}\n`;
+}
+
+/** Set `key=value` in the secrets file at `path` (created if missing), owner-only. */
+export function writeEnvLine(path: string, key: string, value: string): void {
+	writeFileSync(path, withEnvLine(existsSync(path) ? readFileSync(path, "utf8") : "", key, value));
+	chmodSync(path, 0o600);
+}
+
+export const DOCKER_BUNDLE_BIN = "/Applications/Docker.app/Contents/Resources/bin";
+
+/**
+ * Docker Desktop's own helpers (`docker-credential-desktop`, which every image pull asks for) live
+ * in its app bundle, which a non-login shell's PATH does not hold — measured 2026-10-05: the first
+ * build failed "executable file not found in $PATH" on its first `FROM`. Pure.
+ */
+export function withDockerBin(path: string | undefined, bundleBin = DOCKER_BUNDLE_BIN): string {
+	const parts = (path ?? "").split(":").filter(Boolean);
+	return parts.includes(bundleBin) ? parts.join(":") : [...parts, bundleBin].join(":");
+}
+
+/** Docker up, opening Docker Desktop if it is not — wrangler builds the image with it. `what` names the image for the error. */
+export async function dockerUp(env: NodeJS.ProcessEnv, what: string): Promise<void> {
+	const up = (): boolean => spawnSync("docker", ["info"], { stdio: "ignore", env }).status === 0;
+	if (up()) return;
+	console.log("Docker is not running — opening it…");
+	spawnSync("open", ["-a", "Docker"]);
+	for (let i = 0; i < 60 && !up(); i++) await Bun.sleep(2000);
+	if (!up()) throw new Error(`Docker did not come up in two minutes; wrangler builds ${what}'s image with it`);
+}
+
+/** `GET /healthz`, with the bearer token or without. Status 0 when nothing answered. */
+export async function healthz(url: string, token: string | null): Promise<{ status: number; version: string | null }> {
+	try {
+		const res = await fetch(`${url}/healthz`, { headers: token ? { authorization: `Bearer ${token}` } : {}, signal: AbortSignal.timeout(10_000) });
+		const body = (await res.json().catch(() => null)) as { version?: string } | null;
+		return { status: res.status, version: body?.version ?? null };
+	} catch {
+		return { status: 0, version: null };
+	}
+}
+
+/** The Worker's `https://<script>.<account subdomain>.workers.dev` address. */
+export async function workersDevUrl(account: string, token: string, script: string): Promise<string> {
+	const sub = (await (
+		await fetch(`https://api.cloudflare.com/client/v4/accounts/${account}/workers/subdomain`, { headers: { authorization: `Bearer ${token}` } })
+	).json()) as { result?: { subdomain?: string } };
+	if (!sub.result?.subdomain) throw new Error("could not read the account's workers.dev subdomain");
+	return `https://${script}.${sub.result.subdomain}.workers.dev`;
+}
+
+export type ContainerRollout = { id: string; created_at: string; status: string; target_configuration?: { image?: string } };
+
+/** Which container application, on which account, read with which API token. */
+export interface ContainerApplication {
+	account: string;
+	token: string;
+	/** The application's name — wrangler names it `<script>-<class name, lowercased>`. */
+	application: string;
+}
+
+/** The most recently created rollout. Pure. */
+export function newestOf(rollouts: readonly ContainerRollout[]): ContainerRollout | null {
+	return [...rollouts].sort((a, b) => b.created_at.localeCompare(a.created_at))[0] ?? null;
+}
+
+/** The application's newest rollout: `undefined` before the application exists, `null` for one with none. */
+export async function newestRollout(app: ContainerApplication, fetchImpl: typeof fetch = fetch): Promise<ContainerRollout | null | undefined> {
+	const api = `https://api.cloudflare.com/client/v4/accounts/${app.account}/containers/applications`;
+	const headers = { authorization: `Bearer ${app.token}` };
+	const apps = (await (await fetchImpl(api, { headers })).json()) as { result?: { id: string; name: string }[] };
+	const found = apps.result?.find((a) => a.name === app.application);
+	if (!found) return undefined;
+	const rollouts = (await (await fetchImpl(`${api}/${found.id}/rollouts`, { headers })).json()) as { result?: ContainerRollout[] };
+	return newestOf(rollouts.result ?? []);
+}
+
+export type RolloutVerdict = "new-image" | "unchanged" | "created" | "waiting";
+
+/** Is the container on the image this deploy pushed, given the newest rollout before it and now? Pure. */
+export function rolloutVerdict(before: ContainerRollout | null | undefined, now: ContainerRollout | null | undefined): RolloutVerdict {
+	if (before === undefined && now === null) return "created";
+	// An application that has never rolled out, and still has not: the deploy changed no image.
+	if (before === null && now === null) return "unchanged";
+	if (!now) return "waiting";
+	if (now.id === before?.id) return now.status === "completed" ? "unchanged" : "waiting";
+	return now.status === "completed" ? "new-image" : "waiting";
+}
+
+export interface AwaitRolloutOptions extends ContainerApplication {
+	/** The newest rollout read BEFORE `wrangler deploy`. */
+	before: ContainerRollout | null | undefined;
+	/** Start an instance — a rollout only moves while one starts. Must not throw. */
+	poke: () => Promise<void>;
+	/** What the container is called in the log and the error ("encoder", "recorder"). */
+	label: string;
+	timeoutMs?: number;
+	intervalMs?: number;
+	/** For tests. */
+	read?: (app: ContainerApplication) => Promise<ContainerRollout | null | undefined>;
+	sleep?: (ms: number) => Promise<void>;
+	now?: () => number;
+	log?: (line: string) => void;
+}
+
+/** Wait until the rollout settles; throws when it has not in `timeoutMs` (twenty minutes). */
+export async function awaitRollout(o: AwaitRolloutOptions): Promise<{ verdict: Exclude<RolloutVerdict, "waiting">; rollout: ContainerRollout | null | undefined }> {
+	const read = o.read ?? ((app: ContainerApplication) => newestRollout(app));
+	const sleep = o.sleep ?? ((ms: number) => Bun.sleep(ms));
+	const now = o.now ?? Date.now;
+	const log = o.log ?? console.log;
+	const interval = o.intervalMs ?? 10_000;
+	const until = now() + (o.timeoutMs ?? 20 * 60_000);
+	let last = "";
+	while (now() < until) {
+		let r: ContainerRollout | null | undefined;
+		try {
+			r = await read(o);
+		} catch (e) {
+			log(`   (could not read the ${o.label} rollout: ${e instanceof Error ? e.message : String(e)} — looking again)`);
+			await sleep(interval);
+			continue;
+		}
+		const verdict = rolloutVerdict(o.before, r);
+		if (verdict !== "waiting") return { verdict, rollout: r };
+		if (r && r.status !== last) log(`   ${o.label} rollout ${r.id.slice(0, 8)}: ${r.status}`);
+		last = r?.status ?? "";
+		await o.poke();
+		await sleep(interval);
+	}
+	throw new Error(`the ${o.label}'s rollout did not complete in ${Math.round((o.timeoutMs ?? 20 * 60_000) / 60_000)} minutes — the instances may still run the previous image`);
+}

package/src/stageLease.ts

@@ -0,0 +1,176 @@
+/**
+ * `cursedops/stage-lease` — an app's hosted stage has ONE holder at a time.
+ *
+ * 🔴 Why (forge task 3048, 2026-10-11 02:26–02:40Z): the ship was in roms' `test:browser`, which
+ * HOLDS `roms-stage` (a key minted for that run, its digest on the Worker, a throwaway Mac end
+ * linked). A batch in the same minutes ran `worker:deploy -- --env stage` twice and its own
+ * `test:browser` twice — each a new link digest on the Worker, which disconnected the ship's Mac
+ * end. The ship's suite went red on 12 specs that pass alone, its alone-retry refused because the
+ * stage now served the batch's commit, and roms did not ship. Nothing refused either side: the
+ * ship's lane record guards PRODUCTION deploys, not the stage.
+ *
+ * The lease is a file, `$FORGE_STATE/locks/stage/<app>.lock`, created exclusively. Whoever holds the
+ * stage takes it — {@link runWorkerDeploy} for every deploy (a production deploy deploys and walks
+ * the stage first), and each app script that holds or walks its stage on its own — and anyone else
+ * is REFUSED with who holds it and since when.
+ *
+ *   · The holder id rides `FORGE_STAGE_LEASE` into every child, so a `test:browser` that spawns its
+ *     own `worker-stage.ts --hold` is let through by the lease it already holds.
+ *   · A process that runs several holders in a row (`autopilot ship run`'s lane: `worker:deploy
+ *     -- --env stage`, then `test:browser`, then `worker:deploy`) sets `FORGE_STAGE_LEASE` AND
+ *     `FORGE_STAGE_LEASE_PID` (its own pid) for each step: the first step's lease is recorded
+ *     against the LANE's pid and outlives the step, so no stranger slips in between steps. The lane
+ *     removes it when it ends ({@link dropStageLease}); a lane that dies leaves a dead pid.
+ *   · A lease whose pid is dead, or older than {@link STAGE_LEASE_MAX_MS}, is stale and taken over.
+ */
+import { randomUUID } from "node:crypto";
+import { mkdirSync, readFileSync, statSync, unlinkSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+/** The holder id, inherited by every child of the process that took the lease. */
+export const STAGE_LEASE_VAR = "FORGE_STAGE_LEASE";
+/** The pid that OWNS a lease taken under {@link STAGE_LEASE_VAR} — a lane running several holders. */
+export const STAGE_LEASE_PID_VAR = "FORGE_STAGE_LEASE_PID";
+/**
+ * Past this, a lease is stale even with a live pid — a recycled pid must not hold a stage forever.
+ * The longest honest holder is a ship lane: roms' suite is ~8 min, a ship run ~25 min.
+ */
+export const STAGE_LEASE_MAX_MS = 2 * 60 * 60 * 1000;
+/** A lease file that does not parse is a writer mid-write — stale only after this long. */
+const UNREADABLE_GRACE_MS = 10_000;
+
+export interface StageLease {
+	holder: string;
+	pid: number;
+	/** ISO time it was taken. */
+	started: string;
+	/** What took it — `roms worker:deploy --env stage`, `roms test:browser`. */
+	what: string;
+}
+
+export interface StageLeaseSpec {
+	/** The generation's state root (`forgeState`). */
+	state: string;
+	/** The app whose stage this is — `roms`. */
+	app: string;
+	what: string;
+}
+
+export interface StageLeaseDeps {
+	/** Read for the holder id, and WRITTEN with it when a fresh lease is taken. Default `process.env`. */
+	env?: NodeJS.ProcessEnv;
+	pid?: number;
+	now?: () => number;
+	alive?: (pid: number) => boolean;
+}
+
+export type StageLeaseResult =
+	| { ok: true; holder: string; inherited: boolean; release: () => void }
+	| { ok: false; refusal: string; held: StageLease | null };
+
+export function stageLeasePath(state: string, app: string): string {
+	return join(state, "locks", "stage", `${app}.lock`);
+}
+
+/** `kill -0`: ESRCH is dead; EPERM is a live process that is not ours. */
+export function pidAlive(pid: number): boolean {
+	if (!Number.isInteger(pid) || pid <= 0) return false;
+	try {
+		process.kill(pid, 0);
+		return true;
+	} catch (e) {
+		return (e as NodeJS.ErrnoException).code === "EPERM";
+	}
+}
+
+function readLease(path: string): StageLease | null {
+	try {
+		const parsed = JSON.parse(readFileSync(path, "utf8")) as Partial<StageLease>;
+		if (typeof parsed.holder !== "string" || typeof parsed.pid !== "number" || typeof parsed.started !== "string") return null;
+		return { holder: parsed.holder, pid: parsed.pid, started: parsed.started, what: String(parsed.what ?? "") };
+	} catch {
+		return null;
+	}
+}
+
+function minutes(ms: number): string {
+	return ms < 60_000 ? `${Math.max(0, Math.round(ms / 1000))}s` : `${Math.round(ms / 60_000)}m`;
+}
+
+/**
+ * Take `<app>`'s stage, or be refused. A fresh lease puts its holder id in `deps.env` so children
+ * inherit it; `release` removes the file only when this process created it for itself (a lease
+ * recorded against a lane's pid is the lane's to drop).
+ */
+export function takeStageLease(spec: StageLeaseSpec, deps: StageLeaseDeps = {}): StageLeaseResult {
+	const env = deps.env ?? process.env;
+	const now = deps.now ?? Date.now;
+	const alive = deps.alive ?? pidAlive;
+	const self = deps.pid ?? process.pid;
+	const named = env[STAGE_LEASE_VAR]?.trim() || "";
+	const lanePid = Number(env[STAGE_LEASE_PID_VAR]);
+	const owner = named && Number.isInteger(lanePid) && lanePid > 0 ? lanePid : self;
+	const path = stageLeasePath(spec.state, spec.app);
+	mkdirSync(dirname(path), { recursive: true });
+
+	for (let attempt = 0; attempt < 3; attempt++) {
+		const holder = named || randomUUID();
+		const lease: StageLease = { holder, pid: owner, started: new Date(now()).toISOString(), what: spec.what };
+		try {
+			writeFileSync(path, `${JSON.stringify(lease)}\n`, { flag: "wx" });
+			env[STAGE_LEASE_VAR] = holder;
+			const ours = owner === self;
+			return {
+				ok: true,
+				holder,
+				inherited: false,
+				release: () => {
+					if (ours) dropStageLease(spec.state, spec.app, holder);
+				},
+			};
+		} catch (e) {
+			if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e;
+		}
+		const held = readLease(path);
+		if (!held) {
+			let age = 0;
+			try {
+				age = now() - statSync(path).mtimeMs;
+			} catch {
+				continue; // gone between the two calls — try again
+			}
+			if (age < UNREADABLE_GRACE_MS) {
+				return { ok: false, held: null, refusal: `${spec.app}'s stage lease (${path}) is being written right now — try again in a few seconds.` };
+			}
+			dropStageLease(spec.state, spec.app, null);
+			continue;
+		}
+		if (named && held.holder === named) return { ok: true, holder: named, inherited: true, release: () => {} };
+		const age = now() - Date.parse(held.started);
+		if (!alive(held.pid) || !(age < STAGE_LEASE_MAX_MS)) {
+			dropStageLease(spec.state, spec.app, held.holder);
+			continue;
+		}
+		return {
+			ok: false,
+			held,
+			refusal:
+				`${spec.app}'s hosted stage is held by \`${held.what || "an unnamed holder"}\` (pid ${held.pid}, since ${held.started}, ${minutes(age)} ago). ` +
+				`A second holder would replace its Worker or its link key under it and turn its run red for nothing. ` +
+				`Wait for it to finish, then run this again. (The lease: ${path} — it is taken over by itself once pid ${held.pid} has exited.)`,
+		};
+	}
+	return { ok: false, held: readLease(path), refusal: `${spec.app}'s stage lease (${path}) changed hands three times while this run tried to take it — try again.` };
+}
+
+/** Remove `<app>`'s lease — only if `holder` still holds it (`null`: whatever is there). */
+export function dropStageLease(state: string, app: string, holder: string | null): boolean {
+	const path = stageLeasePath(state, app);
+	if (holder !== null && readLease(path)?.holder !== holder) return false;
+	try {
+		unlinkSync(path);
+		return true;
+	} catch {
+		return false;
+	}
+}

package/src/workerDeploy.ts

@@ -60,6 +60,7 @@ import { liveColumns, liveColumnsQueries, missingColumnStatements, schemaTables
 import { wranglerRows } from "cursedops/d1-import";
 import { forgeCodeRoot, forgeState } from "cursedops/roots";
 import { appendLedger, deferral, readShipSetting, type ShipEventLine, type ShipNowWhere, type ShipSetting } from "cursedops/ship";
+import { type StageLeaseResult, takeStageLease } from "cursedops/stage-lease";
 
 /** The two deployments every Worker app here has. */
 export type WorkerEnv = "production" | "stage";
@@ -338,6 +339,12 @@ export interface DeployDeps {
 	shipWhere?: ShipNowWhere;
 	/** Where a deferred production deploy's `ship` line goes. {@link workerDeployDeps} appends it to the ledger. */
 	recordShip?: (line: ShipEventLine) => void;
+	/**
+	 * Take the app's hosted stage for the whole deploy (`./stageLease.ts`) — every deploy touches the
+	 * stage, a production one deploys and walks it first. {@link workerDeployDeps} takes the
+	 * generation's lease; absent (or `null` with no state root), nothing is leased.
+	 */
+	lease?: (app: string, what: string) => StageLeaseResult | null;
 }
 
 /**
@@ -421,6 +428,10 @@ export function workerDeployDeps(cwd: string): DeployDeps {
 		recordShip: (line) => {
 			appendLedger(forgeState(cwd), line);
 		},
+		lease: (app, what) => {
+			const state = forgeState(cwd);
+			return state ? takeStageLease({ state, app, what }) : null;
+		},
 	};
 }
 
@@ -707,7 +718,9 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 	const commit = deps.git(["rev-parse", "HEAD"]);
 	const porcelain = deps.git(["status", "--porcelain"]);
 	const dirty = porcelain.length > 0;
+	let release = (): void => {};
 	const finish = (result: DeployResult): DeployResult => {
+		release();
 		enter("");
 		try {
 			deps.record?.({
@@ -755,6 +768,11 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 	if (spec.env === "production" && dirty) {
 		return stop("clean-tree", `the tree is dirty — production would report ${commit.slice(0, 8)} and run something else:\n${porcelain}`);
 	}
+	// 🔴 One holder per hosted stage (task 3048): a stage deploy under the ship's held suite swapped
+	// the Worker and its link key under it. Taken before anything builds, held until `finish`.
+	const lease = deps.lease?.(spec.app, `${spec.app} worker:deploy${spec.env === "stage" ? " --env stage" : ""} @ ${commit.slice(0, 8)}`);
+	if (lease && !lease.ok) return stop("stage-lease", lease.refusal);
+	if (lease?.ok) release = lease.release;
 	for (const refusal of spec.beforeStage ?? []) {
 		const why = refusal();
 		if (why) return stop("refusal", why);