cursedops 0.10.24 → 0.10.26

package/README.md

@@ -12,7 +12,7 @@ bun add cursedops
 | `cursedops/paths` | the generation's whole-tree laws (`check-paths`, `check-doc-citations`, plus any `--also`) run over ONE repo, from a checkout or a worktree — and its `forge-paths` bin (0.5.0) |
 | `cursedops/launchd` | installing, replacing and removing a macOS launchd user agent, and the port a LIVE job serves on (`livePort`, 0.5.0) |
 | `cursedops/smoke` | the scaffolding of a deployed smoke — the ledger, the fetch, the DNS hint, the exit code — the one check no app owns (every address of a deployment serving the same built client), and since 0.5.0 its VERDICTS: the origin asked on loopback, the smoke's own environment, a network that lies about DNS, and a version that has settled |
-| `cursedops/worker-deploy` | the sequence that ships a Cloudflare Worker — clean tree, stage first, schema, `--var` stamp, secrets, smoke — and the readers it needs (0.5.0); `readWranglerFile(root)` since 0.9.0, lifted from four apps' `readWrangler` (task 2153) |
+| `cursedops/worker-deploy` | the sequence that ships a Cloudflare Worker — clean tree, stage first, schema, `--var` stamp, secrets, smoke — and the readers it needs (0.5.0); `readWranglerFile(root)` since 0.9.0, lifted from four apps' `readWrangler` (task 2153); since 0.10.25 every run, red or green, appends a `kind: "deploy"` line with each leg's wall clock (`install`, `stage`, `walk`, `build`, `schema`, `upload`, `secrets`, `smoke`) to the generation's ledger (task 2246) |
 | `cursedops/worker-deploy` (CPU) | a `wrangler tail` around the traffic a deploy already sends, each request's BILLED CPU charged to its declared route, red on an undeclared, over-budget or uncapped route — and `runWorkerDeploy`'s `stageWalk` + `cpuTail` that wrap the walk and the smoke in it (0.7.0, task 2134, lifted from collections) |
 | `cursedops/worker-secrets` | a Worker holding EXACTLY its deployment's secrets, uploaded over a pipe, read back (0.5.0) |
 | `cursedops/worker-rollback` | a hostname back on its Mac origin: origin first, route second, the route found rather than typed (0.5.0) |

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cursedops",
-	"version": "0.10.24",
+	"version": "0.10.26",
 	"description": "The build-and-ops answers this generation's apps wrote independently and identically: finding a generation's roots — and printing a command that runs when pasted — without knowing a path, the generation's whole-tree laws run over one repo from a checkout or a worktree, macOS launchd agent install/replace/remove and the live port a job serves, the scaffolding and verdicts of a deployed smoke (origin probe, the smoke's own environment, a network that lies about DNS, a settled version), the static-serving helpers eight apps copied — the path-traversal guard among them — the API floor that keeps an unmatched /api/... from ever being answered with the app shell, the commit and dirty flag a checkout-served process reports, the Cloudflare Worker deploy toolkit four apps copied (the deploy sequence, exact-set secrets over a pipe, origin-first rollback, the curl edge fetch, the row-for-row D1 import proof, the billed-CPU tail check around a deploy's walk and smoke, and each app's worker:secrets and worker:smoke main as one function of its data), the relay a Worker fronts a Mac-bound app with (the Durable Object, the frames, the Mac's dialer and key rotation — lifted from station for roms — and the signed-in stage walk's skeleton and the relay app's whole worker:deploy), the Worker import-graph and await-port checks every Worker app's suite runs over its own source, and the public-surface ratchet three published libraries each carried a forked copy of. Mechanism only — no app knows its name from here. Bun, zero runtime dependencies (typescript is an optional peer, for public-surface only), ships source.",
 	"type": "module",
 	"scripts": {

package/src/relayLink.ts

@@ -318,7 +318,7 @@ export async function rotateLinkKey(spec: RotateSpec): Promise<{ ok: true; file:
 	const put =
 		spec.putSecret ??
 		((name: string, value: string) => {
-			const r = spawnSync("bunx", ["wrangler@4", "secret", "put", name, ...(spec.env ? ["--env", spec.env] : [])], {
+			const r = spawnSync("bunx", ["wrangler", "secret", "put", name, ...(spec.env ? ["--env", spec.env] : [])], {
 				cwd: spec.appDir,
 				input: value,
 				env: { ...process.env, ...spec.credential },

package/src/relayStage.ts

@@ -54,6 +54,37 @@ export async function until(ms: number, ok: () => Promise<boolean>, clock: Clock
 	return false;
 }
 
+/**
+ * True once `isLinked` has said yes on every ask for a whole `windowMs` (default 15 s) — false if that
+ * never happens within `maxMs` (default 2 min). A throw counts as no, and restarts the window.
+ *
+ * 🔴 Why not a sleep and then {@link until}: a digest put is a new Worker version that reaches the
+ * edge in its own time, and `until` passes the first instant the link is up. Measured 2026-09-28 on
+ * station, beside three other deploys: the walk slept its fixed 10 s, the link came up, then flapped
+ * (`1002 Expected 101` — an edge still on the old digest refusing the new key) and the first
+ * signed-in read got the offline 503. `oldKeyRefused` already records that 15 s was not always
+ * enough. So the walk waits for a link that STAYS up, and how long that took is printed.
+ */
+export async function linkedSteadily(spec: {
+	isLinked: () => Promise<boolean>;
+	windowMs?: number;
+	maxMs?: number;
+	clock?: Clock;
+}): Promise<{ steady: boolean; seconds: number }> {
+	const clock = spec.clock ?? realClock;
+	const window = spec.windowMs ?? 15_000;
+	const started = clock.now();
+	let upSince: number | null = null;
+	while (clock.now() - started < (spec.maxMs ?? 120_000)) {
+		const up = await spec.isLinked().catch(() => false);
+		if (!up) upSince = null;
+		else if (upSince === null) upSince = clock.now();
+		else if (clock.now() - upSince >= window) return { steady: true, seconds: Math.round((clock.now() - started) / 1000) };
+		await clock.sleep(1_000);
+	}
+	return { steady: false, seconds: Math.round((clock.now() - started) / 1000) };
+}
+
 /** The Access service token, as the pair a Mac end's two variables and a roster socket carry. */
 export function accessPair(headers: Record<string, string>): { id: string; secret: string } {
 	return { id: headers["CF-Access-Client-Id"] ?? "", secret: headers["CF-Access-Client-Secret"] ?? "" };
@@ -142,10 +173,11 @@ export interface DigestPut {
 /**
  * Put `key`'s digest on the STAGE Worker (`--env stage`, always — this module has no production
  * path). A per-run key for the walk, and again for the rotation leg. `ok: false` carries wrangler's
- * stderr. A secret put is a new version: the caller gives it a few seconds to reach the edge.
+ * stderr. A secret put is a new version that reaches the edge in its own time: the caller waits for
+ * a link that stays up ({@link linkedSteadily}), never a fixed sleep.
  */
 export async function putStageLinkDigest(put: DigestPut): Promise<{ ok: boolean; stderr: string }> {
-	const done = (put.run ?? realRun)(["bunx", "wrangler@4", "secret", "put", put.secretName, "--env", "stage"], {
+	const done = (put.run ?? realRun)(["bunx", "wrangler", "secret", "put", put.secretName, "--env", "stage"], {
 		cwd: put.cwd,
 		input: await sha256Hex(put.key),
 		env: put.credential,

package/src/workerDeploy.ts

@@ -53,8 +53,9 @@
  * refusal without wrangler, git or a network.
  */
 import { spawn, spawnSync } from "node:child_process";
-import { existsSync, readFileSync } from "node:fs";
-import { join } from "node:path";
+import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { forgeState } from "cursedops/roots";
 
 /** The two deployments every Worker app here has. */
 export type WorkerEnv = "production" | "stage";
@@ -305,6 +306,62 @@ export interface DeployDeps {
 	read?: (path: string) => string;
 	log?: (line: string) => void;
 	error?: (line: string) => void;
+	/** The clock the legs are timed on. Default `Date.now`. */
+	now?: () => number;
+	/** Whether a path in the checkout exists — the local wrangler. Default `existsSync` against the process's cwd. */
+	exists?: (path: string) => boolean;
+	/** The unpublished library builds in `node_modules` ({@link previewMarks}). Default: read the process's cwd. */
+	previews?: () => PreviewMark[];
+	/**
+	 * Where the run's {@link DeployEventLine} goes, on EVERY exit — refused, red or green.
+	 * {@link workerDeployDeps} appends it to the generation's ledger; absent, nothing is recorded.
+	 * A throw here is swallowed: a ledger that can fail a deploy is worse than none.
+	 */
+	record?: (line: DeployEventLine) => void;
+}
+
+/**
+ * One line per deploy into the generation's ledger (`$FORGE_STATE/state/autopilot/events.jsonl`,
+ * the file `autopilot timings` reads), with the wall clock of every LEG.
+ *
+ * 🔴 Why here: until 0.10.25 the whole ledger held 3 `deploy` events, all backfilled by hand, while
+ * `autopilot phases` measured deploys at ~110 agent-minutes a day (2026-09-24 → 28) — the slowest
+ * thing a task did once gates stopped re-proving unchanged bytes. Every app's `worker:deploy`,
+ * stage and production, passes through {@link runWorkerDeploy}, so this is the one seam that sees
+ * all of them. A production deploy runs its stage deploy as a child, which records its OWN line
+ * (`env: "stage"`) — so the `stage` leg here and that line's legs together say where it went.
+ */
+export interface DeployEventLine {
+	at: string;
+	kind: "deploy";
+	what: "cursedops/worker-deploy";
+	repo: string;
+	env: WorkerEnv;
+	ms: number;
+	ok: boolean;
+	/** The step it stopped at, or `done` / `cpu-unmeasured` — {@link DeployResult.step}. */
+	outcome: string;
+	commit: string;
+	dirty: boolean;
+	/**
+	 * Milliseconds per leg, in the order they ran: `prepare` (git and the refusals), `install`,
+	 * `stage` (the stage deploy a production deploy runs first), `walk`, `build`, `schema`,
+	 * `upload` (`wrangler deploy`), `secrets`, `smoke`. A leg that did not run is absent.
+	 */
+	legs: Record<string, number>;
+}
+
+/** Append one line to the ledger under a state root. Never throws; `false` when it did not write. */
+export function recordDeployEvent(state: string | null, line: DeployEventLine): boolean {
+	if (!state) return false;
+	try {
+		const file = join(state, "state", "autopilot", "events.jsonl");
+		mkdirSync(dirname(file), { recursive: true });
+		appendFileSync(file, `${JSON.stringify(line)}\n`);
+		return true;
+	} catch {
+		return false;
+	}
 }
 
 export interface DeployResult {
@@ -344,6 +401,9 @@ export function workerDeployDeps(cwd: string): DeployDeps {
 		},
 		git: (args) => (spawnSync("git", [...args], { cwd, encoding: "utf8" }).stdout ?? "").trim(),
 		read: (path) => readFileSync(join(cwd, path), "utf8"),
+		record: (line) => {
+			recordDeployEvent(forgeState(cwd), line);
+		},
 	};
 }
 
@@ -492,19 +552,136 @@ export function judgeSchemaApply(code: number, stdout: string): { applied: boole
  * Run the sequence in the header's order, stopping at the first red with a sentence. Never
  * exits — the caller does, with {@link DeployResult.code}.
  */
+/** Where an app's OWN wrangler is, once `bun install` has run. */
+export const LOCAL_WRANGLER = "node_modules/.bin/wrangler";
+
+/**
+ * Refuse a deploy whose app has no wrangler of its own — `null` when it has one.
+ *
+ * 🔴 Why (task 2260): with no local install, `bunx wrangler` resolves into ONE directory shared by
+ * every process on the Mac (`$TMPDIR/bunx-<uid>-wrangler@latest`), and bunx refreshes it in place.
+ * On 2026-09-28 it was rewritten at 12:17 while four deploys ran, and station's stage walk died on
+ * `Cannot find module './lib/websocket-server'` — a half-extracted `ws`. A version spec
+ * (`wrangler@4`) is worse: bunx then skips a local install even when there is one. So every app
+ * declares `wrangler` in its own `devDependencies`, locked by its own `bun.lock`, and nothing here
+ * passes bunx a version (`workerDeploy.test.ts` scans `src/` for one).
+ */
+export function localWranglerRefusal(exists: (path: string) => boolean): string | null {
+	if (exists(LOCAL_WRANGLER)) return null;
+	return `this app has no wrangler of its own (${LOCAL_WRANGLER} is missing), so \`bunx wrangler\` would use the one install every deploy on this Mac shares — and concurrent deploys have corrupted it mid-run. Run \`bun add -d wrangler@^4\` in the app and commit package.json and bun.lock. Nothing was deployed.`;
+}
+
+/** The file `tools/lib-preview.ts` writes INSIDE a package it overlaid with an unpublished build. */
+export const PREVIEW_MARK = ".lib-preview.json";
+
+/** One unpublished library build sitting in `node_modules` — what its {@link PREVIEW_MARK} says. */
+export interface PreviewMark {
+	lib: string;
+	/** The version in the library's own tree — never the one `bun.lock` names. */
+	version: string;
+	/** The library tree's HEAD when it was packed, `-dirty` when it had uncommitted changes. */
+	commit: string;
+}
+
+/**
+ * Every package under `nodeModules` (top level and `@scope/name`) carrying a {@link PREVIEW_MARK}.
+ *
+ * The mark lives inside the package's own directory, never beside it, so it cannot outlive the bytes
+ * it describes: an install that replaces the package (a lockfile bump, `--force`) removes it too.
+ * `bun install --frozen-lockfile` leaves an overlay whose `package.json` version matches the lock, so
+ * the preview writes the locked version there and its real one here (measured 2026-09-28).
+ */
+export function previewMarks(nodeModules = "node_modules"): PreviewMark[] {
+	const marks: PreviewMark[] = [];
+	const read = (dir: string): void => {
+		const file = join(dir, PREVIEW_MARK);
+		if (!existsSync(file)) return;
+		try {
+			const mark = JSON.parse(readFileSync(file, "utf8")) as Partial<PreviewMark>;
+			marks.push({ lib: mark.lib ?? dir, version: mark.version ?? "?", commit: mark.commit ?? "?" });
+		} catch {
+			marks.push({ lib: dir, version: "?", commit: "unreadable mark" });
+		}
+	};
+	let names: string[] = [];
+	try {
+		names = readdirSync(nodeModules);
+	} catch {
+		return marks;
+	}
+	for (const name of names) {
+		if (name.startsWith("@")) {
+			let scoped: string[] = [];
+			try {
+				scoped = readdirSync(join(nodeModules, name));
+			} catch {}
+			for (const inner of scoped) read(join(nodeModules, name, inner));
+		} else if (!name.startsWith(".")) read(join(nodeModules, name));
+	}
+	return marks;
+}
+
+/** `<lib>@<version> (<commit>)`, joined — how a deploy names what it would ship unpublished. */
+export function describePreviews(marks: readonly PreviewMark[]): string {
+	return marks.map((m) => `${m.lib}@${m.version} (${m.commit})`).join(", ");
+}
+
+/**
+ * Production never ships an unpublished library (task 2252) — `null` when nothing is previewed. A
+ * stage deploy ships it, says so, and stamps itself dirty: `/healthz` then says the bytes are not
+ * the commit's, which is true.
+ */
+export function previewRefusal(env: WorkerEnv, marks: readonly PreviewMark[]): string | null {
+	if (env !== "production" || marks.length === 0) return null;
+	return `node_modules holds an UNPUBLISHED build of ${describePreviews(marks)} — production ships only published libraries. Publish it (\`tools/publish-libs.sh\`), bump this app to it, or put the locked version back with \`bun $FORGE/tools/lib-preview.ts --restore\`. Nothing was deployed.`;
+}
+
 /** What {@link runWorkerDeploy} runs first: `node_modules` exactly as `bun.lock` says. */
 export const INSTALL: readonly string[] = ["bun", "install", "--frozen-lockfile"];
 
 export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): DeployResult {
 	const log = deps.log ?? ((line: string) => console.log(line));
 	const error = deps.error ?? ((line: string) => console.error(line));
+	const now = deps.now ?? Date.now;
+	const started = now();
+	const legs: Record<string, number> = {};
+	let leg = "prepare";
+	let legFrom = started;
+	/** Close the leg in flight and open `next` — see {@link DeployEventLine.legs}. */
+	const enter = (next: string): void => {
+		const at = now();
+		legs[leg] = (legs[leg] ?? 0) + (at - legFrom);
+		leg = next;
+		legFrom = at;
+	};
 	const tag = `[${spec.app}]`;
 	const commit = deps.git(["rev-parse", "HEAD"]);
 	const porcelain = deps.git(["status", "--porcelain"]);
 	const dirty = porcelain.length > 0;
+	const finish = (result: DeployResult): DeployResult => {
+		enter("");
+		try {
+			deps.record?.({
+				at: new Date(started).toISOString(),
+				kind: "deploy",
+				what: "cursedops/worker-deploy",
+				repo: spec.app,
+				env: spec.env,
+				ms: now() - started,
+				ok: result.code !== 1,
+				outcome: result.step,
+				commit: commit.slice(0, 8),
+				dirty,
+				legs,
+			});
+		} catch {
+			// never the deploy's verdict — see DeployDeps.record
+		}
+		return result;
+	};
 	const stop = (step: string, detail: string): DeployResult => {
 		error(`\n🔴 ${tag} ${detail}`);
-		return { code: 1, step, detail, commit, dirty };
+		return finish({ code: 1, step, detail, commit, dirty });
 	};
 	const step = (message: string): void => log(`\n▸ ${message}`);
 	/** What passed under a CPU tail that never attached — decision 6 of the CPU check. */
@@ -538,17 +715,27 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 	 * no-op when they already agree, and refuses, rather than rewriting the lock, when they cannot.
 	 */
 	step("make node_modules match bun.lock");
+	enter("install");
 	if (deps.run(INSTALL, {}) !== 0) {
 		return stop("install", `\`${INSTALL.join(" ")}\` failed — node_modules cannot be made to match bun.lock, so nothing was built or deployed.`);
 	}
+	const noWrangler = localWranglerRefusal(deps.exists ?? existsSync);
+	if (noWrangler) return stop("wrangler", noWrangler);
+	const previews = (deps.previews ?? previewMarks)();
+	const previewed = previewRefusal(spec.env, previews);
+	if (previewed) return stop("preview", previewed);
+	if (previews.length > 0) log(`\n🟡 ${tag} this stage deploy ships an UNPUBLISHED ${describePreviews(previews)} — stamped dirty; production will refuse it until it is published`);
+	const shipsDirty = dirty || previews.length > 0;
 	if (spec.env === "production") {
 		for (const command of spec.stageFirst ?? []) {
 			step(`the stage first — production waits on it: ${command.join(" ")}`);
+			enter("stage");
 			if (deps.run(command, {}) !== 0) return stop("stage", `\`${command.join(" ")}\` failed — production was NOT touched.`);
 		}
 		for (const walk of spec.stageWalk ?? []) {
 			const command = spec.cpuTail ? underCpuTail(spec.cpuTail, "stage", walk) : [...walk];
 			step(`walk the stage signed in${spec.cpuTail ? ", under a CPU tail" : ""}: ${command.join(" ")}`);
+			enter("walk");
 			const walked = deps.run(command, {});
 			if (spec.cpuTail && walked === TAIL_UNATTACHED_EXIT) unmeasured.push("the stage walk");
 			else if (walked !== 0) return stop("stage", `\`${command.join(" ")}\` failed — production was NOT touched.`);
@@ -556,8 +743,10 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 	}
 	if (spec.build) {
 		step("build");
+		enter("build");
 		if (deps.run(spec.build, {}) !== 0) return stop("build", "the build failed — nothing was deployed.");
 	}
+	enter("prepare");
 	for (const refusal of spec.beforeShip ?? []) {
 		const why = refusal();
 		if (why) return stop("refusal", `${why} — nothing was deployed.`);
@@ -566,6 +755,7 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 	if (spec.databaseName) {
 		const schema = spec.schemaFile ?? "db/schema.sql";
 		step(`apply ${schema} to D1 \`${spec.databaseName}\``);
+		enter("schema");
 		let commands: string[];
 		try {
 			commands = schemaCommands((deps.read ?? ((path: string) => readFileSync(path, "utf8")))(schema));
@@ -588,19 +778,22 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 			log(`  ${verdict.line}`);
 		}
 	}
-	step(`deploy ${spec.workerName} @ ${commit.slice(0, 8)}${dirty ? " (dirty — stage only)" : ""}`);
+	step(`deploy ${spec.workerName} @ ${commit.slice(0, 8)}${shipsDirty ? " (dirty — stage only)" : ""}`);
 	const stamp = spec.stampVars
-		? ["--var", `${spec.stampVars.commit}:${commit}`, "--var", `${spec.stampVars.dirty}:${dirty ? "true" : "false"}`]
+		? ["--var", `${spec.stampVars.commit}:${commit}`, "--var", `${spec.stampVars.dirty}:${shipsDirty ? "true" : "false"}`]
 		: [];
+	enter("upload");
 	if (deps.run(["bunx", "wrangler", "deploy", ...envArgs, ...stamp], spec.credential) !== 0) return stop("deploy", "wrangler deploy failed.");
 	if (spec.secrets) {
 		step("attach the secrets — exactly this deployment's set, over a pipe");
+		enter("secrets");
 		if (deps.run(spec.secrets, spec.credential) !== 0) {
 			return stop("secrets", "the Worker is deployed and its secrets are NOT attached — it serves nothing until they are.");
 		}
 	}
 	if (spec.smoke) {
 		step(`smoke every address this deployment answers on${spec.cpuTail ? ", under a CPU tail" : ""}`);
+		enter("smoke");
 		const smoke = spec.cpuTail ? underCpuTail(spec.cpuTail, spec.env, spec.smoke) : spec.smoke;
 		const smoked = deps.run(smoke, {});
 		if (spec.cpuTail && smoked === TAIL_UNATTACHED_EXIT) unmeasured.push("the smoke");
@@ -617,10 +810,10 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 			`${detail} Worker CPU UNMEASURED — the tail never attached around ${unmeasured.join(" and ")}, which PASSED. ` +
 			"Nothing is wrong with the deployment and nothing needs rolling back; rerun the smoke under the app's CPU wrapper to measure it.";
 		error(`\n🟡 ${tag} ${why}`);
-		return { code: TAIL_UNATTACHED_EXIT, step: "cpu-unmeasured", detail: why, commit, dirty };
+		return finish({ code: TAIL_UNATTACHED_EXIT, step: "cpu-unmeasured", detail: why, commit, dirty });
 	}
 	log(`\n✅ ${detail}`);
-	return { code: 0, step: "done", detail, commit, dirty };
+	return finish({ code: 0, step: "done", detail, commit, dirty });
 }
 
 /**
@@ -1046,13 +1239,13 @@ export interface WorkerCpuDeps {
 	error: (line: string) => void;
 }
 
-/** The real {@link WorkerCpuDeps}: `bunx wrangler@4 tail`, pinned to the major whose JSON shape the parser reads. */
+/** The real {@link WorkerCpuDeps}: `bunx wrangler tail` — the APP's own wrangler, whose major (4) its `devDependencies` pin: {@link localWranglerRefusal}. */
 export function workerCpuDeps(): WorkerCpuDeps {
 	return {
 		startTail: (workerName, cwd, env) => {
 			let out = "";
 			let err = "";
-			const child = spawn("bunx", ["wrangler@4", "tail", workerName, "--format", "json"], {
+			const child = spawn("bunx", ["wrangler", "tail", workerName, "--format", "json"], {
 				cwd,
 				env: { ...process.env, ...env },
 				stdio: ["ignore", "pipe", "pipe"],