cursedops 0.11.2 → 0.12.1

package/README.md

@@ -25,7 +25,7 @@ bun add cursedops
 | `cursedops/bound-lists` | the static check that no SQL builds an `IN (…)` list one `?` per value — the shape D1 500s on past 100 ids. Test-time only; the recorded third exception, see below |
 | `cursedops/staged-client` | the IMMUTABLE client a checkout-served app serves — a build staged into `<APP_DATA_DIR>/client/<commit>/` behind `CURRENT`, so another agent's `vite build` in the checkout never changes live bytes — and its `forge-client stamp` / `forge-client stage` bin (0.6.0, lifted from family, flix, roms, station) |
 | `cursedops/deploy-tree` | the two refusals a checkout-served deploy needs — never deploy a dirty tree, never `git revert` in one (`readTreeStatus`, `refuseToDeploy`, `refuseToRevert`; 0.6.0, lifted from nine apps) |
-| `cursedops/ship` | the owner's FAST SHIPPING switch as every deploy reads it — `readShipSetting` (`$FORGE_STATE/state/autopilot/ship.json`; absent or unreadable is `fast`, no generation is `each`), `deferral`/`shipDeferral` (the sentence a production deploy prints instead of deploying, or `null`), `FORGE_SHIP_NOW=1` to ship now whatever it says. `runWorkerDeploy` asks it before installing anything; a Mac-side `bun run deploy` calls `shipDeferral` first (0.11.0) |
+| `cursedops/ship` | the owner's FAST SHIPPING switch as every deploy reads it — `readShipSetting` (`$FORGE_STATE/state/autopilot/ship.json`; absent or unreadable is `fast`, no generation is `each`), `deferral`/`shipDeferral` (the sentence a production deploy prints instead of deploying, or `null`), `FORGE_SHIP_NOW=1` to ship now whatever it says. `runWorkerDeploy` asks it before installing anything; a Mac-side `bun run deploy` calls `shipDeferral` first (0.11.0). Since 0.12.0 `shipNowVerdict` honours ship-now under `$AUTOPILOT_BATCH` only for a batch holding a live claim on the ship task or a `prod-down:` incident, with `FORGE_SHIP_NOW_WHY` set, and every ask is a `kind: "ship-now"` ledger row (task 2384) |
 | `cursedops/relay` | the WORKER half of a relay — a Worker in front of a Mac-bound app over ONE outbound WebSocket: the frames (station's wire, byte for byte), the key-digest door (`acceptLink`), and `RelayLink`, the Durable Object an app subclasses with its name, body ceiling, timeout and (optionally) station's per-cookie offline cache. No `node:` import (0.10.0, lifted from station for roms, task 065) |
 | `cursedops/relay-link` | the MAC half — `dialLink` (answer each frame with the app's own `fetch`, redial 1 s → 60 s on close, no timer otherwise), `readLinkSettings` for `<APP>_LINK_URL`/`<APP>_LINK_KEY`, and `rotateLinkKey` (digest to the Worker FIRST, the key to the 0600 file only if it took) (0.10.0) |
 | `cursedops/relay-stage` | the signed-in STAGE WALK's skeleton both relay apps shared — `stageByteCheck` (SHA-256 of every built file, a stale byte re-asked 4 × 10 s), `putStageLinkDigest` (`--env stage` only), `grantStageOwner` (auth-stage's 2FA grant, never production's), `oldKeyRefused` (the rotation leg: 20 s refused window, 3 min ceiling), `awaitSystemResolver`, `servedPath`, `STAGE_AUTH_URL`. With `relayWorkerDeployCli` in `cursedops/worker-deploy`, a relay app's whole `worker:deploy` as data (0.10.6) |

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cursedops",
-	"version": "0.11.2",
+	"version": "0.12.1",
 	"description": "The build-and-ops answers this generation's apps wrote independently and identically: finding a generation's roots — and printing a command that runs when pasted — without knowing a path, the generation's whole-tree laws run over one repo from a checkout or a worktree, macOS launchd agent install/replace/remove and the live port a job serves, the scaffolding and verdicts of a deployed smoke (origin probe, the smoke's own environment, a network that lies about DNS, a settled version), the static-serving helpers eight apps copied — the path-traversal guard among them — the API floor that keeps an unmatched /api/... from ever being answered with the app shell, the commit and dirty flag a checkout-served process reports, the Cloudflare Worker deploy toolkit four apps copied (the deploy sequence, exact-set secrets over a pipe, origin-first rollback, the curl edge fetch, the row-for-row D1 import proof, the billed-CPU tail check around a deploy's walk and smoke, and each app's worker:secrets and worker:smoke main as one function of its data), the relay a Worker fronts a Mac-bound app with (the Durable Object, the frames, the Mac's dialer and key rotation — lifted from station for roms — and the signed-in stage walk's skeleton and the relay app's whole worker:deploy), the Worker import-graph and await-port checks every Worker app's suite runs over its own source, and the public-surface ratchet three published libraries each carried a forked copy of. Mechanism only — no app knows its name from here. Bun, zero runtime dependencies (typescript is an optional peer, for public-surface only), ships source.",
 	"type": "module",
 	"scripts": {

package/src/ship.ts

@@ -38,10 +38,17 @@
  * deploys, an outage fix, and a deploy the owner asked for in words (his sentence beats a switch he
  * flipped yesterday — the precedence the retired `fastMode.ts` settled on 2026-08-13). A stage
  * deploy is never deferred: it IS the fast way to see a change on a real edge.
+ *
+ * 🔴 **Under a batch (`$AUTOPILOT_BATCH` set) it is honoured only for the ship task or an incident**
+ * ({@link shipNowVerdict}), and only with a reason in `FORGE_SHIP_NOW_WHY`. Until task 2384 any agent
+ * could prefix one variable and turn the owner's switch off for itself — the thing
+ * `~/.claude/CLAUDE.md` §2 says an agent may never do — and no record said it had. Now every ask,
+ * honoured or refused, is a `kind: "ship-now"` ledger row, and `autopilot doctor` lists the ones
+ * with no reason. In the owner's own shell (no batch) it is honoured exactly as before, and recorded.
  */
-import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
+import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync } from "node:fs";
 import { dirname, join } from "node:path";
-import { forgeState } from "cursedops/roots";
+import { forgeCodeRoot, forgeState } from "cursedops/roots";
 
 export const SHIP_MODES = ["fast", "each"] as const;
 export type ShipMode = (typeof SHIP_MODES)[number];
@@ -85,12 +92,152 @@ export function readShipSetting(state: string | null): ShipSetting {
 	}
 }
 
-/** Is `FORGE_SHIP_NOW` set to something that means yes? */
+/** Is `FORGE_SHIP_NOW` set to something that means yes? Asked is not honoured — see {@link shipNowVerdict}. */
 export function shipNowAsked(env: NodeJS.ProcessEnv = process.env): boolean {
 	const v = env[SHIP_NOW_VAR]?.trim().toLowerCase();
 	return v === "1" || v === "true" || v === "yes";
 }
 
+/** Why ship-now was asked — required under a batch, recorded always. */
+export const SHIP_NOW_WHY_VAR = "FORGE_SHIP_NOW_WHY";
+
+/** The variable the runner sets on its batch agent — present means an agent, not the owner. */
+export const BATCH_VAR = "AUTOPILOT_BATCH";
+
+/**
+ * The `escalation-key:` values a claimed task may carry for its batch to ship now: autopilot's
+ * `SHIP_KEY` (`ship.ts`) and the prod watch's `prod-down:<host>` (`prod.ts`). The file shape is
+ * the interface, exactly as `ship.json` is — this module never imports the runner.
+ */
+export const SHIP_NOW_KEYS: readonly RegExp[] = [/^ship$/, /^prod-down:\S+$/];
+
+export interface ShipNowVerdict {
+	/** `FORGE_SHIP_NOW` means yes. */
+	asked: boolean;
+	/** Ship now: asked, and (outside a batch, or on a claim that earns it). */
+	honoured: boolean;
+	/** One sentence: what made it honoured or refused. */
+	why: string;
+	/** `FORGE_SHIP_NOW_WHY`, trimmed — `""` when not given. */
+	reason: string;
+	batch?: string;
+	/** The claimed task file that earned it. */
+	task?: string;
+}
+
+interface BoardClaim {
+	kind?: unknown;
+	pid?: unknown;
+	task?: unknown;
+	batch?: unknown;
+	expiresAt?: unknown;
+}
+
+function pidAlive(pid: number): boolean {
+	try {
+		process.kill(pid, 0);
+		return true;
+	} catch (err) {
+		return (err as NodeJS.ErrnoException).code === "EPERM";
+	}
+}
+
+/** The `escalation-key:` in a task file's front matter, lower-cased, or `null`. */
+export function escalationKey(text: string): string | null {
+	const head = /^---\n([\s\S]*?)\n---/.exec(text.replace(/\r\n/g, "\n"));
+	const m = head ? /^escalation-key:\s*(.+?)\s*$/im.exec(head[1] ?? "") : null;
+	return m?.[1] ? m[1].toLowerCase() : null;
+}
+
+export interface ShipNowWhere {
+	/** `$FORGE_STATE` — the board lives at `state/board/`. */
+	state: string | null;
+	/** `$FORGE` — claimed task files live at `tasks/In_Progress/`. */
+	forge: string | null;
+	now?: number;
+	isAlive?: (pid: number) => boolean;
+}
+
+/**
+ * Is `FORGE_SHIP_NOW` honoured here? Outside a batch, whenever it is asked: that is the owner's own
+ * shell. Under `$AUTOPILOT_BATCH`, only when that batch holds a live `kind: "task"` board claim
+ * (`$FORGE_STATE/state/board/*.json`, the pid alive and not expired) on a task in
+ * `$FORGE/tasks/In_Progress/` whose `escalation-key:` is one of {@link SHIP_NOW_KEYS}, AND
+ * `FORGE_SHIP_NOW_WHY` says why. Anything it cannot read is a refusal — a refused ship-now only
+ * makes the deploy later, which is the safe direction.
+ */
+export function shipNowVerdict(where: ShipNowWhere, env: NodeJS.ProcessEnv = process.env): ShipNowVerdict {
+	const asked = shipNowAsked(env);
+	const reason = env[SHIP_NOW_WHY_VAR]?.trim() ?? "";
+	const batch = env[BATCH_VAR]?.trim() || undefined;
+	const base = { asked, reason, ...(batch ? { batch } : {}) };
+	if (!asked) return { ...base, honoured: false, why: `${SHIP_NOW_VAR} is not set` };
+	if (!batch) return { ...base, honoured: true, why: `no ${BATCH_VAR} — the owner's own shell` };
+	const refuse = (why: string): ShipNowVerdict => ({ ...base, honoured: false, why });
+	if (!reason) return refuse(`under batch ${batch}, ${SHIP_NOW_VAR} needs ${SHIP_NOW_WHY_VAR}="<why this cannot wait for the ship>"`);
+	if (!where.state || !where.forge) return refuse(`under batch ${batch}, no generation to read the board from`);
+	const board = join(where.state, "state", "board");
+	const now = where.now ?? Date.now();
+	const isAlive = where.isAlive ?? pidAlive;
+	let files: string[] = [];
+	try {
+		files = readdirSync(board).filter((f) => f.startsWith("task-") && f.endsWith(".json"));
+	} catch {
+		/* no board → no claim */
+	}
+	const held: string[] = [];
+	for (const f of files) {
+		let c: BoardClaim;
+		try {
+			c = JSON.parse(readFileSync(join(board, f), "utf8")) as BoardClaim;
+		} catch {
+			continue;
+		}
+		if (c.kind !== "task" || c.batch !== batch || typeof c.task !== "string" || typeof c.pid !== "number") continue;
+		if (typeof c.expiresAt === "string" && Date.parse(c.expiresAt) < now) continue;
+		if (!isAlive(c.pid)) continue;
+		held.push(c.task);
+		let key: string | null = null;
+		try {
+			key = escalationKey(readFileSync(join(where.forge, "tasks", "In_Progress", c.task), "utf8"));
+		} catch {
+			continue;
+		}
+		if (key && SHIP_NOW_KEYS.some((k) => k.test(key))) return { ...base, honoured: true, why: `batch ${batch} holds ${c.task} (escalation-key: ${key})`, task: c.task };
+	}
+	return refuse(
+		`batch ${batch} holds ${held.length ? held.join(", ") : "no live task claim"} — ${SHIP_NOW_VAR} is honoured under a batch only for the ship task or a prod incident (escalation-key: ship | prod-down:<host>)`,
+	);
+}
+
+/** The ledger line every ship-now ask leaves — honoured or refused. */
+export interface ShipNowEventLine {
+	at: string;
+	kind: "ship-now";
+	/** The deploy or publish it was asked for. */
+	what: string;
+	ok: boolean;
+	reason: string;
+	detail: string;
+	batch?: string;
+	task?: string;
+}
+
+/** Record one ship-now ask in the ledger. Never throws. */
+export function recordShipNow(state: string | null, what: string, v: ShipNowVerdict, at: Date = new Date()): boolean {
+	const line: ShipNowEventLine = {
+		at: at.toISOString(),
+		kind: "ship-now",
+		what,
+		ok: v.honoured,
+		reason: v.reason,
+		detail: v.why,
+		...(v.batch ? { batch: v.batch } : {}),
+		...(v.task ? { task: v.task } : {}),
+	};
+	return appendLedger(state, line);
+}
+
 /**
  * The sentence a production deploy prints INSTEAD of deploying, or `null` to deploy now.
  *
@@ -98,16 +245,21 @@ export function shipNowAsked(env: NodeJS.ProcessEnv = process.env): boolean {
  * and the runner's ship task — filed when the queue empties or the oldest owed commit is too old —
  * deploys every app that is behind, once. `autopilot ship` lists what is owed.
  */
-export function deferral(what: string, setting: ShipSetting, env: NodeJS.ProcessEnv = process.env): string | null {
-	if (setting.mode !== "fast" || shipNowAsked(env)) return null;
+export function deferral(what: string, setting: ShipSetting, env: NodeJS.ProcessEnv = process.env, where: ShipNowWhere = { state: null, forge: null }): string | null {
+	if (setting.mode !== "fast") return null;
+	const now = shipNowVerdict(where, env);
+	if (now.asked) recordShipNow(where.state, what, now);
+	if (now.honoured) return null;
+	const refused = now.asked ? [`🔴 ${SHIP_NOW_VAR} REFUSED: ${now.why}.`] : [];
 	const by = setting.setBy === "default" ? "the default" : `set by ${setting.setBy}${setting.at ? ` at ${setting.at}` : ""}`;
 	return [
+		...refused,
 		`🚢 ${what}: DEFERRED — fast shipping is on (${by}${setting.unreadable ? `; ship.json unreadable: ${setting.unreadable}` : ""}).`,
 		"   Nothing was deployed, and that is the intended result, not a failure: this app is now owed to",
 		"   the ship, which deploys every app behind its HEAD once, when the queue empties. Carry on with",
 		"   the next task — file this one as done once it is landed and gated, saying the deploy is owed.",
-		`   \`autopilot ship\` lists what is owed. To deploy now anyway (an outage fix, the ship itself, or`,
-		`   the owner asked for it): ${SHIP_NOW_VAR}=1 <the same command>.`,
+		`   \`autopilot ship\` lists what is owed. To deploy now anyway (the ship task or a prod incident`,
+		`   your batch holds, or the owner in his own shell): ${SHIP_NOW_VAR}=1 ${SHIP_NOW_WHY_VAR}="<why>" <the same command>.`,
 	].join("\n");
 }
 
@@ -118,7 +270,8 @@ export function deferral(what: string, setting: ShipSetting, env: NodeJS.Process
  *     if (deferred) { console.log(deferred); process.exit(0); }
  */
 export function shipDeferral(cwd: string, what: string, env: NodeJS.ProcessEnv = process.env): string | null {
-	return deferral(what, readShipSetting(forgeState(cwd, env)), env);
+	const state = forgeState(cwd, env);
+	return deferral(what, readShipSetting(state), env, { state, forge: forgeCodeRoot(cwd, env) });
 }
 
 /** The ledger line a deferred production deploy leaves — what `autopilot ship` counts. */

package/src/workerDeploy.ts

@@ -55,8 +55,8 @@
 import { spawn, spawnSync } from "node:child_process";
 import { existsSync, readdirSync, readFileSync } from "node:fs";
 import { join } from "node:path";
-import { forgeState } from "cursedops/roots";
-import { appendLedger, deferral, readShipSetting, type ShipEventLine, type ShipSetting } from "cursedops/ship";
+import { forgeCodeRoot, forgeState } from "cursedops/roots";
+import { appendLedger, deferral, readShipSetting, type ShipEventLine, type ShipNowWhere, type ShipSetting } from "cursedops/ship";
 
 /** The two deployments every Worker app here has. */
 export type WorkerEnv = "production" | "stage";
@@ -327,6 +327,12 @@ export interface DeployDeps {
 	ship?: () => ShipSetting;
 	/** Where `FORGE_SHIP_NOW` is read from. Default `process.env`. */
 	env?: NodeJS.ProcessEnv;
+	/**
+	 * The generation roots `FORGE_SHIP_NOW` is judged against under a batch — the board, the claimed
+	 * task, and the ledger its `ship-now` row goes to (`shipNowVerdict` in `./ship.ts`). Absent, a batch's
+	 * ship-now is refused and nothing is recorded.
+	 */
+	shipWhere?: ShipNowWhere;
 	/** Where a deferred production deploy's `ship` line goes. {@link workerDeployDeps} appends it to the ledger. */
 	recordShip?: (line: ShipEventLine) => void;
 }
@@ -408,6 +414,7 @@ export function workerDeployDeps(cwd: string): DeployDeps {
 			recordDeployEvent(forgeState(cwd), line);
 		},
 		ship: () => readShipSetting(forgeState(cwd)),
+		shipWhere: { state: forgeState(cwd), forge: forgeCodeRoot(cwd) },
 		recordShip: (line) => {
 			appendLedger(forgeState(cwd), line);
 		},
@@ -523,6 +530,34 @@ export function schemaApplyArgv(databaseName: string, commands: readonly string[
 	return commands.map((command) => ["bunx", "wrangler", "d1", "execute", databaseName, "--remote", "--command", command, "--json", "-y", ...envArgs]);
 }
 
+/**
+ * The reason a non-zero `wrangler … --json` gave, or `null` when it printed no error document.
+ *
+ * With `--json`, wrangler writes its failure to STDOUT as `{"error":{"text":…,"notes":[{"text":…}],
+ * "kind":"error","name":"APIError","code":7403}}` (task 2427: vault's production deploy stopped on
+ * 2026-10-01 with only "wrangler exited 1", and the ship batch rebuilt the call by hand to learn it
+ * was a transient 7403). `api` is whether it is a Cloudflare API error — worth one retry, since the
+ * schema is idempotent and the same commands applied minutes later.
+ */
+export function wranglerError(stdout: string): { reason: string; api: boolean } | null {
+	const start = stdout.search(/\{\s*"error"\s*:/);
+	if (start < 0) return null;
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(stdout.slice(start, stdout.lastIndexOf("}") + 1));
+	} catch {
+		return null;
+	}
+	const error = (parsed as { error?: unknown } | null)?.error;
+	if (!error || typeof error !== "object") return null;
+	const { text, notes, name, kind } = error as { text?: unknown; notes?: unknown; name?: unknown; kind?: unknown };
+	const parts = [typeof text === "string" ? text.trim() : ""];
+	if (Array.isArray(notes)) for (const note of notes) if (typeof (note as { text?: unknown })?.text === "string") parts.push((note as { text: string }).text.trim());
+	const reason = parts.filter(Boolean).join(" — ");
+	if (!reason) return null;
+	return { reason, api: name === "APIError" || (kind === "error" && /cloudflare api/i.test(reason)) };
+}
+
 /**
  * Did one schema chunk APPLY? Its exit code alone does not say.
  *
@@ -537,22 +572,26 @@ export function schemaApplyArgv(databaseName: string, commands: readonly string[
  * one result per statement, every one `success: true`. Anything else — no JSON, an empty array,
  * a failed result — is not applied. Pure, so its failure path is tested without wrangler.
  */
-export function judgeSchemaApply(code: number, stdout: string): { applied: boolean; line: string } {
-	if (code !== 0) return { applied: false, line: `wrangler exited ${code}` };
+export function judgeSchemaApply(code: number, stdout: string): { applied: boolean; line: string; retry: boolean } {
+	if (code !== 0) {
+		const failure = wranglerError(stdout);
+		if (!failure) return { applied: false, line: `wrangler exited ${code}`, retry: false };
+		return { applied: false, line: `wrangler exited ${code}: ${failure.reason}`, retry: failure.api };
+	}
 	// the first `[` that opens a JSON array, never the `[ERROR]`/`[WARNING]` of a banner — as `wranglerRows`
 	const start = stdout.search(/\[\s*[{\]]/);
 	const banner = stdout.trim().slice(0, 160).replace(/\s+/g, " ") || "(nothing)";
-	if (start < 0) return { applied: false, line: `wrangler exited 0 and printed no JSON result — it did not run the query. It printed: ${banner}` };
+	if (start < 0) return { applied: false, retry: true, line: `wrangler exited 0 and printed no JSON result — it did not run the query. It printed: ${banner}` };
 	let parsed: unknown;
 	try {
 		parsed = JSON.parse(stdout.slice(start));
 	} catch (cause) {
-		return { applied: false, line: `wrangler's JSON result did not parse (${(cause as Error).message})` };
+		return { applied: false, retry: true, line: `wrangler's JSON result did not parse (${(cause as Error).message})` };
 	}
-	if (!Array.isArray(parsed) || parsed.length === 0) return { applied: false, line: "wrangler printed an empty result — no statement ran" };
+	if (!Array.isArray(parsed) || parsed.length === 0) return { applied: false, retry: true, line: "wrangler printed an empty result — no statement ran" };
 	const failed = parsed.filter((result) => (result as { success?: unknown } | null)?.success !== true);
-	if (failed.length > 0) return { applied: false, line: `${failed.length} of ${parsed.length} result(s) did not report success` };
-	return { applied: true, line: `applied — ${parsed.length} result(s), every one success` };
+	if (failed.length > 0) return { applied: false, retry: true, line: `${failed.length} of ${parsed.length} result(s) did not report success` };
+	return { applied: true, retry: false, line: `applied — ${parsed.length} result(s), every one success` };
 }
 
 /**
@@ -699,7 +738,7 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 	// runner's ship instead — asked before anything installs or builds, so deferring costs nothing.
 	// Its `ship` line, not a `deploy` one: a deferral is not a deploy, and would drag every median.
 	if (spec.env === "production" && deps.ship) {
-		const deferred = deferral(`${spec.app} production deploy`, deps.ship(), deps.env ?? process.env);
+		const deferred = deferral(`${spec.app} production deploy`, deps.ship(), deps.env ?? process.env, deps.shipWhere);
 		if (deferred) {
 			log(`\n${deferred}`);
 			try {
@@ -785,16 +824,16 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 			return stop("schema", `${schema} could not be read as statements (${(cause as Error).message}) — nothing was deployed.`);
 		}
 		for (const argv of schemaApplyArgv(spec.databaseName, commands, envArgs)) {
-			// a silent exit 0 is intermittent and the schema is idempotent, so ONE retry; a real
-			// failure (non-zero) is not retried — it would fail the same way
-			const attempt = (): { code: number; verdict: ReturnType<typeof judgeSchemaApply> } => {
+			// a silent exit 0 and a Cloudflare API error are intermittent and the schema is idempotent,
+			// so ONE retry; any other failure is not retried — it would fail the same way
+			const attempt = (): ReturnType<typeof judgeSchemaApply> => {
 				const { code, stdout } = deps.capture(argv, spec.credential);
-				return { code, verdict: judgeSchemaApply(code, stdout) };
+				return judgeSchemaApply(code, stdout);
 			};
-			let { code, verdict } = attempt();
-			if (!verdict.applied && code === 0) {
+			let verdict = attempt();
+			if (!verdict.applied && verdict.retry) {
 				error(`  🟡 ${verdict.line} — retrying once`);
-				({ code, verdict } = attempt());
+				verdict = attempt();
 			}
 			if (!verdict.applied) return stop("schema", `the schema did not apply (${verdict.line}) — nothing was deployed.`);
 			log(`  ${verdict.line}`);