cursedops 0.12.1 → 0.13.0

package/README.md

@@ -17,6 +17,7 @@ bun add cursedops
 | `cursedops/worker-secrets` | a Worker holding EXACTLY its deployment's secrets, uploaded over a pipe, read back (0.5.0) — and read again, up to five times, before a set that is not exact is believed: the list lags the puts (0.11.1) |
 | `cursedops/worker-rollback` | a hostname back on its Mac origin: origin first, route second, the route found rather than typed (0.5.0) |
 | `cursedops/edge-fetch` | a request to a deployed Worker as a script must make it — curl pinned past the Mac's negative DNS cache (0.5.0); a request's own `maxTimeSec` over the instance's (0.11.1) |
+| `cursedops/d1-columns` | the schema step's first half: any COLUMN `db/schema.sql` has and a live D1 table lacks, read in one `pragma_table_info` query and added by `ALTER TABLE … ADD COLUMN` — refusing a key, UNIQUE, or NOT NULL-without-DEFAULT column. `runWorkerDeploy` runs it for every app with a `databaseName` (0.13.0, task 2488, lifted from roms) |
 | `cursedops/d1-import` | a cutover's data proof — SQL literals and the row-for-row comparison, never a count (0.5.0); and `d1SchemaText`, the generated `db/schema.sql` that refuses a statement spanning lines, because D1's exec splits on newlines (0.9.0, lifted from five apps, task 2153) — and its `forge-d1-schema` bin (0.9.1), so no app carries its own schema script at all |
 | `cursedops/backups` | `BACKUP_KEEP`, the fleet's one retention for dated snapshots — six apps each exported `KEEP = 14` (0.9.0, task 2153); `BACKUP_STAMP_PATTERN`, `backupStamp` and its inverse `stampTime` — the local-time run name every backup script writes and station's audit reads (0.11.2) |
 | `cursedops/serve` | the static tier's four helpers — the path-traversal guard, the MIME table, the hashed-asset test, the crash handlers |

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cursedops",
-	"version": "0.12.1",
+	"version": "0.13.0",
 	"description": "The build-and-ops answers this generation's apps wrote independently and identically: finding a generation's roots — and printing a command that runs when pasted — without knowing a path, the generation's whole-tree laws run over one repo from a checkout or a worktree, macOS launchd agent install/replace/remove and the live port a job serves, the scaffolding and verdicts of a deployed smoke (origin probe, the smoke's own environment, a network that lies about DNS, a settled version), the static-serving helpers eight apps copied — the path-traversal guard among them — the API floor that keeps an unmatched /api/... from ever being answered with the app shell, the commit and dirty flag a checkout-served process reports, the Cloudflare Worker deploy toolkit four apps copied (the deploy sequence, exact-set secrets over a pipe, origin-first rollback, the curl edge fetch, the row-for-row D1 import proof, the billed-CPU tail check around a deploy's walk and smoke, and each app's worker:secrets and worker:smoke main as one function of its data), the relay a Worker fronts a Mac-bound app with (the Durable Object, the frames, the Mac's dialer and key rotation — lifted from station for roms — and the signed-in stage walk's skeleton and the relay app's whole worker:deploy), the Worker import-graph and await-port checks every Worker app's suite runs over its own source, and the public-surface ratchet three published libraries each carried a forked copy of. Mechanism only — no app knows its name from here. Bun, zero runtime dependencies (typescript is an optional peer, for public-surface only), ships source.",
 	"type": "module",
 	"scripts": {
@@ -97,6 +97,12 @@
 			"source": "./src/edgeFetch.ts",
 			"import": "./src/edgeFetch.ts"
 		},
+		"./d1-columns": {
+			"types": "./src/d1Columns.ts",
+			"bun": "./src/d1Columns.ts",
+			"source": "./src/d1Columns.ts",
+			"import": "./src/d1Columns.ts"
+		},
 		"./d1-import": {
 			"types": "./src/d1Import.ts",
 			"bun": "./src/d1Import.ts",

package/src/d1Columns.ts

@@ -0,0 +1,111 @@
+/**
+ * A COLUMN added to a store reaches a live D1 — the step `db/schema.sql` cannot take.
+ *
+ * `runWorkerDeploy` applies `db/schema.sql` on every ship, and every line is `CREATE … IF NOT
+ * EXISTS`: right for a new table, and a silent no-op for a new column on a table D1 already has.
+ * roms found it first (task 2208: `rom_uploads.held_parts`, every upload a 500) and fixed it in the
+ * app; music found it again on 2026-10-01 (task 2488: `tracks.audio_hash` reached production by hand,
+ * never `music-stage`, and the schema's index on it stopped the ship). So the step lives here, in the
+ * one deploy every D1 app runs, for whatever `spec.databaseName` is.
+ *
+ * Before the schema applies, each table's live columns are read in ONE ordinary query
+ * ({@link liveColumnsQueries} — `pragma_table_info`, which D1's query API serves) and compared with
+ * the schema file's; a missing one is added. Only what `ALTER TABLE … ADD COLUMN` can do safely: a
+ * column that is a key, UNIQUE, or NOT NULL without a DEFAULT is REFUSED — that is a real migration,
+ * and the deploy stops rather than half-apply it. A table D1 does not have yet is left to the schema
+ * file, which creates it whole.
+ */
+
+export interface SchemaTable {
+	table: string;
+	/** Each column's name and its whole definition, as the CREATE TABLE line gives it. */
+	columns: Array<{ name: string; definition: string }>;
+}
+
+/** Split a column list on its top-level commas (a `CHECK (a IN ('x','y'))` holds commas of its own). */
+function topLevelParts(body: string): string[] {
+	const parts: string[] = [];
+	let depth = 0;
+	let quoted = false;
+	let current = "";
+	for (const ch of body) {
+		if (ch === "'") quoted = !quoted;
+		if (!quoted && ch === "(") depth++;
+		if (!quoted && ch === ")") depth--;
+		if (!quoted && depth === 0 && ch === ",") {
+			parts.push(current.trim());
+			current = "";
+		} else current += ch;
+	}
+	if (current.trim()) parts.push(current.trim());
+	return parts;
+}
+
+const CONSTRAINT = /^(PRIMARY\s+KEY|UNIQUE|CHECK|FOREIGN\s+KEY|CONSTRAINT)\b/i;
+
+/** Every `CREATE TABLE` in a schema file (one statement per line, as `d1SchemaText` writes it), with its columns. */
+export function schemaTables(schemaSql: string): SchemaTable[] {
+	const tables: SchemaTable[] = [];
+	for (const line of schemaSql.split("\n")) {
+		const match = /^CREATE TABLE IF NOT EXISTS\s+("?)(\w+)\1\s*\((.*)\)\s*;?\s*$/i.exec(line.trim());
+		if (!match) continue;
+		const columns = topLevelParts(match[3] as string)
+			.filter((part) => !CONSTRAINT.test(part))
+			.map((definition) => ({ name: (definition.split(/\s+/, 1)[0] as string).replace(/"/g, ""), definition }));
+		tables.push({ table: match[2] as string, columns });
+	}
+	return tables;
+}
+
+/**
+ * The read: `SELECT '<t>' AS t, name FROM pragma_table_info('<t>') UNION ALL …`, packed into as few
+ * queries as `maxBytes` allows — one wrangler call for a whole schema, not one per table.
+ */
+export function liveColumnsQueries(tables: readonly SchemaTable[], maxBytes: number): string[] {
+	const queries: string[] = [];
+	let current = "";
+	for (const { table } of tables) {
+		const part = `SELECT '${table}' AS t, name FROM pragma_table_info('${table}')`;
+		const next = current ? `${current} UNION ALL ${part}` : part;
+		if (current && Buffer.byteLength(`${next};`) > maxBytes) {
+			queries.push(`${current};`);
+			current = part;
+		} else current = next;
+	}
+	if (current) queries.push(`${current};`);
+	return queries;
+}
+
+/** {@link liveColumnsQueries}' rows, as table → its column names. A table D1 lacks has no rows, so no entry. */
+export function liveColumns(rows: ReadonlyArray<Record<string, unknown>>): Map<string, Set<string>> {
+	const live = new Map<string, Set<string>>();
+	for (const row of rows) {
+		const table = String(row.t);
+		const names = live.get(table) ?? new Set<string>();
+		names.add(String(row.name));
+		live.set(table, names);
+	}
+	return live;
+}
+
+/**
+ * The `ALTER TABLE … ADD COLUMN` statements that bring `live` up to the schema. `live` maps a table
+ * to its column names; a table absent from it is skipped (the schema file creates it). Throws on a
+ * column ADD COLUMN cannot add safely — see the header.
+ */
+export function missingColumnStatements(tables: readonly SchemaTable[], live: Map<string, Set<string>>): string[] {
+	const statements: string[] = [];
+	for (const { table, columns } of tables) {
+		const have = live.get(table);
+		if (!have || have.size === 0) continue;
+		for (const { name, definition } of columns) {
+			if (have.has(name)) continue;
+			const upper = definition.toUpperCase();
+			if (/\bPRIMARY\s+KEY\b|\bUNIQUE\b/.test(upper) || (/\bNOT\s+NULL\b/.test(upper) && !/\bDEFAULT\b/.test(upper))) {
+				throw new Error(`${table}.${name} (${definition}) is missing from D1 and cannot be added by ADD COLUMN — write a real migration`);
+			}
+			statements.push(`ALTER TABLE ${table} ADD COLUMN ${definition}`);
+		}
+	}
+	return statements;
+}

package/src/workerDeploy.ts

@@ -39,7 +39,8 @@
  *   4. **the STAGE first, for production** — `collections` went live 2026-09-22 with every anonymous
  *      check green and its owner unable to sign in; the stage walk is what found why, and a step
  *      in the sequence is one a person in a hurry cannot skip;
- *   5. build → 6. **schema** (`IF NOT EXISTS`, the database named explicitly, never the binding a
+ *   5. build → 6. **schema** (first any COLUMN a live table lacks — `cursedops/d1-columns`, since
+ *      `IF NOT EXISTS` never adds one — then `IF NOT EXISTS`, the database named explicitly, never the binding a
  *      forgotten `--env` would resolve; ordinary `--command` queries, never the blocking
  *      `--file` import — {@link schemaApplyArgv}) → 7. **deploy with the `--var` stamp** (commit AND dirty
  *      flag, so `/healthz` publishes what is running) → 8. **secrets** (after the deploy: a secret
@@ -55,6 +56,8 @@
 import { spawn, spawnSync } from "node:child_process";
 import { existsSync, readdirSync, readFileSync } from "node:fs";
 import { join } from "node:path";
+import { liveColumns, liveColumnsQueries, missingColumnStatements, schemaTables } from "cursedops/d1-columns";
+import { wranglerRows } from "cursedops/d1-import";
 import { forgeCodeRoot, forgeState } from "cursedops/roots";
 import { appendLedger, deferral, readShipSetting, type ShipEventLine, type ShipNowWhere, type ShipSetting } from "cursedops/ship";
 
@@ -818,23 +821,53 @@ export function runWorkerDeploy(spec: WorkerDeploySpec, deps: DeployDeps): Deplo
 		step(`apply ${schema} to D1 \`${spec.databaseName}\``);
 		enter("schema");
 		let commands: string[];
+		let text: string;
 		try {
-			commands = schemaCommands((deps.read ?? ((path: string) => readFileSync(path, "utf8")))(schema));
+			text = (deps.read ?? ((path: string) => readFileSync(path, "utf8")))(schema);
+			commands = schemaCommands(text);
 		} catch (cause) {
 			return stop("schema", `${schema} could not be read as statements (${(cause as Error).message}) — nothing was deployed.`);
 		}
-		for (const argv of schemaApplyArgv(spec.databaseName, commands, envArgs)) {
-			// a silent exit 0 and a Cloudflare API error are intermittent and the schema is idempotent,
-			// so ONE retry; any other failure is not retried — it would fail the same way
-			const attempt = (): ReturnType<typeof judgeSchemaApply> => {
+		// a silent exit 0 and a Cloudflare API error are intermittent and every query here is
+		// idempotent, so ONE retry; any other failure is not retried — it would fail the same way
+		const query = (argv: readonly string[]): ReturnType<typeof judgeSchemaApply> & { stdout: string } => {
+			const attempt = () => {
 				const { code, stdout } = deps.capture(argv, spec.credential);
-				return judgeSchemaApply(code, stdout);
+				return { ...judgeSchemaApply(code, stdout), stdout };
 			};
 			let verdict = attempt();
 			if (!verdict.applied && verdict.retry) {
 				error(`  🟡 ${verdict.line} — retrying once`);
 				verdict = attempt();
 			}
+			return verdict;
+		};
+		// Columns first (task 2488): `IF NOT EXISTS` never adds one to a table D1 already has, and the
+		// schema below may index it (`./d1Columns.ts`).
+		const tables = schemaTables(text);
+		if (tables.length > 0) {
+			const rows: Array<Record<string, unknown>> = [];
+			for (const argv of schemaApplyArgv(spec.databaseName, liveColumnsQueries(tables, SCHEMA_COMMAND_BYTES), envArgs)) {
+				const verdict = query(argv);
+				if (!verdict.applied) return stop("schema", `D1's live columns could not be read (${verdict.line}) — nothing was deployed.`);
+				rows.push(...wranglerRows(verdict.stdout));
+			}
+			let added: string[];
+			try {
+				added = missingColumnStatements(tables, liveColumns(rows));
+			} catch (cause) {
+				return stop("schema", `${(cause as Error).message} — nothing was deployed.`);
+			}
+			if (added.length > 0) {
+				for (const argv of schemaApplyArgv(spec.databaseName, schemaCommands(added.map((sql) => `${sql};`).join("\n")), envArgs)) {
+					const verdict = query(argv);
+					if (!verdict.applied) return stop("schema", `the missing columns were not added (${verdict.line}) — nothing was deployed.`);
+				}
+				log(`  added ${added.length} column(s): ${added.join("; ")}`);
+			} else log(`  every column ${schema} names is in D1 (or its table is new)`);
+		}
+		for (const argv of schemaApplyArgv(spec.databaseName, commands, envArgs)) {
+			const verdict = query(argv);
 			if (!verdict.applied) return stop("schema", `the schema did not apply (${verdict.line}) — nothing was deployed.`);
 			log(`  ${verdict.line}`);
 		}