@lobomfz/db 0.4.1 → 0.5.0

package/src/migration/diff.ts

@@ -14,6 +14,10 @@ import {
 import type { FtsTokenizer } from "../types.js";
 import type {
 	ColumnSchema,
+	DesiredSchema,
+	DesiredTable,
+	DestructiveChange,
+	ExistingSchema,
 	IntrospectedFts,
 	IntrospectedTable,
 	IntrospectedVectorIndex,
@@ -21,38 +25,33 @@ import type {
 	MigrationOp,
 } from "./types.js";
 
-export interface DesiredColumn extends ColumnSchema {
-	addable: boolean;
-	columnDef: string;
-}
-
-export type DesiredIndex = {
-	name: string;
-	columns: string[];
-	sql: string;
-};
-
-export type DesiredTable = {
-	name: string;
-	sql: string;
-	columns: DesiredColumn[];
-	indexes?: DesiredIndex[];
-};
-
 export class Differ {
 	private ops: MigrationOp[] = [];
 	private desiredNames: Set<string>;
 	private rebuiltTables = new Set<string>();
 
+	private desired: DesiredTable[];
+	private existing: Map<string, IntrospectedTable>;
+	private desiredFts: ResolvedFtsPlan[];
+	private existingFts: Map<string, IntrospectedFts>;
+	private desiredVectorIndexes: ResolvedVectorIndex[];
+	private existingVectorIndexes: Map<string, IntrospectedVectorIndex>;
+
+	readonly destructive: DestructiveChange[] = [];
+
 	constructor(
-		private desired: DesiredTable[],
-		private existing: Map<string, IntrospectedTable>,
-		private desiredFts: ResolvedFtsPlan[] = [],
-		private existingFts: Map<string, IntrospectedFts> = new Map(),
-		private desiredVectorIndexes: ResolvedVectorIndex[] = [],
-		private existingVectorIndexes: Map<string, IntrospectedVectorIndex> = new Map(),
+		desired: DesiredSchema,
+		existing: ExistingSchema,
+		private options: { guided?: boolean } = {},
 	) {
-		this.desiredNames = new Set(desired.map((t) => t.name));
+		this.desired = desired.desiredTables;
+		this.existing = existing.existing;
+		this.desiredFts = desired.desiredFts ?? [];
+		this.existingFts = existing.existingFts ?? new Map();
+		this.desiredVectorIndexes = desired.desiredVectorIndexes ?? [];
+		this.existingVectorIndexes = existing.existingVectorIndexes ?? new Map();
+
+		this.desiredNames = new Set(this.desired.map((t) => t.name));
 	}
 
 	diff() {
@@ -106,6 +105,16 @@ export class Differ {
 	}
 
 	private buildRebuild(table: DesiredTable, existingTable: IntrospectedTable) {
+		if (this.options.guided) {
+			this.buildGuidedRebuild(table, existingTable);
+			return;
+		}
+
+		const desiredNames = new Set(table.columns.map((c) => c.name));
+		const droppedColumns = [...existingTable.columns.keys()].filter(
+			(name) => !desiredNames.has(name),
+		);
+		const typeChanged: string[] = [];
 		const columnCopies: ColumnCopy[] = [];
 
 		for (const col of table.columns) {
@@ -122,6 +131,7 @@ export class Differ {
 					);
 				}
 
+				typeChanged.push(col.name);
 				continue;
 			}
 
@@ -141,6 +151,41 @@ export class Differ {
 			}
 		}
 
+		if (existingTable.hasData && droppedColumns.length > 0) {
+			this.destructive.push({ table: table.name, kind: "DropColumn", columns: droppedColumns });
+		}
+
+		if (existingTable.hasData && typeChanged.length > 0) {
+			this.destructive.push({ table: table.name, kind: "TypeChange", columns: typeChanged });
+		}
+
+		this.ops.push({
+			type: "RebuildTable",
+			table: table.name,
+			createSql: table.sql,
+			columnCopies,
+		});
+		this.rebuiltTables.add(table.name);
+	}
+
+	private buildGuidedRebuild(table: DesiredTable, existingTable: IntrospectedTable) {
+		const columnCopies: ColumnCopy[] = [];
+
+		for (const col of table.columns) {
+			if (!existingTable.columns.has(col.name)) {
+				continue;
+			}
+
+			if (col.notnull && col.defaultValue !== null) {
+				columnCopies.push({
+					name: col.name,
+					expr: `COALESCE("${col.name}", ${col.defaultValue})`,
+				});
+			} else {
+				columnCopies.push({ name: col.name, expr: `"${col.name}"` });
+			}
+		}
+
 		this.ops.push({
 			type: "RebuildTable",
 			table: table.name,
@@ -193,9 +238,15 @@ export class Differ {
 	}
 
 	private dropOrphans() {
-		for (const [name] of this.existing) {
-			if (!this.desiredNames.has(name)) {
-				this.ops.push({ type: "DropTable", table: name });
+		for (const [name, table] of this.existing) {
+			if (this.desiredNames.has(name)) {
+				continue;
+			}
+
+			this.ops.push({ type: "DropTable", table: name });
+
+			if (!this.options.guided && table.hasData) {
+				this.destructive.push({ table: name, kind: "DropTable" });
 			}
 		}
 	}

package/src/migration/drift-error.ts

@@ -0,0 +1,9 @@
+export class MigrationDriftError extends Error {
+	constructor(readonly id: string) {
+		super(
+			`Migration "${id}" was already applied with a different body. ` +
+				`Migrations are immutable once applied: revert the change, or give the new logic a fresh id.`,
+		);
+		this.name = "MigrationDriftError";
+	}
+}

package/src/migration/execute.ts

@@ -14,36 +14,14 @@ export class Executor {
 		}
 
 		const hasRebuild = this.ops.some((op) => op.type === "RebuildTable");
-
-		let restoreFk = false;
-
-		if (hasRebuild) {
-			const { foreign_keys } = this.db.prepare("PRAGMA foreign_keys").get() as {
-				foreign_keys: number;
-			};
-
-			if (foreign_keys === 1) {
-				this.db.run("PRAGMA foreign_keys = OFF");
-				restoreFk = true;
-			}
-		}
+		const restoreFk = hasRebuild && Executor.disableForeignKeys(this.db);
 
 		try {
 			this.db.transaction(() => {
-				for (const op of this.ops) {
-					try {
-						this.executeOp(op);
-					} catch (e) {
-						throw this.parseError(e, op);
-					}
-				}
+				this.applyWithin();
 
-				if (restoreFk) {
-					const violations = this.db.prepare("PRAGMA foreign_key_check").all();
-
-					if (violations.length > 0) {
-						throw new Error("Foreign key check failed after rebuild");
-					}
+				if (hasRebuild) {
+					Executor.assertForeignKeysIntact(this.db);
 				}
 			})();
 		} finally {
@@ -53,6 +31,38 @@ export class Executor {
 		}
 	}
 
+	applyWithin() {
+		for (const op of this.ops) {
+			try {
+				this.executeOp(op);
+			} catch (e) {
+				throw this.parseError(e, op);
+			}
+		}
+	}
+
+	static disableForeignKeys(db: Database): boolean {
+		const { foreign_keys } = db.prepare("PRAGMA foreign_keys").get() as {
+			foreign_keys: number;
+		};
+
+		if (foreign_keys !== 1) {
+			return false;
+		}
+
+		db.run("PRAGMA foreign_keys = OFF");
+
+		return true;
+	}
+
+	static assertForeignKeysIntact(db: Database) {
+		const violations = db.prepare("PRAGMA foreign_key_check").all();
+
+		if (violations.length > 0) {
+			throw new Error("Foreign key check failed after rebuild");
+		}
+	}
+
 	private executeOp(op: MigrationOp) {
 		switch (op.type) {
 			case "CreateTable": {

package/src/migration/guided.ts

@@ -0,0 +1,136 @@
+import type { Database } from "bun:sqlite";
+
+import type { Kysely } from "kysely";
+
+import type { DataMigration } from "../types.js";
+import { Differ } from "./diff.js";
+import { Executor } from "./execute.js";
+import { Introspector } from "./introspect.js";
+import type { MigrationRegistry } from "./registry.js";
+import type {
+	DesiredSchema,
+	ExistingSchema,
+	IntrospectedTable,
+	MigrationOp,
+} from "./types.js";
+
+export type GuidedMigrationDeps = {
+	sqlite: Database;
+	kysely: Kysely<any>;
+	desired: DesiredSchema;
+	existing: Map<string, IntrospectedTable>;
+	migration: DataMigration<any, any, any>;
+	checksum: string;
+	registry: MigrationRegistry;
+};
+
+export class GuidedMigrationRunner {
+	constructor(private deps: GuidedMigrationDeps) {}
+
+	async run() {
+		const expandOps = this.expandOps();
+
+		const restoreFk = Executor.disableForeignKeys(this.deps.sqlite);
+
+		this.deps.sqlite.run("BEGIN");
+
+		try {
+			new Executor(this.deps.sqlite, expandOps).applyWithin();
+
+			await this.deps.migration.run({ db: this.deps.kysely });
+
+			const { ops, existing } = this.contractPlan();
+
+			this.logDrops(ops, existing);
+
+			new Executor(this.deps.sqlite, ops).applyWithin();
+
+			this.deps.registry.record(this.deps.migration.id, this.deps.checksum);
+
+			Executor.assertForeignKeysIntact(this.deps.sqlite);
+
+			this.deps.sqlite.run("COMMIT");
+		} catch (e) {
+			this.deps.sqlite.run("ROLLBACK");
+			throw e;
+		} finally {
+			if (restoreFk) {
+				this.deps.sqlite.run("PRAGMA foreign_keys = ON");
+			}
+		}
+	}
+
+	private expandOps() {
+		const creates: MigrationOp[] = [];
+		const adds: MigrationOp[] = [];
+
+		for (const table of this.deps.desired.desiredTables) {
+			const existing = this.deps.existing.get(table.name);
+
+			if (!existing) {
+				creates.push({ type: "CreateTable", table: table.name, sql: table.sql });
+				continue;
+			}
+
+			for (const col of table.columns) {
+				if (!existing.columns.has(col.name)) {
+					adds.push({ type: "AddColumn", table: table.name, columnDef: col.nullableAddDef });
+				}
+			}
+		}
+
+		return [...creates, ...adds];
+	}
+
+	private contractPlan() {
+		const introspector = new Introspector(this.deps.sqlite);
+		const existing = introspector.introspect();
+
+		const existingState: ExistingSchema = {
+			existing,
+			existingFts: introspector.introspectFts(),
+			existingVectorIndexes: introspector.introspectVectorIndexes(),
+		};
+
+		const ops = new Differ(this.deps.desired, existingState, { guided: true }).diff();
+
+		return { ops, existing };
+	}
+
+	private logDrops(ops: MigrationOp[], existing: Map<string, IntrospectedTable>) {
+		const drops: string[] = [];
+
+		for (const op of ops) {
+			if (op.type === "DropTable") {
+				drops.push(`table "${op.table}"`);
+				continue;
+			}
+
+			if (op.type !== "RebuildTable") {
+				continue;
+			}
+
+			const before = existing.get(op.table);
+
+			if (!before) {
+				continue;
+			}
+
+			const kept = new Set(op.columnCopies.map((c) => c.name));
+
+			for (const name of before.columns.keys()) {
+				if (!kept.has(name)) {
+					drops.push(`column "${name}" from "${op.table}"`);
+				}
+			}
+		}
+
+		if (drops.length === 0) {
+			return;
+		}
+
+		console.warn(
+			`Migration "${this.deps.migration.id}" will drop:\n${drops.map((d) => `  - ${d}`).join("\n")}`,
+		);
+	}
+}

package/src/migration/introspect.ts

@@ -2,12 +2,13 @@ import type { Database } from "bun:sqlite";
 
 import { FTS_TRIGGER_PREFIX } from "../fts/sql.js";
 import { VEC_TRIGGER_PREFIX } from "../vector-index/sql.js";
-import type {
-	IntrospectedColumn,
-	IntrospectedFts,
-	IntrospectedIndex,
-	IntrospectedTable,
-	IntrospectedVectorIndex,
+import {
+	MIGRATIONS_TABLE,
+	type IntrospectedColumn,
+	type IntrospectedFts,
+	type IntrospectedIndex,
+	type IntrospectedTable,
+	type IntrospectedVectorIndex,
 } from "./types.js";
 
 type TableListRow = {
@@ -60,6 +61,10 @@ export class Introspector {
 				continue;
 			}
 
+			if (row.name === MIGRATIONS_TABLE) {
+				continue;
+			}
+
 			if (virtualPrefixes.some((prefix) => row.name.startsWith(prefix))) {
 				continue;
 			}

package/src/migration/registry.ts

@@ -0,0 +1,43 @@
+import type { Database } from "bun:sqlite";
+
+import { MigrationDriftError } from "./drift-error.js";
+import { MIGRATIONS_TABLE } from "./types.js";
+
+export class MigrationRegistry {
+	constructor(private db: Database) {}
+
+	ensureTable() {
+		this.db.run(
+			`CREATE TABLE IF NOT EXISTS "${MIGRATIONS_TABLE}" ` +
+				`("id" TEXT PRIMARY KEY, "checksum" TEXT NOT NULL, "applied_at" INTEGER NOT NULL)`,
+		);
+	}
+
+	appliedChecksum(id: string) {
+		const row = this.db
+			.prepare(`SELECT "checksum" FROM "${MIGRATIONS_TABLE}" WHERE "id" = ?`)
+			.get(id) as { checksum: string } | null;
+
+		return row?.checksum ?? null;
+	}
+
+	assertNoDrift(id: string, checksum: string) {
+		const applied = this.appliedChecksum(id);
+
+		if (applied !== null && applied !== checksum) {
+			throw new MigrationDriftError(id);
+		}
+	}
+
+	record(id: string, checksum: string) {
+		this.db
+			.prepare(
+				`INSERT INTO "${MIGRATIONS_TABLE}" ("id", "checksum", "applied_at") VALUES (?, ?, ?)`,
+			)
+			.run(id, checksum, Date.now());
+	}
+
+	static checksum(run: (...args: any[]) => unknown) {
+		return new Bun.CryptoHasher("sha256").update(run.toString()).digest("hex");
+	}
+}

package/src/migration/types.ts

@@ -1,3 +1,6 @@
+import type { ResolvedFtsPlan } from "../fts/resolve.js";
+import type { ResolvedVectorIndex } from "../vector-index/resolve.js";
+
 export type CreateTableOp = {
 	type: "CreateTable";
 	table: string;
@@ -101,6 +104,14 @@ export type MigrationOp =
 	| DropVectorIndexOp
 	| RebuildVectorIndexOp;
 
+export const MIGRATIONS_TABLE = "__migrations";
+
+export type DestructiveChange = {
+	table: string;
+	kind: "DropTable" | "DropColumn" | "TypeChange";
+	columns?: string[];
+};
+
 export type ColumnSchema = {
 	name: string;
 	type: string;
@@ -139,3 +150,34 @@ export type IntrospectedVectorIndex = {
 	sql: string;
 	triggers: Map<string, string>;
 };
+
+export interface DesiredColumn extends ColumnSchema {
+	addable: boolean;
+	columnDef: string;
+	nullableAddDef: string;
+}
+
+export type DesiredIndex = {
+	name: string;
+	columns: string[];
+	sql: string;
+};
+
+export type DesiredTable = {
+	name: string;
+	sql: string;
+	columns: DesiredColumn[];
+	indexes?: DesiredIndex[];
+};
+
+export type DesiredSchema = {
+	desiredTables: DesiredTable[];
+	desiredFts?: ResolvedFtsPlan[];
+	desiredVectorIndexes?: ResolvedVectorIndex[];
+};
+
+export type ExistingSchema = {
+	existing: Map<string, IntrospectedTable>;
+	existingFts?: Map<string, IntrospectedFts>;
+	existingVectorIndexes?: Map<string, IntrospectedVectorIndex>;
+};