@lobomfz/db 0.4.0 → 0.5.0

package/src/migration/diff.ts

@@ -1,39 +1,65 @@
-import type { ColumnSchema, IntrospectedTable, ColumnCopy, MigrationOp } from "./types.js";
-
-export interface DesiredColumn extends ColumnSchema {
-	addable: boolean;
-	columnDef: string;
-}
-
-export type DesiredIndex = {
-	name: string;
-	columns: string[];
-	sql: string;
-};
-
-export type DesiredTable = {
-	name: string;
-	sql: string;
-	columns: DesiredColumn[];
-	indexes?: DesiredIndex[];
-};
+import type { ResolvedFtsPlan } from "../fts/resolve.js";
+import {
+	generateCreateFtsSql,
+	generateFtsTriggers,
+	generateRebuildSelect,
+} from "../fts/sql.js";
+import { tokenizeClause } from "../fts/tokenizer.js";
+import type { ResolvedVectorIndex } from "../vector-index/resolve.js";
+import {
+	generateCreateVecSql,
+	generateRebuildVectorSql,
+	generateVectorTriggers,
+} from "../vector-index/sql.js";
+import type { FtsTokenizer } from "../types.js";
+import type {
+	ColumnSchema,
+	DesiredSchema,
+	DesiredTable,
+	DestructiveChange,
+	ExistingSchema,
+	IntrospectedFts,
+	IntrospectedTable,
+	IntrospectedVectorIndex,
+	ColumnCopy,
+	MigrationOp,
+} from "./types.js";
 
 export class Differ {
 	private ops: MigrationOp[] = [];
 	private desiredNames: Set<string>;
 	private rebuiltTables = new Set<string>();
 
+	private desired: DesiredTable[];
+	private existing: Map<string, IntrospectedTable>;
+	private desiredFts: ResolvedFtsPlan[];
+	private existingFts: Map<string, IntrospectedFts>;
+	private desiredVectorIndexes: ResolvedVectorIndex[];
+	private existingVectorIndexes: Map<string, IntrospectedVectorIndex>;
+
+	readonly destructive: DestructiveChange[] = [];
+
 	constructor(
-		private desired: DesiredTable[],
-		private existing: Map<string, IntrospectedTable>,
+		desired: DesiredSchema,
+		existing: ExistingSchema,
+		private options: { guided?: boolean } = {},
 	) {
-		this.desiredNames = new Set(desired.map((t) => t.name));
+		this.desired = desired.desiredTables;
+		this.existing = existing.existing;
+		this.desiredFts = desired.desiredFts ?? [];
+		this.existingFts = existing.existingFts ?? new Map();
+		this.desiredVectorIndexes = desired.desiredVectorIndexes ?? [];
+		this.existingVectorIndexes = existing.existingVectorIndexes ?? new Map();
+
+		this.desiredNames = new Set(this.desired.map((t) => t.name));
 	}
 
 	diff() {
 		this.diffTables();
 		this.dropOrphans();
 		this.diffIndexes();
+		this.diffFts();
+		this.diffVectorIndexes();
 		return this.ops;
 	}
 
@@ -79,6 +105,16 @@ export class Differ {
 	}
 
 	private buildRebuild(table: DesiredTable, existingTable: IntrospectedTable) {
+		if (this.options.guided) {
+			this.buildGuidedRebuild(table, existingTable);
+			return;
+		}
+
+		const desiredNames = new Set(table.columns.map((c) => c.name));
+		const droppedColumns = [...existingTable.columns.keys()].filter(
+			(name) => !desiredNames.has(name),
+		);
+		const typeChanged: string[] = [];
 		const columnCopies: ColumnCopy[] = [];
 
 		for (const col of table.columns) {
@@ -95,6 +131,7 @@ export class Differ {
 					);
 				}
 
+				typeChanged.push(col.name);
 				continue;
 			}
 
@@ -114,6 +151,41 @@ export class Differ {
 			}
 		}
 
+		if (existingTable.hasData && droppedColumns.length > 0) {
+			this.destructive.push({ table: table.name, kind: "DropColumn", columns: droppedColumns });
+		}
+
+		if (existingTable.hasData && typeChanged.length > 0) {
+			this.destructive.push({ table: table.name, kind: "TypeChange", columns: typeChanged });
+		}
+
+		this.ops.push({
+			type: "RebuildTable",
+			table: table.name,
+			createSql: table.sql,
+			columnCopies,
+		});
+		this.rebuiltTables.add(table.name);
+	}
+
+	private buildGuidedRebuild(table: DesiredTable, existingTable: IntrospectedTable) {
+		const columnCopies: ColumnCopy[] = [];
+
+		for (const col of table.columns) {
+			if (!existingTable.columns.has(col.name)) {
+				continue;
+			}
+
+			if (col.notnull && col.defaultValue !== null) {
+				columnCopies.push({
+					name: col.name,
+					expr: `COALESCE("${col.name}", ${col.defaultValue})`,
+				});
+			} else {
+				columnCopies.push({ name: col.name, expr: `"${col.name}"` });
+			}
+		}
+
 		this.ops.push({
 			type: "RebuildTable",
 			table: table.name,
@@ -166,9 +238,15 @@ export class Differ {
 	}
 
 	private dropOrphans() {
-		for (const [name] of this.existing) {
-			if (!this.desiredNames.has(name)) {
-				this.ops.push({ type: "DropTable", table: name });
+		for (const [name, table] of this.existing) {
+			if (this.desiredNames.has(name)) {
+				continue;
+			}
+
+			this.ops.push({ type: "DropTable", table: name });
+
+			if (!this.options.guided && table.hasData) {
+				this.destructive.push({ table: name, kind: "DropTable" });
 			}
 		}
 	}
@@ -217,4 +295,170 @@ export class Differ {
 			}
 		}
 	}
+
+	private diffFts() {
+		const desiredNames = new Set(this.desiredFts.map((p) => p.name));
+
+		for (const plan of this.desiredFts) {
+			const existing = this.existingFts.get(plan.name);
+			const triggers = generateFtsTriggers(plan);
+			const createSql = generateCreateFtsSql(plan);
+			const rebuildSql = generateRebuildSelect(plan);
+
+			if (!existing) {
+				this.ops.push({
+					type: "CreateFts",
+					name: plan.name,
+					createSql,
+					triggers,
+				});
+				this.ops.push({
+					type: "RebuildFts",
+					name: plan.name,
+					rebuildSql,
+				});
+				continue;
+			}
+
+			const desiredCols = plan.fields.map((f) => f.name);
+			const colsMatch =
+				desiredCols.length === existing.columns.length &&
+				desiredCols.every((c, i) => existing.columns[i] === c);
+			const triggersMatch =
+				existing.triggers.size === triggers.length &&
+				triggers.every(
+					(t) => normalizeSql(existing.triggers.get(t.name) ?? "") === normalizeSql(t.sql),
+				);
+			const tokenizerMatch = ftsTokenizerMatches(existing.sql, plan.tokenizer);
+
+			if (colsMatch && triggersMatch && tokenizerMatch) {
+				continue;
+			}
+
+			this.ops.push({
+				type: "DropFts",
+				name: plan.name,
+				triggerNames: [...existing.triggers.keys()],
+			});
+			this.ops.push({
+				type: "CreateFts",
+				name: plan.name,
+				createSql,
+				triggers,
+			});
+			this.ops.push({
+				type: "RebuildFts",
+				name: plan.name,
+				rebuildSql,
+			});
+		}
+
+		for (const [name, existing] of this.existingFts) {
+			if (desiredNames.has(name)) {
+				continue;
+			}
+
+			this.ops.push({
+				type: "DropFts",
+				name,
+				triggerNames: [...existing.triggers.keys()],
+			});
+		}
+	}
+
+	private diffVectorIndexes() {
+		const desiredNames = new Set(this.desiredVectorIndexes.map((p) => p.name));
+
+		for (const plan of this.desiredVectorIndexes) {
+			const existing = this.existingVectorIndexes.get(plan.name);
+			const triggers = generateVectorTriggers(plan);
+			const createSql = generateCreateVecSql(plan);
+			const rebuildSql = generateRebuildVectorSql(plan);
+
+			if (!existing) {
+				this.ops.push({
+					type: "CreateVectorIndex",
+					name: plan.name,
+					createSql,
+					triggers,
+				});
+				this.ops.push({
+					type: "RebuildVectorIndex",
+					name: plan.name,
+					rebuildSql,
+				});
+				continue;
+			}
+
+			const triggersMatch =
+				existing.triggers.size === triggers.length &&
+				triggers.every(
+					(t) => normalizeSql(existing.triggers.get(t.name) ?? "") === normalizeSql(t.sql),
+				);
+			const shapeMatch = vectorShapeMatches(existing.sql, plan);
+
+			if (triggersMatch && shapeMatch) {
+				continue;
+			}
+
+			this.ops.push({
+				type: "DropVectorIndex",
+				name: plan.name,
+				triggerNames: [...existing.triggers.keys()],
+			});
+			this.ops.push({
+				type: "CreateVectorIndex",
+				name: plan.name,
+				createSql,
+				triggers,
+			});
+			this.ops.push({
+				type: "RebuildVectorIndex",
+				name: plan.name,
+				rebuildSql,
+			});
+		}
+
+		for (const [name, existing] of this.existingVectorIndexes) {
+			if (desiredNames.has(name)) {
+				continue;
+			}
+
+			this.ops.push({
+				type: "DropVectorIndex",
+				name,
+				triggerNames: [...existing.triggers.keys()],
+			});
+		}
+	}
+}
+
+function ftsTokenizerMatches(existingSql: string, desired: FtsTokenizer) {
+	const match = existingSql.match(/tokenize\s*=\s*'((?:''|[^'])*)'/i);
+
+	if (!match) {
+		return false;
+	}
+
+	return match[1]!.replaceAll("''", "'") === tokenizeClause(desired);
+}
+
+function vectorShapeMatches(existingSql: string, desired: ResolvedVectorIndex) {
+	const dimMatch = existingSql.match(/float\s*\[\s*(\d+)\s*\]/i);
+	const metricMatch = existingSql.match(/distance_metric\s*=\s*([a-zA-Z0-9_]+)/i);
+	const colMatch = existingSql.match(/"([^"]+)"\s+float\s*\[/i);
+
+	if (!dimMatch || !metricMatch || !colMatch) {
+		return false;
+	}
+
+	return (
+		Number.parseInt(dimMatch[1]!, 10) === desired.dim &&
+		metricMatch[1]!.toLowerCase() === desired.metric &&
+		colMatch[1] === desired.column
+	);
+}
+
+function normalizeSql(sql: string): string {
+	return sql.replaceAll(/\s+/g, " ").trim();
 }

package/src/migration/drift-error.ts

@@ -0,0 +1,9 @@
+export class MigrationDriftError extends Error {
+	constructor(readonly id: string) {
+		super(
+			`Migration "${id}" was already applied with a different body. ` +
+				`Migrations are immutable once applied: revert the change, or give the new logic a fresh id.`,
+		);
+		this.name = "MigrationDriftError";
+	}
+}

package/src/migration/execute.ts

@@ -14,36 +14,14 @@ export class Executor {
 		}
 
 		const hasRebuild = this.ops.some((op) => op.type === "RebuildTable");
-
-		let restoreFk = false;
-
-		if (hasRebuild) {
-			const { foreign_keys } = this.db.prepare("PRAGMA foreign_keys").get() as {
-				foreign_keys: number;
-			};
-
-			if (foreign_keys === 1) {
-				this.db.run("PRAGMA foreign_keys = OFF");
-				restoreFk = true;
-			}
-		}
+		const restoreFk = hasRebuild && Executor.disableForeignKeys(this.db);
 
 		try {
 			this.db.transaction(() => {
-				for (const op of this.ops) {
-					try {
-						this.executeOp(op);
-					} catch (e) {
-						throw this.parseError(e, op);
-					}
-				}
-
-				if (restoreFk) {
-					const violations = this.db.prepare("PRAGMA foreign_key_check").all();
+				this.applyWithin();
 
-					if (violations.length > 0) {
-						throw new Error("Foreign key check failed after rebuild");
-					}
+				if (hasRebuild) {
+					Executor.assertForeignKeysIntact(this.db);
 				}
 			})();
 		} finally {
@@ -53,6 +31,38 @@ export class Executor {
 		}
 	}
 
+	applyWithin() {
+		for (const op of this.ops) {
+			try {
+				this.executeOp(op);
+			} catch (e) {
+				throw this.parseError(e, op);
+			}
+		}
+	}
+
+	static disableForeignKeys(db: Database): boolean {
+		const { foreign_keys } = db.prepare("PRAGMA foreign_keys").get() as {
+			foreign_keys: number;
+		};
+
+		if (foreign_keys !== 1) {
+			return false;
+		}
+
+		db.run("PRAGMA foreign_keys = OFF");
+
+		return true;
+	}
+
+	static assertForeignKeysIntact(db: Database) {
+		const violations = db.prepare("PRAGMA foreign_key_check").all();
+
+		if (violations.length > 0) {
+			throw new Error("Foreign key check failed after rebuild");
+		}
+	}
+
 	private executeOp(op: MigrationOp) {
 		switch (op.type) {
 			case "CreateTable": {
@@ -73,6 +83,48 @@ export class Executor {
 			case "DropIndex": {
 				return this.db.run(`DROP INDEX "${op.index}"`);
 			}
+			case "CreateFts": {
+				this.db.run(op.createSql);
+
+				for (const trigger of op.triggers) {
+					this.db.run(trigger.sql);
+				}
+
+				return;
+			}
+			case "DropFts": {
+				for (const name of op.triggerNames) {
+					this.db.run(`DROP TRIGGER IF EXISTS "${name}"`);
+				}
+
+				return this.db.run(`DROP TABLE IF EXISTS "${op.name}"`);
+			}
+			case "RebuildFts": {
+				this.db.run(`DELETE FROM "${op.name}"`);
+
+				return this.db.run(op.rebuildSql);
+			}
+			case "CreateVectorIndex": {
+				this.db.run(op.createSql);
+
+				for (const trigger of op.triggers) {
+					this.db.run(trigger.sql);
+				}
+
+				return;
+			}
+			case "DropVectorIndex": {
+				for (const name of op.triggerNames) {
+					this.db.run(`DROP TRIGGER IF EXISTS "${name}"`);
+				}
+
+				return this.db.run(`DROP TABLE IF EXISTS "${op.name}"`);
+			}
+			case "RebuildVectorIndex": {
+				this.db.run(`DELETE FROM "${op.name}"`);
+
+				return this.db.run(op.rebuildSql);
+			}
 		}
 	}
 

package/src/migration/guided.ts

@@ -0,0 +1,136 @@
+import type { Database } from "bun:sqlite";
+
+import type { Kysely } from "kysely";
+
+import type { DataMigration } from "../types.js";
+import { Differ } from "./diff.js";
+import { Executor } from "./execute.js";
+import { Introspector } from "./introspect.js";
+import type { MigrationRegistry } from "./registry.js";
+import type {
+	DesiredSchema,
+	ExistingSchema,
+	IntrospectedTable,
+	MigrationOp,
+} from "./types.js";
+
+export type GuidedMigrationDeps = {
+	sqlite: Database;
+	kysely: Kysely<any>;
+	desired: DesiredSchema;
+	existing: Map<string, IntrospectedTable>;
+	migration: DataMigration<any, any, any>;
+	checksum: string;
+	registry: MigrationRegistry;
+};
+
+export class GuidedMigrationRunner {
+	constructor(private deps: GuidedMigrationDeps) {}
+
+	async run() {
+		const expandOps = this.expandOps();
+
+		const restoreFk = Executor.disableForeignKeys(this.deps.sqlite);
+
+		this.deps.sqlite.run("BEGIN");
+
+		try {
+			new Executor(this.deps.sqlite, expandOps).applyWithin();
+
+			await this.deps.migration.run({ db: this.deps.kysely });
+
+			const { ops, existing } = this.contractPlan();
+
+			this.logDrops(ops, existing);
+
+			new Executor(this.deps.sqlite, ops).applyWithin();
+
+			this.deps.registry.record(this.deps.migration.id, this.deps.checksum);
+
+			Executor.assertForeignKeysIntact(this.deps.sqlite);
+
+			this.deps.sqlite.run("COMMIT");
+		} catch (e) {
+			this.deps.sqlite.run("ROLLBACK");
+			throw e;
+		} finally {
+			if (restoreFk) {
+				this.deps.sqlite.run("PRAGMA foreign_keys = ON");
+			}
+		}
+	}
+
+	private expandOps() {
+		const creates: MigrationOp[] = [];
+		const adds: MigrationOp[] = [];
+
+		for (const table of this.deps.desired.desiredTables) {
+			const existing = this.deps.existing.get(table.name);
+
+			if (!existing) {
+				creates.push({ type: "CreateTable", table: table.name, sql: table.sql });
+				continue;
+			}
+
+			for (const col of table.columns) {
+				if (!existing.columns.has(col.name)) {
+					adds.push({ type: "AddColumn", table: table.name, columnDef: col.nullableAddDef });
+				}
+			}
+		}
+
+		return [...creates, ...adds];
+	}
+
+	private contractPlan() {
+		const introspector = new Introspector(this.deps.sqlite);
+		const existing = introspector.introspect();
+
+		const existingState: ExistingSchema = {
+			existing,
+			existingFts: introspector.introspectFts(),
+			existingVectorIndexes: introspector.introspectVectorIndexes(),
+		};
+
+		const ops = new Differ(this.deps.desired, existingState, { guided: true }).diff();
+
+		return { ops, existing };
+	}
+
+	private logDrops(ops: MigrationOp[], existing: Map<string, IntrospectedTable>) {
+		const drops: string[] = [];
+
+		for (const op of ops) {
+			if (op.type === "DropTable") {
+				drops.push(`table "${op.table}"`);
+				continue;
+			}
+
+			if (op.type !== "RebuildTable") {
+				continue;
+			}
+
+			const before = existing.get(op.table);
+
+			if (!before) {
+				continue;
+			}
+
+			const kept = new Set(op.columnCopies.map((c) => c.name));
+
+			for (const name of before.columns.keys()) {
+				if (!kept.has(name)) {
+					drops.push(`column "${name}" from "${op.table}"`);
+				}
+			}
+		}
+
+		if (drops.length === 0) {
+			return;
+		}
+
+		console.warn(
+			`Migration "${this.deps.migration.id}" will drop:\n${drops.map((d) => `  - ${d}`).join("\n")}`,
+		);
+	}
+}