@lobomfz/db 0.4.1 → 0.5.1

package/README.md

@@ -13,7 +13,7 @@ bun add @lobomfz/db
 ```typescript
 import { Database, generated, type } from "@lobomfz/db";
 
-const db = new Database({
+const db = await Database.open({
 	path: "data.db",
 	schema: {
 		tables: {
@@ -68,7 +68,7 @@ type("number.integer").configure({ references: "users.id", onDelete: "cascade" }
 JSON columns are validated against the schema on write by default. To also validate on read:
 
 ```typescript
-new Database({
+await Database.open({
 	// ...
 	validation: { onRead: true }, // default: { onRead: false }
 });
@@ -76,7 +76,7 @@ new Database({
 
 ## Migrations
 
-Schema changes are applied automatically on startup. Every time `new Database(...)` runs, the library compares your Arktype schema against the actual SQLite database and applies the minimum set of operations to bring them in sync. No migration files, no version tracking — the database itself is the source of truth.
+Schema changes are applied automatically on startup. `Database.open(...)` is async because migrations run before it resolves: the library compares your Arktype schema against the actual SQLite database and applies the minimum set of operations to bring them in sync. No migration files, no version tracking — the database itself is the source of truth.
 
 ### What's supported
 
@@ -102,7 +102,58 @@ Table rebuilds follow SQLite's [recommended procedure](https://www.sqlite.org/la
 - Adding a NOT NULL column without DEFAULT to a table **with data** throws an error
 - Changing a nullable column to NOT NULL without DEFAULT throws if any row has NULL in that column
 - Nullable-to-NOT-NULL with DEFAULT uses `COALESCE` to fill existing NULLs
-- Column renames are treated as drop + add (data in the old column is not preserved)
+- Without a guided migration, column renames are treated as drop + add (data in the old column is not preserved)
+
+### Refusing destructive changes
+
+By default the diff applies whatever brings the database in sync, including drops. Set `failOnDataLoss` to turn dropping a table, dropping a column, or a data-discarding type change into a thrown `DataLossError` instead — the database is left untouched.
+
+```typescript
+await Database.open({
+	// ...
+	failOnDataLoss: true,
+});
+```
+
+Configuring a `migration` (see below) routes through the guided path instead, where the drops are intrinsic to the restructure. The gate does not apply there; the drops are logged via `console.warn` so they are still visible.
+
+### Guided data migrations
+
+The automatic diff can rename or restructure a table only by dropping the old shape. When you need to preserve and transform the data — rename a column, split one column into two, convert a type — pass a `migration` with a stable `id` and a `run` function. It receives the same typed Kysely client and runs inside the migration transaction.
+
+```typescript
+await Database.open({
+	path: "data.db",
+	schema: {
+		tables: {
+			users: type({ id: generated("autoincrement"), name: "string", email: "string" }),
+		},
+	},
+	migration: {
+		id: "rename-user-fields",
+		run: async ({ db }) => {
+			const rows = await db
+				.selectFrom("users")
+				.select(["id", sql<string>`full_name`.as("full_name")])
+				.execute();
+
+			for (const row of rows) {
+				await db.updateTable("users").set({ name: row.full_name }).where("id", "=", row.id).execute();
+			}
+		},
+	},
+});
+```
+
+The migration runs as expand then contract, all in one transaction:
+
+1. New tables and new columns are added (new columns nullable, so the backfill can write them).
+2. Your `run` backfills and transforms the data, reading old columns with raw `sql` and writing the new ones through the typed client.
+3. The schema is reconciled to the target: leftover columns and tables are dropped (logged via `console.warn`) and `NOT NULL` is tightened.
+
+`run` already executes inside the migration transaction, so do not open your own with `db.transaction(...)`. Your reads and writes are already atomic with the rest of the migration; if `run` throws, the whole transaction rolls back and the database is left exactly as it was. Referential integrity is checked with `PRAGMA foreign_key_check` before the commit, even when `foreign_keys` is disabled.
+
+Each `id` runs at most once. The applied set lives in a `__migrations` table; a fresh database records the `id` as a baseline without running it (the schema is created directly). Editing the body of an already-applied migration throws `MigrationDriftError` — migrations are immutable once applied, so give new logic a fresh `id`.
 
 ## License
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lobomfz/db",
-	"version": "0.4.1",
+	"version": "0.5.1",
 	"description": "Bun SQLite database with Arktype schemas and typed Kysely client",
 	"keywords": [
 		"arktype",

package/src/database.ts

@@ -4,20 +4,23 @@ import type { Type } from "arktype";
 import { Kysely } from "kysely";
 
 import { BunSqliteDialect } from "./dialect/dialect.js";
-import type { DbFieldMeta } from "./env.js";
 import { resolveFtsPlan, type FtsTableInfo, type ResolvedFtsPlan } from "./fts/resolve.js";
 import { generateRebuildSelect } from "./fts/sql.js";
-import type { GeneratedPreset } from "./generated.js";
-import { Differ, type DesiredTable } from "./migration/diff.js";
+import { DataLossError } from "./migration/data-loss-error.js";
+import { Differ } from "./migration/diff.js";
 import { Executor } from "./migration/execute.js";
+import { GuidedMigrationRunner } from "./migration/guided.js";
 import { Introspector } from "./migration/introspect.js";
+import { MigrationRegistry } from "./migration/registry.js";
+import type { DesiredSchema, DesiredTable, ExistingSchema } from "./migration/types.js";
 import { ResultHydrationPlugin } from "./plugin.js";
+import { SchemaCompiler } from "./schema-compiler.js";
 import type {
+	DataMigration,
 	DatabaseOptions,
 	DatabasePragmas,
 	FtsApi,
 	FtsConfig,
-	IndexDefinition,
 	SchemaRecord,
 	TablesFromSchemas,
 	VectorIndexApi,
@@ -33,49 +36,6 @@ import {
 import { generateRebuildVectorSql } from "./vector-index/sql.js";
 import { WriteValidationPlugin } from "./write-validation-plugin.js";
 
-type ArkBranch = {
-	domain?: string;
-	proto?: unknown;
-	unit?: unknown;
-	structure?: unknown;
-	inner?: { divisor?: unknown };
-	meta?: DbFieldMeta;
-};
-
-type StructureProp = {
-	key: string;
-	required: boolean;
-	value: Type & {
-		branches: ArkBranch[];
-		proto?: unknown;
-		meta: DbFieldMeta;
-	};
-	inner: { default?: unknown };
-};
-
-type Prop = {
-	key: string;
-	kind: "required" | "optional";
-	domain?: string;
-	nullable?: boolean;
-	isBoolean?: boolean;
-	isInteger?: boolean;
-	isDate?: boolean;
-	isBlob?: boolean;
-	isJson?: boolean;
-	isVector?: boolean;
-	vectorDim?: number;
-	jsonSchema?: Type;
-	meta?: DbFieldMeta;
-	generated?: GeneratedPreset;
-	defaultValue?: unknown;
-};
-
-const typeMap: Record<string, string> = {
-	string: "TEXT",
-	number: "REAL",
-};
-
 const defaultPragmas: DatabasePragmas = {
 	foreign_keys: true,
 };
@@ -86,6 +46,7 @@ export class Database<
 	V extends VectorIndexConfig<T> = VectorIndexConfig<T>,
 > {
 	private sqlite: BunDatabase;
+	private compiler = new SchemaCompiler();
 	private ftsPlans = new Map<string, ResolvedFtsPlan>();
 	private vectorIndexPlans = new Map<string, ResolvedVectorIndex>();
 
@@ -97,12 +58,24 @@ export class Database<
 
 	readonly vectorIndex: VectorIndexApi<V>;
 
-	constructor(private options: DatabaseOptions<T, F, V>) {
+	static async open<
+		T extends SchemaRecord,
+		F extends FtsConfig<T> = FtsConfig<T>,
+		V extends VectorIndexConfig<T> = VectorIndexConfig<T>,
+	>(options: DatabaseOptions<T, F, V>) {
+		const db = new Database(options);
+
+		await db.runMigrations();
+
+		return db;
+	}
+
+	private constructor(private options: DatabaseOptions<T, F, V>) {
 		const tableSchemas = new Map<string, Type>(Object.entries(options.schema.tables));
 		const writeSchemas = new Map<string, Type>(
 			Object.entries(options.schema.tables).map(([table, schema]) => [
 				table,
-				this.createWriteSchema(schema),
+				this.compiler.createWriteSchema(schema),
 			]),
 		);
 
@@ -112,8 +85,6 @@ export class Database<
 		this.assertVectorIndexesHaveExtension();
 		this.loadConfiguredExtensions();
 
-		this.migrate();
-
 		const validation = {
 			onRead: options.validation?.onRead ?? false,
 		};
@@ -191,271 +162,18 @@ export class Database<
 		}
 	}
 
-	private normalizeProp(structureProp: StructureProp, parentSchema: Type) {
-		const { key, value: v, inner } = structureProp;
-		const kind: Prop["kind"] = structureProp.required ? "required" : "optional";
-		const defaultValue = inner.default;
-
-		const concrete = v.branches.filter((b) => b.unit !== null && b.domain !== "undefined");
-		const nullable = concrete.length < v.branches.length;
-
-		const branchMeta = v.branches.find((b) => b.meta && Object.keys(b.meta).length > 0)?.meta;
-		const meta = { ...branchMeta, ...v.meta };
-		const generated = meta._generated === "vector" ? undefined : meta._generated;
-
-		if (meta._generated === "vector") {
-			return {
-				key,
-				kind,
-				nullable,
-				isVector: true,
-				vectorDim: meta._vectorDim,
-				meta,
-				defaultValue,
-			};
-		}
-
-		if (v.proto === Date || concrete.some((b) => b.proto === Date)) {
-			return { key, kind, nullable, isDate: true, generated, defaultValue };
-		}
-
-		if (v.proto === Uint8Array || concrete.some((b) => b.proto === Uint8Array)) {
-			return {
-				key,
-				kind,
-				nullable,
-				isBlob: true,
-				meta,
-				generated,
-				defaultValue,
-			};
-		}
-
-		if (concrete.length > 0 && concrete.every((b) => b.domain === "boolean")) {
-			return { key, kind, nullable, isBoolean: true, generated, defaultValue };
-		}
-
-		if (concrete.some((b) => !!b.structure)) {
-			return {
-				key,
-				kind,
-				nullable,
-				isJson: true,
-				jsonSchema: (parentSchema as any).get(key) as Type,
-				meta,
-				generated,
-				defaultValue,
-			};
-		}
-
-		const branch = concrete[0];
-
-		return {
-			key,
-			kind,
-			nullable,
-			domain: branch?.domain,
-			isInteger: !!branch?.inner?.divisor,
-			meta,
-			generated,
-			defaultValue,
-		};
-	}
-
-	private sqlType(prop: Prop) {
-		if (prop.isJson) {
-			return "TEXT";
-		}
-
-		if (prop.isBlob || prop.isVector) {
-			return "BLOB";
-		}
-
-		if (prop.isDate || prop.isBoolean || prop.isInteger) {
-			return "INTEGER";
-		}
-
-		if (prop.domain) {
-			return typeMap[prop.domain] ?? "TEXT";
-		}
-
-		return "TEXT";
-	}
-
-	private columnConstraint(prop: Prop) {
-		if (prop.generated === "autoincrement") {
-			return "PRIMARY KEY AUTOINCREMENT";
-		}
-
-		if (prop.meta?.primaryKey) {
-			return "PRIMARY KEY";
-		}
-
-		if (prop.kind === "required" && !prop.nullable) {
-			return "NOT NULL";
-		}
-
-		return null;
-	}
-
-	private defaultClause(prop: Prop) {
-		if (prop.generated === "now") {
-			return "DEFAULT (CAST(unixepoch('subsec') * 1000 AS INTEGER))";
-		}
-
-		if (prop.defaultValue === undefined || prop.generated === "autoincrement") {
-			return null;
-		}
-
-		if (prop.defaultValue === null) {
-			return "DEFAULT NULL";
-		}
-
-		if (typeof prop.defaultValue === "string") {
-			return `DEFAULT '${prop.defaultValue}'`;
-		}
-
-		if (typeof prop.defaultValue === "number" || typeof prop.defaultValue === "boolean") {
-			return `DEFAULT ${prop.defaultValue}`;
-		}
-
-		throw new Error(
-			`Unsupported default value type: ${typeof prop.defaultValue} ${JSON.stringify(prop)}`,
-		);
-	}
-
-	private columnDef(prop: Prop) {
-		return [
-			`"${prop.key}"`,
-			this.sqlType(prop),
-			this.columnConstraint(prop),
-			prop.meta?.unique ? "UNIQUE" : null,
-			this.defaultClause(prop),
-		]
-			.filter(Boolean)
-			.join(" ");
-	}
-
-	private foreignKey(prop: Prop) {
-		const ref = prop.meta?.references;
-
-		if (!ref) {
-			return null;
-		}
-
-		const [table, column] = ref.split(".");
-
-		let fk = `FOREIGN KEY ("${prop.key}") REFERENCES "${table}"("${column}")`;
-
-		const onDelete = prop.meta?.onDelete;
-
-		if (onDelete) {
-			fk += ` ON DELETE ${onDelete.toUpperCase()}`;
-		}
-
-		return fk;
-	}
-
-	private addColumnDef(prop: Prop) {
-		let def = this.columnDef(prop);
-
-		const ref = prop.meta?.references;
-
-		if (ref) {
-			const [table, column] = ref.split(".");
-			def += ` REFERENCES "${table}"("${column}")`;
-
-			if (prop.meta?.onDelete) {
-				def += ` ON DELETE ${prop.meta.onDelete.toUpperCase()}`;
-			}
-		}
-
-		return def;
-	}
-
-	private parseSchemaProps(schema: Type) {
-		const structureProps = (schema as any).structure?.props as StructureProp[] | undefined;
-
-		if (!structureProps) {
-			return [];
-		}
-
-		return structureProps.map((p) => this.normalizeProp(p, schema));
-	}
-
-	private createWriteSchema(schema: Type) {
-		const autoIncrementColumns = this.parseSchemaProps(schema)
-			.filter((prop) => prop.generated === "autoincrement")
-			.map((prop) => prop.key);
-
-		if (autoIncrementColumns.length === 0) {
-			return schema;
-		}
-
-		return (schema as any).omit(...autoIncrementColumns) as Type;
-	}
-
-	private generateCreateTableSQL(tableName: string, props: Prop[]) {
-		const columns: string[] = [];
-		const fks: string[] = [];
-
-		for (const prop of props) {
-			columns.push(this.columnDef(prop));
-
-			const fk = this.foreignKey(prop);
-
-			if (fk) {
-				fks.push(fk);
-			}
-		}
-
-		return `CREATE TABLE IF NOT EXISTS "${tableName}" (${columns.concat(fks).join(", ")})`;
-	}
-
-	private migrate() {
+	private buildDesiredSchema(): DesiredSchema {
 		const desiredTables: DesiredTable[] = [];
 		const tableInfos = new Map<string, FtsTableInfo>();
 		const vectorAnchors = new Map<string, VectorAnchorInfo>();
 		const schemaIndexes = this.options.schema.indexes;
 
 		for (const [name, schema] of Object.entries(this.options.schema.tables)) {
-			const props = this.parseSchemaProps(schema);
-
-			const columns = props.map((prop) => {
-				const isNotNull = this.columnConstraint(prop) === "NOT NULL";
-				const defaultClause = this.defaultClause(prop);
-				const hasLiteralDefault = prop.generated !== "now" && defaultClause !== null;
-
-				return {
-					name: prop.key,
-					addable: !isNotNull || hasLiteralDefault,
-					columnDef: this.addColumnDef(prop),
-					type: this.sqlType(prop),
-					notnull: isNotNull,
-					defaultValue: defaultClause
-						? defaultClause.replace("DEFAULT ", "").replace(/^\((.+)\)$/, "$1")
-						: null,
-					unique: !!prop.meta?.unique,
-					references: prop.meta?.references ?? null,
-					onDelete: prop.meta?.onDelete?.toUpperCase() ?? null,
-				};
-			});
-
-			const indexes = (schemaIndexes?.[name] ?? []).map((indexDef) => ({
-				name: this.generateIndexName(name, indexDef.columns, indexDef.unique ?? false),
-				columns: indexDef.columns,
-				sql: this.generateCreateIndexSQL(name, indexDef),
-			}));
-
-			desiredTables.push({
-				name,
-				sql: this.generateCreateTableSQL(name, props),
-				columns,
-				indexes,
-			});
-
-			tableInfos.set(name, this.buildTableInfo(props));
-			vectorAnchors.set(name, this.buildVectorAnchorInfo(props));
+			const compiled = this.compiler.compileTable(name, schema, schemaIndexes?.[name] ?? []);
+
+			desiredTables.push(compiled.table);
+			tableInfos.set(name, compiled.tableInfo);
+			vectorAnchors.set(name, compiled.vectorAnchor);
 		}
 
 		const desiredFts: ResolvedFtsPlan[] = [];
@@ -479,21 +197,27 @@ export class Database<
 			}
 		}
 
+		return { desiredTables, desiredFts, desiredVectorIndexes };
+	}
+
+	private async runMigrations() {
+		const desired = this.buildDesiredSchema();
+
 		const introspector = new Introspector(this.sqlite);
-		const existing = introspector.introspect();
-		const existingFts = introspector.introspectFts();
 		const existingVectorIndexes = introspector.introspectVectorIndexes();
-
-		const ops = new Differ(
-			desiredTables,
-			existing,
-			desiredFts,
-			existingFts,
-			desiredVectorIndexes,
+		const existingState: ExistingSchema = {
+			existing: introspector.introspect(),
+			existingFts: introspector.introspectFts(),
 			existingVectorIndexes,
-		).diff();
+		};
 
-		new Executor(this.sqlite, ops).execute();
+		const migration = this.options.migration;
+
+		if (migration) {
+			await this.applyWithMigration(desired, existingState, migration);
+		} else {
+			this.applyDiff(desired, existingState);
+		}
 
 		for (const name of existingVectorIndexes.keys()) {
 			if (!this.vectorIndexPlans.has(name)) {
@@ -502,6 +226,54 @@ export class Database<
 		}
 	}
 
+	private applyDiff(desired: DesiredSchema, existingState: ExistingSchema) {
+		const differ = new Differ(desired, existingState);
+
+		const ops = differ.diff();
+
+		if (this.options.failOnDataLoss && differ.destructive.length > 0) {
+			throw new DataLossError(differ.destructive);
+		}
+
+		new Executor(this.sqlite, ops).execute();
+	}
+
+	private async applyWithMigration(
+		desired: DesiredSchema,
+		existingState: ExistingSchema,
+		migration: DataMigration<T, F, V>,
+	) {
+		const registry = new MigrationRegistry(this.sqlite);
+		registry.ensureTable();
+
+		const checksum = MigrationRegistry.checksum(migration.run);
+
+		registry.assertNoDrift(migration.id, checksum);
+
+		if (registry.appliedChecksum(migration.id) !== null) {
+			this.applyDiff(desired, existingState);
+			return;
+		}
+
+		const fresh = !desired.desiredTables.some((table) => existingState.existing.has(table.name));
+
+		if (fresh) {
+			this.applyDiff(desired, existingState);
+			registry.record(migration.id, checksum);
+			return;
+		}
+
+		await new GuidedMigrationRunner({
+			sqlite: this.sqlite,
+			kysely: this.kysely,
+			desired,
+			existing: existingState.existing,
+			migration,
+			checksum,
+			registry,
+		}).run();
+	}
+
 	private rebuildFts(name: string) {
 		const plan = this.ftsPlans.get(name);
 
@@ -528,70 +300,6 @@ export class Database<
 		})();
 	}
 
-	private buildTableInfo(props: Prop[]): FtsTableInfo {
-		const pkProp = props.find(
-			(p) => p.generated === "autoincrement" || p.meta?.primaryKey === true,
-		);
-
-		return {
-			columns: new Set(props.map((p) => p.key)),
-			pkColumn: pkProp?.key ?? null,
-			pkIsInteger: pkProp ? this.sqlType(pkProp) === "INTEGER" : false,
-			foreignKeys: props.flatMap((p) => {
-				const ref = p.meta?.references;
-
-				if (!ref) {
-					return [];
-				}
-
-				const [referencedTable, referencedColumn] = ref.split(".");
-
-				if (!referencedTable || !referencedColumn) {
-					throw new Error(
-						`Invalid references format "${ref}" for column "${p.key}": expected "table.column"`,
-					);
-				}
-
-				return [{ column: p.key, referencedTable, referencedColumn }];
-			}),
-		};
-	}
-
-	private buildVectorAnchorInfo(props: Prop[]): VectorAnchorInfo {
-		const pkProp = props.find(
-			(p) => p.generated === "autoincrement" || p.meta?.primaryKey === true,
-		);
-
-		const vectorColumns = new Map<string, number>();
-
-		for (const p of props) {
-			if (p.isVector && typeof p.vectorDim === "number") {
-				vectorColumns.set(p.key, p.vectorDim);
-			}
-		}
-
-		return {
-			columns: new Set(props.map((p) => p.key)),
-			pkColumn: pkProp?.key ?? null,
-			pkIsInteger: pkProp ? this.sqlType(pkProp) === "INTEGER" : false,
-			vectorColumns,
-		};
-	}
-
-	private generateIndexName(tableName: string, columns: string[], unique: boolean) {
-		const prefix = unique ? "ux" : "ix";
-
-		return `${prefix}_${tableName}_${columns.join("_")}`;
-	}
-
-	private generateCreateIndexSQL(tableName: string, indexDef: IndexDefinition) {
-		const indexName = this.generateIndexName(tableName, indexDef.columns, indexDef.unique ?? false);
-		const unique = indexDef.unique ? "UNIQUE " : "";
-		const columns = indexDef.columns.map((c) => `"${c}"`).join(", ");
-
-		return `CREATE ${unique}INDEX "${indexName}" ON "${tableName}" (${columns})`;
-	}
-
 	reset(table?: keyof T & string): void {
 		const tables = table ? [table] : Object.keys(this.options.schema.tables);
 

package/src/index.ts

@@ -4,6 +4,8 @@ export { vector } from "./vector.js";
 export { nearest, type NearestOptions } from "./vector-index/nearest.js";
 export { ExtensionLoadError } from "./vector-index/load.js";
 export { JsonParseError } from "./errors.js";
+export { DataLossError } from "./migration/data-loss-error.js";
+export { MigrationDriftError } from "./migration/drift-error.js";
 export { jsonArrayFrom, jsonObjectFrom } from "./json.js";
 export {
 	sql,
@@ -18,6 +20,7 @@ export { configure } from "arktype/config";
 export { ValidationError } from "./validation-error.js";
 export type { DbFieldMeta } from "./env.js";
 export type {
+	DataMigration,
 	DatabaseOptions,
 	DatabaseExtension,
 	SchemaRecord,

package/src/migration/data-loss-error.ts

@@ -0,0 +1,29 @@
+import type { DestructiveChange } from "./types.js";
+
+export class DataLossError extends Error {
+	constructor(readonly changes: DestructiveChange[]) {
+		super(
+			`Refusing to apply a migration that loses data (failOnDataLoss is on):\n${changes
+				.map(formatChange)
+				.map((line) => `  - ${line}`)
+				.join("\n")}`,
+		);
+		this.name = "DataLossError";
+	}
+}
+
+function formatChange(change: DestructiveChange) {
+	const columns = (change.columns ?? []).map((c) => `"${c}"`).join(", ");
+
+	switch (change.kind) {
+		case "DropTable": {
+			return `drop table "${change.table}"`;
+		}
+		case "DropColumn": {
+			return `drop column ${columns} from "${change.table}"`;
+		}
+		case "TypeChange": {
+			return `change type of column ${columns} in "${change.table}" (discards data)`;
+		}
+	}
+}