@lobomfz/db 0.4.0 → 0.4.1

package/src/fts/sql.ts

@@ -0,0 +1,148 @@
+import type { ResolvedFtsPlan } from "./resolve.js";
+import { tokenizeOption } from "./tokenizer.js";
+
+export const FTS_TRIGGER_PREFIX = "__fts_sync_";
+
+export type FtsTriggerEvent = "AI" | "AU" | "AD";
+
+export function ftsTriggerName(fts: string, sourceTable: string, event: FtsTriggerEvent) {
+	return `${FTS_TRIGGER_PREFIX}${fts}_${sourceTable}_${event}`;
+}
+
+export type GeneratedTrigger = { name: string; sql: string };
+
+export function generateCreateFtsSql(plan: ResolvedFtsPlan) {
+	const cols = plan.fields.map((f) => `"${f.name}"`).join(", ");
+	const tokenize = tokenizeOption(plan.tokenizer);
+
+	return `CREATE VIRTUAL TABLE "${plan.name}" USING fts5(${cols}, ${tokenize})`;
+}
+
+function insertValueFor(field: ResolvedFtsPlan["fields"][number], anchor: string) {
+	if (field.sourceTable === anchor) {
+		return `COALESCE(NEW."${field.sourceColumn}", '')`;
+	}
+
+	return `''`;
+}
+
+function anchorTriggers(plan: ResolvedFtsPlan): GeneratedTrigger[] {
+	const fts = plan.name;
+	const anchor = plan.anchor;
+	const pk = plan.anchorPkColumn;
+
+	const allCols = plan.fields.map((f) => `"${f.name}"`).join(", ");
+	const insertValues = plan.fields.map((f) => insertValueFor(f, anchor)).join(", ");
+
+	const anchorFields = plan.fields.filter((f) => f.sourceTable === anchor);
+
+	const aiName = ftsTriggerName(fts, anchor, "AI");
+	const adName = ftsTriggerName(fts, anchor, "AD");
+
+	const triggers: GeneratedTrigger[] = [
+		{
+			name: aiName,
+			sql:
+				`CREATE TRIGGER "${aiName}" AFTER INSERT ON "${anchor}" BEGIN ` +
+				`INSERT INTO "${fts}"(rowid, ${allCols}) VALUES (NEW."${pk}", ${insertValues}); ` +
+				`END`,
+		},
+		{
+			name: adName,
+			sql:
+				`CREATE TRIGGER "${adName}" AFTER DELETE ON "${anchor}" BEGIN ` +
+				`DELETE FROM "${fts}" WHERE rowid = OLD."${pk}"; ` +
+				`END`,
+		},
+	];
+
+	if (anchorFields.length > 0) {
+		const trackedCols = anchorFields.map((f) => `"${f.sourceColumn}"`).join(", ");
+		const setClause = anchorFields
+			.map((f) => `"${f.name}" = COALESCE(NEW."${f.sourceColumn}", '')`)
+			.join(", ");
+		const auName = ftsTriggerName(fts, anchor, "AU");
+
+		triggers.push({
+			name: auName,
+			sql:
+				`CREATE TRIGGER "${auName}" AFTER UPDATE OF ${trackedCols} ON "${anchor}" BEGIN ` +
+				`UPDATE "${fts}" SET ${setClause} WHERE rowid = NEW."${pk}"; ` +
+				`END`,
+		});
+	}
+
+	return triggers;
+}
+
+function sourceTriggers(plan: ResolvedFtsPlan): GeneratedTrigger[] {
+	const triggers: GeneratedTrigger[] = [];
+	const fts = plan.name;
+
+	for (const source of plan.sources) {
+		const fields = plan.fields.filter((f) => f.sourceTable === source.table);
+
+		if (fields.length === 0) {
+			continue;
+		}
+
+		const trackedCols = fields.map((f) => `"${f.sourceColumn}"`).join(", ");
+		const setNew = fields
+			.map((f) => `"${f.name}" = COALESCE(NEW."${f.sourceColumn}", '')`)
+			.join(", ");
+		const clearSet = fields.map((f) => `"${f.name}" = ''`).join(", ");
+
+		const aiName = ftsTriggerName(fts, source.table, "AI");
+		const auName = ftsTriggerName(fts, source.table, "AU");
+		const adName = ftsTriggerName(fts, source.table, "AD");
+
+		triggers.push({
+			name: aiName,
+			sql:
+				`CREATE TRIGGER "${aiName}" AFTER INSERT ON "${source.table}" BEGIN ` +
+				`UPDATE "${fts}" SET ${setNew} WHERE rowid = NEW."${source.fkColumn}"; ` +
+				`END`,
+		});
+
+		triggers.push({
+			name: auName,
+			sql:
+				`CREATE TRIGGER "${auName}" AFTER UPDATE OF ${trackedCols} ON "${source.table}" BEGIN ` +
+				`UPDATE "${fts}" SET ${setNew} WHERE rowid = NEW."${source.fkColumn}"; ` +
+				`END`,
+		});
+
+		triggers.push({
+			name: adName,
+			sql:
+				`CREATE TRIGGER "${adName}" AFTER DELETE ON "${source.table}" BEGIN ` +
+				`UPDATE "${fts}" SET ${clearSet} WHERE rowid = OLD."${source.fkColumn}"; ` +
+				`END`,
+		});
+	}
+
+	return triggers;
+}
+
+export function generateFtsTriggers(plan: ResolvedFtsPlan) {
+	return [...anchorTriggers(plan), ...sourceTriggers(plan)];
+}
+
+export function generateRebuildSelect(plan: ResolvedFtsPlan) {
+	const cols = plan.fields.map((f) => `"${f.name}"`).join(", ");
+	const selectExprs = plan.fields
+		.map((f) => `COALESCE("${f.sourceTable}"."${f.sourceColumn}", '')`)
+		.join(", ");
+	const joins = plan.sources
+		.map(
+			(s) =>
+				`LEFT JOIN "${s.table}" ON "${s.table}"."${s.fkColumn}" = "${plan.anchor}"."${plan.anchorPkColumn}"`,
+		)
+		.join(" ");
+
+	const from = joins
+		? `FROM "${plan.anchor}" ${joins}`
+		: `FROM "${plan.anchor}"`;
+
+	return `INSERT INTO "${plan.name}"(rowid, ${cols}) SELECT "${plan.anchor}"."${plan.anchorPkColumn}", ${selectExprs} ${from}`;
+}

package/src/fts/tokenizer.ts

@@ -0,0 +1,27 @@
+import type { FtsTokenizer } from "../types.js";
+
+export const DEFAULT_TOKENIZER: FtsTokenizer = { type: "unicode61", removeDiacritics: 2 };
+
+export function tokenizeClause(tokenizer: FtsTokenizer) {
+	if (tokenizer.type === "trigram") {
+		return "trigram";
+	}
+
+	const parts: string[] = ["unicode61"];
+
+	if (tokenizer.removeDiacritics !== undefined) {
+		parts.push(`remove_diacritics ${tokenizer.removeDiacritics}`);
+	}
+
+	if (tokenizer.categories) {
+		parts.push(`categories '${tokenizer.categories}'`);
+	}
+
+	return parts.join(" ");
+}
+
+export function tokenizeOption(tokenizer: FtsTokenizer | undefined) {
+	const clause = tokenizeClause(tokenizer ?? DEFAULT_TOKENIZER);
+
+	return `tokenize = '${clause.replaceAll("'", "''")}'`;
+}

package/src/index.ts

@@ -1,7 +1,10 @@
 export { Database } from "./database.js";
 export { generated, type GeneratedPreset } from "./generated.js";
+export { vector } from "./vector.js";
+export { nearest, type NearestOptions } from "./vector-index/nearest.js";
+export { ExtensionLoadError } from "./vector-index/load.js";
 export { JsonParseError } from "./errors.js";
-export { jsonArrayFrom, jsonObjectFrom } from "kysely/helpers/sqlite";
+export { jsonArrayFrom, jsonObjectFrom } from "./json.js";
 export {
 	sql,
 	type Selectable,
@@ -16,6 +19,7 @@ export { ValidationError } from "./validation-error.js";
 export type { DbFieldMeta } from "./env.js";
 export type {
 	DatabaseOptions,
+	DatabaseExtension,
 	SchemaRecord,
 	TablesFromSchemas,
 	InferTableType,
@@ -25,4 +29,14 @@ export type {
 	DatabaseSchema,
 	JsonValidation,
 	SqliteMasterRow,
+	FtsTokenizer,
+	FtsConfig,
+	FtsEntry,
+	FtsFieldRef,
+	FtsRowType,
+	FtsApi,
+	VectorIndexApi,
+	VectorIndexConfig,
+	VectorIndexEntry,
+	VectorMetric,
 } from "./types.js";

package/src/json.ts

@@ -0,0 +1,13 @@
+import type { RawBuilder, Simplify } from "kysely";
+import {
+	jsonArrayFrom as baseJsonArrayFrom,
+	jsonObjectFrom as baseJsonObjectFrom,
+} from "kysely/helpers/sqlite";
+
+export const jsonArrayFrom: <O>(
+	expr: Parameters<typeof baseJsonArrayFrom<O>>[0],
+) => RawBuilder<Simplify<O>[]> = baseJsonArrayFrom as never;
+
+export const jsonObjectFrom: <O>(
+	expr: Parameters<typeof baseJsonObjectFrom<O>>[0],
+) => RawBuilder<Simplify<O> | null> = baseJsonObjectFrom as never;

package/src/migration/diff.ts

@@ -1,4 +1,25 @@
-import type { ColumnSchema, IntrospectedTable, ColumnCopy, MigrationOp } from "./types.js";
+import type { ResolvedFtsPlan } from "../fts/resolve.js";
+import {
+	generateCreateFtsSql,
+	generateFtsTriggers,
+	generateRebuildSelect,
+} from "../fts/sql.js";
+import { tokenizeClause } from "../fts/tokenizer.js";
+import type { ResolvedVectorIndex } from "../vector-index/resolve.js";
+import {
+	generateCreateVecSql,
+	generateRebuildVectorSql,
+	generateVectorTriggers,
+} from "../vector-index/sql.js";
+import type { FtsTokenizer } from "../types.js";
+import type {
+	ColumnSchema,
+	IntrospectedFts,
+	IntrospectedTable,
+	IntrospectedVectorIndex,
+	ColumnCopy,
+	MigrationOp,
+} from "./types.js";
 
 export interface DesiredColumn extends ColumnSchema {
 	addable: boolean;
@@ -26,6 +47,10 @@ export class Differ {
 	constructor(
 		private desired: DesiredTable[],
 		private existing: Map<string, IntrospectedTable>,
+		private desiredFts: ResolvedFtsPlan[] = [],
+		private existingFts: Map<string, IntrospectedFts> = new Map(),
+		private desiredVectorIndexes: ResolvedVectorIndex[] = [],
+		private existingVectorIndexes: Map<string, IntrospectedVectorIndex> = new Map(),
 	) {
 		this.desiredNames = new Set(desired.map((t) => t.name));
 	}
@@ -34,6 +59,8 @@ export class Differ {
 		this.diffTables();
 		this.dropOrphans();
 		this.diffIndexes();
+		this.diffFts();
+		this.diffVectorIndexes();
 		return this.ops;
 	}
 
@@ -217,4 +244,170 @@ export class Differ {
 			}
 		}
 	}
+
+	private diffFts() {
+		const desiredNames = new Set(this.desiredFts.map((p) => p.name));
+
+		for (const plan of this.desiredFts) {
+			const existing = this.existingFts.get(plan.name);
+			const triggers = generateFtsTriggers(plan);
+			const createSql = generateCreateFtsSql(plan);
+			const rebuildSql = generateRebuildSelect(plan);
+
+			if (!existing) {
+				this.ops.push({
+					type: "CreateFts",
+					name: plan.name,
+					createSql,
+					triggers,
+				});
+				this.ops.push({
+					type: "RebuildFts",
+					name: plan.name,
+					rebuildSql,
+				});
+				continue;
+			}
+
+			const desiredCols = plan.fields.map((f) => f.name);
+			const colsMatch =
+				desiredCols.length === existing.columns.length &&
+				desiredCols.every((c, i) => existing.columns[i] === c);
+			const triggersMatch =
+				existing.triggers.size === triggers.length &&
+				triggers.every(
+					(t) => normalizeSql(existing.triggers.get(t.name) ?? "") === normalizeSql(t.sql),
+				);
+			const tokenizerMatch = ftsTokenizerMatches(existing.sql, plan.tokenizer);
+
+			if (colsMatch && triggersMatch && tokenizerMatch) {
+				continue;
+			}
+
+			this.ops.push({
+				type: "DropFts",
+				name: plan.name,
+				triggerNames: [...existing.triggers.keys()],
+			});
+			this.ops.push({
+				type: "CreateFts",
+				name: plan.name,
+				createSql,
+				triggers,
+			});
+			this.ops.push({
+				type: "RebuildFts",
+				name: plan.name,
+				rebuildSql,
+			});
+		}
+
+		for (const [name, existing] of this.existingFts) {
+			if (desiredNames.has(name)) {
+				continue;
+			}
+
+			this.ops.push({
+				type: "DropFts",
+				name,
+				triggerNames: [...existing.triggers.keys()],
+			});
+		}
+	}
+
+	private diffVectorIndexes() {
+		const desiredNames = new Set(this.desiredVectorIndexes.map((p) => p.name));
+
+		for (const plan of this.desiredVectorIndexes) {
+			const existing = this.existingVectorIndexes.get(plan.name);
+			const triggers = generateVectorTriggers(plan);
+			const createSql = generateCreateVecSql(plan);
+			const rebuildSql = generateRebuildVectorSql(plan);
+
+			if (!existing) {
+				this.ops.push({
+					type: "CreateVectorIndex",
+					name: plan.name,
+					createSql,
+					triggers,
+				});
+				this.ops.push({
+					type: "RebuildVectorIndex",
+					name: plan.name,
+					rebuildSql,
+				});
+				continue;
+			}
+
+			const triggersMatch =
+				existing.triggers.size === triggers.length &&
+				triggers.every(
+					(t) => normalizeSql(existing.triggers.get(t.name) ?? "") === normalizeSql(t.sql),
+				);
+			const shapeMatch = vectorShapeMatches(existing.sql, plan);
+
+			if (triggersMatch && shapeMatch) {
+				continue;
+			}
+
+			this.ops.push({
+				type: "DropVectorIndex",
+				name: plan.name,
+				triggerNames: [...existing.triggers.keys()],
+			});
+			this.ops.push({
+				type: "CreateVectorIndex",
+				name: plan.name,
+				createSql,
+				triggers,
+			});
+			this.ops.push({
+				type: "RebuildVectorIndex",
+				name: plan.name,
+				rebuildSql,
+			});
+		}
+
+		for (const [name, existing] of this.existingVectorIndexes) {
+			if (desiredNames.has(name)) {
+				continue;
+			}
+
+			this.ops.push({
+				type: "DropVectorIndex",
+				name,
+				triggerNames: [...existing.triggers.keys()],
+			});
+		}
+	}
+}
+
+function ftsTokenizerMatches(existingSql: string, desired: FtsTokenizer) {
+	const match = existingSql.match(/tokenize\s*=\s*'((?:''|[^'])*)'/i);
+
+	if (!match) {
+		return false;
+	}
+
+	return match[1]!.replaceAll("''", "'") === tokenizeClause(desired);
+}
+
+function vectorShapeMatches(existingSql: string, desired: ResolvedVectorIndex) {
+	const dimMatch = existingSql.match(/float\s*\[\s*(\d+)\s*\]/i);
+	const metricMatch = existingSql.match(/distance_metric\s*=\s*([a-zA-Z0-9_]+)/i);
+	const colMatch = existingSql.match(/"([^"]+)"\s+float\s*\[/i);
+
+	if (!dimMatch || !metricMatch || !colMatch) {
+		return false;
+	}
+
+	return (
+		Number.parseInt(dimMatch[1]!, 10) === desired.dim &&
+		metricMatch[1]!.toLowerCase() === desired.metric &&
+		colMatch[1] === desired.column
+	);
+}
+
+function normalizeSql(sql: string): string {
+	return sql.replaceAll(/\s+/g, " ").trim();
 }

package/src/migration/execute.ts

@@ -73,6 +73,48 @@ export class Executor {
 			case "DropIndex": {
 				return this.db.run(`DROP INDEX "${op.index}"`);
 			}
+			case "CreateFts": {
+				this.db.run(op.createSql);
+
+				for (const trigger of op.triggers) {
+					this.db.run(trigger.sql);
+				}
+
+				return;
+			}
+			case "DropFts": {
+				for (const name of op.triggerNames) {
+					this.db.run(`DROP TRIGGER IF EXISTS "${name}"`);
+				}
+
+				return this.db.run(`DROP TABLE IF EXISTS "${op.name}"`);
+			}
+			case "RebuildFts": {
+				this.db.run(`DELETE FROM "${op.name}"`);
+
+				return this.db.run(op.rebuildSql);
+			}
+			case "CreateVectorIndex": {
+				this.db.run(op.createSql);
+
+				for (const trigger of op.triggers) {
+					this.db.run(trigger.sql);
+				}
+
+				return;
+			}
+			case "DropVectorIndex": {
+				for (const name of op.triggerNames) {
+					this.db.run(`DROP TRIGGER IF EXISTS "${name}"`);
+				}
+
+				return this.db.run(`DROP TABLE IF EXISTS "${op.name}"`);
+			}
+			case "RebuildVectorIndex": {
+				this.db.run(`DELETE FROM "${op.name}"`);
+
+				return this.db.run(op.rebuildSql);
+			}
 		}
 	}
 

package/src/migration/introspect.ts

@@ -1,6 +1,14 @@
 import type { Database } from "bun:sqlite";
 
-import type { IntrospectedColumn, IntrospectedIndex, IntrospectedTable } from "./types.js";
+import { FTS_TRIGGER_PREFIX } from "../fts/sql.js";
+import { VEC_TRIGGER_PREFIX } from "../vector-index/sql.js";
+import type {
+	IntrospectedColumn,
+	IntrospectedFts,
+	IntrospectedIndex,
+	IntrospectedTable,
+	IntrospectedVectorIndex,
+} from "./types.js";
 
 type TableListRow = {
 	name: string;
@@ -12,6 +20,7 @@ type TableXInfoRow = {
 	type: string;
 	notnull: number;
 	dflt_value: string | null;
+	hidden: number;
 };
 
 type IndexListRow = {
@@ -31,18 +40,30 @@ type ForeignKeyListRow = {
 	on_delete: string;
 };
 
+type MasterRow = {
+	name: string;
+	sql: string | null;
+};
+
 export class Introspector {
 	constructor(private db: Database) {}
 
 	introspect() {
 		const tables = new Map<string, IntrospectedTable>();
 		const tableRows = this.db.prepare("PRAGMA table_list").all() as TableListRow[];
+		const virtualPrefixes = tableRows
+			.filter((r) => r.type === "virtual")
+			.map((r) => `${r.name}_`);
 
 		for (const row of tableRows) {
 			if (row.type !== "table" || row.name.startsWith("sqlite_")) {
 				continue;
 			}
 
+			if (virtualPrefixes.some((prefix) => row.name.startsWith(prefix))) {
+				continue;
+			}
+
 			const indexRows = this.db.prepare(`PRAGMA index_list("${row.name}")`).all() as IndexListRow[];
 			const uniqueCols = this.uniqueColumns(indexRows);
 			const fkMap = this.foreignKeys(row.name);
@@ -56,6 +77,117 @@ export class Introspector {
 		return tables;
 	}
 
+	hasVec0Tables() {
+		const row = this.db
+			.prepare(
+				"SELECT 1 FROM sqlite_master WHERE type = 'table' AND sql LIKE '%USING vec0%' LIMIT 1",
+			)
+			.get();
+
+		return row !== null;
+	}
+
+	introspectFts() {
+		const fts = new Map<string, IntrospectedFts>();
+		const tableRows = this.db.prepare("PRAGMA table_list").all() as TableListRow[];
+		const libraryTriggers = this.triggersByPrefix(FTS_TRIGGER_PREFIX);
+
+		for (const row of tableRows) {
+			if (row.type !== "virtual" || row.name.startsWith("sqlite_")) {
+				continue;
+			}
+
+			const master = this.db
+				.prepare(`SELECT name, sql FROM sqlite_master WHERE name = ? AND type = 'table'`)
+				.get(row.name) as MasterRow | null;
+
+			if (!master?.sql || !/USING\s+fts5/i.test(master.sql)) {
+				continue;
+			}
+
+			const colRows = this.db.prepare(`PRAGMA table_xinfo("${row.name}")`).all() as TableXInfoRow[];
+			const columns = colRows.filter((c) => c.hidden !== 1).map((c) => c.name);
+
+			fts.set(row.name, {
+				name: row.name,
+				columns,
+				sql: master.sql,
+				triggers: new Map(),
+			});
+		}
+
+		this.assignTriggersByPrefix(libraryTriggers, FTS_TRIGGER_PREFIX, fts);
+
+		return fts;
+	}
+
+	introspectVectorIndexes() {
+		const indexes = new Map<string, IntrospectedVectorIndex>();
+		const tableRows = this.db.prepare("PRAGMA table_list").all() as TableListRow[];
+		const libraryTriggers = this.triggersByPrefix(VEC_TRIGGER_PREFIX);
+
+		for (const row of tableRows) {
+			if (row.type !== "virtual" || row.name.startsWith("sqlite_")) {
+				continue;
+			}
+
+			const master = this.db
+				.prepare(`SELECT name, sql FROM sqlite_master WHERE name = ? AND type = 'table'`)
+				.get(row.name) as MasterRow | null;
+
+			if (!master?.sql || !/USING\s+vec0/i.test(master.sql)) {
+				continue;
+			}
+
+			indexes.set(row.name, {
+				name: row.name,
+				sql: master.sql,
+				triggers: new Map(),
+			});
+		}
+
+		this.assignTriggersByPrefix(libraryTriggers, VEC_TRIGGER_PREFIX, indexes);
+
+		return indexes;
+	}
+
+	private assignTriggersByPrefix(
+		triggers: Map<string, string>,
+		prefix: string,
+		target: Map<string, { triggers: Map<string, string> }>,
+	) {
+		const sortedNames = [...target.keys()].sort((a, b) => b.length - a.length);
+
+		for (const [triggerName, triggerSql] of triggers) {
+			const withoutPrefix = triggerName.slice(prefix.length);
+
+			for (const name of sortedNames) {
+				if (withoutPrefix.startsWith(`${name}_`)) {
+					target.get(name)!.triggers.set(triggerName, triggerSql);
+					break;
+				}
+			}
+		}
+	}
+
+	private triggersByPrefix(prefix: string) {
+		const rows = this.db
+			.prepare(
+				`SELECT name, sql FROM sqlite_master WHERE type = 'trigger' AND name LIKE '${prefix}%'`,
+			)
+			.all() as MasterRow[];
+
+		const map = new Map<string, string>();
+
+		for (const row of rows) {
+			if (row.sql !== null) {
+				map.set(row.name, row.sql);
+			}
+		}
+
+		return map;
+	}
+
 	private uniqueColumns(indexRows: IndexListRow[]) {
 		const unique = new Set<string>();