@lobomfz/db 0.4.0 → 0.5.0

package/src/fts/resolve.ts

@@ -0,0 +1,208 @@
+import type { FtsEntry, FtsFieldRef, FtsTokenizer } from "../types.js";
+import { DEFAULT_TOKENIZER } from "./tokenizer.js";
+
+export type FtsTableInfo = {
+	columns: Set<string>;
+	pkColumn: string | null;
+	pkIsInteger: boolean;
+	foreignKeys: Array<{
+		column: string;
+		referencedTable: string;
+		referencedColumn: string;
+	}>;
+};
+
+export type ResolvedFtsField = {
+	name: string;
+	sourceTable: string;
+	sourceColumn: string;
+};
+
+export type ResolvedFtsSource = {
+	table: string;
+	fkColumn: string;
+};
+
+export type ResolvedFtsPlan = {
+	name: string;
+	anchor: string;
+	anchorPkColumn: string;
+	fields: ResolvedFtsField[];
+	sources: ResolvedFtsSource[];
+	tokenizer: FtsTokenizer;
+};
+
+function parseFieldRef(name: string, fieldName: string, ref: FtsFieldRef) {
+	const source = typeof ref === "string" ? ref : ref.source;
+	const [sourceTable, sourceColumn, ...rest] = source.split(".");
+
+	if (!sourceTable || !sourceColumn || rest.length > 0) {
+		throw new Error(
+			`FTS "${name}": field "${fieldName}" has invalid reference "${source}", expected "table.column"`,
+		);
+	}
+
+	return {
+		sourceTable,
+		sourceColumn,
+		via: typeof ref === "string" ? undefined : ref.via,
+	};
+}
+
+function parseVia(name: string, fieldName: string, via: string) {
+	const [table, column, ...rest] = via.split(".");
+
+	if (!table || !column || rest.length > 0) {
+		throw new Error(
+			`FTS "${name}": field "${fieldName}" has invalid via "${via}", expected "table.column"`,
+		);
+	}
+
+	return { table, column };
+}
+
+function resolveCrossTableFk(
+	name: string,
+	fieldName: string,
+	sourceTable: string,
+	sourceInfo: FtsTableInfo,
+	anchor: string,
+	anchorPk: string,
+	via: string | undefined,
+) {
+	if (via) {
+		const parsed = parseVia(name, fieldName, via);
+
+		if (parsed.table !== sourceTable) {
+			throw new Error(
+				`FTS "${name}": field "${fieldName}" via "${via}" must reference a column on source table "${sourceTable}"`,
+			);
+		}
+
+		if (!sourceInfo.columns.has(parsed.column)) {
+			throw new Error(
+				`FTS "${name}": field "${fieldName}" via "${via}" column does not exist`,
+			);
+		}
+
+		const fk = sourceInfo.foreignKeys.find((f) => f.column === parsed.column);
+
+		if (!fk || fk.referencedTable !== anchor || fk.referencedColumn !== anchorPk) {
+			throw new Error(
+				`FTS "${name}": field "${fieldName}" via "${via}" does not reference "${anchor}.${anchorPk}"`,
+			);
+		}
+
+		return parsed.column;
+	}
+
+	const matches = sourceInfo.foreignKeys.filter(
+		(f) => f.referencedTable === anchor && f.referencedColumn === anchorPk,
+	);
+
+	if (matches.length === 0) {
+		throw new Error(
+			`FTS "${name}": field "${fieldName}" source table "${sourceTable}" has no foreign key to anchor "${anchor}.${anchorPk}"; add \`via\` to disambiguate`,
+		);
+	}
+
+	if (matches.length > 1) {
+		const cols = matches.map((m) => m.column).join(", ");
+
+		throw new Error(
+			`FTS "${name}": field "${fieldName}" source table "${sourceTable}" has multiple foreign keys to anchor "${anchor}.${anchorPk}" (${cols}); add \`via\` to disambiguate`,
+		);
+	}
+
+	return matches[0]!.column;
+}
+
+export function resolveFtsPlan(
+	name: string,
+	entry: FtsEntry<any>,
+	tables: Map<string, FtsTableInfo>,
+) {
+	const anchor = tables.get(entry.anchor);
+
+	if (!anchor) {
+		throw new Error(
+			`FTS "${name}": anchor table "${entry.anchor}" is not declared in schema`,
+		);
+	}
+
+	if (!anchor.pkColumn) {
+		throw new Error(
+			`FTS "${name}": anchor table "${entry.anchor}" has no primary key; anchor PK must be an integer`,
+		);
+	}
+
+	if (!anchor.pkIsInteger) {
+		throw new Error(
+			`FTS "${name}": anchor table "${entry.anchor}" primary key "${anchor.pkColumn}" must be an integer`,
+		);
+	}
+
+	const fields: ResolvedFtsField[] = [];
+	const sources = new Map<string, ResolvedFtsSource>();
+	const fieldEntries = Object.entries(entry.fields);
+
+	if (fieldEntries.length === 0) {
+		throw new Error(`FTS "${name}": must declare at least one field`);
+	}
+
+	for (const [fieldName, ref] of fieldEntries) {
+		const parsed = parseFieldRef(name, fieldName, ref);
+		const sourceTable = tables.get(parsed.sourceTable);
+
+		if (!sourceTable) {
+			throw new Error(
+				`FTS "${name}": field "${fieldName}" references unknown table "${parsed.sourceTable}"`,
+			);
+		}
+
+		if (!sourceTable.columns.has(parsed.sourceColumn)) {
+			throw new Error(
+				`FTS "${name}": field "${fieldName}" references unknown column "${parsed.sourceTable}.${parsed.sourceColumn}"`,
+			);
+		}
+
+		if (parsed.sourceTable !== entry.anchor) {
+			const fkColumn = resolveCrossTableFk(
+				name,
+				fieldName,
+				parsed.sourceTable,
+				sourceTable,
+				entry.anchor,
+				anchor.pkColumn,
+				parsed.via,
+			);
+
+			const existing = sources.get(parsed.sourceTable);
+
+			if (existing && existing.fkColumn !== fkColumn) {
+				throw new Error(
+					`FTS "${name}": field "${fieldName}" resolves to via "${parsed.sourceTable}.${fkColumn}" but source already bound to "${parsed.sourceTable}.${existing.fkColumn}"`,
+				);
+			}
+
+			if (!existing) {
+				sources.set(parsed.sourceTable, { table: parsed.sourceTable, fkColumn });
+			}
+		}
+
+		fields.push({
+			name: fieldName,
+			sourceTable: parsed.sourceTable,
+			sourceColumn: parsed.sourceColumn,
+		});
+	}
+
+	return {
+		name,
+		anchor: entry.anchor,
+		anchorPkColumn: anchor.pkColumn,
+		fields,
+		sources: [...sources.values()],
+		tokenizer: entry.tokenizer ?? DEFAULT_TOKENIZER,
+	};
+}

package/src/fts/sql.ts

@@ -0,0 +1,148 @@
+import type { ResolvedFtsPlan } from "./resolve.js";
+import { tokenizeOption } from "./tokenizer.js";
+
+export const FTS_TRIGGER_PREFIX = "__fts_sync_";
+
+export type FtsTriggerEvent = "AI" | "AU" | "AD";
+
+export function ftsTriggerName(fts: string, sourceTable: string, event: FtsTriggerEvent) {
+	return `${FTS_TRIGGER_PREFIX}${fts}_${sourceTable}_${event}`;
+}
+
+export type GeneratedTrigger = { name: string; sql: string };
+
+export function generateCreateFtsSql(plan: ResolvedFtsPlan) {
+	const cols = plan.fields.map((f) => `"${f.name}"`).join(", ");
+	const tokenize = tokenizeOption(plan.tokenizer);
+
+	return `CREATE VIRTUAL TABLE "${plan.name}" USING fts5(${cols}, ${tokenize})`;
+}
+
+function insertValueFor(field: ResolvedFtsPlan["fields"][number], anchor: string) {
+	if (field.sourceTable === anchor) {
+		return `COALESCE(NEW."${field.sourceColumn}", '')`;
+	}
+
+	return `''`;
+}
+
+function anchorTriggers(plan: ResolvedFtsPlan): GeneratedTrigger[] {
+	const fts = plan.name;
+	const anchor = plan.anchor;
+	const pk = plan.anchorPkColumn;
+
+	const allCols = plan.fields.map((f) => `"${f.name}"`).join(", ");
+	const insertValues = plan.fields.map((f) => insertValueFor(f, anchor)).join(", ");
+
+	const anchorFields = plan.fields.filter((f) => f.sourceTable === anchor);
+
+	const aiName = ftsTriggerName(fts, anchor, "AI");
+	const adName = ftsTriggerName(fts, anchor, "AD");
+
+	const triggers: GeneratedTrigger[] = [
+		{
+			name: aiName,
+			sql:
+				`CREATE TRIGGER "${aiName}" AFTER INSERT ON "${anchor}" BEGIN ` +
+				`INSERT INTO "${fts}"(rowid, ${allCols}) VALUES (NEW."${pk}", ${insertValues}); ` +
+				`END`,
+		},
+		{
+			name: adName,
+			sql:
+				`CREATE TRIGGER "${adName}" AFTER DELETE ON "${anchor}" BEGIN ` +
+				`DELETE FROM "${fts}" WHERE rowid = OLD."${pk}"; ` +
+				`END`,
+		},
+	];
+
+	if (anchorFields.length > 0) {
+		const trackedCols = anchorFields.map((f) => `"${f.sourceColumn}"`).join(", ");
+		const setClause = anchorFields
+			.map((f) => `"${f.name}" = COALESCE(NEW."${f.sourceColumn}", '')`)
+			.join(", ");
+		const auName = ftsTriggerName(fts, anchor, "AU");
+
+		triggers.push({
+			name: auName,
+			sql:
+				`CREATE TRIGGER "${auName}" AFTER UPDATE OF ${trackedCols} ON "${anchor}" BEGIN ` +
+				`UPDATE "${fts}" SET ${setClause} WHERE rowid = NEW."${pk}"; ` +
+				`END`,
+		});
+	}
+
+	return triggers;
+}
+
+function sourceTriggers(plan: ResolvedFtsPlan): GeneratedTrigger[] {
+	const triggers: GeneratedTrigger[] = [];
+	const fts = plan.name;
+
+	for (const source of plan.sources) {
+		const fields = plan.fields.filter((f) => f.sourceTable === source.table);
+
+		if (fields.length === 0) {
+			continue;
+		}
+
+		const trackedCols = fields.map((f) => `"${f.sourceColumn}"`).join(", ");
+		const setNew = fields
+			.map((f) => `"${f.name}" = COALESCE(NEW."${f.sourceColumn}", '')`)
+			.join(", ");
+		const clearSet = fields.map((f) => `"${f.name}" = ''`).join(", ");
+
+		const aiName = ftsTriggerName(fts, source.table, "AI");
+		const auName = ftsTriggerName(fts, source.table, "AU");
+		const adName = ftsTriggerName(fts, source.table, "AD");
+
+		triggers.push({
+			name: aiName,
+			sql:
+				`CREATE TRIGGER "${aiName}" AFTER INSERT ON "${source.table}" BEGIN ` +
+				`UPDATE "${fts}" SET ${setNew} WHERE rowid = NEW."${source.fkColumn}"; ` +
+				`END`,
+		});
+
+		triggers.push({
+			name: auName,
+			sql:
+				`CREATE TRIGGER "${auName}" AFTER UPDATE OF ${trackedCols} ON "${source.table}" BEGIN ` +
+				`UPDATE "${fts}" SET ${setNew} WHERE rowid = NEW."${source.fkColumn}"; ` +
+				`END`,
+		});
+
+		triggers.push({
+			name: adName,
+			sql:
+				`CREATE TRIGGER "${adName}" AFTER DELETE ON "${source.table}" BEGIN ` +
+				`UPDATE "${fts}" SET ${clearSet} WHERE rowid = OLD."${source.fkColumn}"; ` +
+				`END`,
+		});
+	}
+
+	return triggers;
+}
+
+export function generateFtsTriggers(plan: ResolvedFtsPlan) {
+	return [...anchorTriggers(plan), ...sourceTriggers(plan)];
+}
+
+export function generateRebuildSelect(plan: ResolvedFtsPlan) {
+	const cols = plan.fields.map((f) => `"${f.name}"`).join(", ");
+	const selectExprs = plan.fields
+		.map((f) => `COALESCE("${f.sourceTable}"."${f.sourceColumn}", '')`)
+		.join(", ");
+	const joins = plan.sources
+		.map(
+			(s) =>
+				`LEFT JOIN "${s.table}" ON "${s.table}"."${s.fkColumn}" = "${plan.anchor}"."${plan.anchorPkColumn}"`,
+		)
+		.join(" ");
+
+	const from = joins
+		? `FROM "${plan.anchor}" ${joins}`
+		: `FROM "${plan.anchor}"`;
+
+	return `INSERT INTO "${plan.name}"(rowid, ${cols}) SELECT "${plan.anchor}"."${plan.anchorPkColumn}", ${selectExprs} ${from}`;
+}

package/src/fts/tokenizer.ts

@@ -0,0 +1,27 @@
+import type { FtsTokenizer } from "../types.js";
+
+export const DEFAULT_TOKENIZER: FtsTokenizer = { type: "unicode61", removeDiacritics: 2 };
+
+export function tokenizeClause(tokenizer: FtsTokenizer) {
+	if (tokenizer.type === "trigram") {
+		return "trigram";
+	}
+
+	const parts: string[] = ["unicode61"];
+
+	if (tokenizer.removeDiacritics !== undefined) {
+		parts.push(`remove_diacritics ${tokenizer.removeDiacritics}`);
+	}
+
+	if (tokenizer.categories) {
+		parts.push(`categories '${tokenizer.categories}'`);
+	}
+
+	return parts.join(" ");
+}
+
+export function tokenizeOption(tokenizer: FtsTokenizer | undefined) {
+	const clause = tokenizeClause(tokenizer ?? DEFAULT_TOKENIZER);
+
+	return `tokenize = '${clause.replaceAll("'", "''")}'`;
+}

package/src/index.ts

@@ -1,7 +1,12 @@
 export { Database } from "./database.js";
 export { generated, type GeneratedPreset } from "./generated.js";
+export { vector } from "./vector.js";
+export { nearest, type NearestOptions } from "./vector-index/nearest.js";
+export { ExtensionLoadError } from "./vector-index/load.js";
 export { JsonParseError } from "./errors.js";
-export { jsonArrayFrom, jsonObjectFrom } from "kysely/helpers/sqlite";
+export { DataLossError } from "./migration/data-loss-error.js";
+export { MigrationDriftError } from "./migration/drift-error.js";
+export { jsonArrayFrom, jsonObjectFrom } from "./json.js";
 export {
 	sql,
 	type Selectable,
@@ -15,7 +20,9 @@ export { configure } from "arktype/config";
 export { ValidationError } from "./validation-error.js";
 export type { DbFieldMeta } from "./env.js";
 export type {
+	DataMigration,
 	DatabaseOptions,
+	DatabaseExtension,
 	SchemaRecord,
 	TablesFromSchemas,
 	InferTableType,
@@ -25,4 +32,14 @@ export type {
 	DatabaseSchema,
 	JsonValidation,
 	SqliteMasterRow,
+	FtsTokenizer,
+	FtsConfig,
+	FtsEntry,
+	FtsFieldRef,
+	FtsRowType,
+	FtsApi,
+	VectorIndexApi,
+	VectorIndexConfig,
+	VectorIndexEntry,
+	VectorMetric,
 } from "./types.js";

package/src/json.ts

@@ -0,0 +1,13 @@
+import type { RawBuilder, Simplify } from "kysely";
+import {
+	jsonArrayFrom as baseJsonArrayFrom,
+	jsonObjectFrom as baseJsonObjectFrom,
+} from "kysely/helpers/sqlite";
+
+export const jsonArrayFrom: <O>(
+	expr: Parameters<typeof baseJsonArrayFrom<O>>[0],
+) => RawBuilder<Simplify<O>[]> = baseJsonArrayFrom as never;
+
+export const jsonObjectFrom: <O>(
+	expr: Parameters<typeof baseJsonObjectFrom<O>>[0],
+) => RawBuilder<Simplify<O> | null> = baseJsonObjectFrom as never;

package/src/migration/data-loss-error.ts

@@ -0,0 +1,29 @@
+import type { DestructiveChange } from "./types.js";
+
+export class DataLossError extends Error {
+	constructor(readonly changes: DestructiveChange[]) {
+		super(
+			`Refusing to apply a migration that loses data (failOnDataLoss is on):\n${changes
+				.map(formatChange)
+				.map((line) => `  - ${line}`)
+				.join("\n")}`,
+		);
+		this.name = "DataLossError";
+	}
+}
+
+function formatChange(change: DestructiveChange) {
+	const columns = (change.columns ?? []).map((c) => `"${c}"`).join(", ");
+
+	switch (change.kind) {
+		case "DropTable": {
+			return `drop table "${change.table}"`;
+		}
+		case "DropColumn": {
+			return `drop column ${columns} from "${change.table}"`;
+		}
+		case "TypeChange": {
+			return `change type of column ${columns} in "${change.table}" (discards data)`;
+		}
+	}
+}