@lobomfz/db 0.3.9 → 0.4.1

package/README.md

@@ -5,7 +5,7 @@ SQLite database with Arktype schemas and typed Kysely client for Bun.
 ## Install
 
 ```bash
-bun add @lobomfz/db arktype kysely
+bun add @lobomfz/db
 ```
 
 ## Usage
@@ -65,12 +65,12 @@ type("string").configure({ unique: true }); // UNIQUE
 type("number.integer").configure({ references: "users.id", onDelete: "cascade" }); // FK
 ```
 
-JSON columns are validated against the schema on write by default. To also validate on read, or to disable write validation:
+JSON columns are validated against the schema on write by default. To also validate on read:
 
 ```typescript
 new Database({
 	// ...
-	validation: { onRead: true }, // default: { onRead: false, onWrite: true }
+	validation: { onRead: true }, // default: { onRead: false }
 });
 ```
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lobomfz/db",
-	"version": "0.3.9",
+	"version": "0.4.1",
 	"description": "Bun SQLite database with Arktype schemas and typed Kysely client",
 	"keywords": [
 		"arktype",
@@ -32,15 +32,18 @@
 	},
 	"dependencies": {},
 	"devDependencies": {
-		"@types/bun": "^1.3.11",
-		"@typescript/native-preview": "^7.0.0-dev.20260324.1",
-		"oxfmt": "^0.42.0",
-		"oxlint": "^1.57.0"
+		"@types/bun": "^1.3.12",
+		"@typescript/native-preview": "^7.0.0-dev.20260417.1",
+		"oxfmt": "^0.45.0",
+		"oxlint": "^1.60.0"
 	},
 	"peerDependencies": {
 		"@ark/schema": ">=0.56.0",
 		"@ark/util": ">=0.56.0",
 		"arktype": "^2.1.29",
 		"kysely": "^0.28.14"
+	},
+	"optionalDependencies": {
+		"sqlite-vec": "^0.1.9"
 	}
 }

package/src/database.ts

@@ -5,18 +5,32 @@ import { Kysely } from "kysely";
 
 import { BunSqliteDialect } from "./dialect/dialect.js";
 import type { DbFieldMeta } from "./env.js";
+import { resolveFtsPlan, type FtsTableInfo, type ResolvedFtsPlan } from "./fts/resolve.js";
+import { generateRebuildSelect } from "./fts/sql.js";
 import type { GeneratedPreset } from "./generated.js";
 import { Differ, type DesiredTable } from "./migration/diff.js";
 import { Executor } from "./migration/execute.js";
 import { Introspector } from "./migration/introspect.js";
-import { ResultHydrationPlugin, type ColumnCoercion, type ColumnsMap } from "./plugin.js";
+import { ResultHydrationPlugin } from "./plugin.js";
 import type {
 	DatabaseOptions,
+	DatabasePragmas,
+	FtsApi,
+	FtsConfig,
 	IndexDefinition,
 	SchemaRecord,
 	TablesFromSchemas,
-	DatabasePragmas,
+	VectorIndexApi,
+	VectorIndexConfig,
 } from "./types.js";
+import { loadExtensions } from "./vector-index/load.js";
+import { registerVectorIndex, unregisterVectorIndex } from "./vector-index/nearest.js";
+import {
+	resolveVectorIndex,
+	type ResolvedVectorIndex,
+	type VectorAnchorInfo,
+} from "./vector-index/resolve.js";
+import { generateRebuildVectorSql } from "./vector-index/sql.js";
 import { WriteValidationPlugin } from "./write-validation-plugin.js";
 
 type ArkBranch = {
@@ -25,7 +39,7 @@ type ArkBranch = {
 	unit?: unknown;
 	structure?: unknown;
 	inner?: { divisor?: unknown };
-	meta?: DbFieldMeta & { _generated?: GeneratedPreset };
+	meta?: DbFieldMeta;
 };
 
 type StructureProp = {
@@ -34,7 +48,7 @@ type StructureProp = {
 	value: Type & {
 		branches: ArkBranch[];
 		proto?: unknown;
-		meta: DbFieldMeta & { _generated?: GeneratedPreset };
+		meta: DbFieldMeta;
 	};
 	inner: { default?: unknown };
 };
@@ -49,6 +63,8 @@ type Prop = {
 	isDate?: boolean;
 	isBlob?: boolean;
 	isJson?: boolean;
+	isVector?: boolean;
+	vectorDim?: number;
 	jsonSchema?: Type;
 	meta?: DbFieldMeta;
 	generated?: GeneratedPreset;
@@ -64,35 +80,97 @@ const defaultPragmas: DatabasePragmas = {
 	foreign_keys: true,
 };
 
-export class Database<T extends SchemaRecord> {
+export class Database<
+	T extends SchemaRecord,
+	F extends FtsConfig<T> = FtsConfig<T>,
+	V extends VectorIndexConfig<T> = VectorIndexConfig<T>,
+> {
 	private sqlite: BunDatabase;
+	private ftsPlans = new Map<string, ResolvedFtsPlan>();
+	private vectorIndexPlans = new Map<string, ResolvedVectorIndex>();
 
-	private columns: ColumnsMap = new Map();
-	private tableColumns = new Map<string, Set<string>>();
+	readonly infer: TablesFromSchemas<T, F, V> = undefined as any;
 
-	readonly infer: TablesFromSchemas<T> = undefined as any;
+	readonly kysely: Kysely<TablesFromSchemas<T, F, V>>;
 
-	readonly kysely: Kysely<TablesFromSchemas<T>>;
+	readonly fts: FtsApi<F>;
+
+	readonly vectorIndex: VectorIndexApi<V>;
+
+	constructor(private options: DatabaseOptions<T, F, V>) {
+		const tableSchemas = new Map<string, Type>(Object.entries(options.schema.tables));
+		const writeSchemas = new Map<string, Type>(
+			Object.entries(options.schema.tables).map(([table, schema]) => [
+				table,
+				this.createWriteSchema(schema),
+			]),
+		);
 
-	constructor(private options: DatabaseOptions<T>) {
 		this.sqlite = new BunDatabase(options.path);
 
 		this.applyPragmas();
+		this.assertVectorIndexesHaveExtension();
+		this.loadConfiguredExtensions();
 
 		this.migrate();
 
 		const validation = {
 			onRead: options.validation?.onRead ?? false,
-			onWrite: options.validation?.onWrite ?? true,
 		};
 
-		this.kysely = new Kysely<TablesFromSchemas<T>>({
+		this.kysely = new Kysely<TablesFromSchemas<T, F, V>>({
 			dialect: new BunSqliteDialect({ database: this.sqlite }),
 			plugins: [
-				new WriteValidationPlugin(this.columns, validation),
-				new ResultHydrationPlugin(this.columns, this.tableColumns, validation),
+				new WriteValidationPlugin(writeSchemas),
+				new ResultHydrationPlugin(tableSchemas, validation),
 			],
 		});
+
+		this.fts = {
+			rebuild: (name: string) => this.rebuildFts(name),
+		} as FtsApi<F>;
+
+		this.vectorIndex = {
+			rebuild: (name: string) => this.rebuildVectorIndex(name),
+		} as VectorIndexApi<V>;
+	}
+
+	private loadConfiguredExtensions() {
+		const extensions = this.options.extensions ?? [];
+
+		loadExtensions(this.sqlite, extensions);
+
+		const hasSqliteVec = extensions.some((e) => e.kind === "sqlite-vec");
+
+		if (hasSqliteVec) {
+			return;
+		}
+
+		const probe = new Introspector(this.sqlite);
+
+		if (probe.hasVec0Tables()) {
+			loadExtensions(this.sqlite, [{ kind: "sqlite-vec" }]);
+		}
+	}
+
+	private assertVectorIndexesHaveExtension() {
+		const vectorIndexes = this.options.schema.vectorIndexes;
+
+		if (!vectorIndexes || Object.keys(vectorIndexes).length === 0) {
+			return;
+		}
+
+		const hasSqliteVec = (this.options.extensions ?? []).some((e) => e.kind === "sqlite-vec");
+
+		if (hasSqliteVec) {
+			return;
+		}
+
+		const [first] = Object.keys(vectorIndexes);
+
+		throw new Error(
+			`vectorIndex "${first}" requires the sqlite-vec extension. Add { kind: "sqlite-vec" } to DatabaseOptions.extensions.`,
+		);
 	}
 
 	private applyPragmas() {
@@ -123,7 +201,19 @@ export class Database<T extends SchemaRecord> {
 
 		const branchMeta = v.branches.find((b) => b.meta && Object.keys(b.meta).length > 0)?.meta;
 		const meta = { ...branchMeta, ...v.meta };
-		const generated = meta._generated;
+		const generated = meta._generated === "vector" ? undefined : meta._generated;
+
+		if (meta._generated === "vector") {
+			return {
+				key,
+				kind,
+				nullable,
+				isVector: true,
+				vectorDim: meta._vectorDim,
+				meta,
+				defaultValue,
+			};
+		}
 
 		if (v.proto === Date || concrete.some((b) => b.proto === Date)) {
 			return { key, kind, nullable, isDate: true, generated, defaultValue };
@@ -177,7 +267,7 @@ export class Database<T extends SchemaRecord> {
 			return "TEXT";
 		}
 
-		if (prop.isBlob) {
+		if (prop.isBlob || prop.isVector) {
 			return "BLOB";
 		}
 
@@ -210,7 +300,7 @@ export class Database<T extends SchemaRecord> {
 
 	private defaultClause(prop: Prop) {
 		if (prop.generated === "now") {
-			return "DEFAULT (unixepoch())";
+			return "DEFAULT (CAST(unixepoch('subsec') * 1000 AS INTEGER))";
 		}
 
 		if (prop.defaultValue === undefined || prop.generated === "autoincrement") {
@@ -293,30 +383,16 @@ export class Database<T extends SchemaRecord> {
 		return structureProps.map((p) => this.normalizeProp(p, schema));
 	}
 
-	private registerColumns(tableName: string, props: Prop[]) {
-		this.tableColumns.set(tableName, new Set(props.map((p) => p.key)));
-
-		const colMap = new Map<string, ColumnCoercion>();
-
-		for (const prop of props) {
-			if (prop.isBoolean) {
-				colMap.set(prop.key, "boolean");
-				continue;
-			}
-
-			if (prop.isDate) {
-				colMap.set(prop.key, "date");
-				continue;
-			}
+	private createWriteSchema(schema: Type) {
+		const autoIncrementColumns = this.parseSchemaProps(schema)
+			.filter((prop) => prop.generated === "autoincrement")
+			.map((prop) => prop.key);
 
-			if (prop.isJson && prop.jsonSchema) {
-				colMap.set(prop.key, { type: "json", schema: prop.jsonSchema });
-			}
+		if (autoIncrementColumns.length === 0) {
+			return schema;
 		}
 
-		if (colMap.size > 0) {
-			this.columns.set(tableName, colMap);
-		}
+		return (schema as any).omit(...autoIncrementColumns) as Type;
 	}
 
 	private generateCreateTableSQL(tableName: string, props: Prop[]) {
@@ -338,13 +414,13 @@ export class Database<T extends SchemaRecord> {
 
 	private migrate() {
 		const desiredTables: DesiredTable[] = [];
+		const tableInfos = new Map<string, FtsTableInfo>();
+		const vectorAnchors = new Map<string, VectorAnchorInfo>();
 		const schemaIndexes = this.options.schema.indexes;
 
 		for (const [name, schema] of Object.entries(this.options.schema.tables)) {
 			const props = this.parseSchemaProps(schema);
 
-			this.registerColumns(name, props);
-
 			const columns = props.map((prop) => {
 				const isNotNull = this.columnConstraint(prop) === "NOT NULL";
 				const defaultClause = this.defaultClause(prop);
@@ -377,12 +453,129 @@ export class Database<T extends SchemaRecord> {
 				columns,
 				indexes,
 			});
+
+			tableInfos.set(name, this.buildTableInfo(props));
+			vectorAnchors.set(name, this.buildVectorAnchorInfo(props));
 		}
 
-		const existing = new Introspector(this.sqlite).introspect();
-		const ops = new Differ(desiredTables, existing).diff();
+		const desiredFts: ResolvedFtsPlan[] = [];
+
+		if (this.options.schema.fts) {
+			for (const [name, entry] of Object.entries(this.options.schema.fts)) {
+				const plan = resolveFtsPlan(name, entry, tableInfos);
+				desiredFts.push(plan);
+				this.ftsPlans.set(name, plan);
+			}
+		}
+
+		const desiredVectorIndexes: ResolvedVectorIndex[] = [];
+
+		if (this.options.schema.vectorIndexes) {
+			for (const [name, entry] of Object.entries(this.options.schema.vectorIndexes)) {
+				const plan = resolveVectorIndex(name, entry as any, vectorAnchors);
+				desiredVectorIndexes.push(plan);
+				this.vectorIndexPlans.set(name, plan);
+				registerVectorIndex(plan);
+			}
+		}
+
+		const introspector = new Introspector(this.sqlite);
+		const existing = introspector.introspect();
+		const existingFts = introspector.introspectFts();
+		const existingVectorIndexes = introspector.introspectVectorIndexes();
+
+		const ops = new Differ(
+			desiredTables,
+			existing,
+			desiredFts,
+			existingFts,
+			desiredVectorIndexes,
+			existingVectorIndexes,
+		).diff();
 
 		new Executor(this.sqlite, ops).execute();
+
+		for (const name of existingVectorIndexes.keys()) {
+			if (!this.vectorIndexPlans.has(name)) {
+				unregisterVectorIndex(name);
+			}
+		}
+	}
+
+	private rebuildFts(name: string) {
+		const plan = this.ftsPlans.get(name);
+
+		if (!plan) {
+			throw new Error(`FTS "${name}" is not declared in the schema`);
+		}
+
+		this.sqlite.transaction(() => {
+			this.sqlite.run(`DELETE FROM "${name}"`);
+			this.sqlite.run(generateRebuildSelect(plan));
+		})();
+	}
+
+	private rebuildVectorIndex(name: string) {
+		const plan = this.vectorIndexPlans.get(name);
+
+		if (!plan) {
+			throw new Error(`vector index "${name}" is not declared in the schema`);
+		}
+
+		this.sqlite.transaction(() => {
+			this.sqlite.run(`DELETE FROM "${name}"`);
+			this.sqlite.run(generateRebuildVectorSql(plan));
+		})();
+	}
+
+	private buildTableInfo(props: Prop[]): FtsTableInfo {
+		const pkProp = props.find(
+			(p) => p.generated === "autoincrement" || p.meta?.primaryKey === true,
+		);
+
+		return {
+			columns: new Set(props.map((p) => p.key)),
+			pkColumn: pkProp?.key ?? null,
+			pkIsInteger: pkProp ? this.sqlType(pkProp) === "INTEGER" : false,
+			foreignKeys: props.flatMap((p) => {
+				const ref = p.meta?.references;
+
+				if (!ref) {
+					return [];
+				}
+
+				const [referencedTable, referencedColumn] = ref.split(".");
+
+				if (!referencedTable || !referencedColumn) {
+					throw new Error(
+						`Invalid references format "${ref}" for column "${p.key}": expected "table.column"`,
+					);
+				}
+
+				return [{ column: p.key, referencedTable, referencedColumn }];
+			}),
+		};
+	}
+
+	private buildVectorAnchorInfo(props: Prop[]): VectorAnchorInfo {
+		const pkProp = props.find(
+			(p) => p.generated === "autoincrement" || p.meta?.primaryKey === true,
+		);
+
+		const vectorColumns = new Map<string, number>();
+
+		for (const p of props) {
+			if (p.isVector && typeof p.vectorDim === "number") {
+				vectorColumns.set(p.key, p.vectorDim);
+			}
+		}
+
+		return {
+			columns: new Set(props.map((p) => p.key)),
+			pkColumn: pkProp?.key ?? null,
+			pkIsInteger: pkProp ? this.sqlType(pkProp) === "INTEGER" : false,
+			vectorColumns,
+		};
 	}
 
 	private generateIndexName(tableName: string, columns: string[], unique: boolean) {

package/src/dialect/serialize.ts

@@ -1,6 +1,6 @@
 export function serializeParam(p: unknown): unknown {
 	if (p instanceof Date) {
-		return Math.floor(p.getTime() / 1000);
+		return p.getTime();
 	}
 
 	if (typeof p === "object" && p !== null && !ArrayBuffer.isView(p)) {

package/src/env.ts

@@ -3,7 +3,8 @@ export type DbFieldMeta = {
 	unique?: boolean;
 	references?: `${string}.${string}`;
 	onDelete?: "cascade" | "set null" | "restrict";
-	_generated?: "autoincrement" | "now";
+	_generated?: "autoincrement" | "now" | "vector";
+	_vectorDim?: number;
 };
 
 declare global {

package/src/fts/resolve.ts

@@ -0,0 +1,208 @@
+import type { FtsEntry, FtsFieldRef, FtsTokenizer } from "../types.js";
+import { DEFAULT_TOKENIZER } from "./tokenizer.js";
+
+export type FtsTableInfo = {
+	columns: Set<string>;
+	pkColumn: string | null;
+	pkIsInteger: boolean;
+	foreignKeys: Array<{
+		column: string;
+		referencedTable: string;
+		referencedColumn: string;
+	}>;
+};
+
+export type ResolvedFtsField = {
+	name: string;
+	sourceTable: string;
+	sourceColumn: string;
+};
+
+export type ResolvedFtsSource = {
+	table: string;
+	fkColumn: string;
+};
+
+export type ResolvedFtsPlan = {
+	name: string;
+	anchor: string;
+	anchorPkColumn: string;
+	fields: ResolvedFtsField[];
+	sources: ResolvedFtsSource[];
+	tokenizer: FtsTokenizer;
+};
+
+function parseFieldRef(name: string, fieldName: string, ref: FtsFieldRef) {
+	const source = typeof ref === "string" ? ref : ref.source;
+	const [sourceTable, sourceColumn, ...rest] = source.split(".");
+
+	if (!sourceTable || !sourceColumn || rest.length > 0) {
+		throw new Error(
+			`FTS "${name}": field "${fieldName}" has invalid reference "${source}", expected "table.column"`,
+		);
+	}
+
+	return {
+		sourceTable,
+		sourceColumn,
+		via: typeof ref === "string" ? undefined : ref.via,
+	};
+}
+
+function parseVia(name: string, fieldName: string, via: string) {
+	const [table, column, ...rest] = via.split(".");
+
+	if (!table || !column || rest.length > 0) {
+		throw new Error(
+			`FTS "${name}": field "${fieldName}" has invalid via "${via}", expected "table.column"`,
+		);
+	}
+
+	return { table, column };
+}
+
+function resolveCrossTableFk(
+	name: string,
+	fieldName: string,
+	sourceTable: string,
+	sourceInfo: FtsTableInfo,
+	anchor: string,
+	anchorPk: string,
+	via: string | undefined,
+) {
+	if (via) {
+		const parsed = parseVia(name, fieldName, via);
+
+		if (parsed.table !== sourceTable) {
+			throw new Error(
+				`FTS "${name}": field "${fieldName}" via "${via}" must reference a column on source table "${sourceTable}"`,
+			);
+		}
+
+		if (!sourceInfo.columns.has(parsed.column)) {
+			throw new Error(
+				`FTS "${name}": field "${fieldName}" via "${via}" column does not exist`,
+			);
+		}
+
+		const fk = sourceInfo.foreignKeys.find((f) => f.column === parsed.column);
+
+		if (!fk || fk.referencedTable !== anchor || fk.referencedColumn !== anchorPk) {
+			throw new Error(
+				`FTS "${name}": field "${fieldName}" via "${via}" does not reference "${anchor}.${anchorPk}"`,
+			);
+		}
+
+		return parsed.column;
+	}
+
+	const matches = sourceInfo.foreignKeys.filter(
+		(f) => f.referencedTable === anchor && f.referencedColumn === anchorPk,
+	);
+
+	if (matches.length === 0) {
+		throw new Error(
+			`FTS "${name}": field "${fieldName}" source table "${sourceTable}" has no foreign key to anchor "${anchor}.${anchorPk}"; add \`via\` to disambiguate`,
+		);
+	}
+
+	if (matches.length > 1) {
+		const cols = matches.map((m) => m.column).join(", ");
+
+		throw new Error(
+			`FTS "${name}": field "${fieldName}" source table "${sourceTable}" has multiple foreign keys to anchor "${anchor}.${anchorPk}" (${cols}); add \`via\` to disambiguate`,
+		);
+	}
+
+	return matches[0]!.column;
+}
+
+export function resolveFtsPlan(
+	name: string,
+	entry: FtsEntry<any>,
+	tables: Map<string, FtsTableInfo>,
+) {
+	const anchor = tables.get(entry.anchor);
+
+	if (!anchor) {
+		throw new Error(
+			`FTS "${name}": anchor table "${entry.anchor}" is not declared in schema`,
+		);
+	}
+
+	if (!anchor.pkColumn) {
+		throw new Error(
+			`FTS "${name}": anchor table "${entry.anchor}" has no primary key; anchor PK must be an integer`,
+		);
+	}
+
+	if (!anchor.pkIsInteger) {
+		throw new Error(
+			`FTS "${name}": anchor table "${entry.anchor}" primary key "${anchor.pkColumn}" must be an integer`,
+		);
+	}
+
+	const fields: ResolvedFtsField[] = [];
+	const sources = new Map<string, ResolvedFtsSource>();
+	const fieldEntries = Object.entries(entry.fields);
+
+	if (fieldEntries.length === 0) {
+		throw new Error(`FTS "${name}": must declare at least one field`);
+	}
+
+	for (const [fieldName, ref] of fieldEntries) {
+		const parsed = parseFieldRef(name, fieldName, ref);
+		const sourceTable = tables.get(parsed.sourceTable);
+
+		if (!sourceTable) {
+			throw new Error(
+				`FTS "${name}": field "${fieldName}" references unknown table "${parsed.sourceTable}"`,
+			);
+		}
+
+		if (!sourceTable.columns.has(parsed.sourceColumn)) {
+			throw new Error(
+				`FTS "${name}": field "${fieldName}" references unknown column "${parsed.sourceTable}.${parsed.sourceColumn}"`,
+			);
+		}
+
+		if (parsed.sourceTable !== entry.anchor) {
+			const fkColumn = resolveCrossTableFk(
+				name,
+				fieldName,
+				parsed.sourceTable,
+				sourceTable,
+				entry.anchor,
+				anchor.pkColumn,
+				parsed.via,
+			);
+
+			const existing = sources.get(parsed.sourceTable);
+
+			if (existing && existing.fkColumn !== fkColumn) {
+				throw new Error(
+					`FTS "${name}": field "${fieldName}" resolves to via "${parsed.sourceTable}.${fkColumn}" but source already bound to "${parsed.sourceTable}.${existing.fkColumn}"`,
+				);
+			}
+
+			if (!existing) {
+				sources.set(parsed.sourceTable, { table: parsed.sourceTable, fkColumn });
+			}
+		}
+
+		fields.push({
+			name: fieldName,
+			sourceTable: parsed.sourceTable,
+			sourceColumn: parsed.sourceColumn,
+		});
+	}
+
+	return {
+		name,
+		anchor: entry.anchor,
+		anchorPkColumn: anchor.pkColumn,
+		fields,
+		sources: [...sources.values()],
+		tokenizer: entry.tokenizer ?? DEFAULT_TOKENIZER,
+	};
+}