@lobomfz/db 0.3.9 → 0.4.0

package/README.md

@@ -5,7 +5,7 @@ SQLite database with Arktype schemas and typed Kysely client for Bun.
 ## Install
 
 ```bash
-bun add @lobomfz/db arktype kysely
+bun add @lobomfz/db
 ```
 
 ## Usage
@@ -65,12 +65,12 @@ type("string").configure({ unique: true }); // UNIQUE
 type("number.integer").configure({ references: "users.id", onDelete: "cascade" }); // FK
 ```
 
-JSON columns are validated against the schema on write by default. To also validate on read, or to disable write validation:
+JSON columns are validated against the schema on write by default. To also validate on read:
 
 ```typescript
 new Database({
 	// ...
-	validation: { onRead: true }, // default: { onRead: false, onWrite: true }
+	validation: { onRead: true }, // default: { onRead: false }
 });
 ```
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lobomfz/db",
-	"version": "0.3.9",
+	"version": "0.4.0",
 	"description": "Bun SQLite database with Arktype schemas and typed Kysely client",
 	"keywords": [
 		"arktype",

package/src/database.ts

@@ -9,7 +9,7 @@ import type { GeneratedPreset } from "./generated.js";
 import { Differ, type DesiredTable } from "./migration/diff.js";
 import { Executor } from "./migration/execute.js";
 import { Introspector } from "./migration/introspect.js";
-import { ResultHydrationPlugin, type ColumnCoercion, type ColumnsMap } from "./plugin.js";
+import { ResultHydrationPlugin } from "./plugin.js";
 import type {
 	DatabaseOptions,
 	IndexDefinition,
@@ -67,14 +67,19 @@ const defaultPragmas: DatabasePragmas = {
 export class Database<T extends SchemaRecord> {
 	private sqlite: BunDatabase;
 
-	private columns: ColumnsMap = new Map();
-	private tableColumns = new Map<string, Set<string>>();
-
 	readonly infer: TablesFromSchemas<T> = undefined as any;
 
 	readonly kysely: Kysely<TablesFromSchemas<T>>;
 
 	constructor(private options: DatabaseOptions<T>) {
+		const tableSchemas = new Map<string, Type>(Object.entries(options.schema.tables));
+		const writeSchemas = new Map<string, Type>(
+			Object.entries(options.schema.tables).map(([table, schema]) => [
+				table,
+				this.createWriteSchema(schema),
+			]),
+		);
+
 		this.sqlite = new BunDatabase(options.path);
 
 		this.applyPragmas();
@@ -83,14 +88,13 @@ export class Database<T extends SchemaRecord> {
 
 		const validation = {
 			onRead: options.validation?.onRead ?? false,
-			onWrite: options.validation?.onWrite ?? true,
 		};
 
 		this.kysely = new Kysely<TablesFromSchemas<T>>({
 			dialect: new BunSqliteDialect({ database: this.sqlite }),
 			plugins: [
-				new WriteValidationPlugin(this.columns, validation),
-				new ResultHydrationPlugin(this.columns, this.tableColumns, validation),
+				new WriteValidationPlugin(writeSchemas),
+				new ResultHydrationPlugin(tableSchemas, validation),
 			],
 		});
 	}
@@ -293,30 +297,16 @@ export class Database<T extends SchemaRecord> {
 		return structureProps.map((p) => this.normalizeProp(p, schema));
 	}
 
-	private registerColumns(tableName: string, props: Prop[]) {
-		this.tableColumns.set(tableName, new Set(props.map((p) => p.key)));
-
-		const colMap = new Map<string, ColumnCoercion>();
-
-		for (const prop of props) {
-			if (prop.isBoolean) {
-				colMap.set(prop.key, "boolean");
-				continue;
-			}
-
-			if (prop.isDate) {
-				colMap.set(prop.key, "date");
-				continue;
-			}
+	private createWriteSchema(schema: Type) {
+		const autoIncrementColumns = this.parseSchemaProps(schema)
+			.filter((prop) => prop.generated === "autoincrement")
+			.map((prop) => prop.key);
 
-			if (prop.isJson && prop.jsonSchema) {
-				colMap.set(prop.key, { type: "json", schema: prop.jsonSchema });
-			}
+		if (autoIncrementColumns.length === 0) {
+			return schema;
 		}
 
-		if (colMap.size > 0) {
-			this.columns.set(tableName, colMap);
-		}
+		return (schema as any).omit(...autoIncrementColumns) as Type;
 	}
 
 	private generateCreateTableSQL(tableName: string, props: Prop[]) {
@@ -343,8 +333,6 @@ export class Database<T extends SchemaRecord> {
 		for (const [name, schema] of Object.entries(this.options.schema.tables)) {
 			const props = this.parseSchemaProps(schema);
 
-			this.registerColumns(name, props);
-
 			const columns = props.map((prop) => {
 				const isNotNull = this.columnConstraint(prop) === "NOT NULL";
 				const defaultClause = this.defaultClause(prop);

package/src/generated.ts

@@ -7,7 +7,7 @@ const generatedTypes = {
 	now: () =>
 		type("Date")
 			.configure({ _generated: "now" })
-			.default(() => new Date(0)),
+			.default(() => new Date()),
 };
 
 export function generated<P extends GeneratedPreset>(

package/src/index.ts

@@ -12,7 +12,7 @@ export {
 } from "kysely";
 export { type, type Type } from "arktype";
 export { configure } from "arktype/config";
-export { JsonValidationError } from "./validation-error.js";
+export { ValidationError } from "./validation-error.js";
 export type { DbFieldMeta } from "./env.js";
 export type {
 	DatabaseOptions,

package/src/plugin.ts

@@ -18,10 +18,10 @@ import {
 } from "kysely";
 
 import { JsonParseError } from "./errors.js";
-import { JsonValidationError } from "./validation-error.js";
+import type { StructureProp } from "./types.js";
+import { ValidationError } from "./validation-error.js";
 
-export type ColumnCoercion = "boolean" | "date" | { type: "json"; schema: Type };
-export type ColumnsMap = Map<string, Map<string, ColumnCoercion>>;
+type ColumnCoercion = "boolean" | "date" | { type: "json"; schema: Type };
 
 type ResolvedCoercion = { table: string; coercion: ColumnCoercion };
 
@@ -79,13 +79,70 @@ const jsonObjectFromFragments = [
 const typePreservingAggregateFunctions = new Set(["max", "min"]);
 
 export class ResultHydrationPlugin implements KyselyPlugin {
+	private columns = new Map<string, Map<string, ColumnCoercion>>();
+	private tableColumns = new Map<string, Set<string>>();
 	private queryPlans = new WeakMap<QueryId, QueryPlan>();
 
 	constructor(
-		private columns: ColumnsMap,
-		private tableColumns: Map<string, Set<string>>,
+		schemas: Map<string, Type>,
 		private validation: { onRead: boolean },
-	) {}
+	) {
+		this.registerSchemas(schemas);
+	}
+
+	private registerSchemas(schemas: Map<string, Type>) {
+		for (const [table, schema] of schemas) {
+			const structureProps = (schema as any).structure?.props as StructureProp[] | undefined;
+
+			if (!structureProps) {
+				continue;
+			}
+
+			this.tableColumns.set(
+				table,
+				new Set(structureProps.map((prop) => prop.key)),
+			);
+
+			const columns = new Map<string, ColumnCoercion>();
+
+			for (const prop of structureProps) {
+				const coercion = this.getColumnCoercion(prop, schema);
+
+				if (!coercion) {
+					continue;
+				}
+
+				columns.set(prop.key, coercion);
+			}
+
+			if (columns.size > 0) {
+				this.columns.set(table, columns);
+			}
+		}
+	}
+
+	private getColumnCoercion(prop: StructureProp, parentSchema: Type) {
+		const concrete = prop.value.branches.filter(
+			(branch) => branch.unit !== null && branch.domain !== "undefined",
+		);
+
+		if (prop.value.proto === Date || concrete.some((branch) => branch.proto === Date)) {
+			return "date" satisfies ColumnCoercion;
+		}
+
+		if (concrete.length > 0 && concrete.every((branch) => branch.domain === "boolean")) {
+			return "boolean" satisfies ColumnCoercion;
+		}
+
+		if (concrete.some((branch) => !!branch.structure)) {
+			return {
+				type: "json",
+				schema: (parentSchema as any).get(prop.key) as Type,
+			} satisfies ColumnCoercion;
+		}
+
+		return null;
+	}
 
 	transformQuery: KyselyPlugin["transformQuery"] = (args) => {
 		this.queryPlans.set(args.queryId, {
@@ -141,7 +198,7 @@ export class ResultHydrationPlugin implements KyselyPlugin {
 		const result = schema(value);
 
 		if (result instanceof type.errors) {
-			throw new JsonValidationError(table, col, result.summary);
+			throw new ValidationError(table, result.summary, col);
 		}
 	}
 

package/src/types.ts

@@ -1,6 +1,23 @@
 import type { Generated } from "kysely";
 import type { Type } from "arktype";
 
+export type ArkBranch = {
+	domain?: string;
+	proto?: unknown;
+	unit?: unknown;
+	structure?: unknown;
+};
+
+export type StructureProp = {
+	key: string;
+	required: boolean;
+	inner: { default?: unknown };
+	value: Type & {
+		branches: ArkBranch[];
+		proto?: unknown;
+	};
+};
+
 type ExtractInput<T> = T extends { inferIn: infer I } ? I : never;
 type ExtractOutput<T> = T extends { infer: infer O } ? O : never;
 
@@ -60,7 +77,6 @@ export type DatabaseSchema<T extends SchemaRecord> = {
 
 export type JsonValidation = {
 	onRead?: boolean;
-	onWrite?: boolean;
 };
 
 export type DatabaseOptions<T extends SchemaRecord> = {

package/src/validation-error.ts

@@ -1,11 +1,15 @@
-export class JsonValidationError extends Error {
+export class ValidationError extends Error {
 	constructor(
 		readonly table: string,
-		readonly column: string,
 		readonly summary: string,
+		readonly column: string | null = null,
 	) {
-		super(`JSON validation failed for ${table}.${column}: ${summary}`);
+		super(
+			column
+				? `Validation failed for ${table}.${column}: ${summary}`
+				: `Validation failed for ${table}: ${summary}`,
+		);
 
-		this.name = "JsonValidationError";
+		this.name = "ValidationError";
 	}
 }

package/src/write-validation-plugin.ts

@@ -1,35 +1,84 @@
 import { type, type Type } from "arktype";
 import {
 	type KyselyPlugin,
+	type InsertQueryNode as KyselyInsertQueryNode,
+	type UpdateQueryNode as KyselyUpdateQueryNode,
+	type OnConflictNode as KyselyOnConflictNode,
+	type ColumnUpdateNode as KyselyColumnUpdateNode,
+	type ValuesItemNode,
+	type OperationNode,
 	type RootOperationNode,
 	ColumnNode,
+	ColumnUpdateNode,
 	DefaultInsertValueNode,
+	InsertQueryNode,
+	OnConflictNode,
+	PrimitiveValueListNode,
+	ReferenceNode,
 	TableNode,
+	ValueListNode,
 	ValueNode,
 	ValuesNode,
 } from "kysely";
 
-import type { ColumnsMap } from "./plugin.js";
-import { JsonValidationError } from "./validation-error.js";
+import type { StructureProp } from "./types.js";
+import { ValidationError } from "./validation-error.js";
+
+type TableWriteSchema = {
+	schema: Type;
+	columns: string[];
+	columnSet: Set<string>;
+	optionalNonNullColumns: Set<string>;
+	insertNullColumns: Set<string>;
+};
+
+type InsertRow = {
+	values: Record<string, unknown>;
+	passthrough: Map<string, OperationNode>;
+};
 
 export class WriteValidationPlugin implements KyselyPlugin {
-	constructor(
-		private columns: ColumnsMap,
-		private validation: { onWrite: boolean },
-	) {}
+	private schemas = new Map<string, TableWriteSchema>();
 
-	transformQuery: KyselyPlugin["transformQuery"] = (args) => {
-		if (this.validation.onWrite) {
-			this.validateWriteNode(args.node);
-		}
+	constructor(schemas: Map<string, Type>) {
+		this.registerSchemas(schemas);
+	}
 
-		return args.node;
+	transformQuery: KyselyPlugin["transformQuery"] = (args) => {
+		return this.transformWriteNode(args.node);
 	};
 
 	transformResult: KyselyPlugin["transformResult"] = async (args) => {
 		return args.result;
 	};
 
+	private registerSchemas(schemas: Map<string, Type>) {
+		for (const [table, schema] of schemas) {
+			const structureProps = (schema as any).structure?.props as StructureProp[] | undefined;
+			const columns = structureProps?.map((prop) => prop.key) ?? [];
+
+			this.schemas.set(table, {
+				schema,
+				columns,
+				columnSet: new Set(columns),
+				optionalNonNullColumns: new Set(
+					structureProps
+						?.filter((prop) => !prop.required && !this.acceptsNull(prop.value))
+						.map((prop) => prop.key) ?? [],
+				),
+				insertNullColumns: new Set(
+					structureProps
+						?.filter((prop) => this.acceptsNull(prop.value) && prop.inner.default === undefined)
+						.map((prop) => prop.key) ?? [],
+				),
+			});
+		}
+	}
+
+	private acceptsNull(field: StructureProp["value"]) {
+		return field.branches.some((branch) => branch.unit === null || branch.domain === "null");
+	}
+
 	private getTableFromNode(node: RootOperationNode) {
 		switch (node.kind) {
 			case "InsertQueryNode":
@@ -48,90 +97,316 @@ export class WriteValidationPlugin implements KyselyPlugin {
 		}
 	}
 
-	private validateJsonValue(table: string, col: string, value: unknown, schema: Type) {
-		if (value === null || value === undefined) {
-			return;
+	private firstErrorColumn(result: { [index: number]: { path: ArrayLike<unknown> } | undefined }) {
+		const column = result[0]?.path[0];
+
+		if (typeof column === "string") {
+			return column;
 		}
 
+		return null;
+	}
+
+	private morph(table: string, schema: Type, value: Record<string, unknown>) {
 		const result = schema(value);
 
 		if (result instanceof type.errors) {
-			throw new JsonValidationError(table, col, result.summary);
+			throw new ValidationError(table, result.summary, this.firstErrorColumn(result));
 		}
+
+		return result as Record<string, unknown>;
+	}
+
+	private pickSchema(schema: Type, columns: Iterable<string>) {
+		return (schema as any).pick(...columns) as Type;
 	}
 
-	private validateWriteNode(node: RootOperationNode) {
-		if (node.kind !== "InsertQueryNode" && node.kind !== "UpdateQueryNode") {
-			return;
+	private stripNullOptionalFields(tableSchema: TableWriteSchema, value: Record<string, unknown>) {
+		const stripped = { ...value };
+
+		for (const column of tableSchema.optionalNonNullColumns) {
+			if (stripped[column] === null) {
+				delete stripped[column];
+			}
 		}
 
+		return stripped;
+	}
+
+	private prepareInsertValue(tableSchema: TableWriteSchema, value: Record<string, unknown>) {
+		const prepared = this.stripNullOptionalFields(tableSchema, value);
+
+		for (const column of tableSchema.insertNullColumns) {
+			if (!Object.prototype.hasOwnProperty.call(prepared, column)) {
+				prepared[column] = null;
+			}
+		}
+
+		return prepared;
+	}
+
+	private transformWriteNode(node: RootOperationNode) {
 		const table = this.getTableFromNode(node);
 
 		if (!table) {
-			return;
+			return node;
+		}
+
+		if (node.kind === "InsertQueryNode") {
+			return this.transformInsert(node, table);
 		}
 
-		const cols = this.columns.get(table);
+		if (node.kind === "UpdateQueryNode") {
+			return this.transformUpdate(node, table);
+		}
+
+		return node;
+	}
+
+	private getInsertRow(columns: string[], valueList: ValuesItemNode) {
+		const row: InsertRow = {
+			values: {},
+			passthrough: new Map(),
+		};
+
+		if (PrimitiveValueListNode.is(valueList)) {
+			for (let i = 0; i < columns.length; i++) {
+				row.values[columns[i]!] = valueList.values[i];
+			}
 
-		if (!cols) {
-			return;
+			return row;
 		}
 
-		for (const [col, value] of this.writeValues(node)) {
-			const coercion = cols.get(col);
+		if (!ValueListNode.is(valueList)) {
+			return null;
+		}
 
-			if (!coercion || typeof coercion === "string") {
+		for (let i = 0; i < columns.length; i++) {
+			const column = columns[i]!;
+			const value = valueList.values[i];
+
+			if (!value || DefaultInsertValueNode.is(value)) {
 				continue;
 			}
 
-			this.validateJsonValue(table, col, value, coercion.schema);
+			if (ValueNode.is(value)) {
+				row.values[column] = value.value;
+				continue;
+			}
+
+			row.passthrough.set(column, value);
 		}
+
+		return row;
 	}
 
-	private *writeValues(node: RootOperationNode) {
-		if (node.kind === "InsertQueryNode") {
-			const columns = node.columns?.map((column) => column.column.name);
+	private morphInsertRow(table: string, tableSchema: TableWriteSchema, row: InsertRow) {
+		const schemaLiteralValues = Object.fromEntries(
+			Object.entries(row.values).filter(([column]) => tableSchema.columnSet.has(column)),
+		);
+		const nonSchemaLiteralValues = Object.fromEntries(
+			Object.entries(row.values).filter(([column]) => !tableSchema.columnSet.has(column)),
+		);
+		const schemaLiteralColumns = Object.keys(schemaLiteralValues);
 
-			if (!columns || !node.values || !ValuesNode.is(node.values)) {
-				return;
+		if (schemaLiteralColumns.length === 0) {
+			return row;
+		}
+
+		const morphedSchemaValues = this.morph(
+			table,
+			this.pickSchema(tableSchema.schema, schemaLiteralColumns),
+			this.prepareInsertValue(tableSchema, schemaLiteralValues),
+		);
+
+		return {
+			values: { ...nonSchemaLiteralValues, ...morphedSchemaValues },
+			passthrough: row.passthrough,
+		};
+	}
+
+	private getInsertColumns(
+		tableSchema: TableWriteSchema,
+		originalColumns: string[],
+		rows: InsertRow[],
+	) {
+		const columns = [...originalColumns];
+
+		for (const column of tableSchema.columns) {
+			if (columns.includes(column)) {
+				continue;
 			}
 
-			for (const valueList of node.values.values) {
-				for (let i = 0; i < columns.length; i++) {
-					const col = columns[i]!;
+			if (rows.some((row) => Object.prototype.hasOwnProperty.call(row.values, column))) {
+				columns.push(column);
+			}
+		}
+
+		for (const row of rows) {
+			for (const column of Object.keys(row.values)) {
+				if (!columns.includes(column)) {
+					columns.push(column);
+				}
+			}
+		}
 
-					if (valueList.kind === "PrimitiveValueListNode") {
-						yield [col, valueList.values[i]] as [string, unknown];
-						continue;
-					}
+		return columns;
+	}
 
-					const raw = valueList.values[i];
+	private createInsertValueList(columns: string[], row: InsertRow) {
+		return ValueListNode.create(
+			columns.map((column) => {
+				const passthrough = row.passthrough.get(column);
 
-					if (!raw || DefaultInsertValueNode.is(raw)) {
-						continue;
-					}
+				if (passthrough) {
+					return passthrough;
+				}
 
-					yield [col, ValueNode.is(raw) ? raw.value : raw] as [string, unknown];
+				if (Object.prototype.hasOwnProperty.call(row.values, column)) {
+					return ValueNode.create(row.values[column]);
 				}
+
+				return DefaultInsertValueNode.create();
+			}),
+		);
+	}
+
+	private transformInsert(node: KyselyInsertQueryNode, table: string) {
+		const onConflict = node.onConflict
+			? this.transformOnConflict(table, node.onConflict)
+			: undefined;
+		const tableSchema = this.schemas.get(table);
+
+		if (!tableSchema) {
+			if (onConflict) {
+				return InsertQueryNode.cloneWith(node, { onConflict });
+			}
+
+			return node;
+		}
+
+		const columns = node.columns?.map((column) => column.column.name);
+
+		if (!columns || !node.values || !ValuesNode.is(node.values)) {
+			if (onConflict) {
+				return InsertQueryNode.cloneWith(node, { onConflict });
 			}
 
-			for (const update of node.onConflict?.updates ?? []) {
-				if (ColumnNode.is(update.column) && ValueNode.is(update.value)) {
-					yield [update.column.column.name, update.value.value] as [string, unknown];
+			return node;
+		}
+
+		const rows: InsertRow[] = [];
+
+		for (const valueList of node.values.values) {
+			const row = this.getInsertRow(columns, valueList);
+
+			if (!row) {
+				if (onConflict) {
+					return InsertQueryNode.cloneWith(node, { onConflict });
 				}
+
+				return node;
+			}
+
+			rows.push(this.morphInsertRow(table, tableSchema, row));
+		}
+
+		const insertColumns = this.getInsertColumns(tableSchema, columns, rows);
+
+		return InsertQueryNode.cloneWith(node, {
+			columns: insertColumns.map((column) => ColumnNode.create(column)),
+			values: ValuesNode.create(rows.map((row) => this.createInsertValueList(insertColumns, row))),
+			onConflict,
+		});
+	}
+
+	private transformUpdates(table: string, updates: readonly KyselyColumnUpdateNode[]) {
+		const tableSchema = this.schemas.get(table);
+
+		if (!tableSchema) {
+			return updates;
+		}
+
+		const literalColumns = new Set<string>();
+		const literalValues: Record<string, unknown> = {};
+		const nullOptionalColumns = new Set<string>();
+
+		for (const update of updates) {
+			if (!ColumnNode.is(update.column) || !ValueNode.is(update.value)) {
+				continue;
+			}
+
+			const column = update.column.column.name;
+
+			if (!tableSchema.columnSet.has(column)) {
+				continue;
+			}
+
+			if (update.value.value === null && tableSchema.optionalNonNullColumns.has(column)) {
+				nullOptionalColumns.add(column);
+				continue;
 			}
 
-			return;
+			literalColumns.add(column);
+			literalValues[column] = update.value.value;
 		}
 
-		if (node.kind !== "UpdateQueryNode" || !node.updates) {
-			return;
+		if (literalColumns.size === 0 && nullOptionalColumns.size === 0) {
+			return updates;
 		}
 
-		for (const update of node.updates) {
-			if (ColumnNode.is(update.column) && ValueNode.is(update.value)) {
-				yield [update.column.column.name, update.value.value] as [string, unknown];
+		const morphed =
+			literalColumns.size === 0
+				? {}
+				: this.morph(
+						table,
+						this.pickSchema(tableSchema.schema, literalColumns),
+						this.stripNullOptionalFields(tableSchema, literalValues),
+					);
+
+		return updates.flatMap((update) => {
+			if (!ColumnNode.is(update.column) || !ValueNode.is(update.value)) {
+				return [update];
 			}
+
+			const column = update.column.column.name;
+
+			if (!literalColumns.has(column) || !tableSchema.columnSet.has(column)) {
+				if (nullOptionalColumns.has(column)) {
+					return [ColumnUpdateNode.create(update.column, ValueNode.create(null))];
+				}
+
+				return [update];
+			}
+
+			if (!Object.prototype.hasOwnProperty.call(morphed, column)) {
+				return [
+					ColumnUpdateNode.create(update.column, ReferenceNode.create(ColumnNode.create(column))),
+				];
+			}
+
+			return [ColumnUpdateNode.create(update.column, ValueNode.create(morphed[column]))];
+		});
+	}
+
+	private transformOnConflict(table: string, node: KyselyOnConflictNode) {
+		if (!node.updates) {
+			return node;
+		}
+
+		return OnConflictNode.cloneWith(node, {
+			updates: this.transformUpdates(table, node.updates),
+		});
+	}
+
+	private transformUpdate(node: KyselyUpdateQueryNode, table: string) {
+		if (!node.updates) {
+			return node;
 		}
+
+		return Object.freeze({
+			...node,
+			updates: this.transformUpdates(table, node.updates),
+		});
 	}
 }