@lix-js/sdk 0.4.7 → 0.5.0-preview.0

package/src/database/init-db.ts

@@ -1,44 +1,103 @@
-import { Kysely } from "kysely";
+import { Kysely, ParseJSONResultsPlugin } from "kysely";
 import { createDialect, type SqliteWasmDatabase } from "sqlite-wasm-kysely";
 import { v7 as uuid_v7, v4 as uuid_v4 } from "uuid";
-import type { LixDatabaseSchema } from "./schema.js";
-import { applySchema } from "./apply-schema.js";
-import { validateFilePath } from "../file/validate-file-path.js";
-import { jsonSha256 } from "../snapshot/json-sha-256.js";
-import { ParseJsonBPluginV1 } from "./kysely-plugin/parse-jsonb-plugin-v1.js";
-import { SerializeJsonBPlugin } from "./kysely-plugin/serialize-jsonb-plugin.js";
-import { createSession } from "./mutation-log/lix-session.js";
-import { applyOwnChangeControlTriggers } from "../own-change-control/database-triggers.js";
+import type { LixDatabaseSchema, LixInternalDatabaseSchema } from "./schema.js";
 import { humanId } from "human-id";
 import { nanoid } from "./nano-id.js";
+import { JSONColumnPlugin } from "./kysely-plugin/json-column-plugin.js";
+import { ViewInsertReturningErrorPlugin } from "./kysely-plugin/view-insert-returning-error-plugin.js";
+import { LixSchemaViewMap } from "./schema.js";
+import { isJsonType } from "../schema-definition/json-type.js";
+// Schema imports
+import { applyLogDatabaseSchema } from "../log/schema.js";
+import { applyChangeDatabaseSchema } from "../change/schema.js";
+import { applyChangeSetDatabaseSchema } from "../change-set/schema.js";
+import { applyVersionDatabaseSchema } from "../version/schema.js";
+import { applySnapshotDatabaseSchema } from "../snapshot/schema.js";
+import { applyStoredSchemaDatabaseSchema } from "../stored-schema/schema.js";
+import { applyKeyValueDatabaseSchema } from "../key-value/schema.js";
+import { applyStateDatabaseSchema } from "../state/schema.js";
+import { applyChangeAuthorDatabaseSchema } from "../change-author/schema.js";
+import { applyLabelDatabaseSchema } from "../label/schema.js";
+import { applyThreadDatabaseSchema } from "../thread/schema.js";
+import { applyAccountDatabaseSchema } from "../account/schema.js";
+import { applyStateHistoryDatabaseSchema } from "../state-history/schema.js";
+import type { LixHooks } from "../hooks/create-hooks.js";
+
+// dynamically computes the json columns for each view
+// via the json schemas.
+const ViewsWithJsonColumns = {
+	state: ["snapshot_content"],
+	state_all: ["snapshot_content"],
+	state_history: ["snapshot_content"],
+	change: ["snapshot_content"],
+	...(() => {
+		const result: Record<string, string[]> = {};
+		for (const [viewName, schema] of Object.entries(LixSchemaViewMap)) {
+			// Check if schema is an object and has properties
+			if (typeof schema === "boolean" || !schema.properties) continue;
+			const jsonColumns = Object.entries(schema.properties)
+				.filter(([, def]) => isJsonType(def))
+				.map(([key]) => key);
+			if (jsonColumns.length) {
+				result[viewName] = jsonColumns;
+				// Also add the _all variant view with the same JSON columns
+				result[viewName + "_all"] = jsonColumns;
+			}
+		}
+		return result;
+	})(),
+};
 
 export function initDb(args: {
 	sqlite: SqliteWasmDatabase;
+	hooks: LixHooks;
 }): Kysely<LixDatabaseSchema> {
-	initFunctions({ sqlite: args.sqlite });
-	applySchema({ sqlite: args.sqlite });
 	const db = new Kysely<LixDatabaseSchema>({
+		// log: ["error", "query"],
 		dialect: createDialect({
 			database: args.sqlite,
 		}),
 		plugins: [
-			ParseJsonBPluginV1({
-				// jsonb columns
-				file: ["metadata"],
-				file_queue: ["metadata_before", "metadata_after"],
-				snapshot: ["content"],
-				mutation_log: ["row_id"],
-			}),
-			SerializeJsonBPlugin(),
+			// needed for things like `jsonArrayFrom()`
+			new ParseJSONResultsPlugin(),
+			JSONColumnPlugin(ViewsWithJsonColumns),
+			new ViewInsertReturningErrorPlugin(Object.keys(LixSchemaViewMap)),
 		],
 	});
+	initFunctions({
+		sqlite: args.sqlite,
+		db: db as unknown as Kysely<LixInternalDatabaseSchema>,
+	});
+
+	// Apply all database schemas first (tables, views, triggers)
+	applyStateDatabaseSchema(
+		args.sqlite,
+		db as unknown as Kysely<LixInternalDatabaseSchema>,
+		args.hooks
+	);
+	applySnapshotDatabaseSchema(args.sqlite);
+	applyChangeDatabaseSchema(args.sqlite);
+	applyChangeSetDatabaseSchema(args.sqlite);
+
+	applyStoredSchemaDatabaseSchema(args.sqlite);
+	applyAccountDatabaseSchema(args.sqlite);
+	applyVersionDatabaseSchema(args.sqlite);
+	applyKeyValueDatabaseSchema(args.sqlite);
+	applyChangeAuthorDatabaseSchema(args.sqlite);
+	applyLabelDatabaseSchema(args.sqlite);
+	applyThreadDatabaseSchema(args.sqlite);
+	applyStateHistoryDatabaseSchema(args.sqlite);
+	// applyFileDatabaseSchema will be called later when lix is fully constructed
+	applyLogDatabaseSchema(args.sqlite);
 
-	// need to apply it here because db object needs to be available
-	applyOwnChangeControlTriggers(args.sqlite, db);
 	return db;
 }
 
-function initFunctions(args: { sqlite: SqliteWasmDatabase }) {
+function initFunctions(args: {
+	sqlite: SqliteWasmDatabase;
+	db: Kysely<LixInternalDatabaseSchema>;
+}) {
 	args.sqlite.createFunction({
 		name: "uuid_v7",
 		arity: 0,
@@ -51,49 +110,6 @@ function initFunctions(args: { sqlite: SqliteWasmDatabase }) {
 		xFunc: () => uuid_v4(),
 	});
 
-	args.sqlite.createFunction({
-		name: "json_sha256",
-		arity: 1,
-		xFunc: (_ctx: number, value) => {
-			if (!value) {
-				return "no-content";
-			}
-
-			const json = args.sqlite.exec("SELECT json(?)", {
-				bind: [value],
-				returnValue: "resultRows",
-			})[0]![0];
-
-			const parsed = JSON.parse(json as string);
-
-			return jsonSha256(parsed);
-		},
-		deterministic: true,
-	});
-
-	args.sqlite.createFunction({
-		name: "is_valid_file_path",
-		arity: 1,
-		xFunc: (_ctx: number, value) => {
-			return validateFilePath(value as string) as unknown as string;
-		},
-		deterministic: true,
-	});
-
-	const lixSession = createSession();
-
-	args.sqlite.createFunction({
-		name: "lix_session",
-		arity: 0,
-		xFunc: () => lixSession.id(),
-	});
-
-	args.sqlite.createFunction({
-		name: "lix_session_clock_tick",
-		arity: 0,
-		xFunc: () => lixSession.sessionClockTick(),
-	});
-
 	args.sqlite.createFunction({
 		name: "human_id",
 		arity: 0,
@@ -102,7 +118,7 @@ function initFunctions(args: { sqlite: SqliteWasmDatabase }) {
 
 	args.sqlite.createFunction({
 		name: "nano_id",
-		arity: 1,
+		arity: -1,
 		// @ts-expect-error - not sure why this is not working
 		xFunc: (_ctx: number, length: number) => {
 			return nanoid(length);

package/src/database/kysely-plugin/json-column-plugin.ts

@@ -0,0 +1,215 @@
+import {
+	ColumnNode,
+	ColumnUpdateNode,
+	InsertQueryNode,
+	OnConflictNode,
+	OperationNodeTransformer,
+	ValueListNode,
+	ValueNode,
+	ValuesNode,
+	type KyselyPlugin,
+	type PluginTransformQueryArgs,
+} from "kysely";
+
+export function JSONColumnPlugin(
+	jsonColumns: Record<string, string[]>
+): KyselyPlugin {
+	const jsonColumnNames = Object.keys(jsonColumns).flatMap(
+		(key) => jsonColumns[key]!
+	);
+
+	return {
+		transformResult: async (args) => {
+			for (const row of args.result.rows) {
+				for (const col of jsonColumnNames) {
+					const text = row[col];
+					try {
+						row[col] = JSON.parse(text as string);
+					} catch {
+						continue;
+					}
+				}
+			}
+
+			return args.result;
+		},
+		transformQuery(args: PluginTransformQueryArgs) {
+			if (args.node.kind === "InsertQueryNode") {
+				const tableNode = (args.node as InsertQueryNode).into;
+				const table =
+					tableNode && tableNode.kind === "TableNode"
+						? tableNode.table.identifier.name
+						: undefined;
+				const columns = (args.node as InsertQueryNode).columns;
+
+				const transformer = new SerializeJsonbTransformer(
+					jsonColumns,
+					table,
+					columns
+				);
+				return transformer.transformNode(args.node);
+			}
+			if (args.node.kind === "UpdateQueryNode") {
+				const tableNode = args.node.table;
+				let table: string | undefined = undefined;
+				if (tableNode && tableNode.kind === "TableNode") {
+					// @ts-expect-error - kysely type narrowing
+					table = tableNode.table.identifier.name;
+				}
+
+				const transformer = new SerializeJsonbTransformer(
+					jsonColumns,
+					table,
+					undefined
+				);
+				return transformer.transformNode(args.node);
+			}
+			return args.node;
+		},
+	};
+}
+
+class SerializeJsonbTransformer extends OperationNodeTransformer {
+	private readonly jsonbColumns: Record<string, string[]>;
+	private readonly table: string | undefined;
+	private readonly columns: ReadonlyArray<ColumnNode> | undefined;
+
+	constructor(
+		jsonbColumns: Record<string, string[]>,
+		table: string | undefined,
+		columns: ReadonlyArray<ColumnNode> | undefined
+	) {
+		super();
+		this.jsonbColumns = jsonbColumns;
+		this.table = table;
+		this.columns = columns;
+	}
+
+	private isJsonbColumn(columnName: string): boolean {
+		if (!this.table || !columnName) return false;
+		return this.jsonbColumns[this.table]?.includes(columnName) ?? false;
+	}
+
+	override transformOnConflict(node: OnConflictNode): OnConflictNode {
+		if (!node.updates) {
+			return node;
+		}
+
+		// Map the updates array directly
+		const newUpdates = node.updates.map((updateItem) => {
+			// Check if it's a ColumnUpdateNode first
+			if (updateItem.kind === "ColumnUpdateNode") {
+				// @ts-expect-error - kysely type narrowing
+				const columnName = updateItem.column.column.name;
+				if (this.isJsonbColumn(columnName)) {
+					const valueNode = updateItem.value;
+					// Check if the value part is specifically a ValueNode
+					if (valueNode.kind === "ValueNode") {
+						// If it is, serialize it
+						return {
+							...updateItem,
+							// @ts-expect-error - kysely type narrowing
+							value: this.serializeValue(valueNode),
+						};
+					}
+				}
+			}
+			// Return unchanged if not a ColumnUpdateNode or not a JSONB column or value is not ValueNode
+			return updateItem;
+		});
+
+		// Return a new node with the transformed updates
+		return {
+			...node,
+			updates: newUpdates,
+		};
+	}
+
+	public override transformNode(node: any): any {
+		if (!node) {
+			return node;
+		}
+		return super.transformNode(node);
+	}
+
+	// Serialize JSONB values in updates for ColumnUpdateNodes
+	protected override transformColumnUpdate(
+		node: ColumnUpdateNode
+	): ColumnUpdateNode {
+		// @ts-expect-error - kysely type narrowing
+		const columnName = node.column.column.name;
+		if (this.isJsonbColumn(columnName)) {
+			return {
+				...node,
+				// @ts-expect-error - kysely type narrowing
+				value: this.serializeValue(node.value),
+			} as ColumnUpdateNode;
+		}
+		return super.transformColumnUpdate(node);
+	}
+
+	// Support .set("col", value) syntax (SetOperationNode)
+	protected override transformSetOperation(node: any): any {
+		// node.column.column.name is the column name for SetOperationNode
+		const columnName = node.column?.column?.name;
+		if (columnName && this.isJsonbColumn(columnName)) {
+			return {
+				...node,
+				value: this.serializeValue(node.value),
+			};
+		}
+		return super.transformSetOperation(node);
+	}
+
+	protected override transformValues(node: ValuesNode): ValuesNode {
+		const newValues = node.values.map((row) => {
+			// Handle literal JS values row
+			if (row.kind === "PrimitiveValueListNode") {
+				const newValueNodes = row.values.map((val, idx) => {
+					const colNode = this.columns?.[idx];
+					const valNode: ValueNode = { kind: "ValueNode", value: val };
+					if (
+						colNode?.kind === "ColumnNode" &&
+						this.isJsonbColumn(colNode.column.name)
+					) {
+						return this.serializeValue(valNode);
+					}
+					return valNode;
+				});
+				return {
+					kind: "ValueListNode",
+					values: newValueNodes,
+				} as ValueListNode;
+			}
+			// Handle AST rows
+			if (row.kind === "ValueListNode") {
+				const newValuesList = row.values.map((valNode, idx) => {
+					const colNode = this.columns?.[idx];
+					if (
+						colNode?.kind === "ColumnNode" &&
+						this.isJsonbColumn(colNode.column.name)
+					) {
+						return this.serializeValue(valNode as ValueNode);
+					}
+					return valNode;
+				});
+				return { ...row, values: newValuesList } as ValueListNode;
+			}
+			return row;
+		});
+		return { ...node, values: newValues };
+	}
+
+	private serializeValue(node: ValueNode): any {
+		const val = node.value;
+		if (val instanceof ArrayBuffer || ArrayBuffer.isView(val) || val === null) {
+			return node;
+		}
+		const jsonText = JSON.stringify(val);
+		return {
+			kind: "FunctionNode",
+			func: "json",
+			arguments: [{ kind: "ValueNode", value: jsonText }],
+		};
+	}
+}

package/src/database/kysely-plugin/parse-jsonb-plugin-v1.ts

@@ -23,26 +23,21 @@ export function ParseJsonBPluginV1(
 
 			for (const row of args.result.rows) {
 				for (const col of jsonColumnNames) {
-					if (!row[col]) {
-						// result does not have a whitelisted json column
-						// ! this is a heursitic. ideally, we invest into
-						// ! building ParseJsonBPluginV2 that detects
-						// ! json columns from the query itself
-						// see https://github.com/opral/lix-sdk/issues/145
+					const raw = row[col];
+					// Only parse binary JSONB blobs
+					if (!(raw instanceof Uint8Array)) {
 						continue;
 					}
 					try {
-						// sqlite has it's own jsonb format
+						// sqlite has its own jsonb format
 						// hence, need to query sqlite to convert
-						// to json
 						const json = sqlite.exec("SELECT json(?)", {
-							bind: [row[col] as Uint8Array],
+							bind: [raw],
 							returnValue: "resultRows",
 						})[0]![0];
 
 						row[col] = JSON.parse(json as string);
 					} catch {
-						// it's not a json binary
 						continue;
 					}
 				}

package/src/database/kysely-plugin/parse-jsonb-plugin-v2.ts

@@ -159,7 +159,7 @@ function transformSelections(
 					kind: "AliasNode",
 					node: {
 						kind: "FunctionNode",
-						func: "json",
+						func: "jsonb",
 						arguments: [
 							{
 								kind: "ReferenceNode",

package/src/database/kysely-plugin/serialize-jsonb-plugin.test.ts

@@ -2,6 +2,7 @@ import { Kysely, sql, type Generated } from "kysely";
 import { expect, test } from "vitest";
 import { createDialect, createInMemoryDatabase } from "sqlite-wasm-kysely";
 import { SerializeJsonBPlugin } from "./serialize-jsonb-plugin.js";
+import { ParseJsonBPluginV1 } from "./parse-jsonb-plugin-v1.js";
 
 test("inserts should work", async () => {
 	const db = await mockDatabase();
@@ -56,6 +57,44 @@ test("updates should work", async () => {
 	});
 });
 
+// JSONB serialization for updates is only supported when using the object form: .set({ col: value }).
+// The string-key form .set('col', value) will not be intercepted for JSONB serialization.
+//
+// (took to long to get this working, should work in the future if people ask for it though.)
+test.skip("works with kysely column syntax", async () => {
+	const db = await mockDatabase();
+
+	await db
+		.insertInto("mock_table")
+		.values({
+			id: 1,
+			metadata: "value1",
+			other: "other",
+			just_blob: new Uint8Array(),
+		})
+		.execute();
+
+	// update
+	await db
+		.updateTable("mock_table")
+		.set("metadata", "value2")
+		.where("id", "=", 1)
+		.execute();
+
+	const result = await db
+		.selectFrom("mock_table")
+		.where("id", "=", 1)
+		.selectAll()
+		.executeTakeFirstOrThrow();
+
+	expect(result).toEqual({
+		id: 1,
+		metadata: "value2",
+		other: "other",
+		just_blob: new Uint8Array(),
+	});
+});
+
 test("onConflict should work", async () => {
 	const db = await mockDatabase();
 
@@ -91,13 +130,142 @@ test("onConflict should work", async () => {
 	});
 });
 
+test("null leads to a nulled column, not nulled json", async () => {
+	const db = await mockDatabase();
+
+	await db
+		.insertInto("mock_table")
+		.values({
+			metadata: null,
+			other: "value",
+			just_blob: new Uint8Array(),
+		})
+		.executeTakeFirstOrThrow();
+
+	const result =
+		await sql`SELECT jsonb(metadata) as metadata FROM mock_table;`.execute(db);
+
+	expect(result.rows).toEqual([{ metadata: null }]);
+});
+
+test("json number primitive works", async () => {
+	const db = await mockDatabase();
+
+	const result = await db
+		.insertInto("mock_table")
+		.values({
+			metadata: 42,
+			other: "value",
+			just_blob: new Uint8Array(),
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	expect(result).toEqual({
+		id: 1,
+		metadata: 42,
+		other: "value",
+		just_blob: new Uint8Array(),
+	});
+});
+
+test("json string primitive works", async () => {
+	const db = await mockDatabase();
+
+	const result = await db
+		.insertInto("mock_table")
+		.values({
+			metadata: "hello",
+			other: "value",
+			just_blob: new Uint8Array(),
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	expect(result).toEqual({
+		id: 1,
+		metadata: "hello",
+		other: "value",
+		just_blob: new Uint8Array(),
+	});
+});
+
+test("json boolean primitive works", async () => {
+	const db = await mockDatabase();
+
+	const result = await db
+		.insertInto("mock_table")
+		.values({
+			metadata: true,
+			other: "value",
+			just_blob: new Uint8Array(),
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	expect(result).toEqual({
+		id: 1,
+		metadata: true,
+		other: "value",
+		just_blob: new Uint8Array(),
+	});
+});
+
+test("json null primitive works", async () => {
+	const db = await mockDatabase();
+
+	const result = await db
+		.insertInto("mock_table")
+		.values({
+			metadata: null,
+			other: "value",
+			just_blob: new Uint8Array(),
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	expect(result).toEqual({
+		id: 1,
+		metadata: null,
+		other: "value",
+		just_blob: new Uint8Array(),
+	});
+});
+
+test("json array works", async () => {
+	const db = await mockDatabase();
+
+	const result = await db
+		.insertInto("mock_table")
+		.values({
+			metadata: [1, 2, "3"],
+			other: "value",
+			just_blob: new Uint8Array(),
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	expect(result).toEqual({
+		id: 1,
+		metadata: [1, 2, "3"],
+		other: "value",
+		just_blob: new Uint8Array(),
+	});
+});
+
 const mockDatabase = async () => {
 	const database = await createInMemoryDatabase({ readOnly: false });
 
 	const db = new Kysely<{
 		mock_table: {
 			id: Generated<number>;
-			metadata: Record<string, any>;
+			metadata:
+				| Record<string, any>
+				| string
+				| number
+				| boolean
+				| null
+				| Array<any>;
 			other: string;
 			just_blob: Uint8Array;
 		};
@@ -109,7 +277,14 @@ const mockDatabase = async () => {
 		dialect: createDialect({
 			database,
 		}),
-		plugins: [SerializeJsonBPlugin()],
+		plugins: [
+			SerializeJsonBPlugin({
+				mock_table: ["metadata"],
+			}),
+			ParseJsonBPluginV1({
+				mock_table: ["metadata"],
+			}),
+		],
 	});
 
 	await sql`