@lix-js/sdk 0.4.7 → 0.5.0-preview.1

package/src/database/init-db.ts

@@ -1,111 +1,195 @@
-import { Kysely } from "kysely";
+import { Kysely, ParseJSONResultsPlugin } from "kysely";
 import { createDialect, type SqliteWasmDatabase } from "sqlite-wasm-kysely";
-import { v7 as uuid_v7, v4 as uuid_v4 } from "uuid";
-import type { LixDatabaseSchema } from "./schema.js";
-import { applySchema } from "./apply-schema.js";
-import { validateFilePath } from "../file/validate-file-path.js";
-import { jsonSha256 } from "../snapshot/json-sha-256.js";
-import { ParseJsonBPluginV1 } from "./kysely-plugin/parse-jsonb-plugin-v1.js";
-import { SerializeJsonBPlugin } from "./kysely-plugin/serialize-jsonb-plugin.js";
-import { createSession } from "./mutation-log/lix-session.js";
-import { applyOwnChangeControlTriggers } from "../own-change-control/database-triggers.js";
-import { humanId } from "human-id";
-import { nanoid } from "./nano-id.js";
+import type { LixDatabaseSchema, LixInternalDatabaseSchema } from "./schema.js";
+import { JSONColumnPlugin } from "./kysely-plugin/json-column-plugin.js";
+import { ViewInsertReturningErrorPlugin } from "./kysely-plugin/view-insert-returning-error-plugin.js";
+import { LixSchemaViewMap } from "./schema.js";
+import { isJsonType } from "../schema-definition/json-type.js";
+// Schema imports
+import { applyLogDatabaseSchema } from "../log/schema.js";
+import { applyChangeDatabaseSchema } from "../change/schema.js";
+import { applyChangeSetDatabaseSchema } from "../change-set/schema.js";
+import { applyCommitDatabaseSchema } from "../commit/schema.js";
+import { applyVersionDatabaseSchema } from "../version/schema.js";
+import { applySnapshotDatabaseSchema } from "../snapshot/schema.js";
+import { applyStoredSchemaDatabaseSchema } from "../stored-schema/schema.js";
+import { applyKeyValueDatabaseSchema } from "../key-value/schema.js";
+import { applyStateDatabaseSchema } from "../state/schema.js";
+import { applyChangeAuthorDatabaseSchema } from "../change-author/schema.js";
+import { applyLabelDatabaseSchema } from "../label/schema.js";
+import { applyThreadDatabaseSchema } from "../thread/schema.js";
+import { applyAccountDatabaseSchema } from "../account/schema.js";
+import { applyStateHistoryDatabaseSchema } from "../state-history/schema.js";
+import type { LixHooks } from "../hooks/create-hooks.js";
+import type { Lix } from "../lix/open-lix.js";
+import { timestamp, uuidV7, generateHumanId } from "../deterministic/index.js";
+import { nanoId } from "../deterministic/nano-id.js";
+import { applyEntityDatabaseSchema } from "../entity/schema.js";
+import { applyEntityThreadDatabaseSchema } from "../entity/thread/schema.js";
+
+/**
+ * Configuration for JSON columns in database views.
+ *
+ * Specifies which columns contain JSON data and what types of JSON values they accept.
+ * This is used by the JSONColumnPlugin to properly serialize/deserialize JSON data.
+ *
+ * Column types:
+ * - `type: 'object'` - Column only accepts JSON objects. String values are assumed to be
+ *   pre-serialized JSON to prevent double serialization when data flows between views.
+ * - `type: ['string', 'number', 'boolean', 'object', 'array', 'null']` - Column accepts
+ *   any valid JSON value. All values are properly serialized.
+ *
+ * @example
+ * ```typescript
+ * {
+ *   change: {
+ *     snapshot_content: { type: 'object' } // Only objects, prevents double serialization
+ *   },
+ *   key_value: {
+ *     value: { type: ['string', 'number', 'boolean', 'object', 'array', 'null'] } // Any JSON
+ *   }
+ * }
+ * ```
+ */
+const ViewsWithJsonColumns = (() => {
+	const result: Record<
+		string,
+		Record<
+			string,
+			{
+				type:
+					| "object"
+					| Array<
+							"string" | "number" | "boolean" | "object" | "array" | "null"
+					  >;
+			}
+		>
+	> = {};
+
+	// Hardcoded object-only columns
+	const hardcodedViews = {
+		state: { snapshot_content: { type: "object" as const } },
+		state_all: { snapshot_content: { type: "object" as const } },
+		state_history: { snapshot_content: { type: "object" as const } },
+		change: { snapshot_content: { type: "object" as const } },
+	};
+
+	// Add the hardcoded columns first
+	Object.assign(result, hardcodedViews);
+
+	// Process schema-based columns
+	for (const [viewName, schema] of Object.entries(LixSchemaViewMap)) {
+		// Check if schema is an object and has properties
+		if (typeof schema === "boolean" || !schema.properties) continue;
+
+		const jsonColumns: Record<
+			string,
+			{
+				type: Array<
+					"string" | "number" | "boolean" | "object" | "array" | "null"
+				>;
+			}
+		> = {};
+
+		for (const [key, def] of Object.entries(schema.properties)) {
+			if (isJsonType(def)) {
+				// All schema-based JSON columns accept any JSON value
+				jsonColumns[key] = {
+					type: ["string", "number", "boolean", "object", "array", "null"],
+				};
+			}
+		}
+
+		if (Object.keys(jsonColumns).length > 0) {
+			result[viewName] = jsonColumns;
+			// Also add the _all variant view with the same JSON columns
+			result[viewName + "_all"] = jsonColumns;
+		}
+	}
+
+	return result;
+})();
 
 export function initDb(args: {
 	sqlite: SqliteWasmDatabase;
+	hooks: LixHooks;
 }): Kysely<LixDatabaseSchema> {
-	initFunctions({ sqlite: args.sqlite });
-	applySchema({ sqlite: args.sqlite });
 	const db = new Kysely<LixDatabaseSchema>({
+		// log: ["error", "query"],
 		dialect: createDialect({
 			database: args.sqlite,
 		}),
 		plugins: [
-			ParseJsonBPluginV1({
-				// jsonb columns
-				file: ["metadata"],
-				file_queue: ["metadata_before", "metadata_after"],
-				snapshot: ["content"],
-				mutation_log: ["row_id"],
-			}),
-			SerializeJsonBPlugin(),
+			// needed for things like `jsonArrayFrom()`
+			new ParseJSONResultsPlugin(),
+			JSONColumnPlugin(ViewsWithJsonColumns),
+			new ViewInsertReturningErrorPlugin(Object.keys(LixSchemaViewMap)),
 		],
 	});
 
-	// need to apply it here because db object needs to be available
-	applyOwnChangeControlTriggers(args.sqlite, db);
-	return db;
-}
-
-function initFunctions(args: { sqlite: SqliteWasmDatabase }) {
-	args.sqlite.createFunction({
-		name: "uuid_v7",
-		arity: 0,
-		xFunc: () => uuid_v7(),
-	});
+	const lix = { sqlite: args.sqlite, db } as unknown as Lix;
 
-	args.sqlite.createFunction({
-		name: "uuid_v4",
-		arity: 0,
-		xFunc: () => uuid_v4(),
+	initFunctions({
+		sqlite: args.sqlite,
+		db: db as unknown as Kysely<LixInternalDatabaseSchema>,
 	});
 
-	args.sqlite.createFunction({
-		name: "json_sha256",
-		arity: 1,
-		xFunc: (_ctx: number, value) => {
-			if (!value) {
-				return "no-content";
-			}
-
-			const json = args.sqlite.exec("SELECT json(?)", {
-				bind: [value],
-				returnValue: "resultRows",
-			})[0]![0];
-
-			const parsed = JSON.parse(json as string);
-
-			return jsonSha256(parsed);
-		},
-		deterministic: true,
-	});
+	// Apply all database schemas first (tables, views, triggers)
+	applySnapshotDatabaseSchema(args.sqlite);
+	applyChangeDatabaseSchema(args.sqlite);
+	applyStateDatabaseSchema(
+		args.sqlite,
+		db as unknown as Kysely<LixInternalDatabaseSchema>,
+		args.hooks
+	);
+	applyEntityDatabaseSchema(lix);
+	applyChangeSetDatabaseSchema(args.sqlite, db);
+	applyCommitDatabaseSchema(lix);
+	applyStoredSchemaDatabaseSchema(args.sqlite);
+	applyVersionDatabaseSchema(lix);
+	applyAccountDatabaseSchema(args.sqlite, db);
+	applyKeyValueDatabaseSchema(lix);
+	applyChangeAuthorDatabaseSchema(lix);
+	applyLabelDatabaseSchema(lix);
+	applyThreadDatabaseSchema(lix);
+	applyEntityThreadDatabaseSchema(lix);
+	applyStateHistoryDatabaseSchema(lix);
+	// applyFileDatabaseSchema will be called later when lix is fully constructed
+	applyLogDatabaseSchema(lix);
 
-	args.sqlite.createFunction({
-		name: "is_valid_file_path",
-		arity: 1,
-		xFunc: (_ctx: number, value) => {
-			return validateFilePath(value as string) as unknown as string;
-		},
-		deterministic: true,
-	});
+	return db;
+}
 
-	const lixSession = createSession();
+function initFunctions(args: {
+	sqlite: SqliteWasmDatabase;
+	db: Kysely<LixInternalDatabaseSchema>;
+}) {
+	const lix = { sqlite: args.sqlite, db: args.db } as unknown as Lix;
 
 	args.sqlite.createFunction({
-		name: "lix_session",
+		name: "lix_uuid_v7",
 		arity: 0,
-		xFunc: () => lixSession.id(),
+		xFunc: () => uuidV7({ lix }),
 	});
 
 	args.sqlite.createFunction({
-		name: "lix_session_clock_tick",
+		name: "human_id",
 		arity: 0,
-		xFunc: () => lixSession.sessionClockTick(),
+		xFunc: () => generateHumanId({ lix, separator: "-", capitalize: false }),
 	});
 
 	args.sqlite.createFunction({
-		name: "human_id",
+		name: "lix_timestamp",
 		arity: 0,
-		xFunc: () => humanId({ separator: "-", capitalize: false }),
+		xFunc: () => timestamp({ lix }),
 	});
 
 	args.sqlite.createFunction({
-		name: "nano_id",
-		arity: 1,
+		name: "lix_nano_id",
+		arity: -1,
 		// @ts-expect-error - not sure why this is not working
 		xFunc: (_ctx: number, length: number) => {
-			return nanoid(length);
+			return nanoId({ lix, length });
 		},
 	});
 }

package/src/database/kysely-plugin/json-column-plugin.test.ts

@@ -0,0 +1,404 @@
+import { Kysely, sql, type Generated } from "kysely";
+import { expect, test } from "vitest";
+import { createDialect, createInMemoryDatabase } from "sqlite-wasm-kysely";
+import { JSONColumnPlugin } from "./json-column-plugin.js";
+
+test("properly serializes string values for any-json columns", async () => {
+	const db = await mockDatabase();
+
+	// Insert with a string that looks like JSON
+	const jsonLikeString = '{"id":"test_id","name":"test_name","value":123}';
+
+	await db
+		.insertInto("mock_table")
+		.values({
+			data: jsonLikeString,
+			other: "value",
+		})
+		.execute();
+
+	// Verify the data was stored as a JSON string (since column accepts any JSON type)
+	const result = await sql<{ stored_data: string }>`
+		SELECT json(data) as stored_data FROM mock_table
+	`.execute(db);
+
+	// The stored data should be the string as a JSON string
+	expect(result.rows[0]?.stored_data).toBe(JSON.stringify(jsonLikeString));
+
+	// Verify it can be parsed correctly
+	const parsed = JSON.parse(result.rows[0]?.stored_data as string);
+	expect(parsed).toBe(jsonLikeString);
+});
+
+test("correctly serializes JavaScript objects", async () => {
+	const db = await mockDatabase();
+
+	// Insert with a JavaScript object
+	const jsObject = { id: "test_id", name: "test_name", value: 123 };
+
+	await db
+		.insertInto("mock_table")
+		.values({
+			data: jsObject,
+			other: "value",
+		})
+		.execute();
+
+	// Verify the data was serialized and stored correctly
+	const result = await sql<{ stored_data: string }>`
+		SELECT json(data) as stored_data FROM mock_table
+	`.execute(db);
+
+	// The stored data should be the serialized object
+	const parsed = JSON.parse(result.rows[0]?.stored_data as string);
+	expect(parsed).toEqual(jsObject);
+});
+
+test("handles null values correctly", async () => {
+	const db = await mockDatabase();
+
+	await db
+		.insertInto("mock_table")
+		.values({
+			data: null,
+			other: "value",
+		})
+		.execute();
+
+	const result = await sql<{ data: any }>`
+		SELECT data FROM mock_table
+	`.execute(db);
+
+	expect(result.rows[0]?.data).toBe(null);
+});
+
+test("handles updates with string values", async () => {
+	const db = await mockDatabase();
+
+	// First insert some data
+	await db
+		.insertInto("mock_table")
+		.values({
+			data: { initial: "data" },
+			other: "value",
+		})
+		.execute();
+
+	// Update with a string that looks like JSON
+	const jsonLikeString = '{"updated":"value","count":42}';
+
+	await db.updateTable("mock_table").set({ data: jsonLikeString }).execute();
+
+	// Verify the update properly serialized the string
+	const result = await sql<{ stored_data: string }>`
+		SELECT json(data) as stored_data FROM mock_table
+	`.execute(db);
+
+	expect(result.rows[0]?.stored_data).toBe(JSON.stringify(jsonLikeString));
+});
+
+test("object-only columns prevent double serialization", async () => {
+	const database = await createInMemoryDatabase({ readOnly: false });
+
+	const db = new Kysely<{
+		mock_table: {
+			id: Generated<number>;
+			object_only: Record<string, any> | null;
+			any_json: any;
+		};
+	}>({
+		dialect: createDialect({ database }),
+		plugins: [
+			JSONColumnPlugin({
+				mock_table: {
+					object_only: { type: "object" },
+					any_json: {
+						type: ["string", "number", "boolean", "object", "array", "null"],
+					},
+				},
+			}),
+		],
+	});
+
+	await sql`
+		CREATE TABLE mock_table (
+			id INTEGER PRIMARY KEY AUTOINCREMENT,
+			object_only TEXT,
+			any_json TEXT
+		) strict;
+	`.execute(db);
+
+	// For object-only columns, pre-serialized JSON strings should not be double-serialized
+	const preSerializedObject = '{"id":"test","value":123}';
+
+	await db
+		.insertInto("mock_table")
+		.values({
+			// @ts-expect-error - dynamic types
+			object_only: preSerializedObject,
+			any_json: "plain string",
+		})
+		.execute();
+
+	const result = await sql<{ object_only_data: string; any_json_data: string }>`
+		SELECT json(object_only) as object_only_data, json(any_json) as any_json_data FROM mock_table
+	`.execute(db);
+
+	// object_only should not be double-serialized
+	expect(result.rows[0]?.object_only_data).toBe(preSerializedObject);
+	expect(JSON.parse(result.rows[0]?.object_only_data as string)).toEqual({
+		id: "test",
+		value: 123,
+	});
+
+	// any_json should be properly serialized as a JSON string
+	expect(result.rows[0]?.any_json_data).toBe('"plain string"');
+});
+
+test("handles onConflict with string values", async () => {
+	const db = await mockDatabase();
+
+	// Insert initial data
+	await db
+		.insertInto("mock_table")
+		.values({
+			id: 1,
+			data: { initial: "data" },
+			other: "value",
+		})
+		.execute();
+
+	// Try to insert with same id, using onConflict with a string
+	const jsonLikeString = '{"conflict":"resolved","value":999}';
+
+	await db
+		.insertInto("mock_table")
+		.values({
+			id: 1,
+			data: { should: "not_be_used" },
+			other: "value",
+		})
+		.onConflict((oc) => oc.doUpdateSet({ data: jsonLikeString }))
+		.execute();
+
+	// Verify the conflict update properly serialized the string
+	const result = await sql<{ stored_data: string }>`
+		SELECT json(data) as stored_data FROM mock_table WHERE id = 1
+	`.execute(db);
+
+	expect(result.rows[0]?.stored_data).toBe(JSON.stringify(jsonLikeString));
+});
+
+const mockDatabase = async () => {
+	const database = await createInMemoryDatabase({ readOnly: false });
+
+	const db = new Kysely<{
+		mock_table: {
+			id: Generated<number>;
+			data: Record<string, any> | string | null;
+			other: string;
+		};
+	}>({
+		dialect: createDialect({
+			database,
+		}),
+		plugins: [
+			JSONColumnPlugin({
+				mock_table: {
+					data: {
+						type: ["string", "number", "boolean", "object", "array", "null"],
+					},
+				},
+			}),
+		],
+	});
+
+	await sql`
+		CREATE TABLE mock_table (
+			id INTEGER PRIMARY KEY AUTOINCREMENT,
+			data TEXT,
+			other TEXT
+		) strict;
+	`.execute(db);
+
+	return db;
+};
+
+test("SQL expressions in JSON columns should not be wrapped in json()", async () => {
+	const db = await mockDatabase();
+
+	// Insert initial data
+	await db
+		.insertInto("mock_table")
+		.values({
+			data: {
+				enabled: true,
+				settings: {
+					theme: "dark",
+					language: "en",
+				},
+			},
+			other: "value",
+		})
+		.execute();
+
+	// Try to update using json_set SQL expression
+	const updateQuery = db
+		.updateTable("mock_table")
+		.set({
+			data: sql`json_set(data, '$.settings.theme', 'light')`,
+		})
+		.where("other", "=", "value")
+		.compile();
+
+	// The SQL should contain json_set directly, not wrapped in json()
+	// Currently this fails - it produces: update "mock_table" set "data" = json(?) where "other" = ?
+	// We want: update "mock_table" set "data" = json_set(data, '$.settings.theme', 'light') where "other" = ?
+	expect(updateQuery.sql).toContain("json_set");
+	expect(updateQuery.sql).not.toContain("json(?)");
+
+	// Execute the update
+	await db
+		.updateTable("mock_table")
+		.set({
+			data: sql`json_set(data, '$.settings.theme', 'light')`,
+		})
+		.where("other", "=", "value")
+		.execute();
+
+	// Verify the update worked correctly
+	const result = await db
+		.selectFrom("mock_table")
+		.select("data")
+		.executeTakeFirst();
+
+	expect(result?.data).toMatchObject({
+		enabled: true,
+		settings: {
+			theme: "light", // Changed from "dark"
+			language: "en",
+		},
+	});
+});
+
+test("Multiple SQL expressions in JSON columns", async () => {
+	const db = await mockDatabase();
+
+	// Insert initial data
+	await db
+		.insertInto("mock_table")
+		.values({
+			data: {
+				count: 10,
+				active: true,
+				nested: {
+					value: 100,
+				},
+			},
+			other: "test",
+		})
+		.execute();
+
+	// Update multiple properties using json_set
+	const updateQuery = db
+		.updateTable("mock_table")
+		.set({
+			data: sql`json_set(data, '$.count', 20, '$.active', false, '$.nested.value', 200)`,
+		})
+		.compile();
+
+	// Should not wrap the SQL expression
+	expect(updateQuery.sql).toContain("json_set");
+	expect(updateQuery.sql).not.toContain("json(?)");
+
+	await db
+		.updateTable("mock_table")
+		.set({
+			data: sql`json_set(data, '$.count', 20, '$.active', false, '$.nested.value', 200)`,
+		})
+		.execute();
+
+	const result = await db
+		.selectFrom("mock_table")
+		.select("data")
+		.executeTakeFirst();
+
+	expect(result?.data).toMatchObject({
+		count: 20,
+		active: 0, // SQLite stores false as 0
+		nested: {
+			value: 200,
+		},
+	});
+});
+
+test("SQL expressions in onConflict updates", async () => {
+	const db = await mockDatabase();
+
+	// Insert initial data
+	await db
+		.insertInto("mock_table")
+		.values({
+			id: 1,
+			data: {
+				version: 1,
+				name: "original",
+				metadata: {
+					updated: false,
+				},
+			},
+			other: "test",
+		})
+		.execute();
+
+	// Try to insert with same id, using onConflict with SQL expression
+	const insertQuery = db
+		.insertInto("mock_table")
+		.values({
+			id: 1,
+			data: { version: 2, name: "ignored" },
+			other: "test",
+		})
+		.onConflict((oc) =>
+			oc.doUpdateSet({
+				data: sql`json_set(data, '$.version', 2, '$.metadata.updated', true)`,
+			})
+		)
+		.compile();
+
+	// The INSERT values will be wrapped in json(?), but the UPDATE should use json_set
+	expect(insertQuery.sql).toContain("json_set");
+	expect(insertQuery.sql).toContain("on conflict do update set");
+	// Verify json_set is in the UPDATE part, not wrapped in json()
+	expect(insertQuery.sql).toMatch(/do update set.*json_set.*\$\.version/);
+
+	// Execute the insert with conflict
+	await db
+		.insertInto("mock_table")
+		.values({
+			id: 1,
+			data: { version: 2, name: "ignored" },
+			other: "test",
+		})
+		.onConflict((oc) =>
+			oc.doUpdateSet({
+				data: sql`json_set(data, '$.version', 2, '$.metadata.updated', true)`,
+			})
+		)
+		.execute();
+
+	// Verify the conflict update worked
+	const result = await db
+		.selectFrom("mock_table")
+		.where("id", "=", 1)
+		.select("data")
+		.executeTakeFirst();
+
+	expect(result?.data).toMatchObject({
+		version: 2, // Updated
+		name: "original", // Kept original
+		metadata: {
+			updated: 1, // SQLite stores true as 1
+		},
+	});
+});