@lix-js/sdk 0.0.1

package/src/database/init-db.test.ts

@@ -0,0 +1,484 @@
+import { createInMemoryDatabase } from "sqlite-wasm-kysely";
+import { test, expect } from "vitest";
+import { initDb } from "./init-db.js";
+import { validate } from "uuid";
+import { mockChange } from "../change/mock-change.js";
+import { jsonSha256 } from "../snapshot/json-sha-256.js";
+import { sql } from "kysely";
+import { openLixInMemory } from "../lix/open-lix-in-memory.js";
+import { updateChangesInVersion } from "../version/update-changes-in-version.js";
+
+test("file ids should default to uuid", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	// init the trigger function (usually defined by lix only)
+	sqlite.createFunction({
+		name: "triggerChangeQueue",
+		arity: 0,
+		// @ts-expect-error - dynamic function
+		xFunc: () => {},
+	});
+
+	const file = await db
+		.insertInto("file")
+		.values({
+			path: "/mock",
+			data: new Uint8Array(),
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	expect(validate(file.id)).toBe(true);
+});
+
+test("change ids should default to uuid", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	const change = await db
+		.insertInto("change")
+		.values({
+			schema_key: "file",
+			entity_id: "value1",
+			file_id: "mock",
+			plugin_key: "mock-plugin",
+			snapshot_id: "no-content",
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	expect(validate(change.id)).toBe(true);
+});
+
+test("snapshot ids should default to sha256", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	const content = { a: "value" };
+
+	const snapshot = await db
+		.insertInto("snapshot")
+		.values({
+			content,
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	expect(snapshot.id).toBe(jsonSha256(content));
+});
+
+test("inserting the same snapshot multiple times should be possible and not lead to duplicates (content addressable)", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	const initialSnapshots = await db
+		.selectFrom("snapshot")
+		.selectAll()
+		.execute();
+
+	const snapshot1 = await db
+		.insertInto("snapshot")
+		.values({
+			content: { a: "some data" },
+		})
+		.onConflict((oc) => oc.doNothing())
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	const snapshot2 = await db
+		.insertInto("snapshot")
+		.values({
+			content: { a: "some data" },
+		})
+		.onConflict((oc) =>
+			oc.doUpdateSet((eb) => ({
+				content: eb.ref("excluded.content"),
+			}))
+		)
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	const snapshots = await db.selectFrom("snapshot").selectAll().execute();
+
+	expect(snapshots).toHaveLength(initialSnapshots.length + 1);
+	expect(snapshot1.id).toBe(snapshot2.id);
+});
+
+test("an empty snapshot should default to the special 'no-content' snapshot to store disk space", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+	const snapshot = await db
+		.insertInto("snapshot")
+		.values({
+			content: null,
+		})
+		.onConflict((oc) =>
+			oc.doUpdateSet((eb) => ({
+				content: eb.ref("excluded.content"),
+			}))
+		)
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	expect(snapshot.id).toBe("no-content");
+});
+
+// https://github.com/opral/lix-sdk/issues/71
+test("files should be able to have metadata", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	sqlite.createFunction({
+		name: "triggerChangeQueue",
+		arity: 0,
+		// @ts-expect-error - dynamic function
+		xFunc: () => {
+			// console.log('test')
+		},
+	});
+
+	const file = await db
+		.insertInto("file")
+		.values({
+			path: "/mock.csv",
+			data: new Uint8Array(),
+			metadata: {
+				primary_key: "email",
+			},
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	expect(file.metadata?.primary_key).toBe("email");
+
+	const updatedFile = await db
+		.updateTable("file")
+		.where("path", "=", "/mock.csv")
+		.set({
+			metadata: {
+				primary_key: "something-else",
+			},
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	expect(updatedFile.metadata?.primary_key).toBe("something-else");
+});
+
+test("change graph edges can't reference themselves", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	await expect(
+		db
+			.insertInto("change_edge")
+			.values({
+				parent_id: "change1",
+				child_id: "change1",
+			})
+			.returningAll()
+			.execute()
+	).rejects.toThrowErrorMatchingInlineSnapshot(
+		`[SQLite3Error: SQLITE_CONSTRAINT_CHECK: sqlite3 result code 275: CHECK constraint failed: parent_id != child_id]`
+	);
+});
+
+test("change set items must be unique", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	await db
+		.insertInto("change_set")
+		.values({
+			id: "change-set-1",
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	await db
+		.insertInto("change")
+		.values(
+			mockChange({
+				id: "change-1",
+			})
+		)
+		.execute();
+
+	await db
+		.insertInto("change_set_element")
+		.values({
+			change_set_id: "change-set-1",
+			change_id: "change-1",
+		})
+		.execute();
+
+	expect(
+		db
+			.insertInto("change_set_element")
+			.values({
+				change_set_id: "change-set-1",
+				change_id: "change-1",
+			})
+			.returningAll()
+			.execute()
+	).rejects.toThrowErrorMatchingInlineSnapshot(
+		`[SQLite3Error: SQLITE_CONSTRAINT_PRIMARYKEY: sqlite3 result code 1555: UNIQUE constraint failed: change_set_element.change_set_id, change_set_element.change_id]`
+	);
+});
+
+test("creating multiple discussions for one change set should be possible", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	const changeSet = await db
+		.insertInto("change_set")
+		.defaultValues()
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	await db
+		.insertInto("discussion")
+		.values([
+			{ id: "discussion-1", change_set_id: changeSet.id },
+			{ id: "discussion-2", change_set_id: changeSet.id },
+		])
+		.returningAll()
+		.execute();
+
+	const discussions = await db
+		.selectFrom("discussion")
+		.selectAll()
+		.where("change_set_id", "=", changeSet.id)
+		.execute();
+
+	expect(discussions).toHaveLength(2);
+});
+
+test("the confirmed label should be created if it doesn't exist", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	const tag = await db
+		.selectFrom("label")
+		.selectAll()
+		.where("name", "=", "confirmed")
+		.executeTakeFirst();
+
+	expect(tag).toMatchObject({
+		name: "confirmed",
+	});
+});
+
+test("a default main version should exist", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	const version = await db
+		.selectFrom("version")
+		.selectAll()
+		.where("name", "=", "main")
+		.executeTakeFirst();
+
+	expect(version).toBeDefined();
+});
+
+test("re-opening the same database shouldn't lead to duplicate insertion of the current version", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	const newversion = await db
+		.insertInto("version")
+		.values({ name: "mock" })
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	await db.updateTable("current_version").set({ id: newversion.id }).execute();
+
+	const db2 = initDb({ sqlite });
+
+	const currentversion = await db2
+		.selectFrom("current_version")
+		.selectAll()
+		.execute();
+
+	expect(currentversion).toHaveLength(1);
+});
+
+test("invalid file paths should be rejected", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	// init the trigger function (usually defined by lix only)
+	sqlite.createFunction({
+		name: "triggerChangeQueue",
+		arity: 0,
+		// @ts-expect-error - dynamic function
+		xFunc: () => {},
+	});
+
+	await expect(
+		db
+			.insertInto("file")
+			.values({
+				path: "invalid-path",
+				data: new Uint8Array(),
+			})
+			.returningAll()
+			.execute()
+	).rejects.toThrowErrorMatchingInlineSnapshot(
+		`[SQLite3Error: SQLITE_ERROR: sqlite3 result code 1: Error: File path must start with a slash.\n\nNot starting a file path with a slash \`/\` leads to ambiguity whether or not the path is a directory or a file.]`
+	);
+});
+
+test("vector clock functions", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	const vectorClockTick1 =
+		await sql`select lix_session() as session, lix_session_clock_tick() as time`.execute(
+			db
+		);
+	const vectorClockTick2 =
+		await sql`select lix_session() as session, lix_session_clock_tick() as time`.execute(
+			db
+		);
+
+	expect((vectorClockTick1.rows[0] as any)["session"]).toEqual(
+		(vectorClockTick2.rows[0] as any)["session"]
+	);
+	expect((vectorClockTick1.rows[0] as any)["time"]).toBeLessThan(
+		(vectorClockTick2.rows[0] as any)["time"]
+	);
+});
+
+test("mutation should only be recorded if sync row is not present", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	await db
+		.insertInto("change")
+		.values({
+			schema_key: "file",
+			entity_id: "value1",
+			file_id: "mock",
+			plugin_key: "mock-plugin",
+			snapshot_id: "no-content",
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+	const mutation = await db.selectFrom("mutation_log").selectAll().execute();
+
+	expect(mutation).toHaveLength(1);
+
+	await db
+		.insertInto("mutation_log")
+		.values({
+			session: "mock",
+			wall_clock: 0,
+			session_time: 0,
+			row_id: { ignored: "ignored" },
+			table_name: "mutation_log",
+			operation: "INSERT",
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	const mutationWithFlag = await db
+		.selectFrom("mutation_log")
+		.selectAll()
+		.execute();
+	expect(mutationWithFlag).toHaveLength(2);
+
+	await db
+		.insertInto("change")
+		.values({
+			schema_key: "file",
+			entity_id: "value1",
+			file_id: "mock",
+			plugin_key: "mock-plugin",
+			snapshot_id: "no-content",
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	const mutationLogAfterIgnoredChange = await db
+		.selectFrom("mutation_log")
+		.selectAll()
+		.execute();
+	expect(mutationLogAfterIgnoredChange).toHaveLength(2);
+
+	// const vectorClockTick1 =
+	// 	await sql`select lix_session() as session, lix_session_clock_tick() as time`.execute(
+	// 		db
+	// 	);
+	// const vectorClockTick2 =
+	// 	await sql`select lix_session() as session, lix_session_clock_tick() as time`.execute(
+	// 		db
+	// 	);
+});
+
+test("deleting a version cascades to version changes", async () => {
+	const lix = await openLixInMemory({});
+	const version = await lix.db
+		.insertInto("version")
+		.values({ name: "mock" })
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	const changes = await lix.db
+		.insertInto("change")
+		.values(
+			mockChange({
+				entity_id: `mock`,
+			})
+		)
+		.returningAll()
+		.execute();
+
+	await updateChangesInVersion({
+		lix,
+		version,
+		changes,
+	});
+
+	await lix.db.deleteFrom("version").where("id", "=", version.id).execute();
+
+	const versionChangesAfterDelete = await lix.db
+		.selectFrom("version_change")
+		.where("version_change.version_id", "=", version.id)
+		.selectAll()
+		.execute();
+
+	expect(versionChangesAfterDelete).toHaveLength(0);
+});

package/src/database/init-db.ts

@@ -0,0 +1,94 @@
+import { Kysely } from "kysely";
+import { createDialect, type SqliteDatabase } from "sqlite-wasm-kysely";
+import { v7 as uuid_v7, v4 as uuid_v4 } from "uuid";
+import type { LixDatabaseSchema } from "./schema.js";
+import { applySchema } from "./apply-schema.js";
+import { validateFilePath } from "../file/validate-file-path.js";
+import { jsonSha256 } from "../snapshot/json-sha-256.js";
+import { ParseJsonBPluginV1 } from "./kysely-plugin/parse-jsonb-plugin-v1.js";
+import { SerializeJsonBPlugin } from "./kysely-plugin/serialize-jsonb-plugin.js";
+import { createSession } from "./mutation-log/lix-session.js";
+import { applyOwnEntityChangeControlTriggers } from "../own-entity-change-control/database-triggers.js";
+
+export function initDb(args: {
+	sqlite: SqliteDatabase;
+}): Kysely<LixDatabaseSchema> {
+	initFunctions({ sqlite: args.sqlite });
+	applySchema({ sqlite: args.sqlite });
+	const db = new Kysely<LixDatabaseSchema>({
+		dialect: createDialect({
+			database: args.sqlite,
+		}),
+		plugins: [
+			ParseJsonBPluginV1({
+				// jsonb columns
+				file: ["metadata"],
+				change_queue: ["metadata_before", "metadata_after"],
+				snapshot: ["content"],
+				mutation_log: ["row_id"],
+			}),
+			SerializeJsonBPlugin(),
+		],
+	});
+
+	// need to apply it here because db object needs to be available
+	applyOwnEntityChangeControlTriggers(args.sqlite, db);
+	return db;
+}
+
+function initFunctions(args: { sqlite: SqliteDatabase }) {
+	args.sqlite.createFunction({
+		name: "uuid_v7",
+		arity: 0,
+		xFunc: () => uuid_v7(),
+	});
+
+	args.sqlite.createFunction({
+		name: "uuid_v4",
+		arity: 0,
+		xFunc: () => uuid_v4(),
+	});
+
+	args.sqlite.createFunction({
+		name: "json_sha256",
+		arity: 1,
+		xFunc: (_ctx: number, value) => {
+			if (!value) {
+				return "no-content";
+			}
+
+			const json = args.sqlite.exec("SELECT json(?)", {
+				bind: [value],
+				returnValue: "resultRows",
+			})[0]![0];
+
+			const parsed = JSON.parse(json as string);
+
+			return jsonSha256(parsed);
+		},
+		deterministic: true,
+	});
+
+	args.sqlite.createFunction({
+		name: "is_valid_file_path",
+		arity: 1,
+		xFunc: (_ctx: number, value) => {
+			return validateFilePath(value as string) as unknown as string;
+		},
+		deterministic: true,
+	});
+
+	const lixSession = createSession();
+
+	args.sqlite.createFunction({
+		name: "lix_session",
+		arity: 0,
+		xFunc: () => lixSession.id(),
+	});
+
+	args.sqlite.createFunction({
+		name: "lix_session_clock_tick",
+		arity: 0,
+		xFunc: () => lixSession.sessionClockTick(),
+	});
+}