@lix-js/sdk 0.0.1

package/src/change-queue/init-change-queue.test.ts

@@ -0,0 +1,457 @@
+import { expect, test, vi } from "vitest";
+import { openLixInMemory } from "../lix/open-lix-in-memory.js";
+import { newLixFile } from "../lix/new-lix.js";
+import type { DetectedChange, LixPlugin } from "../plugin/lix-plugin.js";
+import type { ChangeQueueEntry, LixFile } from "../database/schema.js";
+import { changeQueueSettled } from "./change-queue-settled.js";
+
+test("should use queue and settled correctly", async () => {
+	const mockPlugin: LixPlugin = {
+		key: "mock-plugin",
+		detectChangesGlob: "*",
+		detectChanges: async ({ before, after }) => {
+			const textBefore = before
+				? new TextDecoder().decode(before?.data)
+				: undefined;
+			const textAfter = after
+				? new TextDecoder().decode(after.data)
+				: undefined;
+
+			if (textBefore === textAfter) {
+				return [];
+			}
+
+			return [
+				{
+					schema: {
+						key: "text",
+						type: "json",
+					},
+					entity_id: "test",
+					snapshot: textAfter ? { text: textAfter } : undefined,
+				},
+			];
+		},
+	};
+
+	const lix = await openLixInMemory({
+		blob: await newLixFile(),
+		providePlugins: [mockPlugin],
+	});
+
+	const currentVersion = await lix.db
+		.selectFrom("current_version")
+		.innerJoin("version", "current_version.id", "version.id")
+		.selectAll()
+		.executeTakeFirstOrThrow();
+
+	const enc = new TextEncoder();
+	const dataInitial = enc.encode("insert text");
+	await lix.db
+		.insertInto("file")
+		.values({ id: "test", path: "/test.txt", data: dataInitial })
+		.execute();
+
+	const queue = await lix.db.selectFrom("change_queue").selectAll().execute();
+
+	expect(queue).toEqual([
+		expect.objectContaining({
+			id: 1,
+			file_id: "test",
+			path_after: "/test.txt",
+			data_after: dataInitial,
+			data_before: null,
+		} satisfies Partial<ChangeQueueEntry>),
+	]);
+
+	await changeQueueSettled({ lix });
+
+	expect(
+		(await lix.db.selectFrom("change_queue").selectAll().execute()).length
+	).toBe(0);
+
+	// TODO QUEUE check if the replacement of file_internal was expected
+	const internalFilesAfter = await lix.db
+		.selectFrom("file")
+		.selectAll()
+		.execute();
+
+	expect(internalFilesAfter).toEqual([
+		expect.objectContaining({
+			data: internalFilesAfter[0]!.data,
+			id: "test",
+			path: "/test.txt",
+			metadata: null,
+		}) satisfies LixFile,
+	]);
+
+	const changes = await lix.db
+		.selectFrom("change")
+		.innerJoin("snapshot", "snapshot.id", "change.snapshot_id")
+		.innerJoin("change_author", "change_author.change_id", "change.id")
+		.selectAll("change")
+		.select("change_author.account_id")
+		.select("snapshot.content")
+		.execute();
+
+	expect(changes).toEqual(
+		expect.arrayContaining([
+			expect.objectContaining({
+				entity_id: "test",
+				schema_key: "text",
+				file_id: "test",
+				plugin_key: "mock-plugin",
+				content: {
+					text: "insert text",
+				},
+				// handles the author (which defaults to anonymous)
+				account_id: expect.stringMatching(/^anonymous_/),
+			}),
+		])
+	);
+
+	const dataUpdate1 = enc.encode("updated text");
+	await lix.db
+		.updateTable("file")
+		.set({ data: dataUpdate1 })
+		.where("id", "=", "test")
+		.execute();
+
+	// const beforeQueueTick = await lix.db
+	// 	.selectFrom("change_queue")
+	// 	.selectAll()
+	// 	.execute();
+
+	// expect(beforeQueueTick.length).toBe(1);
+
+	// const afterQueueTick = await lix.db
+	// 	.selectFrom("change_queue")
+	// 	.selectAll()
+	// 	.execute();
+
+	// expect(afterQueueTick.length).toBe(0);
+
+	// update2 is equal to update1
+	const dataUpdate2 = dataUpdate1;
+
+	// insert same file again
+	await lix.db
+		.updateTable("file")
+		.set({ data: dataUpdate2 })
+		.where("id", "=", "test")
+		.execute();
+
+	const dataUpdate3 = enc.encode("second text update");
+
+	await lix.db
+		.updateTable("file")
+		.set({ data: dataUpdate3 })
+		.where("id", "=", "test")
+		.execute();
+
+	const queue2 = await lix.db.selectFrom("change_queue").selectAll().execute();
+
+	expect(queue2).toEqual([
+		// change update 1 is the same as change update 2
+		// hence, only 2 change queue entries are expected
+		expect.objectContaining({
+			id: 3,
+			file_id: "test",
+			path_after: "/test.txt",
+			metadata_after: null,
+			data_before: dataUpdate1,
+			data_after: dataUpdate2,
+		} satisfies Partial<ChangeQueueEntry>),
+		expect.objectContaining({
+			id: 4,
+			file_id: "test",
+			path_after: "/test.txt",
+			metadata_after: null,
+			data_before: dataUpdate2,
+			data_after: dataUpdate3,
+		} satisfies Partial<ChangeQueueEntry>),
+	]);
+
+	await changeQueueSettled({ lix });
+
+	expect(
+		(await lix.db.selectFrom("change_queue").selectAll().execute()).length
+	).toBe(0);
+
+	const updatedChanges = await lix.db
+		.selectFrom("change")
+		.innerJoin("snapshot", "snapshot.id", "change.snapshot_id")
+		.innerJoin("change_author", "change_author.change_id", "change.id")
+		.selectAll("change")
+		.where("schema_key", "=", "text")
+		.select("change_author.account_id")
+		.select("snapshot.content")
+		.execute();
+
+	const updatedEdges = await lix.db
+		.selectFrom("change_edge")
+		.selectAll()
+		.execute();
+
+	const versionChanges = await lix.db
+		.selectFrom("version_change")
+		.where("version_id", "=", currentVersion.id)
+		.selectAll()
+		.execute();
+
+	expect(updatedChanges).toEqual(
+		expect.arrayContaining([
+			expect.objectContaining({
+				entity_id: "test",
+				schema_key: "text",
+				file_id: "test",
+				plugin_key: "mock-plugin",
+				content: {
+					text: "insert text",
+				},
+				account_id: expect.stringMatching(/^anonymous_/),
+			}),
+			expect.objectContaining({
+				entity_id: "test",
+				file_id: "test",
+				plugin_key: "mock-plugin",
+				schema_key: "text",
+				content: {
+					text: "updated text",
+				},
+				account_id: expect.stringMatching(/^anonymous_/),
+			}),
+			expect.objectContaining({
+				file_id: "test",
+				entity_id: "test",
+				plugin_key: "mock-plugin",
+				schema_key: "text",
+				content: {
+					text: "second text update",
+				},
+				account_id: expect.stringMatching(/^anonymous_/),
+			}),
+		])
+	);
+
+	expect(updatedEdges).toEqual(
+		expect.arrayContaining([
+			// 0 is the parent of 1
+			// 1 is the parent of 2
+			expect.objectContaining({
+				parent_id: updatedChanges[0]?.id,
+				child_id: updatedChanges[1]?.id,
+			}),
+			expect.objectContaining({
+				parent_id: updatedChanges[1]?.id,
+				child_id: updatedChanges[2]?.id,
+			}),
+		])
+	);
+
+	// the version change pointers points to the last change
+	expect(versionChanges).toEqual(
+		expect.arrayContaining([
+			expect.objectContaining({
+				version_id: currentVersion.id,
+				change_id: updatedChanges[2]?.id,
+			}),
+		])
+	);
+});
+
+test.todo("changes should contain the author", async () => {
+	const mockPlugin: LixPlugin = {
+		key: "mock-plugin",
+		detectChangesGlob: "*",
+		detectChanges: vi.fn().mockResolvedValue([
+			{
+				schema: {
+					key: "mock",
+					type: "json",
+				},
+				entity_id: "mock",
+				snapshot: {
+					text: "value1",
+				},
+			},
+			{
+				schema: {
+					key: "mock",
+					type: "json",
+				},
+				entity_id: "mock",
+				snapshot: {
+					text: "value2",
+				},
+			},
+		] satisfies DetectedChange[]),
+	};
+	const lix = await openLixInMemory({
+		blob: await newLixFile(),
+		providePlugins: [mockPlugin],
+	});
+
+	// testing an insert
+
+	await lix.db
+		.insertInto("file")
+		.values({
+			id: "mock",
+			data: new Uint8Array(),
+			path: "/mock-file.json",
+		})
+		.execute();
+
+	await changeQueueSettled({ lix });
+
+	// const changes1 = await lix.db.selectFrom("change").selectAll().execute();
+
+	// expect(changes1[0]?.author).toBe("some-id");
+
+	// testing an update
+
+	await lix.db
+		.updateTable("file")
+		.set({
+			data: new Uint8Array(),
+		})
+		.where("id", "=", "mock")
+		.execute();
+
+	await changeQueueSettled({ lix });
+
+	// const changes2 = await lix.db.selectFrom("change").selectAll().execute();
+
+	// expect(changes2[1]?.author).toBe("some-id");
+
+	await lix.db
+		.updateTable("file")
+		.set({
+			data: new Uint8Array(),
+		})
+		.where("id", "=", "mock")
+		.execute();
+
+	await changeQueueSettled({ lix });
+
+	// const changes3 = await lix.db.selectFrom("change").selectAll().execute();
+
+	// expect(changes3.at(-1)?.author).toBe("some-other-id");
+});
+
+// lix own change control handles file deletions
+test("should handle file deletions correctly", async () => {
+	const mockPlugin: LixPlugin = {
+		key: "mock-plugin",
+		detectChangesGlob: "*",
+		detectChanges: async ({ before, after }) => {
+			const txt = new TextDecoder().decode(before ? before.data : after.data);
+
+			return [
+				{
+					schema: {
+						key: "text",
+						type: "json",
+					},
+					entity_id: "test",
+					snapshot: {
+						text: txt,
+					},
+				},
+			];
+		},
+	};
+
+	const lix = await openLixInMemory({
+		blob: await newLixFile(),
+		providePlugins: [mockPlugin],
+	});
+
+	const enc = new TextEncoder();
+	const dataInitial = enc.encode("file to be deleted");
+
+	// Insert initial file
+	await lix.db
+		.insertInto("file")
+		.values({ id: "file0", path: "/delete.txt", data: dataInitial })
+		.execute();
+
+	// Queue deletion
+	await changeQueueSettled({ lix });
+
+	const changesAfterInsert = await lix.db
+		.selectFrom("change")
+		.innerJoin("snapshot", "snapshot.id", "change.snapshot_id")
+		.where("file_id", "=", "file0")
+		.selectAll()
+		.execute();
+
+	expect(changesAfterInsert).toEqual(
+		expect.arrayContaining([
+			expect.objectContaining({
+				entity_id: "test",
+				file_id: "file0",
+				plugin_key: "mock-plugin",
+				schema_key: "text",
+				content: {
+					text: "file to be deleted",
+				},
+			}),
+		])
+	);
+
+	await lix.db.deleteFrom("file").where("id", "=", "file0").execute();
+
+	// Ensure the queue reflects the deletion entry
+	const queue = await lix.db.selectFrom("change_queue").selectAll().execute();
+
+	expect(queue).toEqual([
+		expect.objectContaining({
+			file_id: "file0",
+			path_before: null,
+			data_before: null,
+			path_after: null,
+			data_after: null,
+		} satisfies Partial<ChangeQueueEntry>),
+	]);
+
+	// Run the change queue settlement process
+	await changeQueueSettled({ lix });
+
+	// Verify the change queue is empty
+	expect(
+		(await lix.db.selectFrom("change_queue").selectAll().execute()).length
+	).toBe(0);
+
+	// Verify the changes reflect the deletion
+	const changes = await lix.db
+		.selectFrom("change")
+		.innerJoin("snapshot", "snapshot.id", "change.snapshot_id")
+		.innerJoin("change_author", "change_author.change_id", "change.id")
+		.where("file_id", "=", "file0")
+		.selectAll("change")
+		.select("change_author.account_id")
+		.select("snapshot.content")
+		.execute();
+
+	expect(changes).toEqual(
+		expect.arrayContaining([
+			expect.objectContaining({
+				entity_id: "test",
+				file_id: "file0",
+				plugin_key: "mock-plugin",
+				schema_key: "text",
+				content: null, // Content is null for deletions
+			}),
+		])
+	);
+
+	// Verify the file no longer exists in the database
+	const internalFilesAfter = await lix.db
+		.selectFrom("file")
+		.selectAll()
+		.execute();
+
+	expect(internalFilesAfter).toEqual([]);
+});

package/src/change-queue/init-change-queue.ts

@@ -0,0 +1,99 @@
+import type { SqliteDatabase } from "sqlite-wasm-kysely";
+import {
+	handleFileUpdate,
+	handleFileInsert,
+	handleFileDelete,
+} from "./file-handlers.js";
+import type { Lix } from "../lix/open-lix.js";
+
+export async function initChangeQueue(args: {
+	lix: Pick<Lix, "db" | "plugin" | "sqlite">;
+	rawDatabase: SqliteDatabase;
+}): Promise<void> {
+	args.rawDatabase.createFunction({
+		name: "triggerChangeQueue",
+		arity: 0,
+		// @ts-expect-error - dynamic function
+		xFunc: () => {
+			// TODO: abort current running queue?
+			queueWorker();
+		},
+	});
+
+	let pending: Promise<void> | undefined;
+
+	let resolve: () => void;
+	// run number counts the worker runs in a current batch and is used to prevent race conditions where a trigger is missed because a previous run is just about to reset the hasMoreEntriesSince flag
+	let runNumber = 1;
+	// If a queue trigger happens during an existing queue run we might miss updates and use hasMoreEntriesSince to make sure there is always a final immediate queue worker execution
+	let hasMoreEntriesSince: number | undefined = undefined;
+
+	async function queueWorker(trail = false) {
+		try {
+			if (pending && !trail) {
+				hasMoreEntriesSince = runNumber;
+				return;
+			}
+			runNumber++;
+
+			if (!pending) {
+				pending = new Promise((res) => {
+					resolve = res;
+				});
+			}
+
+			const entry = await args.lix.db
+				.selectFrom("change_queue")
+				.selectAll()
+				.orderBy("id asc")
+				.limit(1)
+				.executeTakeFirst();
+
+			if (entry) {
+				if (entry.data_before && entry.data_after) {
+					await handleFileUpdate({
+						changeQueueEntry: entry,
+						lix: args.lix,
+					});
+				} else if (!entry.data_before && entry.data_after) {
+					await handleFileInsert({
+						changeQueueEntry: entry,
+						lix: args.lix,
+					});
+				} else {
+					await handleFileDelete({
+						changeQueueEntry: entry,
+						lix: args.lix,
+					});
+				}
+			}
+
+			// console.log("getrting { numEntries }");
+
+			const { numEntries } = await args.lix.db
+				.selectFrom("change_queue")
+				.select((eb) => eb.fn.count<number>("id").as("numEntries"))
+				.executeTakeFirstOrThrow();
+
+			// console.log({ numEntries });
+
+			if (
+				!hasMoreEntriesSince ||
+				(numEntries === 0 && hasMoreEntriesSince < runNumber)
+			) {
+				resolve!(); // TODO: fix type
+				hasMoreEntriesSince = undefined;
+				pending = undefined;
+				// console.log("resolving");
+			}
+
+			// TODO: handle endless tries on failing quee entries
+			// we either execute the queue immediately if we know there is more work or fall back to polling
+			setTimeout(() => queueWorker(true), hasMoreEntriesSince ? 0 : 1000);
+		} catch (e) {
+			console.error("change queue failed ", e);
+		}
+	}
+	// start a worker in case there are entries
+	return queueWorker();
+}

package/src/change-queue/with-skip-change-queue.test.ts

@@ -0,0 +1,158 @@
+import { expect, test } from "vitest";
+import type { LixPlugin } from "../plugin/lix-plugin.js";
+import { openLixInMemory } from "../lix/open-lix-in-memory.js";
+import { changeQueueSettled } from "./change-queue-settled.js";
+import { withSkipChangeQueue } from "./with-skip-change-queue.js";
+
+test("skipping the change queue should be possible", async () => {
+	const mockPlugin: LixPlugin = {
+		key: "mock-plugin",
+		detectChangesGlob: "*",
+		detectChanges: async ({ after }) => {
+			const textAfter = after
+				? new TextDecoder().decode(after.data)
+				: undefined;
+			return [
+				{
+					schema: {
+						key: "text",
+						type: "json",
+					},
+					entity_id: "test",
+					snapshot: { text: textAfter },
+				},
+			];
+		},
+	};
+
+	const lix = await openLixInMemory({
+		providePlugins: [mockPlugin],
+	});
+
+	await lix.db
+		.insertInto("file")
+		.values({
+			id: "test",
+			data: new TextEncoder().encode("update0"),
+			path: "/test.txt",
+		})
+		.execute();
+
+	await changeQueueSettled({ lix });
+
+	const changes0 = await lix.db
+		.selectFrom("change")
+		.innerJoin("snapshot", "snapshot.id", "change.snapshot_id")
+		.where("schema_key", "=", "text")
+		.selectAll()
+		.execute();
+
+	expect(changes0).toHaveLength(1);
+	expect(changes0[0]?.content).toEqual({ text: "update0" });
+
+	await withSkipChangeQueue(lix.db, async (trx) => {
+		await trx
+			.updateTable("file")
+			.set({
+				data: new TextEncoder().encode("update1"),
+			})
+			.where("id", "=", "test")
+			.execute();
+	});
+
+	await changeQueueSettled({ lix });
+
+	const changes1 = await lix.db
+		.selectFrom("change")
+		.innerJoin("snapshot", "snapshot.id", "change.snapshot_id")
+		.selectAll()
+		.where("schema_key", "=", "text")
+		.execute();
+
+	// change is skipped, so no new change is created
+	expect(changes1).toHaveLength(1);
+	expect(changes1[0]?.content).toEqual({ text: "update0" });
+});
+
+test("skipping the change queue should be possible with multiple changes", async () => {
+	const mockPlugin: LixPlugin = {
+		key: "mock-plugin",
+		detectChangesGlob: "*",
+		detectChanges: async ({ after }) => {
+			const textAfter = after
+				? new TextDecoder().decode(after.data)
+				: undefined;
+			return [
+				{
+					schema: {
+						key: "text",
+						type: "json",
+					},
+					entity_id: "test",
+					snapshot: { text: textAfter },
+				},
+			];
+		},
+	};
+
+	const lix = await openLixInMemory({
+		providePlugins: [mockPlugin],
+	});
+
+	await lix.db
+		.insertInto("file")
+		.values({
+			id: "test",
+			data: new TextEncoder().encode("initial"),
+			path: "/test.txt",
+		})
+		.execute();
+
+	const manyUpdates = Array.from({ length: 100 }, (_, i) => `update${i}`);
+
+	const first50Updates = manyUpdates.slice(0, 50);
+	const last50Updates = manyUpdates.slice(50);
+
+	await lix.db.transaction().execute(async (trx) => {
+		for (const update of first50Updates) {
+			await trx
+				.updateTable("file")
+				.set({
+					data: new TextEncoder().encode(update),
+				})
+				.where("id", "=", "test")
+				.execute();
+		}
+		for (const update of last50Updates) {
+			await withSkipChangeQueue(trx, async (trx) => {
+				await trx
+					.updateTable("file")
+					.set({
+						data: new TextEncoder().encode(update),
+					})
+					.where("id", "=", "test")
+					.execute();
+			});
+		}
+	});
+
+	await changeQueueSettled({ lix });
+
+	const file = await lix.db
+		.selectFrom("file")
+		.selectAll()
+		.where("id", "=", "test")
+		.executeTakeFirstOrThrow();
+
+	const changes = await lix.db
+		.selectFrom("change")
+		.innerJoin("snapshot", "snapshot.id", "change.snapshot_id")
+		.where("file_id", "=", file.id)
+		.selectAll()
+		.execute();
+
+	// expecting the latest change
+	expect(file.data).toEqual(new TextEncoder().encode("update99"));
+	// and only 50 changes + 1 insert change have been created
+	expect(changes).toHaveLength(51);
+});