@lix-js/sdk 0.4.6 → 0.5.0-preview.0

package/src/file-queue/file-queue-process.test.ts

@@ -1,456 +0,0 @@
-import { expect, test, vi } from "vitest";
-import { openLixInMemory } from "../lix/open-lix-in-memory.js";
-import { newLixFile } from "../lix/new-lix.js";
-import type { DetectedChange, LixPlugin } from "../plugin/lix-plugin.js";
-import type { FileQueueEntry, LixFile } from "../database/schema.js";
-import { fileQueueSettled } from "./file-queue-settled.js";
-
-test("should use queue and settled correctly", async () => {
-	const mockPlugin: LixPlugin = {
-		key: "mock-plugin",
-		detectChangesGlob: "*",
-		detectChanges: async ({ before, after }) => {
-			const textBefore = before
-				? new TextDecoder().decode(before?.data)
-				: undefined;
-			const textAfter = after
-				? new TextDecoder().decode(after.data)
-				: undefined;
-
-			if (textBefore === textAfter) {
-				return [];
-			}
-
-			return [
-				{
-					schema: {
-						key: "text",
-						type: "json",
-					},
-					entity_id: "test",
-					snapshot: textAfter ? { text: textAfter } : undefined,
-				},
-			];
-		},
-	};
-
-	const lix = await openLixInMemory({
-		blob: await newLixFile(),
-		providePlugins: [mockPlugin],
-	});
-
-	const currentVersion = await lix.db
-		.selectFrom("current_version")
-		.innerJoin("version", "current_version.id", "version.id")
-		.selectAll()
-		.executeTakeFirstOrThrow();
-
-	const enc = new TextEncoder();
-	const dataInitial = enc.encode("insert text");
-	await lix.db
-		.insertInto("file")
-		.values({ id: "test", path: "/test.txt", data: dataInitial })
-		.execute();
-
-	const queue = await lix.db.selectFrom("file_queue").selectAll().execute();
-
-	expect(queue).toEqual([
-		expect.objectContaining({
-			id: 1,
-			file_id: "test",
-			path_after: "/test.txt",
-			data_after: dataInitial,
-			data_before: null,
-		} satisfies Partial<FileQueueEntry>),
-	]);
-
-	await fileQueueSettled({ lix });
-
-	expect(
-		(await lix.db.selectFrom("file_queue").selectAll().execute()).length
-	).toBe(0);
-
-	// TODO QUEUE check if the replacement of file_internal was expected
-	const internalFilesAfter = await lix.db
-		.selectFrom("file")
-		.selectAll()
-		.execute();
-
-	expect(internalFilesAfter).toEqual([
-		expect.objectContaining({
-			data: internalFilesAfter[0]!.data,
-			id: "test",
-			path: "/test.txt",
-			metadata: null,
-		}) satisfies LixFile,
-	]);
-
-	const changes = await lix.db
-		.selectFrom("change")
-		.innerJoin("snapshot", "snapshot.id", "change.snapshot_id")
-		.innerJoin("change_author", "change_author.change_id", "change.id")
-		.selectAll("change")
-		.select("change_author.account_id")
-		.select("snapshot.content")
-		.execute();
-
-	expect(changes).toEqual(
-		expect.arrayContaining([
-			expect.objectContaining({
-				entity_id: "test",
-				schema_key: "text",
-				file_id: "test",
-				plugin_key: "mock-plugin",
-				content: {
-					text: "insert text",
-				},
-				account_id: expect.stringMatching(/./),
-			}),
-		])
-	);
-
-	const dataUpdate1 = enc.encode("updated text");
-	await lix.db
-		.updateTable("file")
-		.set({ data: dataUpdate1 })
-		.where("id", "=", "test")
-		.execute();
-
-	// const beforeQueueTick = await lix.db
-	// 	.selectFrom("file_queue")
-	// 	.selectAll()
-	// 	.execute();
-
-	// expect(beforeQueueTick.length).toBe(1);
-
-	// const afterQueueTick = await lix.db
-	// 	.selectFrom("file_queue")
-	// 	.selectAll()
-	// 	.execute();
-
-	// expect(afterQueueTick.length).toBe(0);
-
-	// update2 is equal to update1
-	const dataUpdate2 = dataUpdate1;
-
-	// insert same file again
-	await lix.db
-		.updateTable("file")
-		.set({ data: dataUpdate2 })
-		.where("id", "=", "test")
-		.execute();
-
-	const dataUpdate3 = enc.encode("second text update");
-
-	await lix.db
-		.updateTable("file")
-		.set({ data: dataUpdate3 })
-		.where("id", "=", "test")
-		.execute();
-
-	const queue2 = await lix.db.selectFrom("file_queue").selectAll().execute();
-
-	expect(queue2).toEqual([
-		// change update 1 is the same as change update 2
-		// hence, only 2 fil queue entries are expected
-		expect.objectContaining({
-			id: 3,
-			file_id: "test",
-			path_after: "/test.txt",
-			metadata_after: null,
-			data_before: dataUpdate1,
-			data_after: dataUpdate2,
-		} satisfies Partial<FileQueueEntry>),
-		expect.objectContaining({
-			id: 4,
-			file_id: "test",
-			path_after: "/test.txt",
-			metadata_after: null,
-			data_before: dataUpdate2,
-			data_after: dataUpdate3,
-		} satisfies Partial<FileQueueEntry>),
-	]);
-
-	await fileQueueSettled({ lix });
-
-	expect(
-		(await lix.db.selectFrom("file_queue").selectAll().execute()).length
-	).toBe(0);
-
-	const updatedChanges = await lix.db
-		.selectFrom("change")
-		.innerJoin("snapshot", "snapshot.id", "change.snapshot_id")
-		.innerJoin("change_author", "change_author.change_id", "change.id")
-		.selectAll("change")
-		.where("schema_key", "=", "text")
-		.select("change_author.account_id")
-		.select("snapshot.content")
-		.execute();
-
-	const updatedEdges = await lix.db
-		.selectFrom("change_edge")
-		.selectAll()
-		.execute();
-
-	const versionChanges = await lix.db
-		.selectFrom("version_change")
-		.where("version_id", "=", currentVersion.id)
-		.selectAll()
-		.execute();
-
-	expect(updatedChanges).toEqual(
-		expect.arrayContaining([
-			expect.objectContaining({
-				entity_id: "test",
-				schema_key: "text",
-				file_id: "test",
-				plugin_key: "mock-plugin",
-				content: {
-					text: "insert text",
-				},
-				account_id: expect.stringMatching(/./),
-			}),
-			expect.objectContaining({
-				entity_id: "test",
-				file_id: "test",
-				plugin_key: "mock-plugin",
-				schema_key: "text",
-				content: {
-					text: "updated text",
-				},
-				account_id: expect.stringMatching(/./),
-			}),
-			expect.objectContaining({
-				file_id: "test",
-				entity_id: "test",
-				plugin_key: "mock-plugin",
-				schema_key: "text",
-				content: {
-					text: "second text update",
-				},
-				account_id: expect.stringMatching(/./),
-			}),
-		])
-	);
-
-	expect(updatedEdges).toEqual(
-		expect.arrayContaining([
-			// 0 is the parent of 1
-			// 1 is the parent of 2
-			expect.objectContaining({
-				parent_id: updatedChanges[0]?.id,
-				child_id: updatedChanges[1]?.id,
-			}),
-			expect.objectContaining({
-				parent_id: updatedChanges[1]?.id,
-				child_id: updatedChanges[2]?.id,
-			}),
-		])
-	);
-
-	// the version change pointers points to the last change
-	expect(versionChanges).toEqual(
-		expect.arrayContaining([
-			expect.objectContaining({
-				version_id: currentVersion.id,
-				change_id: updatedChanges[2]?.id,
-			}),
-		])
-	);
-});
-
-test.todo("changes should contain the author", async () => {
-	const mockPlugin: LixPlugin = {
-		key: "mock-plugin",
-		detectChangesGlob: "*",
-		detectChanges: vi.fn().mockResolvedValue([
-			{
-				schema: {
-					key: "mock",
-					type: "json",
-				},
-				entity_id: "mock",
-				snapshot: {
-					text: "value1",
-				},
-			},
-			{
-				schema: {
-					key: "mock",
-					type: "json",
-				},
-				entity_id: "mock",
-				snapshot: {
-					text: "value2",
-				},
-			},
-		] satisfies DetectedChange[]),
-	};
-	const lix = await openLixInMemory({
-		blob: await newLixFile(),
-		providePlugins: [mockPlugin],
-	});
-
-	// testing an insert
-
-	await lix.db
-		.insertInto("file")
-		.values({
-			id: "mock",
-			data: new Uint8Array(),
-			path: "/mock-file.json",
-		})
-		.execute();
-
-	await fileQueueSettled({ lix });
-
-	// const changes1 = await lix.db.selectFrom("change").selectAll().execute();
-
-	// expect(changes1[0]?.author).toBe("some-id");
-
-	// testing an update
-
-	await lix.db
-		.updateTable("file")
-		.set({
-			data: new Uint8Array(),
-		})
-		.where("id", "=", "mock")
-		.execute();
-
-	await fileQueueSettled({ lix });
-
-	// const changes2 = await lix.db.selectFrom("change").selectAll().execute();
-
-	// expect(changes2[1]?.author).toBe("some-id");
-
-	await lix.db
-		.updateTable("file")
-		.set({
-			data: new Uint8Array(),
-		})
-		.where("id", "=", "mock")
-		.execute();
-
-	await fileQueueSettled({ lix });
-
-	// const changes3 = await lix.db.selectFrom("change").selectAll().execute();
-
-	// expect(changes3.at(-1)?.author).toBe("some-other-id");
-});
-
-// lix own change control handles file deletions
-test("should handle file deletions correctly", async () => {
-	const mockPlugin: LixPlugin = {
-		key: "mock-plugin",
-		detectChangesGlob: "*",
-		detectChanges: async ({ before, after }) => {
-			const txt = new TextDecoder().decode(before ? before.data : after.data);
-
-			return [
-				{
-					schema: {
-						key: "text",
-						type: "json",
-					},
-					entity_id: "test",
-					snapshot: {
-						text: txt,
-					},
-				},
-			];
-		},
-	};
-
-	const lix = await openLixInMemory({
-		blob: await newLixFile(),
-		providePlugins: [mockPlugin],
-	});
-
-	const enc = new TextEncoder();
-	const dataInitial = enc.encode("file to be deleted");
-
-	// Insert initial file
-	await lix.db
-		.insertInto("file")
-		.values({ id: "file0", path: "/delete.txt", data: dataInitial })
-		.execute();
-
-	// Queue deletion
-	await fileQueueSettled({ lix });
-
-	const changesAfterInsert = await lix.db
-		.selectFrom("change")
-		.innerJoin("snapshot", "snapshot.id", "change.snapshot_id")
-		.where("file_id", "=", "file0")
-		.selectAll()
-		.execute();
-
-	expect(changesAfterInsert).toEqual(
-		expect.arrayContaining([
-			expect.objectContaining({
-				entity_id: "test",
-				file_id: "file0",
-				plugin_key: "mock-plugin",
-				schema_key: "text",
-				content: {
-					text: "file to be deleted",
-				},
-			}),
-		])
-	);
-
-	await lix.db.deleteFrom("file").where("id", "=", "file0").execute();
-
-	// Ensure the queue reflects the deletion entry
-	const queue = await lix.db.selectFrom("file_queue").selectAll().execute();
-
-	expect(queue).toEqual([
-		expect.objectContaining({
-			file_id: "file0",
-			path_before: null,
-			data_before: null,
-			path_after: null,
-			data_after: null,
-		} satisfies Partial<FileQueueEntry>),
-	]);
-
-	// Run the fil queue settlement process
-	await fileQueueSettled({ lix });
-
-	// Verify the fil queue is empty
-	expect(
-		(await lix.db.selectFrom("file_queue").selectAll().execute()).length
-	).toBe(0);
-
-	// Verify the changes reflect the deletion
-	const changes = await lix.db
-		.selectFrom("change")
-		.innerJoin("snapshot", "snapshot.id", "change.snapshot_id")
-		.innerJoin("change_author", "change_author.change_id", "change.id")
-		.where("file_id", "=", "file0")
-		.selectAll("change")
-		.select("change_author.account_id")
-		.select("snapshot.content")
-		.execute();
-
-	expect(changes).toEqual(
-		expect.arrayContaining([
-			expect.objectContaining({
-				entity_id: "test",
-				file_id: "file0",
-				plugin_key: "mock-plugin",
-				schema_key: "text",
-				content: null, // Content is null for deletions
-			}),
-		])
-	);
-
-	// Verify the file no longer exists in the database
-	const internalFilesAfter = await lix.db
-		.selectFrom("file")
-		.selectAll()
-		.execute();
-
-	expect(internalFilesAfter).toEqual([]);
-});

package/src/file-queue/file-queue-process.ts

@@ -1,108 +0,0 @@
-import {
-	handleFileUpdate,
-	handleFileInsert,
-	handleFileDelete,
-} from "./file-handlers.js";
-import type { Lix } from "../lix/open-lix.js";
-
-export async function initFileQueueProcess(args: {
-	lix: Pick<Lix, "db" | "plugin" | "sqlite">;
-}): Promise<void> {
-	args.lix.sqlite.createFunction({
-		name: "triggerFileQueue",
-		arity: 0,
-		// @ts-expect-error - dynamic function
-		xFunc: () => {
-			// TODO: abort current running queue?
-			queueWorker();
-		},
-	});
-
-	let pending: Promise<void> | undefined;
-
-	let resolve: () => void;
-	// run number counts the worker runs in a current batch and is used to prevent race conditions where a trigger is missed because a previous run is just about to reset the hasMoreEntriesSince flag
-	let runNumber = 1;
-	// If a queue trigger happens during an existing queue run we might miss updates and use hasMoreEntriesSince to make sure there is always a final immediate queue worker execution
-	let hasMoreEntriesSince: number | undefined = undefined;
-
-	async function queueWorker(trail = false) {
-		try {
-			if (args.lix.sqlite.isOpen() === false) {
-				return;
-			}
-			if (pending && !trail) {
-				hasMoreEntriesSince = runNumber;
-				return;
-			}
-			runNumber++;
-
-			if (!pending) {
-				pending = new Promise((res) => {
-					resolve = res;
-				});
-			}
-
-			const entry = await args.lix.db
-				.selectFrom("file_queue")
-				.selectAll()
-				.orderBy("id asc")
-				.limit(1)
-				.executeTakeFirst();
-
-			if (entry) {
-				if (entry.data_before && entry.data_after) {
-					await handleFileUpdate({
-						fileQueueEntry: entry,
-						lix: args.lix,
-					});
-				} else if (!entry.data_before && entry.data_after) {
-					await handleFileInsert({
-						fileQueueEntry: entry,
-						lix: args.lix,
-					});
-				} else {
-					await handleFileDelete({
-						fileQueueEntry: entry,
-						lix: args.lix,
-					});
-				}
-			}
-
-			// console.log("getrting { numEntries }");
-
-			const { numEntries } = await args.lix.db
-				.selectFrom("file_queue")
-				.select((eb) => eb.fn.count<number>("id").as("numEntries"))
-				.executeTakeFirstOrThrow();
-
-			// console.log({ numEntries });
-
-			if (
-				!hasMoreEntriesSince ||
-				(numEntries === 0 && hasMoreEntriesSince < runNumber)
-			) {
-				resolve!();
-				hasMoreEntriesSince = undefined;
-				pending = undefined;
-				// console.log("resolving");
-			} else {
-				// there are more entries to process
-				queueWorker(true);
-			}
-		} catch (e) {
-			// database has been closed, ignore
-			if (
-				e instanceof Error &&
-				(e.message.includes("DB has been closed") ||
-					e.message.includes("driver has already been destroyed"))
-			) {
-				return;
-			}
-			console.error("file queue failed ", e);
-		}
-	}
-	// start a worker in case there are entries
-	queueWorker();
-	return;
-}

package/src/file-queue/file-queue-settled.test.ts

@@ -1,56 +0,0 @@
-import { test, expect } from "vitest";
-import { openLixInMemory } from "../lix/open-lix-in-memory.js";
-import { fileQueueSettled } from "./file-queue-settled.js";
-
-test("should wait until the file queue is settled", async () => {
-	const lix = await openLixInMemory({});
-
-	await lix.db
-		.insertInto("file_queue")
-		.values([
-			{
-				id: 1,
-				file_id: "file1",
-				path_after: "path1",
-				data_after: new TextEncoder().encode("data1"),
-				metadata_after: null,
-			},
-			{
-				id: 2,
-				file_id: "file2",
-				path_after: "path2",
-				data_after: new TextEncoder().encode("data2"),
-				metadata_after: null,
-			},
-		])
-		.execute();
-
-	// Start a background task to remove entries from the file_queue after a delay
-	setTimeout(async () => {
-		await lix.db.deleteFrom("file_queue").where("id", "=", 1).execute();
-		await lix.db.deleteFrom("file_queue").where("id", "=", 2).execute();
-	}, 110);
-
-	await fileQueueSettled({ lix });
-
-	const remainingEntries = await lix.db
-		.selectFrom("file_queue")
-		.selectAll()
-		.execute();
-
-	expect(remainingEntries).toEqual([]);
-});
-
-test("should return immediately if the file queue is already empty", async () => {
-	const lix = await openLixInMemory({});
-
-	await lix.db.deleteFrom("file_queue").execute();
-
-	await fileQueueSettled({ lix });
-
-	const remainingEntries = await lix.db
-		.selectFrom("file_queue")
-		.selectAll()
-		.execute();
-	expect(remainingEntries).toEqual([]);
-});

package/src/file-queue/file-queue-settled.ts

@@ -1,31 +0,0 @@
-import type { Lix } from "../lix/open-lix.js";
-
-/**
- * Waits until the file queue is settled.
- *
- * @example
- *   ```ts
- *   await fileQueueSettled({ lix });
- *   ```
- */
-export async function fileQueueSettled(args: {
-	lix: Pick<Lix, "db">;
-}): Promise<void> {
-	let hasEntries = true;
-
-	while (hasEntries) {
-		const entries = await args.lix.db
-			.selectFrom("file_queue")
-			.selectAll()
-			.limit(1)
-			.execute();
-
-		hasEntries = entries.length > 0;
-
-		if (hasEntries) {
-			// poll again in 50ms. This is a workaround until subscriptions
-			// or another mechanism is implemented to notify when the queue is settled
-			await new Promise((resolve) => setTimeout(resolve, 50));
-		}
-	}
-}

package/src/file-queue/index.ts

@@ -1 +0,0 @@
-export { fileQueueSettled } from "./file-queue-settled.js";