@lix-js/sdk 0.0.1

package/src/change-conflict/resolve-conflict-by-selecting.test.ts

@@ -0,0 +1,118 @@
+import { test, expect, vi } from "vitest";
+import { openLixInMemory } from "../lix/open-lix-in-memory.js";
+import { newLixFile } from "../lix/new-lix.js";
+import type { NewChange, Snapshot } from "../database/schema.js";
+import type { LixPlugin } from "../plugin/lix-plugin.js";
+import { mockJsonSnapshot } from "../snapshot/mock-json-snapshot.js";
+import { resolveChangeConflictBySelecting } from "./resolve-conflict-by-selecting.js";
+
+test("it should resolve a conflict and apply the changes", async () => {
+	const mockSnapshots: Snapshot[] = [
+		mockJsonSnapshot({
+			id: "value1",
+		}),
+		mockJsonSnapshot({
+			id: "value2",
+		}),
+	];
+
+	const mockChanges: NewChange[] = [
+		{
+			plugin_key: "plugin1",
+			schema_key: "mock",
+			file_id: "mock",
+			entity_id: "value1",
+			snapshot_id: mockSnapshots[0]!.id,
+		},
+		{
+			plugin_key: "plugin1",
+			file_id: "mock",
+			entity_id: "value2",
+			schema_key: "mock",
+			snapshot_id: mockSnapshots[1]!.id,
+		},
+	];
+
+	const mockPlugin: LixPlugin = {
+		key: "plugin1",
+		applyChanges: vi.fn().mockResolvedValue({
+			fileData: new TextEncoder().encode(
+				JSON.stringify(mockSnapshots[0]?.content)
+			),
+		}),
+	};
+
+	const lix = await openLixInMemory({
+		blob: await newLixFile(),
+		providePlugins: [mockPlugin],
+	});
+
+	await lix.db
+		.insertInto("file")
+		.values({
+			id: "mock",
+			path: "/mock",
+			data: new Uint8Array(),
+		})
+		.execute();
+
+	const snapshots = await lix.db
+		.insertInto("snapshot")
+		.values(
+			mockSnapshots.map((s) => {
+				return { content: s.content };
+			})
+		)
+		.returningAll()
+		.execute();
+
+	const changes = await lix.db
+		.insertInto("change")
+		.values(mockChanges)
+		.returningAll()
+		.execute();
+
+	const changeSet0 = await lix.db
+		.insertInto("change_set")
+		.defaultValues()
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	const conflict = await lix.db
+		.insertInto("change_conflict")
+		.values({
+			key: "mock",
+			change_set_id: changeSet0.id,
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	await resolveChangeConflictBySelecting({
+		lix: lix,
+		conflict: conflict,
+		select: changes[0]!,
+	});
+
+	const resolvedConflict = await lix.db
+		.selectFrom("change_conflict")
+		.innerJoin(
+			"change_conflict_resolution",
+			"change_conflict_resolution.change_conflict_id",
+			"change_conflict.id"
+		)
+		.where("id", "=", conflict.id)
+		.selectAll()
+		.executeTakeFirstOrThrow();
+
+	// TODO QUEUE check if the replacement of file_internal was expected
+	const fileAfterResolve = await lix.db
+		.selectFrom("file")
+		.selectAll()
+		.where("id", "=", changes[0]!.file_id)
+		.executeTakeFirstOrThrow();
+
+	const parsed = JSON.parse(new TextDecoder().decode(fileAfterResolve.data));
+
+	expect(parsed).toStrictEqual(snapshots[0]!.content);
+	expect(resolvedConflict.resolved_change_id).toBe(changes[0]!.id);
+});

package/src/change-conflict/resolve-conflict-by-selecting.ts

@@ -0,0 +1,66 @@
+import type { Change, ChangeConflict } from "../database/schema.js";
+import type { Lix } from "../lix/open-lix.js";
+import { applyChanges } from "../change/apply-changes.js";
+import { updateChangesInVersion } from "../version/update-changes-in-version.js";
+
+/**
+ * Resolves a conflict by selecting one of the two
+ * changes in the conflict.
+ */
+export async function resolveChangeConflictBySelecting(args: {
+	lix: Lix;
+	conflict: ChangeConflict;
+	select: Change;
+}): Promise<void> {
+	const executeInTransaction = async (trx: Lix["db"]) => {
+		await trx
+			.insertInto("change_conflict_resolution")
+			.values({
+				change_conflict_id: args.conflict.id,
+				resolved_change_id: args.select.id,
+			})
+			// conflict resolution already exists
+			.onConflict((oc) => oc.doNothing())
+			.execute();
+
+		// TODO on-demand apply changes https://linear.app/opral/issue/LIXDK-219/on-demand-applychanges-on-filedata-read
+		await applyChanges({
+			lix: { ...args.lix, db: trx },
+			changes: [args.select],
+		});
+
+		const versionsWithConflicts = await trx
+			.selectFrom("version")
+			.innerJoin(
+				"version_change_conflict",
+				"version_change_conflict.version_id",
+				"version.id"
+			)
+			.where(
+				"version_change_conflict.change_conflict_id",
+				"=",
+				args.conflict.id
+			)
+			.selectAll()
+			.execute();
+
+		for (const version of versionsWithConflicts) {
+			await updateChangesInVersion({
+				lix: { ...args.lix, db: trx },
+				changes: [args.select],
+				version,
+			});
+			// remove the conflict pointer
+			await trx
+				.deleteFrom("version_change_conflict")
+				.where("version_id", "=", version.id)
+				.where("change_conflict_id", "=", args.conflict.id)
+				.execute();
+		}
+	};
+
+	if (args.lix.db.isTransaction) {
+		return executeInTransaction(args.lix.db);
+	}
+	return args.lix.db.transaction().execute(executeInTransaction);
+}

package/src/change-conflict/update-change-conflicts.ts

@@ -0,0 +1,79 @@
+// import { createChangeConflict } from "../change-conflict/create-change-conflict.js";
+// import { detectChangeConflicts } from "../change-conflict/detect-change-conflicts.js";
+// import type { version } from "../database/schema.js";
+// import type { Lix } from "../lix/open-lix.js";
+
+// export async function updateChangeConflicts(args: {
+// 	lix: Pick<Lix, "db" | "plugin">;
+// 	version: Pick<version, "id">;
+// }) {
+// 	const executeInTransaction = async (trx: Lix["db"]) => {
+// 		// const targetversiones = await trx
+// 		// 	.selectFrom("version_target")
+// 		// 	.where("target_version_id", "=", args.version.id)
+// 		// 	.selectAll()
+// 		// 	.execute();
+
+// 		for (const targetversion of [args.version]) {
+// 			const diffingPointers = await getversionChangePointerDiff(trx, {
+// 				sourceversion: { id: targetversion.source_version_id },
+// 				targetversion: { id: targetversion.target_version_id },
+// 			});
+
+// 			// the set all change ids that are diffing
+// 			const diffingChangeIds = new Set<string>();
+
+// 			for (const pointer of diffingPointers) {
+// 				if (pointer.source_change_id !== pointer.target_change_id) {
+// 					if (pointer.source_change_id) {
+// 						diffingChangeIds.add(pointer.source_change_id);
+// 					}
+// 					if (pointer.target_change_id) {
+// 						diffingChangeIds.add(pointer.target_change_id);
+// 					}
+// 				}
+// 			}
+
+// 			// could be queried in a single query with a join
+// 			// but decided to keep it simpler for now
+// 			const diffingChanges = await trx
+// 				.selectFrom("change")
+// 				.select(["id"])
+// 				.where("id", "in", [...diffingChangeIds])
+// 				.selectAll()
+// 				.execute();
+
+// 			const detectedConflicts = await detectChangeConflicts({
+// 				lix: { ...args.lix, db: trx },
+// 				changes: diffingChanges,
+// 			});
+
+// 			// insert the detected conflicts
+// 			// (ignore if the conflict already exists)
+// 			for (const detectedConflict of detectedConflicts) {
+// 				const conflict = await createChangeConflict({
+// 					lix: { ...args.lix, db: trx },
+// 					version: args.version,
+// 					key: detectedConflict.key,
+// 					conflictingChangeIds: detectedConflict.conflictingChangeIds,
+// 				});
+
+// 				// todo move to createChangeConflict
+// 				await trx
+// 					.insertInto("version_change_conflict_pointer")
+// 					.values({
+// 						version_id: args.version.id,
+// 						change_conflict_id: conflict.id,
+// 					})
+// 					.onConflict((oc) => oc.doNothing())
+// 					.execute();
+// 			}
+// 		}
+// 	};
+
+// 	if (args.lix.db.isTransaction) {
+// 		return executeInTransaction(args.lix.db);
+// 	} else {
+// 		return args.lix.db.transaction().execute(executeInTransaction);
+// 	}
+// }

package/src/change-queue/change-queue-settled.test.ts

@@ -0,0 +1,56 @@
+import { test, expect } from "vitest";
+import { openLixInMemory } from "../lix/open-lix-in-memory.js";
+import { changeQueueSettled } from "./change-queue-settled.js";
+
+test("should wait until the change queue is settled", async () => {
+	const lix = await openLixInMemory({});
+
+	await lix.db
+		.insertInto("change_queue")
+		.values([
+			{
+				id: 1,
+				file_id: "file1",
+				path_after: "path1",
+				data_after: new TextEncoder().encode("data1"),
+				metadata_after: null,
+			},
+			{
+				id: 2,
+				file_id: "file2",
+				path_after: "path2",
+				data_after: new TextEncoder().encode("data2"),
+				metadata_after: null,
+			},
+		])
+		.execute();
+
+	// Start a background task to remove entries from the change_queue after a delay
+	setTimeout(async () => {
+		await lix.db.deleteFrom("change_queue").where("id", "=", 1).execute();
+		await lix.db.deleteFrom("change_queue").where("id", "=", 2).execute();
+	}, 110);
+
+	await changeQueueSettled({ lix });
+
+	const remainingEntries = await lix.db
+		.selectFrom("change_queue")
+		.selectAll()
+		.execute();
+
+	expect(remainingEntries).toEqual([]);
+});
+
+test("should return immediately if the change queue is already empty", async () => {
+	const lix = await openLixInMemory({});
+
+	await lix.db.deleteFrom("change_queue").execute();
+
+	await changeQueueSettled({ lix });
+
+	const remainingEntries = await lix.db
+		.selectFrom("change_queue")
+		.selectAll()
+		.execute();
+	expect(remainingEntries).toEqual([]);
+});

package/src/change-queue/change-queue-settled.ts

@@ -0,0 +1,31 @@
+import type { Lix } from "../lix/open-lix.js";
+
+/**
+ * Waits until the change queue is settled.
+ *
+ * @example
+ *   ```ts
+ *   await changeQueueSettled({ lix });
+ *   ```
+ */
+export async function changeQueueSettled(args: {
+	lix: Pick<Lix, "db">;
+}): Promise<void> {
+	let hasEntries = true;
+
+	while (hasEntries) {
+		const entries = await args.lix.db
+			.selectFrom("change_queue")
+			.selectAll()
+			.limit(1)
+			.execute();
+
+		hasEntries = entries.length > 0;
+
+		if (hasEntries) {
+			// poll again in 50ms. This is a workaround until subscriptions
+			// or another mechanism is implemented to notify when the queue is settled
+			await new Promise((resolve) => setTimeout(resolve, 50));
+		}
+	}
+}

package/src/change-queue/file-handlers.ts

@@ -0,0 +1,273 @@
+import type { ChangeQueueEntry } from "../database/schema.js";
+import type { DetectedChange } from "../plugin/lix-plugin.js";
+import type { Lix } from "../lix/open-lix.js";
+import { sql } from "kysely";
+import { createChange } from "../change/create-change.js";
+import { changeIsLeafInVersion } from "../query-filter/change-is-leaf-in-version.js";
+
+// start a new normalize path function that has the absolute minimum implementation.
+function normalizePath(path: string) {
+	if (!path.startsWith("/")) {
+		return "/" + path;
+	}
+	return path;
+}
+
+async function glob(args: {
+	lix: Pick<Lix, "db">;
+	glob: string;
+	path: string;
+}) {
+	const result =
+		await sql`SELECT CASE WHEN ${args.path} GLOB ${args.glob} THEN 1 ELSE 0 END AS matches`.execute(
+			args.lix.db
+		);
+
+	// Extract the result from the response
+
+	return (result.rows[0] as any)?.matches === 1;
+}
+
+// creates initial changes for new files
+export async function handleFileInsert(args: {
+	lix: Pick<Lix, "db" | "plugin" | "sqlite">;
+	changeQueueEntry: ChangeQueueEntry;
+}): Promise<void> {
+	const detectedChanges: Array<DetectedChange & { pluginKey: string }> = [];
+
+	const plugins = await args.lix.plugin.getAll();
+
+	// the path of the file is either the after path or the before path
+	// depending on whether the file was deleted, updated, or created
+	const path =
+		args.changeQueueEntry.path_after ?? args.changeQueueEntry.path_before;
+
+	if (path === null) {
+		throw new Error("Both before and after paths are null");
+	}
+
+	for (const plugin of plugins) {
+		// glob expressions are expressed relative without leading / but path has leading /
+		if (
+			!(await glob({
+				lix: args.lix,
+				path: normalizePath(path),
+				glob: "/" + plugin.detectChangesGlob,
+			}))
+		) {
+			break;
+		}
+
+		if (plugin.detectChanges === undefined) {
+			const error = new Error(
+				"Plugin does not support detecting changes even though the glob matches."
+			);
+			// https://linear.app/opral/issue/LIXDK-195/await-change-queue-to-log-errors
+			console.error(error);
+			throw error;
+		}
+
+		if (args.changeQueueEntry.data_after === null) {
+			throw new Error("Data after is null");
+		}
+
+		for (const change of await plugin.detectChanges({
+			lix: args.lix,
+			before: undefined,
+			after: {
+				id: args.changeQueueEntry.file_id,
+				path,
+				metadata: args.changeQueueEntry.metadata_after,
+				data: args.changeQueueEntry.data_after,
+			},
+		})) {
+			detectedChanges.push({
+				...change,
+				pluginKey: plugin.key,
+			});
+		}
+	}
+
+	await args.lix.db.transaction().execute(async (trx) => {
+		const currentAuthors = await trx
+			.selectFrom("active_account")
+			.selectAll()
+			.execute();
+
+		const currentVersion = await trx
+			.selectFrom("current_version")
+			.innerJoin("version", "current_version.id", "version.id")
+			.selectAll()
+			.executeTakeFirstOrThrow();
+
+		await Promise.all(
+			detectedChanges.map(async (detectedChange) => {
+				await createChange({
+					lix: { ...args.lix, db: trx },
+					authors: currentAuthors,
+					version: currentVersion,
+					entityId: detectedChange.entity_id,
+					fileId: args.changeQueueEntry.file_id,
+					pluginKey: detectedChange.pluginKey,
+					schemaKey: detectedChange.schema.key,
+					snapshotContent: detectedChange.snapshot,
+				});
+			})
+		);
+
+		await trx
+			.deleteFrom("change_queue")
+			.where("id", "=", args.changeQueueEntry.id)
+			.execute();
+	});
+}
+
+export async function handleFileUpdate(args: {
+	lix: Pick<Lix, "db" | "plugin" | "sqlite">;
+	changeQueueEntry: ChangeQueueEntry;
+}): Promise<void> {
+	const detectedChanges: Array<DetectedChange & { pluginKey: string }> = [];
+
+	const plugins = await args.lix.plugin.getAll();
+
+	// the path of the file is either the after path or the before path
+	// depending on whether the file was deleted, updated, or created
+	const path =
+		args.changeQueueEntry.path_after ?? args.changeQueueEntry.path_before;
+
+	if (path === null) {
+		throw new Error("Both before and after paths are null");
+	}
+
+	for (const plugin of plugins) {
+		// glob expressions are expressed relative without leading / but path has leading /
+		if (
+			!(await glob({
+				lix: args.lix,
+				path: normalizePath(path),
+				glob: "/" + plugin.detectChangesGlob,
+			}))
+		) {
+			break;
+		}
+		if (plugin.detectChanges === undefined) {
+			const error = new Error(
+				"Plugin does not support detecting changes even though the glob matches."
+			);
+			// https://linear.app/opral/issue/LIXDK-195/await-change-queue-to-log-errors
+			console.error(error);
+			throw error;
+		}
+		for (const change of await plugin.detectChanges({
+			lix: args.lix,
+			before: args.changeQueueEntry.data_before
+				? {
+						id: args.changeQueueEntry.file_id,
+						path: path,
+						metadata: args.changeQueueEntry.metadata_before,
+						data: args.changeQueueEntry.data_before,
+					}
+				: undefined,
+			after: {
+				id: args.changeQueueEntry.file_id,
+				path,
+				metadata: args.changeQueueEntry.metadata_after,
+				data: args.changeQueueEntry.data_after!,
+			},
+		})) {
+			detectedChanges.push({
+				...change,
+				pluginKey: plugin.key,
+			});
+		}
+	}
+
+	await args.lix.db.transaction().execute(async (trx) => {
+		const currentAuthors = await trx
+			.selectFrom("active_account")
+			.selectAll()
+			.execute();
+
+		const currentVersion = await trx
+			.selectFrom("current_version")
+			.innerJoin("version", "current_version.id", "version.id")
+			.selectAll()
+			.executeTakeFirstOrThrow();
+
+		await Promise.all(
+			detectedChanges.map(async (detectedChange) => {
+				await createChange({
+					lix: { ...args.lix, db: trx },
+					authors: currentAuthors,
+					version: currentVersion,
+					entityId: detectedChange.entity_id,
+					fileId: args.changeQueueEntry.file_id,
+					pluginKey: detectedChange.pluginKey,
+					schemaKey: detectedChange.schema.key,
+					snapshotContent: detectedChange.snapshot,
+				});
+			})
+		);
+
+		await trx
+			.deleteFrom("change_queue")
+			.where("id", "=", args.changeQueueEntry.id)
+			.execute();
+	});
+}
+
+/**
+ * File deletions don't need to invoke a plugin to detect changes.
+ *
+ * Instead, we can simply query the database for all changes that are related to the file
+ * and create the corresponding deletion changes for the current version.
+ *
+ * - simpler plugin API (because deletions don't need to be accounted for)
+ * - faster file deletion (because we don't need to invoke plugins)
+ */
+export async function handleFileDelete(args: {
+	lix: Pick<Lix, "db" | "plugin" | "sqlite">;
+	changeQueueEntry: ChangeQueueEntry;
+}): Promise<void> {
+	await args.lix.db.transaction().execute(async (trx) => {
+		const currentVersion = await trx
+			.selectFrom("current_version")
+			.innerJoin("version", "current_version.id", "version.id")
+			.selectAll()
+			.executeTakeFirstOrThrow();
+
+		const toBeDeletedEntities = await trx
+			.selectFrom("change")
+			.where("file_id", "=", args.changeQueueEntry.file_id)
+			.where(changeIsLeafInVersion(currentVersion))
+			.select("entity_id")
+			.select("schema_key")
+			.select("plugin_key")
+			.execute();
+
+		const currentAuthors = await trx
+			.selectFrom("active_account")
+			.selectAll()
+			.execute();
+
+		await Promise.all(
+			toBeDeletedEntities.map(async (change) => {
+				await createChange({
+					lix: { ...args.lix, db: trx },
+					authors: currentAuthors,
+					version: currentVersion,
+					entityId: change.entity_id,
+					fileId: args.changeQueueEntry.file_id,
+					pluginKey: change.plugin_key,
+					schemaKey: change.schema_key,
+					snapshotContent: null, // Snapshot is null for deletions
+				});
+			})
+		);
+
+		await trx
+			.deleteFrom("change_queue")
+			.where("id", "=", args.changeQueueEntry.id)
+			.execute();
+	});
+}

package/src/change-queue/index.ts

@@ -0,0 +1 @@
+export { changeQueueSettled } from "./change-queue-settled.js";