@lix-js/sdk 0.0.1

package/src/lix/merge.ts

@@ -0,0 +1,244 @@
+import type { Lix } from "./open-lix.js";
+import { getLeafChangesOnlyInSource } from "./merge.get-leaf-changes-only-in-source.js";
+import { withSkipChangeQueue } from "../change-queue/with-skip-change-queue.js";
+import type { DetectedConflict } from "../plugin/lix-plugin.js";
+
+/**
+ * Combined the changes of the source lix into the target lix.
+ */
+export async function merge(args: {
+	targetLix: Lix;
+	sourceLix: Lix;
+	// TODO selectively merge changes
+	// onlyTheseChanges
+}): Promise<void> {
+	// TODO increase performance by using attach mode
+	//      and only get the changes that are not in target.
+	const sourceChangesWithSnapshot = await args.sourceLix.db
+		.selectFrom("change")
+		.innerJoin("snapshot", "snapshot.id", "change.snapshot_id")
+		.selectAll("change")
+		.select("snapshot.content")
+		.execute();
+
+	// TODO don't query the changes again. inefficient.
+	const leafChangesOnlyInSource = await getLeafChangesOnlyInSource({
+		sourceLix: args.sourceLix,
+		targetLix: args.targetLix,
+	});
+
+	// 2. Let the plugin detect conflicts
+
+	const plugins = await args.sourceLix.plugin.getAll();
+	const plugin = plugins[0];
+
+	// TODO function assumes that all changes belong to the same file
+	if (plugins.length > 1) {
+		throw new Error("Unimplemented. Only one plugin is supported for now");
+	}
+
+	const conflicts: DetectedConflict[] =
+		// (await plugin?.detectConflicts?.({
+		// 	sourceLix: args.sourceLix,
+		// 	targetLix: args.targetLix,
+		// })) ??
+
+		[];
+
+	const changesPerFile: Record<string, ArrayBuffer> = {};
+
+	const fileIds = new Set(sourceChangesWithSnapshot.map((c) => c.file_id));
+
+	for (const fileId of fileIds) {
+		// 3. apply non conflicting leaf changes
+		// TODO inefficient double looping
+		const nonConflictingLeafChangesInSourceForFile = leafChangesOnlyInSource
+			.filter((sourceChange) =>
+				conflicts.every(
+					(conflict) =>
+						conflict.conflictingChangeIds.has(sourceChange.id) === false
+				)
+			)
+			.filter((c) => c.file_id === fileId);
+
+		let file = await args.targetLix.db
+			.selectFrom("file")
+			.selectAll()
+			.where("id", "=", fileId)
+			.executeTakeFirst();
+
+		// If the file does not exist in the target lix - we just copy the whole file over
+		if (!file) {
+			file = await args.sourceLix.db
+				.selectFrom("file")
+				.selectAll()
+				.where("id", "=", fileId)
+				.executeTakeFirst();
+
+			if (file) {
+				const fileToInsert = {
+					id: file.id,
+					path: file.path,
+					data: file.data,
+					metadata: file.metadata,
+				};
+
+				await withSkipChangeQueue(args.targetLix.db, async (trx) => {
+					await trx
+						.insertInto("file")
+						.values({ ...fileToInsert })
+						.executeTakeFirst();
+				});
+			}
+		}
+
+		if (!plugin?.applyChanges) {
+			throw new Error("Plugin does not support applying changes");
+		}
+
+		const { fileData } = await plugin.applyChanges({
+			changes: nonConflictingLeafChangesInSourceForFile,
+			// @ts-expect-error - TODO apply changes can be an undefined file
+			file,
+			lix: args.targetLix,
+		});
+
+		changesPerFile[fileId] = fileData;
+	}
+
+	// DISCUSSIONS
+
+	/**
+	 * NOTE:
+	 * this is a naiv implementation that selects just all discussions
+	 * and passes them to the target
+	 **/
+	// selecting all discussion related entries for later upsert
+	const sourceDiscussions = await args.sourceLix.db
+		.selectFrom("discussion")
+		.selectAll()
+		.execute();
+
+	const sourceComments = await args.sourceLix.db
+		.selectFrom("comment")
+		.selectAll()
+		.execute();
+
+	// change graph
+
+	const sourceEdges = await args.sourceLix.db
+		.selectFrom("change_edge")
+		.selectAll()
+		.execute();
+
+	// change sets
+
+	const sourceChangeSets = await args.sourceLix.db
+		.selectFrom("change_set")
+		.selectAll()
+		.execute();
+
+	const sourceChangeSetItems = await args.sourceLix.db
+		.selectFrom("change_set_element")
+		.selectAll()
+		.execute();
+
+	await args.targetLix.db.transaction().execute(async (trx) => {
+		if (sourceChangesWithSnapshot.length > 0) {
+			// copy the snapshots from source
+			await trx
+				.insertInto("snapshot")
+				.values(
+					// https://github.com/opral/inlang-message-sdk/issues/123
+					sourceChangesWithSnapshot.map((change) => ({
+						content: change.content,
+					}))
+				)
+				// ignore if already exists
+				.onConflict((oc) => oc.doNothing())
+				.execute();
+
+			await trx
+				.insertInto("change")
+				.values(
+					sourceChangesWithSnapshot.map((c) => ({
+						...c,
+						content: undefined,
+					}))
+				)
+				// ignore if already exists
+				.onConflict((oc) => oc.doNothing())
+				.execute();
+		}
+
+		// insert the conflicts of those changes
+		if (conflicts.length > 0) {
+			// await trx
+			// 	.insertInto("conflict")
+			// 	.values(conflicts)
+			// 	// ignore if already exists
+			// 	.onConflict((oc) => oc.doNothing())
+			// 	.execute();
+		}
+
+		for (const [fileId, fileData] of Object.entries(changesPerFile)) {
+			// update the file data with the applied changes
+			await withSkipChangeQueue(trx, async (trx) => {
+				await trx
+					.updateTable("file")
+					.set("data", fileData)
+					.where("id", "=", fileId)
+					.execute();
+			});
+		}
+
+		// copy edges
+		if (sourceEdges.length > 0) {
+			await trx
+				.insertInto("change_edge")
+				.values(sourceEdges)
+				// ignore if already exists
+				.onConflict((oc) => oc.doNothing())
+				.execute();
+		}
+
+		// add change sets and change_set_memberships
+		if (sourceChangeSets.length > 0) {
+			await trx
+				.insertInto("change_set")
+				.values(sourceChangeSets)
+				// ignore if already exists
+				.onConflict((oc) => oc.doNothing())
+				.execute();
+		}
+		if (sourceChangeSetItems.length > 0) {
+			await trx
+				.insertInto("change_set_element")
+				.values(sourceChangeSetItems)
+				// ignore if already exists
+				.onConflict((oc) => oc.doNothing())
+				.execute();
+		}
+
+		// add discussions, comments and discsussion_change_mappings
+
+		if (sourceDiscussions.length > 0) {
+			await trx
+				.insertInto("discussion")
+				.values(sourceDiscussions)
+				// ignore if already exists
+				.onConflict((oc) => oc.doNothing())
+				.execute();
+		}
+
+		if (sourceComments.length > 0) {
+			await trx
+				.insertInto("comment")
+				.values(sourceComments)
+				// ignore if already exists
+				.onConflict((oc) => oc.doNothing())
+
+				.execute();
+		}
+	});
+}

package/src/lix/new-lix.test.ts

@@ -0,0 +1,23 @@
+import { test, expect } from "vitest";
+import { openLixInMemory } from "./open-lix-in-memory.js";
+import { newLixFile } from "./new-lix.js";
+
+test("inserting a change should auto fill the created_at column", async () => {
+	const lix = await openLixInMemory({ blob: await newLixFile() });
+
+	await lix.db
+		.insertInto("change")
+		.values({
+			id: "test",
+			entity_id: "test",
+			schema_key: "file",
+			file_id: "mock",
+			plugin_key: "mock-plugin",
+			snapshot_id: "no-content",
+		})
+		.execute();
+
+	const changes = await lix.db.selectFrom("change").selectAll().execute();
+	expect(changes).lengthOf(1);
+	expect(changes[0]?.created_at).toBeDefined();
+});

package/src/lix/new-lix.ts

@@ -0,0 +1,30 @@
+import {
+	createInMemoryDatabase,
+	contentFromDatabase,
+} from "sqlite-wasm-kysely";
+import { initDb } from "../database/init-db.js";
+
+/**
+ * Creates a new lix file.
+ *
+ * The app is responsible for saving the project "whereever"
+ * e.g. the user's computer, cloud storage, or OPFS in the browser.
+ */
+export async function newLixFile(): Promise<Blob> {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+
+	// applying the schema etc.
+	const db = initDb({ sqlite });
+
+	try {
+		return new Blob([contentFromDatabase(sqlite)]);
+	} catch (e) {
+		throw new Error(`Failed to create new Lix file: ${e}`, { cause: e });
+	} finally {
+		// in any case destroy the memory db
+		sqlite.close();
+		await db.destroy();
+	}
+}

package/src/lix/open-lix-in-memory.test.ts

@@ -0,0 +1,30 @@
+import { test, expect } from "vitest";
+import { openLixInMemory } from "./open-lix-in-memory.js";
+
+test("it should open a lix in memory", async () => {
+	const lix = await openLixInMemory({});
+	// querying a table to see if the schema is applied
+	const changes = await lix.db.selectFrom("change").selectAll().execute();
+	expect(changes).toEqual([]);
+});
+
+test("it should open a lix in memory from a blob", async () => {
+	const lix1 = await openLixInMemory({});
+	await lix1.db
+		.insertInto("file")
+		.values({
+			id: "1",
+			path: "/a.txt",
+			data: new TextEncoder().encode("hello"),
+		})
+		.execute();
+	const lix2 = await openLixInMemory({ blob: await lix1.toBlob() });
+	const files = await lix2.db.selectFrom("file").selectAll().execute();
+	expect(files).toEqual([
+		expect.objectContaining({
+			id: "1",
+			path: "/a.txt",
+			data: new TextEncoder().encode("hello"),
+		}),
+	]);
+});

package/src/lix/open-lix-in-memory.ts

@@ -0,0 +1,28 @@
+import { createInMemoryDatabase, importDatabase } from "sqlite-wasm-kysely";
+import { openLix } from "./open-lix.js";
+
+/**
+ * Opens a lix in memory.
+ *
+ */
+export async function openLixInMemory(
+	args: {
+		/**
+		 * The lix file to open. If not provided, an empty (new) lix will be opened.
+		 */
+		blob?: Blob;
+	} & Omit<Parameters<typeof openLix>[0], "database">
+): Promise<ReturnType<typeof openLix>> {
+	const database = await createInMemoryDatabase({
+		readOnly: false,
+	});
+
+	if (args.blob) {
+		importDatabase({
+			db: database,
+			content: new Uint8Array(await args.blob.arrayBuffer()),
+		});
+	}
+
+	return openLix({ ...args, database });
+}

package/src/lix/open-lix.test.ts

@@ -0,0 +1,33 @@
+import { expect, test } from "vitest";
+import { openLixInMemory } from "./open-lix-in-memory.js";
+import { newLixFile } from "./new-lix.js";
+import type { LixPlugin } from "../plugin/lix-plugin.js";
+
+test("providing plugins should be possible", async () => {
+	const mockPlugin: LixPlugin = {
+		key: "mock-plugin",
+	};
+	const lix = await openLixInMemory({
+		blob: await newLixFile(),
+		providePlugins: [mockPlugin],
+	});
+	expect(await lix.plugin.getAll()).toContain(mockPlugin);
+});
+
+test("providing key values should be possible", async () => {
+	const lix = await openLixInMemory({
+		blob: await newLixFile(),
+		keyValues: [{ key: "key", value: "value" }],
+	});
+	const value = await lix.db.selectFrom("key_value").selectAll().execute();
+	expect(value).toContainEqual({ key: "key", value: "value" });
+
+	// testing overwriting key values
+	const lix1 = await openLixInMemory({
+		blob: await lix.toBlob(),
+		keyValues: [{ key: "key", value: "value2" }],
+	});
+
+	const value1 = await lix1.db.selectFrom("key_value").selectAll().execute();
+	expect(value1).toContainEqual({ key: "key", value: "value2" });
+});

package/src/lix/open-lix.ts

@@ -0,0 +1,103 @@
+import type { LixPlugin } from "../plugin/lix-plugin.js";
+import { loadPlugins } from "../plugin/load-plugin.js";
+import { contentFromDatabase, type SqliteDatabase } from "sqlite-wasm-kysely";
+import { initDb } from "../database/init-db.js";
+import { initChangeQueue } from "../change-queue/init-change-queue.js";
+import { changeQueueSettled } from "../change-queue/change-queue-settled.js";
+import type { Kysely } from "kysely";
+import type { LixDatabaseSchema } from "../database/schema.js";
+import { initSyncProcess } from "../sync/sync-process.js";
+import type { KeyValue } from "../key-value/database-schema.js";
+
+export type Lix = {
+	/**
+	 * The raw SQLite instance.
+	 *
+	 * Required for advanced use cases that can't be
+	 * expressed with the db API.
+	 *
+	 * Use with caution, automatic transformation of
+	 * results like parsing json (similar to the db API)
+	 * is not guaranteed.
+	 */
+	sqlite: SqliteDatabase;
+	db: Kysely<LixDatabaseSchema>;
+	toBlob: () => Promise<Blob>;
+	plugin: {
+		getAll: () => Promise<LixPlugin[]>;
+	};
+	close: () => Promise<void>;
+};
+
+/**
+ * Common setup between different lix environments.
+ */
+export async function openLix(args: {
+	database: SqliteDatabase;
+	/**
+	 * Usecase are lix apps that define their own file format,
+	 * like inlang (unlike a markdown, csv, or json plugin).
+	 *
+	 * (+) avoids separating app code from plugin code and
+	 *     resulting bundling logic.
+	 *
+	 * (-) such a file format must always be opened with the
+	 *     file format sdk. the file is not portable
+	 *
+	 * @example
+	 *   const lix = await openLixInMemory({ providePlugins: [myPlugin] })
+	 */
+	providePlugins?: LixPlugin[];
+	/**
+	 * Set the key values when opening the lix.
+	 *
+	 * @example
+	 *   const lix = await openLix({ keyValues: [{ key: "#lix_sync", value: "false" }] })
+	 */
+	keyValues?: KeyValue[];
+}): Promise<Lix> {
+	const db = initDb({ sqlite: args.database });
+
+	if (args.keyValues && args.keyValues.length > 0) {
+		await db
+			.insertInto("key_value")
+			.values(args.keyValues)
+			.onConflict((oc) =>
+				oc.doUpdateSet((eb) => ({ value: eb.ref("excluded.value") }))
+			)
+			.execute();
+	}
+
+	const plugins = await loadPlugins(db);
+	if (args.providePlugins && args.providePlugins.length > 0) {
+		plugins.push(...args.providePlugins);
+	}
+
+	const plugin = {
+		getAll: async () => plugins,
+	};
+
+	const toBlob = async () => {
+		await changeQueueSettled({ lix: { db } });
+		return new Blob([contentFromDatabase(args.database)]);
+	};
+
+	initChangeQueue({
+		lix: { db, plugin, sqlite: args.database },
+		rawDatabase: args.database,
+	});
+
+	initSyncProcess({ lix: { db, plugin, toBlob } });
+
+	return {
+		db,
+		sqlite: args.database,
+		toBlob,
+		plugin,
+		close: async () => {
+			await changeQueueSettled({ lix: { db } });
+			// args.database.close();
+			// await db.destroy();
+		},
+	};
+}