@lix-js/sdk 0.0.1

package/src/server-api-handler/routes/push-v1.ts

@@ -0,0 +1,180 @@
+import type * as LixServerApi from "@lix-js/server-api-schema";
+import type { LixServerApiHandlerRoute } from "../create-server-api-handler.js";
+import { mergeTheirState } from "../../sync/merge-state.js";
+import type { Change } from "../../database/schema.js";
+// import { detectChangeConflicts } from "../../change-conflict/detect-change-conflicts.js";
+// import { createChangeConflict } from "../../change-conflict/create-change-conflict.js";
+import { CompiledQuery } from "kysely";
+
+type RequestBody =
+	LixServerApi.paths["/lsa/push-v1"]["post"]["requestBody"]["content"]["application/json"];
+
+type ResponseBody = LixServerApi.paths["/lsa/push-v1"]["post"]["responses"];
+
+export const route: LixServerApiHandlerRoute = async (context) => {
+	const body = (await context.request.json()) as RequestBody;
+	const exists = await context.environment.hasLix({ id: body.lix_id });
+
+	if (!exists) {
+		return new Response(null, { status: 404 });
+	}
+
+	const open = await context.environment.openLix({ id: body.lix_id });
+
+	try {
+		// console.log(
+		// 	"----------- PROCESSING PUSH FROM CLIENT  --s----------- with state:",
+		// 	body.vector_clock
+		// );
+
+		await open.lix.db.transaction().execute(async (trx) => {
+			await trx.executeQuery(
+				CompiledQuery.raw("PRAGMA defer_foreign_keys = ON;")
+			);
+
+			await mergeTheirState({
+				lix: { ...open.lix, db: trx },
+				sourceVectorClock: body.vector_clock,
+				sourceData: body.data,
+			});
+
+			const allIncomingChanges = (body.data.change ?? []) as Change[];
+
+			// retrieve the version change updates from the changes
+			const incomingVersionChanges: Record<string, Change[]> = {};
+
+			for (const change of (body.data.change ?? []) as Change[]) {
+				if (change.schema_key !== "lix_version_change_table") {
+					continue;
+				}
+				const [versionId] = change.entity_id.split(",");
+				if (versionId === undefined) {
+					throw new Error("Expected versionId to be defined");
+				}
+				// safe assignment
+				if (!incomingVersionChanges[versionId]) {
+					incomingVersionChanges[versionId] = [];
+				}
+				incomingVersionChanges[versionId].push(change);
+			}
+
+			for (const [versionId, versionChanges] of Object.entries(
+				incomingVersionChanges
+			)) {
+				const incomingChanges = await Promise.all(
+					versionChanges.map(async (change) => {
+						const [, changeId] = change.entity_id.split(",");
+						const foundChange = allIncomingChanges.find(
+							(c) => c.id === changeId
+						);
+						if (foundChange === undefined) {
+							// TODO unclear why the fallback is needed. waiting for simplification
+							// of sync before investigating further.
+							//
+							// Observation:
+							//
+							// The lix_server_url change existed on the server
+							// but the client didn't send it. Maybe due to the
+							// test environment in the e2e test "should sync versions".
+							const fallback = await trx
+								.selectFrom("change")
+								.where("id", "=", changeId!)
+								.selectAll()
+								.executeTakeFirst();
+							if (fallback) {
+								return fallback;
+							} else {
+								throw new Error(
+									"Change not found. Expected to find a change for the version change in the incoming changes."
+								);
+							}
+						}
+						return foundChange;
+					})
+				);
+				const existingChanges: Change[] = [];
+
+				// manually constructing the symmetric difference
+				// between the incoming version and existing version
+				await Promise.all(
+					incomingChanges.map(async (change) => {
+						const existingChange = await trx
+							.selectFrom("version_change")
+							.innerJoin("change", "version_change.change_id", "change.id")
+							.where("version_change.version_id", "=", versionId)
+							.where("change.file_id", "=", change.file_id)
+							.where("change.schema_key", "=", change.schema_key)
+							.where("change.entity_id", "=", change.entity_id)
+							// only select the change if it differs from the incoming change
+							.where("change.id", "!=", change.id)
+							.selectAll("change")
+							.executeTakeFirst();
+						if (existingChange) {
+							existingChanges.push(existingChange);
+						}
+					})
+				);
+
+				//! Deactivated change conflict detection.
+				//!
+				//! Led to foreign key bugs etc. Will be reactivated
+				//! with the "conflicts" milestone
+				// const detectedConflicts = await detectChangeConflicts({
+				// 	lix: { ...open.lix, db: trx },
+				// 	changes: [...incomingChanges, ...existingChanges],
+				// });
+
+				// for (const conflict of detectedConflicts) {
+				// 	await createChangeConflict({
+				// 		lix: { ...open.lix, db: trx },
+				// 		key: conflict.key,
+				// 		version: { id: versionId },
+				// 		conflictingChangeIds: new Set(conflict.conflictingChangeIds),
+				// 	});
+				// }
+			}
+
+			// const allmlog = await trx
+			// 	.selectFrom("mutation_log")
+			// 	.selectAll()
+			// 	// .select(({ fn }) => {
+			// 	// 	return ["session", fn.max<number>("session_time").as("time")];
+			// 	// })
+			// 	// .groupBy("session")
+			// 	.execute();
+
+			// const vcAfterPush = await trx
+			// 	.selectFrom("mutation_log")
+			// 	.select(({ fn }) => {
+			// 		return ["session", fn.max<number>("session_time").as("time")];
+			// 	})
+			// 	.groupBy("session")
+			// 	.execute();
+
+			// console.log(
+			// 	"----------- DONE PROCESSING PUSH FROM CLIENT  ---- ---------",
+			// 	vcAfterPush,
+			// 	allmlog.length
+			// );
+		});
+
+		await context.environment.closeLix(open);
+
+		return new Response(null, {
+			status: 201,
+		});
+	} catch (error) {
+		return new Response(
+			JSON.stringify({
+				code: "FAILED_TO_INSERT_DATA",
+				message: (error as any)?.message,
+			} satisfies ResponseBody["400"]["content"]["application/json"]),
+			{
+				status: 400,
+				headers: {
+					"Content-Type": "application/json",
+				},
+			}
+		);
+	}
+};

package/src/snapshot/create-snapshot.test.ts

@@ -0,0 +1,68 @@
+import { expect, test } from "vitest";
+import { openLixInMemory } from "../lix/open-lix-in-memory.js";
+import { createSnapshot } from "./create-snapshot.js";
+
+test("should insert a new snapshot", async () => {
+	const lix = await openLixInMemory({});
+	const content = { a: "some data" };
+	const snapshot = await createSnapshot({ lix, content });
+
+	expect(snapshot.content).toEqual(content);
+});
+
+test("should insert a new snapshot with no content", async () => {
+	const lix = await openLixInMemory({});
+	const snapshot = await createSnapshot({ lix, content: undefined });
+
+	const snapshots = await lix.db.selectFrom("snapshot").selectAll().execute();
+
+	expect(snapshots).toHaveLength(1);
+	expect(snapshots[0]).toEqual(snapshot);
+	expect(snapshot.content).toBe(null);
+});
+
+test("should retrieve existing snapshot if content is the same", async () => {
+	const lix = await openLixInMemory({});
+	const content = { a: "some data" };
+	const snapshot1 = await createSnapshot({ lix, content });
+	const snapshot2 = await createSnapshot({ lix, content });
+
+	expect(snapshot1.id).toBe(snapshot2.id);
+	expect(snapshot1.content).toEqual(snapshot2.content);
+});
+
+test("should handle transaction correctly", async () => {
+	const lix = await openLixInMemory({});
+	const content = { a: "some data" };
+
+	await lix.db.transaction().execute(async (trx) => {
+		const snapshot = await createSnapshot({ lix: { db: trx }, content });
+
+		expect(snapshot.content).toEqual(content);
+	});
+});
+
+test("should retrieve existing snapshot within a transaction", async () => {
+	const lix = await openLixInMemory({});
+	const content = { a: "some data" };
+
+	await lix.db.transaction().execute(async (trx) => {
+		const snapshotsBefore = await trx
+			.selectFrom("snapshot")
+			.selectAll()
+			.execute();
+
+		const snapshot1 = await createSnapshot({ lix: { db: trx }, content });
+		const snapshot2 = await createSnapshot({ lix: { db: trx }, content });
+
+		const snapshotsAfter = await trx
+			.selectFrom("snapshot")
+			.selectAll()
+			.execute();
+
+		// one snapshot should be added
+		expect(snapshotsAfter).toHaveLength(snapshotsBefore.length + 1);
+		expect(snapshot1.id).toBe(snapshot2.id);
+		expect(snapshot1.content).toEqual(snapshot2.content);
+	});
+});

package/src/snapshot/create-snapshot.ts

@@ -0,0 +1,40 @@
+import type { Snapshot } from "../database/schema.js";
+import type { Lix } from "../lix/open-lix.js";
+
+/**
+ * Creates a snapshot and inserts it or retrieves the existing snapshot from the database.
+ *
+ * Snapshots are content-addressed to avoid storing the same snapshot multiple times.
+ * Hence, an insert might not actually insert a new snapshot but return an existing one.
+ *
+ * @example
+ *   ```ts
+ *   const snapshot = await createSnapshot({ lix, content });
+ *   ```
+ */
+export async function createSnapshot(args: {
+	lix: Pick<Lix, "db">;
+	content?: Snapshot["content"];
+}): Promise<Snapshot> {
+	const executeInTransaction = async (trx: Lix["db"]) => {
+		const snapshot = await trx
+			.insertInto("snapshot")
+			.values({
+				content: args.content ?? null,
+			})
+			.onConflict((oc) =>
+				oc.doUpdateSet((eb) => ({
+					content: eb.ref("excluded.content"),
+				}))
+			)
+			.returningAll()
+			.executeTakeFirstOrThrow();
+		return snapshot;
+	};
+
+	if (args.lix.db.isTransaction) {
+		return executeInTransaction(args.lix.db);
+	} else {
+		return args.lix.db.transaction().execute(executeInTransaction);
+	}
+}

package/src/snapshot/index.ts

@@ -0,0 +1,2 @@
+export { mockJsonSnapshot } from "./mock-json-snapshot.js";
+export { createSnapshot } from "./create-snapshot.js";

package/src/snapshot/json-sha-256.test.ts

@@ -0,0 +1,12 @@
+import { test, expect } from "vitest";
+import { jsonSha256 } from "./json-sha-256.js";
+
+test("sha256 returns the same hash for the same content irrespective of the property ordering", async () => {
+	const obj1 = { a: "some data", b: 1, d: { f: "prop", e: "nested" } };
+	const obj2 = { b: 1, a: "some data", d: { e: "nested", f: "prop" } };
+
+	const hash1 = jsonSha256(obj1);
+	const hash2 = jsonSha256(obj2);
+
+	expect(hash1).toBe(hash2);
+});

package/src/snapshot/json-sha-256.ts

@@ -0,0 +1,35 @@
+import { sha256 } from "js-sha256";
+
+/**
+ * Creates a sha256 hash from a json object.
+ *
+ * Sorts the object properties to increase consistency,
+ * and de-duplication of snapshots.
+ */
+export function jsonSha256(content: Record<string, unknown>): string {
+	const sortedContent = deepSortObject(content);
+
+	return sha256(JSON.stringify(sortedContent));
+}
+
+// naive deep sorting to receive consistent hash
+function deepSortObject(obj: Record<string, unknown>): any {
+	if (Array.isArray(obj)) {
+		return obj.map((item) =>
+			typeof item === "object" && item !== null
+				? deepSortObject(item as Record<string, unknown>)
+				: item
+		);
+	} else if (typeof obj === "object" && obj !== null) {
+		return Object.keys(obj)
+			.sort()
+			.reduce(
+				(acc, key) => {
+					acc[key] = deepSortObject(obj[key] as Record<string, unknown>);
+					return acc;
+				},
+				{} as Record<string, unknown>
+			);
+	}
+	return obj;
+}

package/src/snapshot/mock-json-snapshot.ts

@@ -0,0 +1,14 @@
+import type { Snapshot } from "../database/schema.js";
+import { jsonSha256 } from "./json-sha-256.js";
+
+/**
+ *
+ * Util function for tests that creates a snapshot that looks like one you got returned from the database after inserting
+ *
+ */
+export function mockJsonSnapshot(content: Record<string, any>): Snapshot {
+	return {
+		id: jsonSha256(content),
+		content: content,
+	};
+}

package/src/sync/get-diffing-rows.ts

@@ -0,0 +1,96 @@
+import type { Lix } from "../lix/open-lix.js";
+import type { VectorClock } from "./merge-state.js";
+
+export async function getDiffingRows(args: {
+	/**
+	 * the lix to merge their state into
+	 */
+	lix: Pick<Lix, "db">;
+	/**
+	 * the the vector clock of the lix to merge in
+	 */
+	targetVectorClock: VectorClock;
+}): Promise<{
+	state: any;
+	upsertedRows: Record<string, any[]>;
+}> {
+	const upsertedRows: Record<string, any[]> = {};
+
+	let state: VectorClock = [];
+
+	const executeInTransaction = async (trx: Lix["db"]) => {
+		state = await trx
+			.selectFrom("mutation_log")
+			.select(({ fn }) => {
+				return ["session", fn.max<number>("session_time").as("time")];
+			})
+			.groupBy("session")
+			.execute();
+		// use the target vector clock to collect all changed rows the target is not aware of
+		let operationsToPush = trx
+			.selectFrom("mutation_log")
+			.selectAll("mutation_log");
+
+		if (args.targetVectorClock.length > 0) {
+			operationsToPush = operationsToPush.where((eb) => {
+				const knownSessions = args.targetVectorClock.map(
+					(sessionTime) => sessionTime.session
+				);
+
+				const ors: any[] = [];
+				ors.push(eb("session", "not in", knownSessions));
+				for (const sessionTime of args.targetVectorClock) {
+					ors.push(
+						eb("session", "=", sessionTime.session).and(
+							"session_time",
+							">",
+							sessionTime.time
+						)
+					);
+				}
+
+				return eb.or(ors) as any;
+			});
+		}
+
+		upsertedRows["mutation_log"] = await operationsToPush.execute();
+
+		// console.log("upsertedRows", upsertedRows);
+
+		for (const operation of upsertedRows["mutation_log"]) {
+			const tableName = operation.table_name;
+			if (upsertedRows[tableName] === undefined) {
+				upsertedRows[tableName] = [];
+			}
+			if (tableName === "snapshot") {
+				// ignore inserted column id
+				upsertedRows[tableName].push(
+					await trx
+						.selectFrom(tableName)
+						.selectAll()
+						.where("id", "=", operation.row_id["id"])
+						.executeTakeFirstOrThrow()
+				);
+			} else {
+				let diffRow = trx.selectFrom(tableName).selectAll();
+
+				for (const [key, value] of Object.entries(operation.row_id)) {
+					diffRow = diffRow.where(key, "=", value);
+				}
+
+				upsertedRows[tableName].push(await diffRow.executeTakeFirstOrThrow());
+			}
+		}
+	};
+
+	if (args.lix.db.isTransaction) {
+		await executeInTransaction(args.lix.db);
+	} else {
+		await args.lix.db.transaction().execute(executeInTransaction);
+	}
+
+	return {
+		state,
+		upsertedRows,
+	};
+}