@lix-js/sdk 0.0.1

package/src/database/kysely-plugin/serialize-jsonb-plugin.test.ts

@@ -0,0 +1,132 @@
+import { Kysely, sql, type Generated } from "kysely";
+import { expect, test } from "vitest";
+import { createDialect, createInMemoryDatabase } from "sqlite-wasm-kysely";
+import { SerializeJsonBPlugin } from "./serialize-jsonb-plugin.js";
+
+test("inserts should work", async () => {
+	const db = await mockDatabase();
+
+	const result = await db
+		.insertInto("mock_table")
+		.values({
+			metadata: { a: 1 },
+			other: "value",
+			just_blob: new Uint8Array(),
+		})
+		.returningAll()
+		// no parsing json plugin is installed, hence select json from db instead
+		.returning(sql`json(metadata)`.as("metadata"))
+		.executeTakeFirstOrThrow();
+
+	expect(result).toEqual({
+		id: 1,
+		metadata: JSON.stringify({ a: 1 }),
+		other: "value",
+		just_blob: new Uint8Array(),
+	});
+});
+
+test("updates should work", async () => {
+	const db = await mockDatabase();
+
+	const insert = await db
+		.insertInto("mock_table")
+		.values({
+			metadata: { a: 1 },
+			other: "value",
+			just_blob: new Uint8Array(),
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	const result = await db
+		.updateTable("mock_table")
+		.set({ metadata: { b: 2 } })
+		.where("id", "=", insert.id)
+		.returningAll()
+		// no parsing json plugin is installed, hence select json from db instead
+		.returning(sql`json(metadata)`.as("metadata"))
+		.executeTakeFirstOrThrow();
+
+	expect(result).toEqual({
+		id: 1,
+		metadata: JSON.stringify({ b: 2 }),
+		other: "value",
+		just_blob: new Uint8Array(),
+	});
+});
+
+test("onConflict should work", async () => {
+	const db = await mockDatabase();
+
+	const insert = await db
+		.insertInto("mock_table")
+		.values({
+			metadata: { a: 1 },
+			other: "value",
+			just_blob: new Uint8Array(),
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	const result = await db
+		.insertInto("mock_table")
+		.values({
+			id: insert.id,
+			metadata: { b: 2 },
+			other: "value",
+			just_blob: new Uint8Array(),
+		})
+		.onConflict((oc) => oc.doUpdateSet({ metadata: { b: 2 } }))
+		.returningAll()
+		// no parsing json plugin is installed, hence select json from db instead
+		.returning(sql`json(metadata)`.as("metadata"))
+		.executeTakeFirstOrThrow();
+
+	expect(result).toEqual({
+		id: 1,
+		metadata: JSON.stringify({ b: 2 }),
+		other: "value",
+		just_blob: new Uint8Array(),
+	});
+});
+
+const mockDatabase = async () => {
+	const database = await createInMemoryDatabase({ readOnly: false });
+
+	const db = new Kysely<{
+		mock_table: {
+			id: Generated<number>;
+			metadata: Record<string, any>;
+			other: string;
+			just_blob: Uint8Array;
+		};
+		other_table: {
+			mock_table_id: number;
+			content: Record<string, any>;
+		};
+	}>({
+		dialect: createDialect({
+			database,
+		}),
+		plugins: [SerializeJsonBPlugin()],
+	});
+
+	await sql`
+		CREATE TABLE mock_table (
+	    id INTEGER PRIMARY KEY AUTOINCREMENT,
+			metadata BLOB,
+			other TEXT,
+			just_blob BLOB
+		) strict;
+
+		CREATE TABLE other_table (
+		  mock_table_id INTEGER,
+			content BLOB,
+
+			FOREIGN KEY (mock_table_id) REFERENCES mock_table (id)
+		) strict;
+	`.execute(db);
+
+	return db;
+};

package/src/database/kysely-plugin/serialize-jsonb-plugin.ts

@@ -0,0 +1,122 @@
+import {
+	OnConflictNode,
+	OperationNodeTransformer,
+	sql,
+	ValueListNode,
+	ValueNode,
+	ValuesNode,
+	type KyselyPlugin,
+} from "kysely";
+
+export function SerializeJsonBPlugin(): KyselyPlugin {
+	const parseJsonTransformer = new SerializeJsonbTransformer();
+
+	return {
+		transformResult: async (args) => args.result,
+		transformQuery(args) {
+			if (
+				args.node.kind === "InsertQueryNode" ||
+				args.node.kind === "UpdateQueryNode"
+			) {
+				const result = parseJsonTransformer.transformNode(args.node);
+
+				return result;
+			}
+			return args.node;
+		},
+	};
+}
+
+class SerializeJsonbTransformer extends OperationNodeTransformer {
+	override transformOnConflict(node: OnConflictNode): OnConflictNode {
+		return super.transformOnConflict({
+			...node,
+			updates: node.updates?.map((updateItem) => {
+				if (updateItem.kind !== "ColumnUpdateNode") {
+					return updateItem;
+				}
+				return {
+					kind: "ColumnUpdateNode",
+					column: updateItem.column,
+					value: this.transformValue(
+						// @ts-expect-error - type mismatch
+						updateItem.value
+					),
+				};
+			}),
+		});
+	}
+
+	override transformValue(node: ValueNode): ValueNode {
+		const { value } = node;
+		const serializedValue = maybeSerializeJson(value);
+		if (value === serializedValue) {
+			return node;
+		}
+		// @ts-expect-error - type mismatch
+		return sql`jsonb(${serializedValue})`.toOperationNode();
+	}
+	/**
+	 * Transforms the value list node by replacing all JSON objects with `jsonb` function calls.
+	 */
+	override transformValueList(node: ValueListNode): ValueListNode {
+		return super.transformValueList({
+			...node,
+			values: node.values.map((listNodeItem) => {
+				if (listNodeItem.kind !== "ValueNode") {
+					return listNodeItem;
+				}
+				// @ts-expect-error - type mismatch
+				const { value } = listNodeItem;
+				const serializedValue = maybeSerializeJson(value);
+
+				if (value === serializedValue) {
+					return listNodeItem;
+				}
+				return sql`jsonb(${serializedValue})`.toOperationNode();
+			}),
+		});
+	}
+
+	/**
+	 * changes PrimitiveValueListNode to ValueListNode to allow kysely to process objects
+	 */
+	override transformValues(node: ValuesNode): ValuesNode {
+		return super.transformValues({
+			...node,
+			values: node.values.map((valueItemNode) => {
+				if (valueItemNode.kind !== "PrimitiveValueListNode") {
+					return valueItemNode;
+				}
+
+				// change PrimitiveValueListNode to ValueListNode
+				return {
+					kind: "ValueListNode",
+					values: valueItemNode.values.map(
+						(value) =>
+							({
+								kind: "ValueNode",
+								value,
+							}) as ValueNode
+					),
+				} as ValueListNode;
+			}),
+		});
+	}
+}
+
+function maybeSerializeJson(value: any): any {
+	if (
+		// binary data
+		value instanceof ArrayBuffer ||
+		// uint8array, etc
+		ArrayBuffer.isView(value) ||
+		value === null ||
+		value === undefined
+	) {
+		return value;
+	} else if (typeof value === "object" || Array.isArray(value)) {
+		return JSON.stringify(value);
+	}
+	return value;
+}

package/src/database/mutation-log/database-schema.ts

@@ -0,0 +1,128 @@
+import type { Insertable, Selectable } from "kysely";
+import type { SqliteDatabase } from "sqlite-wasm-kysely";
+
+export const tablesByDepencies: string[] = [
+	// no dependencies
+	"snapshot",
+	"label",
+	"version",
+	"account",
+	"file",
+	"change_set",
+	"key_value",
+	// Depends on: snapshot
+	"change",
+
+	// Depends on: file
+	"change_queue",
+	// Depends on: change
+	"change_author",
+	// Depends on: change
+	"change_edge",
+	// Depends on: change_set
+	"change_conflict",
+	// Depends on: change_conflict, change
+	"change_conflict_resolution",
+	// Depends on: change_set, change
+	"change_set_element",
+	// Depends on: label, change_set
+	"change_set_label",
+	// Depends on: change_set_label
+	"change_set_label_author",
+	//Depends on: change_set
+	"discussion",
+	// Depends on: account, discussion, comment
+	"comment",
+	// Depends on: version, change
+	"version_change",
+	// Depends on: version, change_conflict
+	"version_change_conflict",
+	// Depends on: version
+	"current_version",
+];
+
+export const tableIdColumns: Record<string, Array<string>> = {
+	// file - File is not synced. Is construcuted from the change table. (see https://github.com/opral/monorepo/pull/3242#discussion_r1863981413)
+	change: ["id"],
+	snapshot: ["id"],
+	change_conflict: ["id"],
+	change_edge: ["parent_id", "child_id"],
+	change_conflict_resolution: ["change_conflict_id", "resolved_change_id"],
+	version_change_conflict: ["version_id", "change_conflict_id"],
+	// version_change: ["version_id", "change_id"],
+};
+
+export function applyMutationLogDatabaseSchema(sqlite: SqliteDatabase): void {
+	// eslint-disable-next-line @typescript-eslint/no-unused-expressions
+	sqlite.exec`
+  -- vector clock
+  CREATE TABLE IF NOT EXISTS mutation_log (
+    row_id BLOB,
+    table_name TEXT,
+    -- TODO SYNC - we might not need the operation as long as we don't expect a delete operation since we utilize upsert queries anyway
+    operation TEXT,
+    session TEXT NOT NULL DEFAULT (lix_session()),
+    session_time INTEGER NOT NULL DEFAULT (lix_session_clock_tick()),
+    wall_clock INTEGER DEFAULT (strftime('%s', 'now') * 1000 + (strftime('%f', 'now') - strftime('%S', 'now')) * 1000)
+  ) STRICT;
+   
+  CREATE TRIGGER IF NOT EXISTS ignore_mutation_during_sync
+  BEFORE INSERT ON mutation_log
+  FOR EACH ROW
+  WHEN EXISTS (SELECT 1 FROM mutation_log WHERE table_name = 'mutation_log')
+  BEGIN
+    SELECT RAISE(IGNORE);
+  END;
+ 
+  `;
+
+	for (const [tableName, idColumns] of Object.entries(tableIdColumns)) {
+		if (idColumns.length === 0) {
+			throw new Error("at least one id required");
+		}
+		sqlite.exec(`
+      -- Trigger for INSERT operations
+      CREATE TRIGGER IF NOT EXISTS ${tableName}_after_insert_log
+      AFTER INSERT ON ${tableName}
+      BEGIN
+          INSERT INTO mutation_log (row_id, table_name, operation)
+          VALUES (JSONB(${toSqliteJson(idColumns)}), '${tableName}', 'INSERT');
+      END;
+  
+      -- Trigger for UPDATE operations
+      CREATE TRIGGER IF NOT EXISTS ${tableName}_after_update_log
+      AFTER UPDATE ON ${tableName}
+      BEGIN
+          INSERT INTO mutation_log (row_id, table_name, operation)
+          VALUES (JSONB(${toSqliteJson(idColumns)}), '${tableName}', 'UPDATE');
+      END;
+    `);
+	}
+}
+
+export type MutationLog = Selectable<MutationLogTable>;
+export type NewMutationLog = Insertable<MutationLogTable>;
+
+export type MutationLogTable = {
+	row_id: Record<string, string>;
+	table_name: string;
+	// -1 = delete, 0 = insert, 1 = update
+	// TODO SYNC - we might not need the operation as long as we don't expect a delete operation since we utilize upsert queries anyway
+	operation: "INSERT" | "UPDATE";
+	session: string;
+	session_time: number;
+	wall_clock: number;
+};
+
+function toSqliteJson(keyColumns: string[]) {
+	if (keyColumns.length === 0) {
+		throw new Error("Key columns array cannot be empty.");
+	}
+
+	const jsonObjectArgs = keyColumns.map(
+		(keyColumnName) => `'${keyColumnName}', NEW.${keyColumnName}`
+	);
+
+	const jsonObjectExpression = "json_object(" + jsonObjectArgs.join(", ") + ")";
+	return jsonObjectExpression;
+}

package/src/database/mutation-log/lix-session.ts

@@ -0,0 +1,19 @@
+import { v4 as uuid_v4 } from "uuid";
+
+export function createSession() {
+	const sessionId = uuid_v4(); // we use v4 here since the order of session should explicetly not matter!
+	let time: number = 0;
+
+	return {
+		/**
+		 * Returns the current lix session id.
+		 */
+		id: (): string => {
+			return sessionId;
+		},
+		sessionClockTick: (): number => {
+			time += 1;
+			return time;
+		},
+	};
+}

package/src/database/schema.ts

@@ -0,0 +1,282 @@
+import type { Generated, Insertable, Selectable, Updateable } from "kysely";
+import type {
+	AccountTable,
+	ActiveAccountTable,
+} from "../account/database-schema.js";
+import type { KeyValueTable } from "../key-value/database-schema.js";
+import type { MutationLogTable } from "./mutation-log/database-schema.js";
+
+export type LixDatabaseSchema = {
+	// account
+	account: AccountTable;
+	active_account: ActiveAccountTable;
+
+	// snapshot
+	snapshot: SnapshotTable;
+	label: LabelTable;
+
+	// file
+	file: LixFileTable;
+	change_queue: ChangeQueueTable;
+
+	// change
+	change: ChangeTable;
+	change_edge: ChangeEdgeTable;
+	change_author: ChangeAuthorTable;
+
+	// change set
+	change_set: ChangeSetTable;
+	change_set_element: ChangeSetElementTable;
+	change_set_label: ChangeSetLabelTable;
+	change_set_label_author: ChangeSetLabelAuthorTable;
+
+	// key value
+	key_value: KeyValueTable;
+
+	// discussion
+	discussion: DiscussionTable;
+	comment: CommentTable;
+
+	// version
+	current_version: CurrentVersionTable;
+	version: VersionTable;
+	version_change: VersionChangeTable;
+	version_change_conflict: VersionChangeConflictTable;
+
+	// change conflicts
+	change_conflict: ChangeConflictTable;
+	change_conflict_resolution: ChangeConflictResolutionTable;
+
+	mutation_log: MutationLogTable;
+};
+
+export type ChangeQueueEntry = Selectable<ChangeQueueTable>;
+export type NewChangeQueueEntry = Insertable<ChangeQueueTable>;
+export type ChangeQueueEntryUpdate = Updateable<ChangeQueueTable>;
+type ChangeQueueTable = {
+	id: Generated<number>;
+	file_id: string;
+	path_before: string | null;
+	path_after: string | null;
+	data_before: ArrayBuffer | null;
+	data_after: ArrayBuffer | null;
+	metadata_before: Record<string, any> | null;
+	metadata_after: Record<string, any> | null;
+};
+
+// named lix file to avoid conflict with built-in file type
+export type LixFile = Selectable<LixFileTable>;
+export type NewLixFile = Insertable<LixFileTable>;
+export type LixFileUpdate = Updateable<LixFileTable>;
+type LixFileTable = {
+	id: Generated<string>;
+	/**
+	 * The path of the file.
+	 *
+	 * The path is currently defined as a subset of RFC 3986.
+	 * Any path can be tested with the `isValidFilePath()` function.
+	 *
+	 * @example
+	 *   - `/path/to/file.txt`
+	 */
+	path: string;
+	data: ArrayBuffer;
+	metadata: Record<string, any> | null;
+};
+
+export type Change = Selectable<ChangeTable>;
+export type NewChange = Insertable<ChangeTable>;
+type ChangeTable = {
+	id: Generated<string>;
+	/**
+	 * The entity the change refers to.
+	 */
+	entity_id: string;
+	file_id: string;
+	/**
+	 * The plugin key that contributed the change.
+	 *
+	 * Exists to ease querying for changes by plugin,
+	 * in case the user changes the plugin configuration.
+	 */
+	plugin_key: string;
+	/**
+	 * The schema key that the change refers to.
+	 */
+	schema_key: string;
+	snapshot_id: string;
+	/**
+	 * The time the change was created.
+	 */
+	created_at: Generated<string>;
+};
+
+export type ChangeEdge = Selectable<ChangeEdgeTable>;
+export type NewChangeEdge = Insertable<ChangeEdgeTable>;
+type ChangeEdgeTable = {
+	parent_id: string;
+	child_id: string;
+};
+
+export type ChangeAuthor = Selectable<ChangeAuthorTable>;
+export type NewChangeAuthor = Insertable<ChangeAuthorTable>;
+type ChangeAuthorTable = {
+	change_id: string;
+	account_id: string;
+};
+
+export type Snapshot = Selectable<SnapshotTable>;
+export type NewSnapshot = Insertable<SnapshotTable>;
+type SnapshotTable = {
+	id: Generated<string>;
+	/**
+	 * The value of the change.
+	 *
+	 * Lix interprets an undefined value as delete operation.
+	 *
+	 * @example
+	 *   - For a csv cell change, the value would be the new cell value.
+	 *   - For an inlang message change, the value would be the new message.
+	 */
+	content: Record<string, any> | null;
+};
+
+// ------ change sets ------
+
+export type ChangeSet = Selectable<ChangeSetTable>;
+export type NewChangeSet = Insertable<ChangeSetTable>;
+export type ChangeSetUpdate = Updateable<ChangeSetTable>;
+type ChangeSetTable = {
+	id: Generated<string>;
+};
+
+export type ChangeSetElement = Selectable<ChangeSetElementTable>;
+export type NewChangeSetElement = Insertable<ChangeSetElementTable>;
+export type ChangeSetElementUpdate = Updateable<ChangeSetElementTable>;
+type ChangeSetElementTable = {
+	change_set_id: string;
+	change_id: string;
+};
+
+// ------ discussions ------
+
+export type Discussion = Selectable<DiscussionTable>;
+export type NewDiscussion = Insertable<DiscussionTable>;
+export type DiscussionUpdate = Updateable<DiscussionTable>;
+type DiscussionTable = {
+	id: Generated<string>;
+	change_set_id: string;
+};
+
+export type Comment = Selectable<CommentTable>;
+export type NewComment = Insertable<CommentTable>;
+export type CommentUpdate = Updateable<CommentTable>;
+type CommentTable = {
+	id: Generated<string>;
+	parent_id: string | null;
+	discussion_id: string;
+	created_at: Generated<string>;
+	created_by: string;
+	content: string;
+};
+
+// ----- tags -----
+
+export type Label = Selectable<LabelTable>;
+export type NewLabel = Insertable<LabelTable>;
+export type LabelUpdate = Updateable<LabelTable>;
+type LabelTable = {
+	id: Generated<string>;
+	name: string;
+};
+
+export type ChangeSetLabel = Selectable<ChangeSetLabelTable>;
+export type NewChangeSetLabel = Insertable<ChangeSetLabelTable>;
+export type ChangeSetLabelUpdate = Updateable<ChangeSetLabelTable>;
+type ChangeSetLabelTable = {
+	change_set_id: string;
+	label_id: string;
+};
+
+export type ChangeSetLabelAuthor = Selectable<ChangeSetLabelAuthorTable>;
+export type NewChangeSetLabelAuthor = Insertable<ChangeSetLabelAuthorTable>;
+export type ChangeSetLabelAuthorUpdate = Updateable<ChangeSetLabelAuthorTable>;
+type ChangeSetLabelAuthorTable = {
+	change_set_id: string;
+	label_id: string;
+	account_id: string;
+};
+
+// ------ versiones ------
+
+export type Version = Selectable<VersionTable>;
+export type Newversion = Insertable<VersionTable>;
+export type VersionUpdate = Updateable<VersionTable>;
+type VersionTable = {
+	id: Generated<string>;
+	name: string | null;
+};
+
+export type VersionChange = Selectable<VersionChangeTable>;
+export type NewVersionChange = Insertable<VersionChangeTable>;
+export type VersionChangeUpdate = Updateable<VersionChangeTable>;
+type VersionChangeTable = {
+	version_id: string;
+	change_id: string;
+};
+
+export type VersionChangeConflict = Selectable<VersionChangeConflictTable>;
+export type NewversionChangeConflict = Insertable<VersionChangeConflictTable>;
+export type VersionChangeConflictUpdate =
+	Updateable<VersionChangeConflictTable>;
+type VersionChangeConflictTable = {
+	version_id: string;
+	change_conflict_id: string;
+};
+
+export type Currentversion = Selectable<CurrentVersionTable>;
+export type NewCurrentversion = Insertable<CurrentVersionTable>;
+export type CurrentversionUpdate = Updateable<CurrentVersionTable>;
+type CurrentVersionTable = {
+	id: string;
+};
+
+// export type versionTarget = Selectable<versionTargetTable>;
+// export type NewversionTarget = Insertable<versionTargetTable>;
+// export type versionTargetUpdate = Updateable<versionTargetTable>;
+// type versionTargetTable = {
+// 	source_version_id: string;
+// 	target_version_id: string;
+// };
+
+// -------- change conflicts --------
+
+export type ChangeConflict = Selectable<ChangeConflictTable>;
+export type NewChangeConflict = Insertable<ChangeConflictTable>;
+export type ChangeConflictUpdate = Updateable<ChangeConflictTable>;
+type ChangeConflictTable = {
+	id: Generated<string>;
+	/**
+	 * The key is used to identify the conflict.
+	 *
+	 * The key should be unique for the plugin and the conflict
+	 * to avoid duplicate conflict reports.
+	 *
+	 * @example
+	 *   - `csv-row-order-changed`
+	 *   - `inlang-message-bundle-foreign-key-violation`
+	 */
+	key: string;
+	change_set_id: string;
+};
+
+export type ChangeConflictResolution =
+	Selectable<ChangeConflictResolutionTable>;
+export type NewChangeConflictResolution =
+	Insertable<ChangeConflictResolutionTable>;
+export type ChangeConflictResolutionUpdate =
+	Updateable<ChangeConflictResolutionTable>;
+type ChangeConflictResolutionTable = {
+	change_conflict_id: string;
+	resolved_change_id: string;
+};

package/src/discussion/create-comment.ts

@@ -0,0 +1,21 @@
+import type { Account } from "../account/database-schema.js";
+import type { Comment } from "../database/schema.js";
+import type { Lix } from "../lix/open-lix.js";
+
+export async function createComment(args: {
+	lix: Pick<Lix, "db">;
+	parentComment: Pick<Comment, "id" | "discussion_id">;
+	content: string;
+	createdBy: Pick<Account, "id">;
+}): Promise<Comment> {
+	return args.lix.db
+		.insertInto("comment")
+		.values({
+			discussion_id: args.parentComment.discussion_id,
+			parent_id: args.parentComment.id,
+			content: args.content,
+			created_by: args.createdBy.id,
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+}