@lix-js/sdk 0.0.1

package/src/database/apply-schema.ts

@@ -0,0 +1,323 @@
+import type { SqliteDatabase } from "sqlite-wasm-kysely";
+import { applyAccountDatabaseSchema } from "../account/database-schema.js";
+import { applyKeyValueDatabaseSchema } from "../key-value/database-schema.js";
+import { applyMutationLogDatabaseSchema } from "./mutation-log/database-schema.js";
+
+/**
+ * Applies the database schema to the given sqlite database.
+ */
+export function applySchema(args: { sqlite: SqliteDatabase }): SqliteDatabase {
+	applyAccountDatabaseSchema(args.sqlite);
+	applyKeyValueDatabaseSchema(args.sqlite);
+
+	// eslint-disable-next-line @typescript-eslint/no-unused-expressions
+	args.sqlite.exec`
+
+  PRAGMA foreign_keys = ON;
+  PRAGMA auto_vacuum = 2; -- incremental https://www.sqlite.org/pragma.html#pragma_auto_vacuum
+ 
+  -- file
+
+  CREATE TABLE IF NOT EXISTS file (
+    id TEXT PRIMARY KEY DEFAULT (uuid_v7()),
+    path TEXT NOT NULL UNIQUE,
+    data BLOB NOT NULL,
+    metadata BLOB,
+    
+
+    CHECK (is_valid_file_path(path))
+  ) STRICT;
+
+  CREATE TABLE IF NOT EXISTS change_queue (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    file_id TEXT NOT NULL,
+    data_before BLOB,
+    data_after BLOB,
+    path_before TEXT,
+    path_after TEXT,
+    metadata_before BLOB,
+    metadata_after BLOB
+  ) STRICT;
+
+  CREATE TRIGGER IF NOT EXISTS file_insert BEFORE INSERT ON file
+  BEGIN
+    INSERT INTO change_queue(
+      file_id, path_after, data_after, metadata_after
+    )
+    VALUES (
+      NEW.id, NEW.path, NEW.data, NEW.metadata
+    );
+    SELECT triggerChangeQueue();
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS file_update BEFORE UPDATE ON file
+  BEGIN
+    INSERT INTO change_queue(
+      file_id, 
+      path_before, data_before, metadata_before, 
+      path_after, data_after, metadata_after
+    )
+
+    VALUES (
+      NEW.id, 
+      OLD.path, OLD.data, OLD.metadata,
+      NEW.path, NEW.data, NEW.metadata
+    );
+
+    SELECT triggerChangeQueue();
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS file_delete BEFORE DELETE ON file
+  BEGIN
+    INSERT INTO change_queue(file_id)
+    VALUES (OLD.id);
+    SELECT triggerChangeQueue();
+  END;
+
+  CREATE TABLE IF NOT EXISTS change (
+    id TEXT PRIMARY KEY DEFAULT (uuid_v7()),
+    entity_id TEXT NOT NULL,
+    schema_key TEXT NOT NULL,
+    file_id TEXT NOT NULL,
+    plugin_key TEXT NOT NULL,
+    snapshot_id TEXT NOT NULL,
+    created_at TEXT DEFAULT CURRENT_TIMESTAMP NOT NULL,
+
+    UNIQUE (id, entity_id, file_id, schema_key),
+    FOREIGN KEY(snapshot_id) REFERENCES snapshot(id)
+  ) STRICT;
+
+  CREATE TABLE IF NOT EXISTS change_author (
+    change_id TEXT NOT NULL,
+    account_id TEXT NOT NULL,
+
+    PRIMARY KEY (change_id, account_id),
+    FOREIGN KEY(change_id) REFERENCES change(id),
+    FOREIGN KEY(account_id) REFERENCES account(id)
+  ) strict;
+
+  CREATE TABLE IF NOT EXISTS change_edge (
+    parent_id TEXT NOT NULL,
+    child_id TEXT NOT NULL,
+    
+    PRIMARY KEY (parent_id, child_id),
+    FOREIGN KEY(parent_id) REFERENCES change(id),
+    FOREIGN KEY(child_id) REFERENCES change(id),
+    -- Prevent self referencing edges
+    CHECK (parent_id != child_id)
+  ) STRICT;
+
+  CREATE TABLE IF NOT EXISTS snapshot (
+    id TEXT GENERATED ALWAYS AS (json_sha256(content)) STORED UNIQUE,
+    content BLOB
+  ) STRICT;
+
+  -- Create the default 'no-content' snapshot
+  -- to avoid foreign key constraint violations in tests
+  INSERT OR IGNORE INTO snapshot (content) VALUES (NULL);
+
+  -- conflicts
+
+  CREATE INDEX IF NOT EXISTS idx_content_hash ON snapshot (id);
+
+  CREATE TABLE IF NOT EXISTS change_conflict (
+    id TEXT PRIMARY KEY DEFAULT (uuid_v7()),
+    
+    key TEXT NOT NULL,
+    change_set_id TEXT NOT NULL,
+    FOREIGN KEY(change_set_id) REFERENCES change_set(id)
+  ) STRICT;
+
+  CREATE TABLE IF NOT EXISTS change_conflict_resolution (
+    change_conflict_id TEXT NOT NULL,
+    resolved_change_id TEXT NOT NULL,
+    
+    -- potential future columns
+    -- resolved_by <account_id>
+    -- resolved_at <timestamp>
+
+    PRIMARY KEY(change_conflict_id, resolved_change_id),
+    FOREIGN KEY(change_conflict_id) REFERENCES change_conflict(id),
+    FOREIGN KEY(resolved_change_id) REFERENCES change(id)
+  ) STRICT;
+
+  -- change sets
+
+  CREATE TABLE IF NOT EXISTS change_set (
+    id TEXT PRIMARY KEY DEFAULT (uuid_v7())
+  ) STRICT;
+
+  CREATE TABLE IF NOT EXISTS change_set_element (
+    
+    change_set_id TEXT NOT NULL,
+    change_id TEXT NOT NULL,
+
+    PRIMARY KEY(change_set_id, change_id),
+    FOREIGN KEY(change_set_id) REFERENCES change_set(id),    
+    FOREIGN KEY(change_id) REFERENCES change(id)
+  ) STRICT;
+
+  CREATE TABLE IF NOT EXISTS change_set_label (
+    label_id TEXT NOT NULL,
+    change_set_id TEXT NOT NULL,
+    PRIMARY KEY(label_id, change_set_id),
+    FOREIGN KEY(label_id) REFERENCES label(id),
+    FOREIGN KEY(change_set_id) REFERENCES change_set(id)
+  ) STRICT;
+
+  CREATE TABLE IF NOT EXISTS change_set_label_author (
+    label_id TEXT NOT NULL,
+    change_set_id TEXT NOT NULL,
+    account_id TEXT NOT NULL,
+
+    PRIMARY KEY(label_id, change_set_id, account_id),
+    FOREIGN KEY(label_id, change_set_id) REFERENCES change_set_label(label_id, change_set_id),
+    FOREIGN KEY(account_id) REFERENCES account(id)
+  ) STRICT;
+
+  -- discussions 
+
+  CREATE TABLE IF NOT EXISTS discussion (
+    id TEXT PRIMARY KEY DEFAULT (uuid_v7()),
+    change_set_id TEXT NOT NULL,
+
+    FOREIGN KEY(change_set_id) REFERENCES change_set(id)
+  ) STRICT;
+
+  CREATE TABLE IF NOT EXISTS comment (
+    id TEXT PRIMARY KEY DEFAULT (uuid_v7()),
+    parent_id TEXT,
+    discussion_id TEXT NULL,
+    created_at TEXT DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    content TEXT NOT NULL,
+    created_by TEXT NOT NULL,
+    
+    FOREIGN KEY(created_by) REFERENCES account(id),
+    FOREIGN KEY(discussion_id) REFERENCES discussion(id),
+    FOREIGN KEY(parent_id) REFERENCES comment(id)
+  ) STRICT;
+
+  -- labels
+  
+  CREATE TABLE IF NOT EXISTS label (
+    id TEXT PRIMARY KEY DEFAULT (uuid_v7()),
+    
+    name TEXT NOT NULL UNIQUE  -- e.g., 'confirmed', 'reviewed'
+    
+  ) STRICT;
+
+  INSERT OR IGNORE INTO label (name) VALUES ('confirmed');
+  INSERT OR IGNORE INTO label (name) VALUES ('grouped');
+
+  -- versions
+
+  CREATE TABLE IF NOT EXISTS version (
+    id TEXT PRIMARY KEY DEFAULT (uuid_v7()),
+
+    -- name is optional. 
+    -- 
+    -- "anonymous" versiones can ease workflows. 
+    -- For example, a user can create a version 
+    -- without a name to experiment with
+    -- changes with no mental overhead of 
+    -- naming the version.
+    name TEXT
+  ) STRICT;
+
+  CREATE TABLE IF NOT EXISTS version_change (
+    version_id TEXT NOT NULL,
+    change_id TEXT NOT NULL,
+
+    PRIMARY KEY (version_id, change_id),
+    FOREIGN KEY (version_id) REFERENCES version(id) ON DELETE CASCADE,
+    FOREIGN KEY (change_id) REFERENCES change(id) ON DELETE CASCADE
+  ) STRICT;
+
+  CREATE TABLE IF NOT EXISTS version_change_conflict (
+    version_id TEXT NOT NULL,
+    change_conflict_id TEXT NOT NULL,
+
+    PRIMARY KEY (version_id, change_conflict_id),
+    FOREIGN KEY (version_id) REFERENCES version(id),
+    FOREIGN KEY (change_conflict_id) REFERENCES change_conflict(id)
+  ) STRICT;
+
+  -- only one version can be active at a time
+  -- hence, the table has only one row
+  CREATE TABLE IF NOT EXISTS current_version (
+    id TEXT NOT NULL PRIMARY KEY,
+
+    FOREIGN KEY(id) REFERENCES version(id)
+  ) STRICT;
+
+  -- Insert the default version if missing
+  -- (this is a workaround for not having a separata creation and migration schema's)
+
+  INSERT INTO version (id, name)
+  SELECT '019328cc-ccb0-7f51-96e8-524df4597ac6', 'main'
+  WHERE NOT EXISTS (SELECT 1 FROM version);
+
+  -- Set the default current version to 'main' if both tables are empty
+  -- (this is a workaround for not having a separata creation and migration schema's)
+  INSERT INTO current_version (id)
+  SELECT '019328cc-ccb0-7f51-96e8-524df4597ac6'
+  WHERE NOT EXISTS (SELECT 1 FROM current_version);
+
+
+  CREATE TEMP TRIGGER IF NOT EXISTS insert_account_if_not_exists_on_change_author
+  BEFORE INSERT ON change_author
+  FOR EACH ROW
+  WHEN NEW.account_id NOT IN (SELECT id FROM account) AND NEW.account_id IN (SELECT id FROM temp.active_account)
+  BEGIN
+    INSERT OR IGNORE INTO account
+      SELECT 
+      *
+      FROM active_account 
+      WHERE id = NEW.account_id;
+  END;
+  
+  CREATE TEMP TRIGGER IF NOT EXISTS insert_account_if_not_exists_on_change_set_label_author
+  BEFORE INSERT ON change_set_label_author
+  FOR EACH ROW
+  WHEN NEW.account_id NOT IN (SELECT id FROM account) AND NEW.account_id IN (SELECT id FROM temp.active_account)
+  BEGIN
+    INSERT OR IGNORE INTO account
+      SELECT 
+      *
+      FROM active_account 
+      WHERE id = NEW.account_id;
+  END;
+    
+  CREATE TEMP TRIGGER IF NOT EXISTS insert_account_if_not_exists_on_comment
+  BEFORE INSERT ON comment
+  FOR EACH ROW
+  WHEN NEW.created_by NOT IN (SELECT id FROM account) AND NEW.created_by IN (SELECT id FROM temp.active_account)
+  BEGIN
+    INSERT OR IGNORE INTO account
+      SELECT 
+      *
+      FROM active_account 
+      WHERE id = NEW.created_by;
+  END;
+  
+
+  `;
+
+	// CREATE TRIGGER IF NOT EXISTS insert_account_if_not_exists_on_change_set_label_author
+	// BEFORE INSERT ON change_set_label_author
+	// FOR EACH ROW
+	// BEGIN
+	//   INSERT OR IGNORE INTO account (id, name)
+	//   VALUES (
+	//     NEW.account_id,
+	//     CASE
+	//       WHEN NEW.account_id LIKE 'anonymous_%' THEN 'anonymous'
+	//       ELSE NEW.account_id
+	//     END
+	//   );
+	// END;
+
+	applyMutationLogDatabaseSchema(args.sqlite);
+
+	return args.sqlite;
+}

package/src/database/execute-sync.test.ts

@@ -0,0 +1,152 @@
+import { expect, test } from "vitest";
+import { openLixInMemory } from "../lix/open-lix-in-memory.js";
+import { executeSync } from "./execute-sync.js";
+import type { Lix } from "../lix/open-lix.js";
+import { mockChange } from "../change/mock-change.js";
+import type { LixFile } from "./schema.js";
+import { sql } from "kysely";
+
+test("executeSync yields the same result as async execute for an insert", async () => {
+	const lix1 = await openLixInMemory({});
+	const lix2 = await openLixInMemory({});
+
+	const query = (lix: Lix) =>
+		lix.db
+			.insertInto("key_value")
+			.values({ key: "foo", value: "bar" })
+			.returningAll();
+
+	const result = executeSync({ lix: lix1, query: query(lix1) });
+	const result2 = await query(lix2).execute();
+
+	expect(result).toEqual(result2);
+});
+
+test("handles joins", async () => {
+	const lix = await openLixInMemory({});
+
+	const mockFile0: LixFile = {
+		id: "file-0",
+		path: "/file-0",
+		data: new ArrayBuffer(0),
+		metadata: {},
+	};
+	const mockChange0 = mockChange({ id: "change-0", file_id: "file-0" });
+
+	await lix.db.insertInto("change").values(mockChange0).execute();
+	await lix.db.insertInto("file").values(mockFile0).execute();
+
+	const query = lix.db
+		.selectFrom("file")
+		.innerJoin("change", "change.file_id", "file.id")
+		.where("file.id", "=", "file-0")
+		.selectAll("change")
+		.select("file.path as file_path");
+
+	const result = executeSync({ lix, query });
+	const result2 = await query.execute();
+
+	expect(result).toEqual(result2);
+});
+
+test("does not transform the query or results (json parsing)", async () => {
+	const lix1 = await openLixInMemory({});
+	const lix2 = await openLixInMemory({});
+
+	const mockFile0: LixFile = {
+		id: "file-0",
+		path: "/file-0",
+		data: new ArrayBuffer(0),
+		metadata: {
+			foo: "bar",
+		},
+	};
+
+	const insertQuery = (lix: Lix) =>
+		lix.db.insertInto("file").values(mockFile0).returningAll();
+
+	const result1 = await insertQuery(lix1).execute();
+	const result2 = executeSync({ lix: lix2, query: insertQuery(lix2) });
+
+	expect(result1).not.toEqual(result2);
+
+	const withManualJson = executeSync({
+		lix: lix2,
+		query: lix2.db
+			.selectFrom("file")
+			.selectAll()
+			.select(sql`json(metadata)`.as("metadata")),
+	});
+
+	for (const row of withManualJson) {
+		row.metadata = JSON.parse(row.metadata as string);
+	}
+
+	expect(result1).toEqual(withManualJson);
+});
+
+// important for function like `createQuery` which are used in triggers and need to be sync
+// but are also used by users where the API is async
+test("using executeSync with a 'fake async' function should work", async () => {
+	const lix = await openLixInMemory({});
+
+	async function fakeAyncQuery(lix: Lix): Promise<any> {
+		const query = lix.db
+			.insertInto("key_value")
+			.values({ key: "foo", value: "bar" })
+			.returningAll();
+		return executeSync({ lix, query }) as any;
+	}
+
+	const result = await fakeAyncQuery(lix);
+
+	expect(result).toEqual([{ key: "foo", value: "bar" }]);
+});
+
+test("it works with kysely transactions", async () => {
+	const lix = await openLixInMemory({});
+
+	// transaction that fails
+	try {
+		await lix.db.transaction().execute(async (trx) => {
+			await trx
+				.insertInto("key_value")
+				.values({ key: "foo", value: "bar" })
+				.execute();
+
+			executeSync({
+				lix,
+				query: trx
+					.insertInto("key_value")
+					.values({ key: "foo2", value: "bar2" }),
+			});
+
+			throw new Error("rollback");
+		});
+	} catch {
+		// ignore;
+	}
+
+	const keyValues = await lix.db.selectFrom("key_value").selectAll().execute();
+
+	expect(keyValues.find((kv) => kv.key === "foo")).toBeUndefined();
+	expect(keyValues.find((kv) => kv.key === "foo2")).toBeUndefined();
+
+	// transaction that succeeds
+	await lix.db.transaction().execute(async (trx) => {
+		await trx
+			.insertInto("key_value")
+			.values({ key: "foo", value: "bar" })
+			.execute();
+
+		executeSync({
+			lix,
+			query: trx.insertInto("key_value").values({ key: "foo2", value: "bar2" }),
+		});
+	});
+
+	const keyValues2 = await lix.db.selectFrom("key_value").selectAll().execute();
+
+	expect(keyValues2.find((kv) => kv.key === "foo")).toBeDefined();
+	expect(keyValues2.find((kv) => kv.key === "foo2")).toBeDefined();
+});

package/src/database/execute-sync.ts

@@ -0,0 +1,47 @@
+import type { Lix } from "../lix/open-lix.js";
+
+/**
+ * Execute a query synchronously.
+ *
+ * WARNING: This function is not recommended for general use.
+ * Only if you need sync queries, like in a trigger for exmaple,
+ * you should use this function. The function is not transforming
+ * the query or the result as the db API does. You get raw SQL.
+ *
+ * @example
+ *   const query = lix.db.selectFrom("key_value").selectAll();
+ *   const result = executeSync({ lix, query }) as KeyValue[];
+ */
+export function executeSync(args: {
+	lix: Pick<Lix, "sqlite">;
+	query: any;
+}): Array<any> {
+	const compiledQuery = args.query.compile();
+
+	const columnNames: string[] = [];
+
+	try {
+		const result = args.lix.sqlite.exec({
+			sql: compiledQuery.sql,
+			bind: compiledQuery.parameters as any[],
+			returnValue: "resultRows",
+			columnNames,
+		});
+
+		return result.map((row) => {
+			const obj: any = {};
+
+			columnNames.forEach((columnName, index) => {
+				obj[columnName] = row[index];
+			});
+
+			return obj;
+		});
+	} catch (e) {
+		console.error(
+			`Error in executeSync for query:\n\n${compiledQuery.sql}\n\n`,
+			e
+		);
+		return [];
+	}
+}

package/src/database/index.ts

@@ -0,0 +1,4 @@
+export * from "./schema.js";
+export { jsonObjectFrom, jsonArrayFrom } from "kysely/helpers/sqlite";
+export { sql } from "kysely";
+export { executeSync } from "./execute-sync.js";