@lix-js/sdk 0.0.1 → 0.2.0

package/src/database/apply-schema.ts

@@ -19,7 +19,7 @@ export function applySchema(args: { sqlite: SqliteDatabase }): SqliteDatabase {
   -- file
 
   CREATE TABLE IF NOT EXISTS file (
-    id TEXT PRIMARY KEY DEFAULT (uuid_v7()),
+    id TEXT PRIMARY KEY DEFAULT (nano_id(10)),
     path TEXT NOT NULL UNIQUE,
     data BLOB NOT NULL,
     metadata BLOB,
@@ -28,7 +28,7 @@ export function applySchema(args: { sqlite: SqliteDatabase }): SqliteDatabase {
     CHECK (is_valid_file_path(path))
   ) STRICT;
 
-  CREATE TABLE IF NOT EXISTS change_queue (
+  CREATE TABLE IF NOT EXISTS file_queue (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     file_id TEXT NOT NULL,
     data_before BLOB,
@@ -41,18 +41,18 @@ export function applySchema(args: { sqlite: SqliteDatabase }): SqliteDatabase {
 
   CREATE TRIGGER IF NOT EXISTS file_insert BEFORE INSERT ON file
   BEGIN
-    INSERT INTO change_queue(
+    INSERT INTO file_queue(
       file_id, path_after, data_after, metadata_after
     )
     VALUES (
       NEW.id, NEW.path, NEW.data, NEW.metadata
     );
-    SELECT triggerChangeQueue();
+    SELECT triggerFileQueue();
   END;
 
   CREATE TRIGGER IF NOT EXISTS file_update BEFORE UPDATE ON file
   BEGIN
-    INSERT INTO change_queue(
+    INSERT INTO file_queue(
       file_id, 
       path_before, data_before, metadata_before, 
       path_after, data_after, metadata_after
@@ -64,14 +64,14 @@ export function applySchema(args: { sqlite: SqliteDatabase }): SqliteDatabase {
       NEW.path, NEW.data, NEW.metadata
     );
 
-    SELECT triggerChangeQueue();
+    SELECT triggerFileQueue();
   END;
 
   CREATE TRIGGER IF NOT EXISTS file_delete BEFORE DELETE ON file
   BEGIN
-    INSERT INTO change_queue(file_id)
+    INSERT INTO file_queue(file_id)
     VALUES (OLD.id);
-    SELECT triggerChangeQueue();
+    SELECT triggerFileQueue();
   END;
 
   CREATE TABLE IF NOT EXISTS change (
@@ -144,7 +144,7 @@ export function applySchema(args: { sqlite: SqliteDatabase }): SqliteDatabase {
   -- change sets
 
   CREATE TABLE IF NOT EXISTS change_set (
-    id TEXT PRIMARY KEY DEFAULT (uuid_v7())
+    id TEXT PRIMARY KEY DEFAULT (nano_id(16))
   ) STRICT;
 
   CREATE TABLE IF NOT EXISTS change_set_element (
@@ -165,34 +165,21 @@ export function applySchema(args: { sqlite: SqliteDatabase }): SqliteDatabase {
     FOREIGN KEY(change_set_id) REFERENCES change_set(id)
   ) STRICT;
 
-  CREATE TABLE IF NOT EXISTS change_set_label_author (
-    label_id TEXT NOT NULL,
-    change_set_id TEXT NOT NULL,
-    account_id TEXT NOT NULL,
-
-    PRIMARY KEY(label_id, change_set_id, account_id),
-    FOREIGN KEY(label_id, change_set_id) REFERENCES change_set_label(label_id, change_set_id),
-    FOREIGN KEY(account_id) REFERENCES account(id)
-  ) STRICT;
-
   -- discussions 
 
   CREATE TABLE IF NOT EXISTS discussion (
-    id TEXT PRIMARY KEY DEFAULT (uuid_v7()),
+    id TEXT PRIMARY KEY DEFAULT (nano_id(12)),
     change_set_id TEXT NOT NULL,
 
     FOREIGN KEY(change_set_id) REFERENCES change_set(id)
   ) STRICT;
 
   CREATE TABLE IF NOT EXISTS comment (
-    id TEXT PRIMARY KEY DEFAULT (uuid_v7()),
+    id TEXT PRIMARY KEY DEFAULT (nano_id(14)),
     parent_id TEXT,
     discussion_id TEXT NULL,
-    created_at TEXT DEFAULT CURRENT_TIMESTAMP NOT NULL,
     content TEXT NOT NULL,
-    created_by TEXT NOT NULL,
     
-    FOREIGN KEY(created_by) REFERENCES account(id),
     FOREIGN KEY(discussion_id) REFERENCES discussion(id),
     FOREIGN KEY(parent_id) REFERENCES comment(id)
   ) STRICT;
@@ -200,7 +187,7 @@ export function applySchema(args: { sqlite: SqliteDatabase }): SqliteDatabase {
   -- labels
   
   CREATE TABLE IF NOT EXISTS label (
-    id TEXT PRIMARY KEY DEFAULT (uuid_v7()),
+    id TEXT PRIMARY KEY DEFAULT (nano_id(8)),
     
     name TEXT NOT NULL UNIQUE  -- e.g., 'confirmed', 'reviewed'
     
@@ -214,23 +201,22 @@ export function applySchema(args: { sqlite: SqliteDatabase }): SqliteDatabase {
   CREATE TABLE IF NOT EXISTS version (
     id TEXT PRIMARY KEY DEFAULT (uuid_v7()),
 
-    -- name is optional. 
-    -- 
-    -- "anonymous" versiones can ease workflows. 
-    -- For example, a user can create a version 
-    -- without a name to experiment with
-    -- changes with no mental overhead of 
-    -- naming the version.
-    name TEXT
+    name TEXT NOT NULL UNIQUE DEFAULT (human_id())
   ) STRICT;
 
   CREATE TABLE IF NOT EXISTS version_change (
     version_id TEXT NOT NULL,
     change_id TEXT NOT NULL,
 
+    entity_id TEXT NOT NULL,
+    schema_key TEXT NOT NULL,
+    file_id TEXT NOT NULL,
+
     PRIMARY KEY (version_id, change_id),
     FOREIGN KEY (version_id) REFERENCES version(id) ON DELETE CASCADE,
-    FOREIGN KEY (change_id) REFERENCES change(id) ON DELETE CASCADE
+    FOREIGN KEY (change_id, entity_id, schema_key, file_id) REFERENCES change(id, entity_id, schema_key, file_id) ON DELETE CASCADE,
+
+    UNIQUE (version_id, entity_id, schema_key, file_id)
   ) STRICT;
 
   CREATE TABLE IF NOT EXISTS version_change_conflict (
@@ -276,47 +262,8 @@ export function applySchema(args: { sqlite: SqliteDatabase }): SqliteDatabase {
       WHERE id = NEW.account_id;
   END;
   
-  CREATE TEMP TRIGGER IF NOT EXISTS insert_account_if_not_exists_on_change_set_label_author
-  BEFORE INSERT ON change_set_label_author
-  FOR EACH ROW
-  WHEN NEW.account_id NOT IN (SELECT id FROM account) AND NEW.account_id IN (SELECT id FROM temp.active_account)
-  BEGIN
-    INSERT OR IGNORE INTO account
-      SELECT 
-      *
-      FROM active_account 
-      WHERE id = NEW.account_id;
-  END;
-    
-  CREATE TEMP TRIGGER IF NOT EXISTS insert_account_if_not_exists_on_comment
-  BEFORE INSERT ON comment
-  FOR EACH ROW
-  WHEN NEW.created_by NOT IN (SELECT id FROM account) AND NEW.created_by IN (SELECT id FROM temp.active_account)
-  BEGIN
-    INSERT OR IGNORE INTO account
-      SELECT 
-      *
-      FROM active_account 
-      WHERE id = NEW.created_by;
-  END;
-  
-
   `;
 
-	// CREATE TRIGGER IF NOT EXISTS insert_account_if_not_exists_on_change_set_label_author
-	// BEFORE INSERT ON change_set_label_author
-	// FOR EACH ROW
-	// BEGIN
-	//   INSERT OR IGNORE INTO account (id, name)
-	//   VALUES (
-	//     NEW.account_id,
-	//     CASE
-	//       WHEN NEW.account_id LIKE 'anonymous_%' THEN 'anonymous'
-	//       ELSE NEW.account_id
-	//     END
-	//   );
-	// END;
-
 	applyMutationLogDatabaseSchema(args.sqlite);
 
 	return args.sqlite;

package/src/database/execute-sync.test.ts

@@ -28,7 +28,7 @@ test("handles joins", async () => {
 	const mockFile0: LixFile = {
 		id: "file-0",
 		path: "/file-0",
-		data: new ArrayBuffer(0),
+		data: new Uint8Array(),
 		metadata: {},
 	};
 	const mockChange0 = mockChange({ id: "change-0", file_id: "file-0" });
@@ -56,7 +56,7 @@ test("does not transform the query or results (json parsing)", async () => {
 	const mockFile0: LixFile = {
 		id: "file-0",
 		path: "/file-0",
-		data: new ArrayBuffer(0),
+		data: new Uint8Array(),
 		metadata: {
 			foo: "bar",
 		},
@@ -100,7 +100,7 @@ test("using executeSync with a 'fake async' function should work", async () => {
 
 	const result = await fakeAyncQuery(lix);
 
-	expect(result).toEqual([{ key: "foo", value: "bar" }]);
+	expect(result).toMatchObject([{ key: "foo", value: "bar" }]);
 });
 
 test("it works with kysely transactions", async () => {

package/src/database/init-db.test.ts

@@ -7,31 +7,28 @@ import { jsonSha256 } from "../snapshot/json-sha-256.js";
 import { sql } from "kysely";
 import { openLixInMemory } from "../lix/open-lix-in-memory.js";
 import { updateChangesInVersion } from "../version/update-changes-in-version.js";
+import { createVersion } from "../version/create-version.js";
+
+// file ids are always in the URL of lix apps
+// to increase sharing, the ids should be as short as possible
+//
+// 129 million file creations will lead to a 1% chance of a collision
+//
+// if someone uses lix to handle 129 million files, we can
+// increase the length of the id :D
+test("file ids should default to nano_id(10)", async () => {
+	const lix = await openLixInMemory({});
 
-test("file ids should default to uuid", async () => {
-	const sqlite = await createInMemoryDatabase({
-		readOnly: false,
-	});
-	const db = initDb({ sqlite });
-
-	// init the trigger function (usually defined by lix only)
-	sqlite.createFunction({
-		name: "triggerChangeQueue",
-		arity: 0,
-		// @ts-expect-error - dynamic function
-		xFunc: () => {},
-	});
-
-	const file = await db
+	const file = await lix.db
 		.insertInto("file")
 		.values({
-			path: "/mock",
+			path: "/mock.txt",
 			data: new Uint8Array(),
 		})
 		.returningAll()
 		.executeTakeFirstOrThrow();
 
-	expect(validate(file.id)).toBe(true);
+	expect(file.id.length).toBe(10);
 });
 
 test("change ids should default to uuid", async () => {
@@ -142,7 +139,7 @@ test("files should be able to have metadata", async () => {
 	const db = initDb({ sqlite });
 
 	sqlite.createFunction({
-		name: "triggerChangeQueue",
+		name: "triggerFileQueue",
 		arity: 0,
 		// @ts-expect-error - dynamic function
 		xFunc: () => {
@@ -243,6 +240,98 @@ test("change set items must be unique", async () => {
 	);
 });
 
+// 8B IDs needed, in order to have a 1% probability of at least one collision.
+test("discussion.id are nano_id(12)", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	const changeSet = await db
+		.insertInto("change_set")
+		.defaultValues()
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	const discussion = await db
+		.insertInto("discussion")
+		.values({
+			change_set_id: changeSet.id,
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	expect(discussion.id.length).toBe(12);
+});
+
+// 499B IDs needed, in order to have a 1% probability of at least one collision.
+test("comment.id are nano_id(14)", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	const changeSet = await db
+		.insertInto("change_set")
+		.defaultValues()
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	const discussion = await db
+		.insertInto("discussion")
+		.values({
+			change_set_id: changeSet.id,
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	const comment = await db
+		.insertInto("comment")
+		.values({
+			discussion_id: discussion.id,
+			content: "mock",
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	expect(comment.id.length).toBe(14);
+});
+
+// 30T IDs needed, in order to have a 1% probability of at least one collision
+test("change_set.id are nano_id(16)", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	const changeSet = await db
+		.insertInto("change_set")
+		.defaultValues()
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	expect(changeSet.id.length).toBe(16);
+});
+
+// 2M IDs needed, in order to have a 1% probability of at least one collision.
+// it is assumed that creating 2 million labels is ... unlikely
+test("label.id is nano_id(8)", async () => {
+	const sqlite = await createInMemoryDatabase({
+		readOnly: false,
+	});
+	const db = initDb({ sqlite });
+
+	const label = await db
+		.insertInto("label")
+		.values({
+			name: "mock",
+		})
+		.returningAll()
+		.executeTakeFirstOrThrow();
+
+	expect(label.id.length).toBe(8);
+});
+
 test("creating multiple discussions for one change set should be possible", async () => {
 	const sqlite = await createInMemoryDatabase({
 		readOnly: false,
@@ -337,7 +426,7 @@ test("invalid file paths should be rejected", async () => {
 
 	// init the trigger function (usually defined by lix only)
 	sqlite.createFunction({
-		name: "triggerChangeQueue",
+		name: "triggerFileQueue",
 		arity: 0,
 		// @ts-expect-error - dynamic function
 		xFunc: () => {},
@@ -482,3 +571,58 @@ test("deleting a version cascades to version changes", async () => {
 
 	expect(versionChangesAfterDelete).toHaveLength(0);
 });
+
+test("version_change must have a unique entity_id, file_id, version_id, and schema_id", async () => {
+	const lix = await openLixInMemory({});
+
+	const version0 = await createVersion({ lix, name: "version0" });
+	const version1 = await createVersion({ lix, name: "version1" });
+
+	const mockChanges = [
+		mockChange({ entity_id: "mock0", file_id: "file0", schema_key: "file" }),
+	] as const;
+
+	await lix.db.insertInto("change").values(mockChanges).execute();
+
+	const versionChange = {
+		change_id: mockChanges[0].id,
+		file_id: mockChanges[0].file_id,
+		entity_id: mockChanges[0].entity_id,
+		schema_key: mockChanges[0].schema_key,
+	};
+
+	await lix.db
+		.insertInto("version_change")
+		.values({ ...versionChange, version_id: version0.id })
+		.execute();
+
+	// inserting the same change again should throw
+	await expect(
+		lix.db
+			.insertInto("version_change")
+			.values({ ...versionChange, version_id: version0.id })
+			.execute()
+	).rejects.toThrowErrorMatchingInlineSnapshot(
+		`[SQLite3Error: SQLITE_CONSTRAINT_UNIQUE: sqlite3 result code 2067: UNIQUE constraint failed: version_change.version_id, version_change.entity_id, version_change.schema_key, version_change.file_id]`
+	);
+
+	// insert into version 1 should work
+	await lix.db
+		.insertInto("version_change")
+		.values({ ...versionChange, version_id: version1.id })
+		.execute();
+});
+
+test("versions have a unique and default human readable name", async () => {
+	const lix = await openLixInMemory({});
+
+	const version0 = await createVersion({ lix });
+	const version1 = await createVersion({ lix });
+
+	expect(version0.name).not.toBe(version1.name);
+
+	await expect(
+		createVersion({ lix, name: version0.name }),
+		"version.name is unique"
+	).rejects.toThrow();
+});

package/src/database/init-db.ts

@@ -8,7 +8,9 @@ import { jsonSha256 } from "../snapshot/json-sha-256.js";
 import { ParseJsonBPluginV1 } from "./kysely-plugin/parse-jsonb-plugin-v1.js";
 import { SerializeJsonBPlugin } from "./kysely-plugin/serialize-jsonb-plugin.js";
 import { createSession } from "./mutation-log/lix-session.js";
-import { applyOwnEntityChangeControlTriggers } from "../own-entity-change-control/database-triggers.js";
+import { applyOwnChangeControlTriggers } from "../own-change-control/database-triggers.js";
+import { humanId } from "human-id";
+import { nanoid } from "./nano-id.js";
 
 export function initDb(args: {
 	sqlite: SqliteDatabase;
@@ -23,7 +25,7 @@ export function initDb(args: {
 			ParseJsonBPluginV1({
 				// jsonb columns
 				file: ["metadata"],
-				change_queue: ["metadata_before", "metadata_after"],
+				file_queue: ["metadata_before", "metadata_after"],
 				snapshot: ["content"],
 				mutation_log: ["row_id"],
 			}),
@@ -32,7 +34,7 @@ export function initDb(args: {
 	});
 
 	// need to apply it here because db object needs to be available
-	applyOwnEntityChangeControlTriggers(args.sqlite, db);
+	applyOwnChangeControlTriggers(args.sqlite, db);
 	return db;
 }
 
@@ -91,4 +93,19 @@ function initFunctions(args: { sqlite: SqliteDatabase }) {
 		arity: 0,
 		xFunc: () => lixSession.sessionClockTick(),
 	});
+
+	args.sqlite.createFunction({
+		name: "human_id",
+		arity: 0,
+		xFunc: () => humanId({ separator: "-", capitalize: false }),
+	});
+
+	args.sqlite.createFunction({
+		name: "nano_id",
+		arity: 1,
+		// @ts-expect-error - not sure why this is not working
+		xFunc: (_ctx: number, length: number) => {
+			return nanoid(length);
+		},
+	});
 }

package/src/database/kysely-plugin/parse-jsonb-plugin-v1.ts

@@ -1,9 +1,12 @@
 import { type KyselyPlugin } from "kysely";
-import { createInMemoryDatabase } from "sqlite-wasm-kysely";
+import {
+	createInMemoryDatabase,
+	type SqliteDatabase,
+} from "sqlite-wasm-kysely";
 
 // workaround for v1. v2 doesn't need to transform
 // jsonb columns during runtime
-const sqlite = await createInMemoryDatabase({});
+let sqlite: SqliteDatabase;
 
 export function ParseJsonBPluginV1(
 	jsonbColumns: Record<string, string[]>
@@ -14,6 +17,10 @@ export function ParseJsonBPluginV1(
 
 	return {
 		transformResult: async (args) => {
+			if (!sqlite) {
+				sqlite = await createInMemoryDatabase({});
+			}
+
 			for (const row of args.result.rows) {
 				for (const col of jsonColumnNames) {
 					if (!row[col]) {

package/src/database/mutation-log/database-schema.ts

@@ -14,7 +14,7 @@ export const tablesByDepencies: string[] = [
 	"change",
 
 	// Depends on: file
-	"change_queue",
+	"file_queue",
 	// Depends on: change
 	"change_author",
 	// Depends on: change
@@ -27,8 +27,6 @@ export const tablesByDepencies: string[] = [
 	"change_set_element",
 	// Depends on: label, change_set
 	"change_set_label",
-	// Depends on: change_set_label
-	"change_set_label_author",
 	//Depends on: change_set
 	"discussion",
 	// Depends on: account, discussion, comment

package/src/database/nano-id.test.ts

@@ -0,0 +1,15 @@
+import { expect, test } from "vitest";
+import { _nanoIdAlphabet, nanoid } from "./nano-id.js";
+
+test("length is obeyed", () => {
+	const id = nanoid(10);
+	expect(id.length).toBe(10);
+});
+
+test("the alphabet does not contain underscores `_` because they are not URL safe", () => {
+	expect(_nanoIdAlphabet).not.toContain("_");
+});
+
+test("the alphabet does not contain dashes `-` because they break selecting the ID from the URL in the browser", () => {
+	expect(_nanoIdAlphabet).not.toContain("-");
+});

package/src/database/nano-id.ts

@@ -0,0 +1,72 @@
+/**
+ * Code taken from the [nanoid](https://github.com/ai/nanoid/blob/main/index.browser.js)
+ * browser implementation. The code is licensed under the MIT license.
+ *
+ */
+
+const random = (bytes: any) => crypto.getRandomValues(new Uint8Array(bytes));
+
+const customRandom = (
+	alphabet: string,
+	defaultSize: number,
+	getRandom: any
+) => {
+	// First, a bitmask is necessary to generate the ID. The bitmask makes bytes
+	// values closer to the alphabet size. The bitmask calculates the closest
+	// `2^31 - 1` number, which exceeds the alphabet size.
+	// For example, the bitmask for the alphabet size 30 is 31 (00011111).
+	// `Math.clz32` is not used, because it is not available in browsers.
+	const mask = (2 << Math.log2(alphabet.length - 1)) - 1;
+	// Though, the bitmask solution is not perfect since the bytes exceeding
+	// the alphabet size are refused. Therefore, to reliably generate the ID,
+	// the random bytes redundancy has to be satisfied.
+
+	// Note: every hardware random generator call is performance expensive,
+	// because the system call for entropy collection takes a lot of time.
+	// So, to avoid additional system calls, extra bytes are requested in advance.
+
+	// Next, a step determines how many random bytes to generate.
+	// The number of random bytes gets decided upon the ID size, mask,
+	// alphabet size, and magic number 1.6 (using 1.6 peaks at performance
+	// according to benchmarks).
+
+	// `-~f => Math.ceil(f)` if f is a float
+	// `-~i => i + 1` if i is an integer
+	const step = -~((1.6 * mask * defaultSize) / alphabet.length);
+
+	return (size = defaultSize) => {
+		let id = "";
+		while (true) {
+			const bytes = getRandom(step);
+			// A compact alternative for `for (var i = 0; i < step; i++)`.
+			let j = step | 0;
+			while (j--) {
+				// Adding `|| ''` refuses a random byte that exceeds the alphabet size.
+				id += alphabet[bytes[j] & mask] || "";
+				if (id.length >= size) return id;
+			}
+		}
+	};
+};
+
+const customAlphabet = (alphabet: string, size = 21) =>
+	customRandom(alphabet, size | 0, random);
+
+/**
+ * Uses every character from nano id except for the `-` and `_` character.
+ *
+ * - Underscore `_` is not URL safe https://github.com/ai/nanoid/issues/347.
+ * - Dash `-` breaks selecting the ID from the URL in the browser.
+ */
+export const _nanoIdAlphabet =
+	"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+
+/**
+ * Generate secure URL-friendly unique ID.
+ *
+ * Use https://zelark.github.io/nano-id-cc/ to calculate the length
+ * of the ID for the use case with the alphabet provided in the
+ * implementation.
+ */
+export const nanoid: (size?: number) => string =
+	customAlphabet(_nanoIdAlphabet);