@lix-js/sdk 0.5.0-preview.0 → 0.5.0-preview.1

package/src/lix/open-lix.test.ts

@@ -2,7 +2,7 @@ import { expect, test } from "vitest";
 import { newLixFile } from "./new-lix.js";
 import type { LixPlugin } from "../plugin/lix-plugin.js";
 import { openLix, usedFileExtensions } from "./open-lix.js";
-import type { Account } from "../account/schema.js";
+import type { LixAccount } from "../account/schema.js";
 
 test("providing plugins should be possible", async () => {
 	const mockPlugin: LixPlugin = {
@@ -44,8 +44,59 @@ test("providing key values should be possible", async () => {
 	expect(value1).toMatchObject({ key: "mock_key", value: "value2" });
 });
 
+test("provided key values in openLix should default to the active version if lixcol_version_id is not specified", async () => {
+	const lix = await openLix({
+		blob: await newLixFile(),
+		keyValues: [
+			{ key: "test_key_1", value: "test_value_1" },
+			{ key: "test_key_2", value: "test_value_2", lixcol_version_id: "global" },
+		],
+	});
+
+	// Get the active version
+	const activeVersion = await lix.db
+		.selectFrom("active_version")
+		.select("version_id")
+		.executeTakeFirst();
+
+	expect(activeVersion).toBeDefined();
+	expect(activeVersion?.version_id).toBeDefined();
+
+	// Check that test_key_1 without lixcol_version_id is associated with the active version
+	const kv1 = await lix.db
+		.selectFrom("key_value_all")
+		.where("key", "=", "test_key_1")
+		.selectAll()
+		.executeTakeFirst();
+
+	expect(kv1?.value).toBe("test_value_1");
+	expect(kv1?.lixcol_version_id).toBe(activeVersion?.version_id);
+
+	// Check that test_key_2 with explicit lixcol_version_id is associated with the global version
+	const kv2 = await lix.db
+		.selectFrom("key_value_all")
+		.where("key", "=", "test_key_2")
+		.selectAll()
+		.executeTakeFirst();
+
+	expect(kv2?.value).toBe("test_value_2");
+	expect(kv2?.lixcol_version_id).toBe("global");
+
+	// The active version should be "main" (not "global")
+	const mainVersion = await lix.db
+		.selectFrom("version")
+		.where("id", "=", activeVersion!.version_id)
+		.selectAll()
+		.executeTakeFirst();
+
+	expect(mainVersion?.name).toBe("main");
+});
+
+// TODO occasional test failures due to timing issues
+// faulty state materialization might be the cause.
+// fix after https://github.com/opral/lix-sdk/issues/308
 test("providing an account should be possible", async () => {
-	const mockAccount: Account = {
+	const mockAccount: LixAccount = {
 		id: "mock-account",
 		name: "peter",
 	};
@@ -55,13 +106,23 @@ test("providing an account should be possible", async () => {
 		blob: await newLixFile(),
 	});
 
-	const accounts = await lix.db
+	const activeAccounts = await lix.db
 		.selectFrom("active_account")
 		.selectAll()
 		.execute();
 
-	expect(accounts, "to be the provided account").toContainEqual(mockAccount);
-	expect(accounts, "no other active account is inserted").lengthOf(1);
+	expect(activeAccounts).toHaveLength(1);
+	expect(activeAccounts[0]?.account_id).toBe(mockAccount.id);
+
+	// Join to verify the actual account was created with correct details
+	const activeAccountWithDetails = await lix.db
+		.selectFrom("active_account as aa")
+		.innerJoin("account_all as a", "a.id", "aa.account_id")
+		.where("a.lixcol_version_id", "=", "global")
+		.select(["a.id", "a.name"])
+		.executeTakeFirst();
+
+	expect(activeAccountWithDetails?.name).toBe(mockAccount.name);
 });
 
 test("usedFileExtensions", async () => {
@@ -145,3 +206,245 @@ test("should default to InMemoryStorage when no storage is provided", async () =
 		data: new TextEncoder().encode("test content"),
 	});
 });
+
+test("providing lix_deterministic_mode = true should lead to deterministic state", async () => {
+	const lix1 = await openLix({
+		keyValues: [
+			{
+				key: "lix_deterministic_mode",
+				value: { enabled: true, bootstrap: true },
+				lixcol_version_id: "global",
+			},
+		],
+	});
+
+	const lix2 = await openLix({
+		keyValues: [
+			{
+				key: "lix_deterministic_mode",
+				value: { enabled: true, bootstrap: true },
+				lixcol_version_id: "global",
+			},
+		],
+	});
+
+	await lix1.db
+		.insertInto("key_value")
+		.values({
+			key: "test_key",
+			value: "test_value",
+		})
+		.execute();
+
+	await lix2.db
+		.insertInto("key_value")
+		.values({
+			key: "test_key",
+			value: "test_value",
+		})
+		.execute();
+
+	const lix1State = await lix1.db.selectFrom("state").selectAll().execute();
+
+	const lix2State = await lix2.db.selectFrom("state").selectAll().execute();
+
+	expect(lix1State).toEqual(lix2State);
+});
+
+test("deterministic mode can be turned on and off", async () => {
+	const deterministicModeKeyValues = [
+		{
+			key: "lix_deterministic_mode",
+			value: { enabled: true, bootstrap: true },
+			lixcol_version_id: "global",
+		},
+	];
+
+	const lix1 = await openLix({
+		keyValues: [...deterministicModeKeyValues],
+	});
+
+	const lix2 = await openLix({
+		keyValues: [...deterministicModeKeyValues],
+	});
+
+	await Promise.all(
+		[lix1, lix2].map(async (lix) => {
+			await lix.db
+				.insertInto("key_value")
+				.values({
+					key: "test_key",
+					value: "test_value",
+				})
+				.execute();
+		})
+	);
+
+	const lix1State = await lix1.db.selectFrom("state").selectAll().execute();
+
+	const lix2State = await lix2.db.selectFrom("state").selectAll().execute();
+
+	expect(lix1State).toEqual(lix2State);
+
+	// now turning off deterministic mode and insert a new key value
+
+	await Promise.all(
+		[lix1, lix2].map(async (lix) => {
+			await lix.db
+				.updateTable("key_value_all")
+				.where("lixcol_version_id", "=", "global")
+				.where("key", "=", "lix_deterministic_mode")
+				.set({ value: { enabled: false } })
+				.execute();
+		})
+	);
+
+	await Promise.all(
+		[lix1, lix2].map(async (lix) => {
+			await lix.db
+				.insertInto("key_value")
+				.values({
+					key: "test_key_2",
+					value: "test_value_2",
+				})
+				.execute();
+		})
+	);
+
+	const [lix1Kv, lix2Kv] = await Promise.all(
+		[lix1, lix2].map((lix) =>
+			lix.db
+				.selectFrom("key_value")
+				.where("key", "=", "test_key_2")
+				.selectAll()
+				.execute()
+		)
+	);
+
+	// the change ids etc should be different given that deterministic mode was turned off
+	expect(lix1Kv).not.toEqual(lix2Kv);
+
+	// turn on deterministic mode again and insert a new key value
+
+	await Promise.all(
+		[lix1, lix2].map(async (lix) => {
+			await lix.db
+				.updateTable("key_value_all")
+				.where("lixcol_version_id", "=", "global")
+				.where("key", "=", "lix_deterministic_mode")
+				.set({ value: { enabled: true } })
+				.execute();
+		})
+	);
+
+	await Promise.all(
+		[lix1, lix2].map(async (lix) => {
+			await lix.db
+				.insertInto("key_value")
+				.values({
+					key: "test_key_3",
+					value: "test_value_3",
+				})
+				.execute();
+		})
+	);
+
+	const [lix1Kv2, lix2Kv2] = await Promise.all(
+		[lix1, lix2].map((lix) =>
+			lix.db
+				.selectFrom("key_value")
+				.where("key", "=", "test_key_3")
+				.selectAll()
+				.executeTakeFirst()
+		)
+	);
+
+	// the change ids etc should be the same given that deterministic mode was turned on again
+	expect(lix1Kv2).toEqual(lix2Kv2);
+});
+
+// reproduction of a real bug
+test("providing deterministic mode with kv false should still insert it", async () => {
+	const lix = await openLix({
+		keyValues: [
+			{
+				key: "lix_deterministic_mode",
+				value: { enabled: false },
+				lixcol_version_id: "global",
+			},
+		],
+	});
+
+	const kvs = await lix.db
+		.selectFrom("key_value_all")
+		.where("key", "=", "lix_deterministic_mode")
+		.where("lixcol_inherited_from_version_id", "is", null)
+		.selectAll()
+		.execute();
+
+	expect(kvs).toContainEqual(
+		expect.objectContaining({
+			key: "lix_deterministic_mode",
+			value: { enabled: false }, // JSON structure is preserved
+			lixcol_version_id: "global",
+		})
+	);
+});
+
+// TODO fix state materialization first
+// state materialization bug. selecting from the active version
+// before querying the key values makes the test pass.
+test("provided key values respect lixcol_untracked setting", async () => {
+	const lix = await openLix({
+		keyValues: [
+			{
+				key: "tracked_key",
+				value: "tracked_value",
+				lixcol_version_id: "global",
+			},
+			{
+				key: "untracked_key",
+				value: "untracked_value",
+				lixcol_version_id: "global",
+				lixcol_untracked: true,
+			},
+		],
+	});
+
+	// Check active version
+	// const activeVersion = await lix.db
+	// 	.selectFrom("active_version")
+	// 	.selectAll()
+	// 	.executeTakeFirst();
+	// console.log("Active version:", activeVersion?.version_id);
+
+	// Check tracked key
+	const trackedKv = await lix.db
+		.selectFrom("key_value_all")
+		.where("key", "=", "tracked_key")
+		.where("lixcol_version_id", "=", "global")
+		.selectAll()
+		.executeTakeFirst();
+
+	expect(trackedKv).toMatchObject({
+		key: "tracked_key",
+		value: "tracked_value",
+		lixcol_version_id: "global",
+		lixcol_untracked: 0, // tracked
+	});
+
+	// Check untracked key
+	const untrackedKv = await lix.db
+		.selectFrom("key_value_all")
+		.where("key", "=", "untracked_key")
+		.where("lixcol_version_id", "=", "global")
+		.selectAll()
+		.executeTakeFirst();
+
+	expect(untrackedKv).toMatchObject({
+		key: "untracked_key",
+		value: "untracked_value",
+		lixcol_version_id: "global",
+		lixcol_untracked: 1, // untracked
+	});
+});

package/src/lix/open-lix.ts

@@ -3,16 +3,23 @@ import { type SqliteWasmDatabase } from "sqlite-wasm-kysely";
 import { initDb } from "../database/init-db.js";
 import { sql, type Kysely } from "kysely";
 import type { LixDatabaseSchema } from "../database/schema.js";
-import type { KeyValue } from "../key-value/schema.js";
+import type { LixKeyValue } from "../key-value/schema.js";
 import { capture } from "../services/telemetry/capture.js";
 import { ENV_VARIABLES } from "../services/env-variables/index.js";
 import { applyFileDatabaseSchema } from "../file/schema.js";
-import type { NewState } from "../entity-views/types.js";
-import type { Account } from "../account/schema.js";
+import type { NewStateAll } from "../entity-views/types.js";
+import type { LixAccount } from "../account/schema.js";
 import { InMemoryStorage } from "./storage/in-memory.js";
 import type { LixStorageAdapter } from "./storage/lix-storage-adapter.js";
 import { createHooks, type LixHooks } from "../hooks/create-hooks.js";
 import { createObserve } from "../observe/create-observe.js";
+import { commitDeterministicSequenceNumber } from "../deterministic/sequence.js";
+import {
+	commitDeterministicRngState,
+	random,
+} from "../deterministic/random.js";
+import { newLixFile } from "./new-lix.js";
+import { switchAccount } from "../account/switch-account.js";
 
 export type Lix = {
 	/**
@@ -66,11 +73,6 @@ export type Lix = {
  * the database is initialized with that data. If a database is provided,
  * uses that database directly.
  *
- * TODO: Add storage abstraction to support:
- *   - OPFS storage (persistent in browser)
- *   - Node.js filesystem storage
- *   - Custom storage adapters
- *
  * @example
  * ```ts
  * // In-memory (default)
@@ -79,9 +81,11 @@ export type Lix = {
  * // From existing data
  * const lix = await openLix({ blob: existingLixFile })
  *
- * // With custom database (current approach)
- * const db = await createInMemoryDatabase({ readOnly: false })
- * const lix = await openLix({ database: db })
+ * // With custom storage adapter
+ * import { MyCustomStorage } from "./my-custom-storage.js"
+ * const lix = await openLix({
+ *   storage: new MyCustomStorage()
+ * })
  * ```
  */
 export async function openLix(args: {
@@ -94,7 +98,7 @@ export async function openLix(args: {
 	 *   const account = localStorage.getItem("account")
 	 *   const lix = await openLix({ account })
 	 */
-	account?: Account;
+	account?: LixAccount;
 	/**
 	 * Lix file data to initialize the database with.
 	 */
@@ -122,66 +126,153 @@ export async function openLix(args: {
 	/**
 	 * Set the key values when opening the lix.
 	 *
+	 * The `lixcol_version_id` defaults to the active version.
+	 *
 	 * @example
 	 *   const lix = await openLix({ keyValues: [{ key: "lix_sync", value: "false" }] })
 	 */
-	keyValues?: NewState<KeyValue>[];
+	keyValues?: NewStateAll<LixKeyValue>[];
 }): Promise<Lix> {
-	const storage = args.storage ?? new InMemoryStorage();
-	const database = await storage.open();
-
-	// Import blob data if provided
-	if (args.blob) {
-		await storage.import(args.blob);
-	}
+	const storage = args.storage ?? (new InMemoryStorage() as LixStorageAdapter);
+	const database = await storage.open({
+		blob: args.blob,
+		createBlob: () => newLixFile({ keyValues: args.keyValues }),
+	});
 
 	// Create hooks before initializing database so they can be used in schema setup
 	const hooks = createHooks();
 
 	const db = initDb({ sqlite: database, hooks });
 
-	// Connect storage to state commit hooks if it supports it
-	if ("onStateCommit" in storage && storage.onStateCommit) {
-		hooks.onStateCommit(() => {
-			storage.onStateCommit!();
-		});
+	// Set up account IMMEDIATELY after db init, before any other operations
+	const persistedState = await storage.getPersistedState?.();
+	const accountToSet = args.account ?? persistedState?.activeAccounts?.[0];
+
+	if (accountToSet) {
+		// First ensure the account exists (create it if it doesn't)
+		const existingAccount = await db
+			.selectFrom("account_all")
+			.where("id", "=", accountToSet.id)
+			.where("lixcol_version_id", "=", "global")
+			.selectAll()
+			.executeTakeFirst();
+
+		if (!existingAccount) {
+			// Create the account in global version
+			await db
+				.insertInto("account_all")
+				.values({
+					id: accountToSet.id,
+					name: accountToSet.name,
+					lixcol_version_id: "global",
+				})
+				.execute();
+		}
+
+		// Use switchAccount to properly set this as the only active account
+		await switchAccount({ lix: { db }, to: [accountToSet] });
+	} else {
+		// Check if there's already an active account
+		const existingActiveAccount = await db
+			.selectFrom("active_account")
+			.selectAll()
+			.executeTakeFirst();
+
+		if (!existingActiveAccount) {
+			// No account provided and no active account exists - create anonymous account
+			const { nanoId } = await import("../deterministic/index.js");
+			const { generateHumanId } = await import(
+				"../deterministic/generate-human-id.js"
+			);
+
+			const activeAccountId = nanoId({
+				lix: { sqlite: database, db: db as any },
+			});
+			const humanName = generateHumanId({
+				lix: { sqlite: database, db: db as any },
+			});
+			const anonymousAccountName = `Anonymous ${humanName}`;
+
+			// Create the anonymous account as untracked
+			await db
+				.insertInto("account_all")
+				.values({
+					id: activeAccountId,
+					name: anonymousAccountName,
+					lixcol_version_id: "global",
+					lixcol_untracked: true,
+				})
+				.execute();
+
+			// Set it as the active account
+			await db
+				.insertInto("active_account")
+				.values({
+					account_id: activeAccountId,
+				})
+				.execute();
+		}
 	}
 
-	if (args.keyValues && args.keyValues.length > 0) {
+	// Only process keyValues if we're opening an existing blob
+	// (newLixFile already handles keyValues during creation)
+	if (args.blob && args.keyValues && args.keyValues.length > 0) {
 		for (const keyValue of args.keyValues) {
-			// Check if the key already exists
-			const existing = await db
-				.selectFrom("key_value")
-				.select("key")
-				.where("key", "=", keyValue.key)
-				.executeTakeFirst();
-
-			if (existing) {
-				// Update existing key
-				await db
-					.updateTable("key_value")
-					.set({ value: keyValue.value })
+			// Determine which table to use based on whether version is specified
+			if (keyValue.lixcol_version_id) {
+				// Check if the key already exists in the specified version
+				const existing = await db
+					.selectFrom("key_value_all")
+					.select("key")
 					.where("key", "=", keyValue.key)
-					.execute();
+					.where("lixcol_version_id", "=", keyValue.lixcol_version_id)
+					.executeTakeFirst();
+
+				if (existing) {
+					// Update existing key in the specified version
+					await db
+						.updateTable("key_value_all")
+						.set({ value: keyValue.value })
+						.where("key", "=", keyValue.key)
+						.where("lixcol_version_id", "=", keyValue.lixcol_version_id)
+						.execute();
+				} else {
+					// Insert new key into the specified version
+					await db
+						.insertInto("key_value_all")
+						.values({
+							key: keyValue.key,
+							value: keyValue.value,
+							lixcol_version_id: keyValue.lixcol_version_id,
+						})
+						.execute();
+				}
 			} else {
-				// Insert new key
-				await db.insertInto("key_value").values(keyValue).execute();
+				// No version specified, use default key_value table (active version)
+				const existing = await db
+					.selectFrom("key_value")
+					.select("key")
+					.where("key", "=", keyValue.key)
+					.executeTakeFirst();
+
+				if (existing) {
+					// Update existing key
+					await db
+						.updateTable("key_value")
+						.set({ value: keyValue.value })
+						.where("key", "=", keyValue.key)
+						.execute();
+				} else {
+					// Insert new key
+					await db
+						.insertInto("key_value")
+						.values({ key: keyValue.key, value: keyValue.value })
+						.execute();
+				}
 			}
 		}
 	}
 
-	if (args.account) {
-		await db.transaction().execute(async (trx) => {
-			// delete the default inserted active account from `initDb`
-			await trx.deleteFrom("active_account").execute();
-			await trx
-				.insertInto("active_account")
-				.values(args.account!)
-				.onConflict((oc) => oc.doUpdateSet(() => ({ ...args.account })))
-				.execute();
-		});
-	}
-
 	const plugins: LixPlugin[] = [];
 	if (args.providePlugins && args.providePlugins.length > 0) {
 		plugins.push(...args.providePlugins);
@@ -192,12 +283,6 @@ export async function openLix(args: {
 		getAllSync: () => plugins,
 	};
 
-	// await initFileQueueProcess({ lix: { db, plugin, sqlite: args.database } });
-
-	// await initSyncProcess({ lix: { db, plugin, sqlite: args.database } });
-
-	captureOpened({ db });
-
 	const observe = createObserve({ hooks });
 
 	const lix = {
@@ -210,19 +295,32 @@ export async function openLix(args: {
 			await storage.close();
 		},
 		toBlob: async () => {
+			commitDeterministicSequenceNumber({ lix: { sqlite: database, db } });
+			commitDeterministicRngState({ lix: { sqlite: database, db } });
 			return storage.export();
 		},
 	};
 
+	// MUST BE AWAITED
+	// The databse queries must run. Otherwise, we get super unexpected behavior
+	// where something is tested and then the async queue kicks in and leads
+	// to unexpected results. For example, cache population!
+	await captureOpened({ lix });
+
 	// Apply file and account schemas now that we have the full lix object with plugins
 	applyFileDatabaseSchema(lix);
 
+	// Connect storage to Lix if the adapter supports it
+	if (storage.connect) {
+		storage.connect({ lix });
+	}
+
 	return lix;
 }
 
-async function captureOpened(args: { db: Kysely<LixDatabaseSchema> }) {
+async function captureOpened(args: { lix: Lix }) {
 	try {
-		const telemetry = await args.db
+		const telemetry = await args.lix.db
 			.selectFrom("key_value")
 			.select("value")
 			.where("key", "=", "lix_telemetry")
@@ -232,21 +330,24 @@ async function captureOpened(args: { db: Kysely<LixDatabaseSchema> }) {
 			return;
 		}
 
-		const activeAccount = await args.db
+		const activeAccount = await args.lix.db
 			.selectFrom("active_account")
-			.select("id")
+			.select("account_id")
 			.executeTakeFirstOrThrow();
 
-		const lixId = await args.db
+		const lixId = await args.lix.db
 			.selectFrom("key_value")
 			.select("value")
 			.where("key", "=", "lix_id")
 			.executeTakeFirstOrThrow();
 
-		const fileExtensions = await usedFileExtensions(args.db);
-		if (Math.random() > 0.1) {
-			await capture("LIX-SDK lix opened", {
-				accountId: activeAccount.id,
+		const fileExtensions = await usedFileExtensions(args.lix.db);
+		if (random({ lix: args.lix }) > 0.1) {
+			// Not awaiting to avoid boot up time and knowing that
+			// no database query is performed here. we dont care if the
+			// server responds with an error or not.
+			void capture("LIX-SDK lix opened", {
+				accountId: activeAccount.account_id,
 				lixId: lixId.value as string,
 				telemetryKeyValue: (telemetry?.value ?? "on") as string,
 				properties: {