@lix-js/sdk 0.4.7 → 0.5.0-preview.1

package/src/lix/open-lix.ts

@@ -1,14 +1,25 @@
 import type { LixPlugin } from "../plugin/lix-plugin.js";
 import { type SqliteWasmDatabase } from "sqlite-wasm-kysely";
 import { initDb } from "../database/init-db.js";
-import { initFileQueueProcess } from "../file-queue/file-queue-process.js";
 import { sql, type Kysely } from "kysely";
 import type { LixDatabaseSchema } from "../database/schema.js";
-import { initSyncProcess } from "../sync/sync-process.js";
-import type { NewKeyValue } from "../key-value/database-schema.js";
+import type { LixKeyValue } from "../key-value/schema.js";
 import { capture } from "../services/telemetry/capture.js";
 import { ENV_VARIABLES } from "../services/env-variables/index.js";
-import type { Account } from "../account/database-schema.js";
+import { applyFileDatabaseSchema } from "../file/schema.js";
+import type { NewStateAll } from "../entity-views/types.js";
+import type { LixAccount } from "../account/schema.js";
+import { InMemoryStorage } from "./storage/in-memory.js";
+import type { LixStorageAdapter } from "./storage/lix-storage-adapter.js";
+import { createHooks, type LixHooks } from "../hooks/create-hooks.js";
+import { createObserve } from "../observe/create-observe.js";
+import { commitDeterministicSequenceNumber } from "../deterministic/sequence.js";
+import {
+	commitDeterministicRngState,
+	random,
+} from "../deterministic/random.js";
+import { newLixFile } from "./new-lix.js";
+import { switchAccount } from "../account/switch-account.js";
 
 export type Lix = {
 	/**
@@ -25,11 +36,57 @@ export type Lix = {
 	db: Kysely<LixDatabaseSchema>;
 	plugin: {
 		getAll: () => Promise<LixPlugin[]>;
+		getAllSync: () => LixPlugin[];
 	};
+	/**
+	 * Hooks for listening to database lifecycle events.
+	 *
+	 * Allows registering callbacks that fire at specific points
+	 * in Lix's execution, such as when state changes are committed.
+	 */
+	hooks: LixHooks;
+	/**
+	 * Closes the lix instance and its storage.
+	 */
+	close: () => Promise<void>;
+	observe: ReturnType<typeof createObserve>;
+	/**
+	 * Serialises the Lix into a {@link Blob}.
+	 *
+	 * Use this helper to persist the current state to disk or send it to a
+	 * server. The blob contains the raw SQLite file representing the Lix
+	 * project.
+	 *
+	 * @example
+	 * ```ts
+	 * const blob = await lix.toBlob()
+	 * download(blob)
+	 * ```
+	 */
+	toBlob: () => Promise<Blob>;
 };
 
 /**
- * Common setup between different lix environments.
+ * Opens a Lix instance.
+ *
+ * Creates an in-memory database by default. If a blob is provided,
+ * the database is initialized with that data. If a database is provided,
+ * uses that database directly.
+ *
+ * @example
+ * ```ts
+ * // In-memory (default)
+ * const lix = await openLix({})
+ *
+ * // From existing data
+ * const lix = await openLix({ blob: existingLixFile })
+ *
+ * // With custom storage adapter
+ * import { MyCustomStorage } from "./my-custom-storage.js"
+ * const lix = await openLix({
+ *   storage: new MyCustomStorage()
+ * })
+ * ```
  */
 export async function openLix(args: {
 	/**
@@ -41,8 +98,17 @@ export async function openLix(args: {
 	 *   const account = localStorage.getItem("account")
 	 *   const lix = await openLix({ account })
 	 */
-	account?: Account;
-	database: SqliteWasmDatabase;
+	account?: LixAccount;
+	/**
+	 * Lix file data to initialize the database with.
+	 */
+	blob?: Blob;
+	/**
+	 * Storage adapter for persisting lix data.
+	 *
+	 * @default InMemoryStorage
+	 */
+	storage?: LixStorageAdapter;
 	/**
 	 * Usecase are lix apps that define their own file format,
 	 * like inlang (unlike a markdown, csv, or json plugin).
@@ -54,39 +120,157 @@ export async function openLix(args: {
 	 *     file format sdk. the file is not portable
 	 *
 	 * @example
-	 *   const lix = await openLixInMemory({ providePlugins: [myPlugin] })
+	 *   const lix = await openLix({ providePlugins: [myPlugin] })
 	 */
 	providePlugins?: LixPlugin[];
 	/**
 	 * Set the key values when opening the lix.
 	 *
+	 * The `lixcol_version_id` defaults to the active version.
+	 *
 	 * @example
 	 *   const lix = await openLix({ keyValues: [{ key: "lix_sync", value: "false" }] })
 	 */
-	keyValues?: NewKeyValue[];
+	keyValues?: NewStateAll<LixKeyValue>[];
 }): Promise<Lix> {
-	const db = initDb({ sqlite: args.database });
-
-	if (args.keyValues && args.keyValues.length > 0) {
-		await db
-			.insertInto("key_value")
-			.values(args.keyValues)
-			.onConflict((oc) =>
-				oc.doUpdateSet((eb) => ({ value: eb.ref("excluded.value") }))
-			)
-			.execute();
-	}
+	const storage = args.storage ?? (new InMemoryStorage() as LixStorageAdapter);
+	const database = await storage.open({
+		blob: args.blob,
+		createBlob: () => newLixFile({ keyValues: args.keyValues }),
+	});
+
+	// Create hooks before initializing database so they can be used in schema setup
+	const hooks = createHooks();
 
-	if (args.account) {
-		await db.transaction().execute(async (trx) => {
-			// delete the default inserted active account from `initDb`
-			await trx.deleteFrom("active_account").execute();
-			await trx
+	const db = initDb({ sqlite: database, hooks });
+
+	// Set up account IMMEDIATELY after db init, before any other operations
+	const persistedState = await storage.getPersistedState?.();
+	const accountToSet = args.account ?? persistedState?.activeAccounts?.[0];
+
+	if (accountToSet) {
+		// First ensure the account exists (create it if it doesn't)
+		const existingAccount = await db
+			.selectFrom("account_all")
+			.where("id", "=", accountToSet.id)
+			.where("lixcol_version_id", "=", "global")
+			.selectAll()
+			.executeTakeFirst();
+
+		if (!existingAccount) {
+			// Create the account in global version
+			await db
+				.insertInto("account_all")
+				.values({
+					id: accountToSet.id,
+					name: accountToSet.name,
+					lixcol_version_id: "global",
+				})
+				.execute();
+		}
+
+		// Use switchAccount to properly set this as the only active account
+		await switchAccount({ lix: { db }, to: [accountToSet] });
+	} else {
+		// Check if there's already an active account
+		const existingActiveAccount = await db
+			.selectFrom("active_account")
+			.selectAll()
+			.executeTakeFirst();
+
+		if (!existingActiveAccount) {
+			// No account provided and no active account exists - create anonymous account
+			const { nanoId } = await import("../deterministic/index.js");
+			const { generateHumanId } = await import(
+				"../deterministic/generate-human-id.js"
+			);
+
+			const activeAccountId = nanoId({
+				lix: { sqlite: database, db: db as any },
+			});
+			const humanName = generateHumanId({
+				lix: { sqlite: database, db: db as any },
+			});
+			const anonymousAccountName = `Anonymous ${humanName}`;
+
+			// Create the anonymous account as untracked
+			await db
+				.insertInto("account_all")
+				.values({
+					id: activeAccountId,
+					name: anonymousAccountName,
+					lixcol_version_id: "global",
+					lixcol_untracked: true,
+				})
+				.execute();
+
+			// Set it as the active account
+			await db
 				.insertInto("active_account")
-				.values(args.account!)
-				.onConflict((oc) => oc.doUpdateSet(() => ({ ...args.account })))
+				.values({
+					account_id: activeAccountId,
+				})
 				.execute();
-		});
+		}
+	}
+
+	// Only process keyValues if we're opening an existing blob
+	// (newLixFile already handles keyValues during creation)
+	if (args.blob && args.keyValues && args.keyValues.length > 0) {
+		for (const keyValue of args.keyValues) {
+			// Determine which table to use based on whether version is specified
+			if (keyValue.lixcol_version_id) {
+				// Check if the key already exists in the specified version
+				const existing = await db
+					.selectFrom("key_value_all")
+					.select("key")
+					.where("key", "=", keyValue.key)
+					.where("lixcol_version_id", "=", keyValue.lixcol_version_id)
+					.executeTakeFirst();
+
+				if (existing) {
+					// Update existing key in the specified version
+					await db
+						.updateTable("key_value_all")
+						.set({ value: keyValue.value })
+						.where("key", "=", keyValue.key)
+						.where("lixcol_version_id", "=", keyValue.lixcol_version_id)
+						.execute();
+				} else {
+					// Insert new key into the specified version
+					await db
+						.insertInto("key_value_all")
+						.values({
+							key: keyValue.key,
+							value: keyValue.value,
+							lixcol_version_id: keyValue.lixcol_version_id,
+						})
+						.execute();
+				}
+			} else {
+				// No version specified, use default key_value table (active version)
+				const existing = await db
+					.selectFrom("key_value")
+					.select("key")
+					.where("key", "=", keyValue.key)
+					.executeTakeFirst();
+
+				if (existing) {
+					// Update existing key
+					await db
+						.updateTable("key_value")
+						.set({ value: keyValue.value })
+						.where("key", "=", keyValue.key)
+						.execute();
+				} else {
+					// Insert new key
+					await db
+						.insertInto("key_value")
+						.values({ key: keyValue.key, value: keyValue.value })
+						.execute();
+				}
+			}
+		}
 	}
 
 	const plugins: LixPlugin[] = [];
@@ -96,24 +280,47 @@ export async function openLix(args: {
 
 	const plugin = {
 		getAll: async () => plugins,
+		getAllSync: () => plugins,
 	};
 
-	await initFileQueueProcess({ lix: { db, plugin, sqlite: args.database } });
-
-	await initSyncProcess({ lix: { db, plugin, sqlite: args.database } });
-
-	captureOpened({ db });
+	const observe = createObserve({ hooks });
 
-	return {
+	const lix = {
 		db,
-		sqlite: args.database,
+		sqlite: database,
 		plugin,
+		hooks,
+		observe,
+		close: async () => {
+			await storage.close();
+		},
+		toBlob: async () => {
+			commitDeterministicSequenceNumber({ lix: { sqlite: database, db } });
+			commitDeterministicRngState({ lix: { sqlite: database, db } });
+			return storage.export();
+		},
 	};
+
+	// MUST BE AWAITED
+	// The databse queries must run. Otherwise, we get super unexpected behavior
+	// where something is tested and then the async queue kicks in and leads
+	// to unexpected results. For example, cache population!
+	await captureOpened({ lix });
+
+	// Apply file and account schemas now that we have the full lix object with plugins
+	applyFileDatabaseSchema(lix);
+
+	// Connect storage to Lix if the adapter supports it
+	if (storage.connect) {
+		storage.connect({ lix });
+	}
+
+	return lix;
 }
 
-async function captureOpened(args: { db: Kysely<LixDatabaseSchema> }) {
+async function captureOpened(args: { lix: Lix }) {
 	try {
-		const telemetry = await args.db
+		const telemetry = await args.lix.db
 			.selectFrom("key_value")
 			.select("value")
 			.where("key", "=", "lix_telemetry")
@@ -123,23 +330,26 @@ async function captureOpened(args: { db: Kysely<LixDatabaseSchema> }) {
 			return;
 		}
 
-		const activeAccount = await args.db
+		const activeAccount = await args.lix.db
 			.selectFrom("active_account")
-			.select("id")
+			.select("account_id")
 			.executeTakeFirstOrThrow();
 
-		const lixId = await args.db
+		const lixId = await args.lix.db
 			.selectFrom("key_value")
 			.select("value")
 			.where("key", "=", "lix_id")
 			.executeTakeFirstOrThrow();
 
-		const fileExtensions = await usedFileExtensions(args.db);
-		if (Math.random() > 0.1) {
-			await capture("LIX-SDK lix opened", {
-				accountId: activeAccount.id,
-				lixId: lixId.value,
-				telemetryKeyValue: telemetry?.value ?? "on",
+		const fileExtensions = await usedFileExtensions(args.lix.db);
+		if (random({ lix: args.lix }) > 0.1) {
+			// Not awaiting to avoid boot up time and knowing that
+			// no database query is performed here. we dont care if the
+			// server responds with an error or not.
+			void capture("LIX-SDK lix opened", {
+				accountId: activeAccount.account_id,
+				lixId: lixId.value as string,
+				telemetryKeyValue: (telemetry?.value ?? "on") as string,
 				properties: {
 					lix_sdk_version: ENV_VARIABLES.LIX_SDK_VERSION,
 					stored_file_extensions: fileExtensions,

package/src/lix/storage/in-memory.test.ts

@@ -0,0 +1,106 @@
+import { expect, test } from "vitest";
+import { InMemoryStorage } from "./in-memory.js";
+import { openLix } from "../open-lix.js";
+import { newLixFile } from "../new-lix.js";
+
+test("should open a database with new lix on first call", async () => {
+	const storage = new InMemoryStorage();
+	const lix = await openLix({ storage });
+
+	const lixId = await lix.db
+		.selectFrom("key_value")
+		.select("value")
+		.where("key", "=", "lix_id")
+		.executeTakeFirstOrThrow();
+
+	expect(lixId.value).toBeDefined();
+	expect(typeof lixId.value).toBe("string");
+});
+
+test("should return same database instance on subsequent calls", async () => {
+	const storage = new InMemoryStorage();
+	const database1 = await storage.open({ createBlob: newLixFile });
+	const database2 = await storage.open({ createBlob: newLixFile });
+
+	expect(database1).toBe(database2);
+});
+
+test("should export current database state as blob", async () => {
+	const storage = new InMemoryStorage();
+	const lix = await openLix({ storage });
+
+	// Add some test data
+	await lix.db
+		.insertInto("file")
+		.values({
+			id: "test-file",
+			path: "/test.txt",
+			data: new TextEncoder().encode("test content"),
+		})
+		.execute();
+
+	const blob = await storage.export();
+
+	// Create new storage and import the blob
+	const newStorage = new InMemoryStorage();
+	const newLix = await openLix({ storage: newStorage, blob });
+
+	// Verify the data was exported and imported correctly
+	const files = await newLix.db.selectFrom("file").selectAll().execute();
+	expect(files).toHaveLength(1);
+	expect(files[0]).toMatchObject({
+		id: "test-file",
+		path: "/test.txt",
+		data: new TextEncoder().encode("test content"),
+	});
+});
+
+test("should close database connection", async () => {
+	const storage = new InMemoryStorage();
+	const database1 = await storage.open({ createBlob: newLixFile });
+
+	await storage.close();
+
+	// Opening again should create a new database
+	const database2 = await storage.open({ createBlob: newLixFile });
+	expect(database1).not.toBe(database2);
+});
+
+test("should handle export before open", async () => {
+	const storage = new InMemoryStorage();
+
+	// Export should throw an error if we haven't opened yet
+	await expect(storage.export()).rejects.toThrow(
+		"Database has not been opened yet"
+	);
+});
+
+test("multiple storage instances should be independent", async () => {
+	const storage1 = new InMemoryStorage();
+	const storage2 = new InMemoryStorage();
+
+	const lix1 = await openLix({ storage: storage1 });
+	const lix2 = await openLix({ storage: storage2 });
+
+	// Add data to first instance
+	await lix1.db
+		.insertInto("file")
+		.values({
+			id: "file1",
+			path: "/test1.txt",
+			data: new TextEncoder().encode("data1"),
+		})
+		.execute();
+
+	// Second instance should not have the data
+	const files2 = await lix2.db.selectFrom("file").selectAll().execute();
+	expect(files2).toHaveLength(0);
+
+	// First instance should still have the data
+	const files1 = await lix1.db.selectFrom("file").selectAll().execute();
+	expect(files1).toHaveLength(1);
+	expect(files1[0]).toMatchObject({
+		id: "file1",
+		path: "/test1.txt",
+	});
+});

package/src/lix/storage/in-memory.ts

@@ -0,0 +1,64 @@
+import {
+	type SqliteWasmDatabase,
+	createInMemoryDatabase,
+	importDatabase,
+	contentFromDatabase,
+} from "sqlite-wasm-kysely";
+import type { LixStorageAdapter } from "./lix-storage-adapter.js";
+
+/**
+ * In-memory storage adapter for Lix.
+ *
+ * Data is stored only for the lifetime of the JavaScript context.
+ * When the page is refreshed or the application is closed, all data is lost.
+ */
+export class InMemoryStorage implements LixStorageAdapter {
+	private database?: SqliteWasmDatabase;
+
+	/**
+	 * Opens an in-memory SQLite database.
+	 *
+	 * Creates a new database and optionally initializes it with the provided blob.
+	 * Returns the same database instance on subsequent calls.
+	 */
+	async open(args: {
+		blob?: Blob;
+		createBlob: () => Promise<Blob>;
+	}): Promise<SqliteWasmDatabase> {
+		if (!this.database) {
+			this.database = await createInMemoryDatabase({ readOnly: false });
+
+			// Initialize with provided blob or create new one
+			const blob = args.blob ?? (await args.createBlob());
+			importDatabase({
+				db: this.database,
+				content: new Uint8Array(await blob.arrayBuffer()),
+			});
+		}
+
+		return this.database;
+	}
+
+	/**
+	 * Closes the database connection.
+	 *
+	 * Note: For in-memory databases, this just clears the reference.
+	 * The data is lost when the database is no longer referenced.
+	 */
+	async close(): Promise<void> {
+		this.database = undefined;
+	}
+
+	/**
+	 * Exports the current database state as a blob.
+	 *
+	 * @throws Error if the database has not been opened yet
+	 */
+	async export(): Promise<Blob> {
+		if (!this.database) {
+			throw new Error("Database has not been opened yet");
+		}
+		const content = contentFromDatabase(this.database);
+		return new Blob([content]);
+	}
+}

package/src/lix/storage/lix-storage-adapter.ts

@@ -0,0 +1,52 @@
+/**
+ * Storage adapter interface for Lix.
+ */
+import type { SqliteWasmDatabase } from "sqlite-wasm-kysely";
+import type { LixAccount } from "../../account/schema.js";
+import type { Lix } from "../open-lix.js";
+
+export interface LixStorageAdapter {
+	/**
+	 * Opens and returns the database instance.
+	 *
+	 * @param args - Arguments for opening the storage
+	 * @param args.blob - Optional blob to initialize the storage with (takes precedence over existing data)
+	 * @param args.createBlob - Callback to create a new blob if no existing data is found
+	 */
+	open(args: {
+		blob?: Blob;
+		createBlob: () => Promise<Blob>;
+	}): Promise<SqliteWasmDatabase>;
+
+	/**
+	 * Closes the database and cleans up resources.
+	 */
+	close(): Promise<void>;
+
+	/**
+	 * Exports the current storage state as a blob.
+	 */
+	export(): Promise<Blob>;
+
+	/**
+	 * Called after the Lix instance is fully initialized.
+	 * Allows storage adapters to set up observers, hooks, etc.
+	 *
+	 * @param args - Object containing the Lix instance and future options
+	 */
+	connect?(args: { lix: Lix }): void;
+
+	/**
+	 * Returns any persisted state that should be restored.
+	 * Called during Lix initialization to restore saved state.
+	 *
+	 * @returns Promise resolving to persisted state, or undefined if none
+	 */
+	getPersistedState?(): Promise<
+		| {
+				activeAccounts?: Pick<LixAccount, "id" | "name">[];
+				// Future: Add more persisted state here
+		  }
+		| undefined
+	>;
+}