@lix-js/sdk 0.5.0-preview.1 → 0.5.0

package/src/lix/open-lix.ts

@@ -1,42 +1,24 @@
 import type { LixPlugin } from "../plugin/lix-plugin.js";
-import { type SqliteWasmDatabase } from "sqlite-wasm-kysely";
-import { initDb } from "../database/init-db.js";
-import { sql, type Kysely } from "kysely";
-import type { LixDatabaseSchema } from "../database/schema.js";
-import type { LixKeyValue } from "../key-value/schema.js";
+import { Kysely, sql } from "kysely";
+import { type LixDatabaseSchema } from "../database/schema.js";
+import type { LixKeyValue } from "../key-value/schema-definition.js";
 import { capture } from "../services/telemetry/capture.js";
 import { ENV_VARIABLES } from "../services/env-variables/index.js";
-import { applyFileDatabaseSchema } from "../file/schema.js";
-import type { NewStateAll } from "../entity-views/types.js";
-import type { LixAccount } from "../account/schema.js";
-import { InMemoryStorage } from "./storage/in-memory.js";
-import type { LixStorageAdapter } from "./storage/lix-storage-adapter.js";
+import type { NewStateByVersion } from "../engine/entity-views/types.js";
+import type { LixAccount } from "../account/schema-definition.js";
 import { createHooks, type LixHooks } from "../hooks/create-hooks.js";
 import { createObserve } from "../observe/create-observe.js";
-import { commitDeterministicSequenceNumber } from "../deterministic/sequence.js";
-import {
-	commitDeterministicRngState,
-	random,
-} from "../deterministic/random.js";
-import { newLixFile } from "./new-lix.js";
-import { switchAccount } from "../account/switch-account.js";
+import type { EngineEvent, LixEngine } from "../engine/boot.js";
+import type { LixEnvironment } from "../environment/api.js";
+import { createEnvironmentDialect } from "../database/sqlite/environment-dialect.js";
+import { createDefaultPlugins } from "../database/kysely/index.js";
+import { random } from "../engine/functions/random.js";
+import { loadPluginFromString } from "../environment/load-from-string.js";
 
 export type Lix = {
-	/**
-	 * The raw SQLite instance.
-	 *
-	 * Required for advanced use cases that can't be
-	 * expressed with the db API.
-	 *
-	 * Use with caution, automatic transformation of
-	 * results like parsing json (similar to the db API)
-	 * is not guaranteed.
-	 */
-	sqlite: SqliteWasmDatabase;
 	db: Kysely<LixDatabaseSchema>;
 	plugin: {
 		getAll: () => Promise<LixPlugin[]>;
-		getAllSync: () => LixPlugin[];
 	};
 	/**
 	 * Hooks for listening to database lifecycle events.
@@ -50,6 +32,14 @@ export type Lix = {
 	 */
 	close: () => Promise<void>;
 	observe: ReturnType<typeof createObserve>;
+
+	/**
+	 * Calls a named engine function and returns its result.
+	 *
+	 * Preferred entrypoint for invoking engine functions.
+	 */
+	call: (name: string, args?: unknown) => Promise<unknown>;
+
 	/**
 	 * Serialises the Lix into a {@link Blob}.
 	 *
@@ -64,6 +54,51 @@ export type Lix = {
 	 * ```
 	 */
 	toBlob: () => Promise<Blob>;
+
+	/**
+	 * In‑process engine context bound to the live database.
+	 *
+	 * When Lix runs in‑process (same thread as the database),
+	 * `engine` is available and tests can directly call helpers
+	 * that accept `{ engine }`.
+	 *
+	 * When Lix runs out‑of‑process (for example inside a Worker), the engine
+	 * is not accessible from the main thread and will be `undefined`. In those
+	 * environments, use `lix.call` to invoke engine functions across the
+	 * boundary.
+	 *
+	 * Guidance:
+	 * - Prefer `lix.db` for normal queries and `lix.call` for engine
+	 *   operations. Reserve `lix.engine` for unit tests or internal SDK code
+	 *   that explicitly requires in‑process access alongside the database.
+	 * - Do not rely on `lix.engine` in application/business logic, since it
+	 *   may be `undefined` in worker/remote environments.
+	 *
+	 * Unit test in the same process
+	 *
+	 * @example
+	 *
+	 * ```ts
+	 * test("example test", async () => {
+	 *   // InMemory environment runs in the same process (in‑process engine).
+	 *   const lix = await openLix({
+	 *      environment: new InMemoryenvironment()
+	 *   });
+	 *   executeSync({
+	 *     engine: lix.engine!,
+	 *     data: [...],
+	 *   });
+	 * });
+	 * ```
+	 *
+	 * Worker/remote environment – prefer router calls
+	 *
+	 * @example
+	 * ```ts
+	 * await lix.call("lix_insert_transaction_state", { timestamp, data });
+	 * ```
+	 */
+	engine?: LixEngine;
 };
 
 /**
@@ -103,26 +138,45 @@ export async function openLix(args: {
 	 * Lix file data to initialize the database with.
 	 */
 	blob?: Blob;
+
+	environment?: LixEnvironment;
+
 	/**
-	 * Storage adapter for persisting lix data.
+	 * Provide plugin instances directly (in-process environments only).
+	 *
+	 * Use this when executing in the same thread as the engine, e.g. tests or
+	 * scripts running with {@link InMemoryEnvironment}. Plugins supplied this
+	 * way are not transferable across worker boundaries.
 	 *
-	 * @default InMemoryStorage
+	 * Prefer {@link providePluginsRaw} when the environment can access bundled
+	 * plugin source (for example via `?raw` import). Raw strings load on the
+	 * target thread and work in both workers and main-thread contexts, but are
+	 * usually unavailable in unit tests.
+	 *
+	 * @example
+	 * ```ts
+	 * const lix = await openLix({
+	 *   providePlugins: [jsonPlugin]
+	 * })
+	 * ```
 	 */
-	storage?: LixStorageAdapter;
+	providePlugins?: LixPlugin[];
+
 	/**
-	 * Usecase are lix apps that define their own file format,
-	 * like inlang (unlike a markdown, csv, or json plugin).
+	 * Provide plugins as stringified ESM modules.
 	 *
-	 * (+) avoids separating app code from plugin code and
-	 *     resulting bundling logic.
-	 *
-	 * (-) such a file format must always be opened with the
-	 *     file format sdk. the file is not portable
+	 * This is the portable format for worker or cross-thread environments where
+	 * functions cannot be structured cloned. Each string is evaluated via
+	 * {@link loadPluginFromString} inside the target environment.
 	 *
 	 * @example
-	 *   const lix = await openLix({ providePlugins: [myPlugin] })
+	 * ```ts
+	 * const jsonPlugin = await fetch("/plugins/json.js").then((res) => res.text());
+	 * const lix = await openLix({ providePluginsRaw: [jsonPlugin] });
+	 * ```
 	 */
-	providePlugins?: LixPlugin[];
+	providePluginsRaw?: string[];
+
 	/**
 	 * Set the key values when opening the lix.
 	 *
@@ -131,194 +185,98 @@ export async function openLix(args: {
 	 * @example
 	 *   const lix = await openLix({ keyValues: [{ key: "lix_sync", value: "false" }] })
 	 */
-	keyValues?: NewStateAll<LixKeyValue>[];
+	keyValues?: NewStateByVersion<LixKeyValue>[];
 }): Promise<Lix> {
-	const storage = args.storage ?? (new InMemoryStorage() as LixStorageAdapter);
-	const database = await storage.open({
-		blob: args.blob,
-		createBlob: () => newLixFile({ keyValues: args.keyValues }),
-	});
-
-	// Create hooks before initializing database so they can be used in schema setup
 	const hooks = createHooks();
+	const blob = args.blob;
+	let engine: LixEngine | undefined;
+	const hostPlugins: LixPlugin[] = [];
 
-	const db = initDb({ sqlite: database, hooks });
-
-	// Set up account IMMEDIATELY after db init, before any other operations
-	const persistedState = await storage.getPersistedState?.();
-	const accountToSet = args.account ?? persistedState?.activeAccounts?.[0];
-
-	if (accountToSet) {
-		// First ensure the account exists (create it if it doesn't)
-		const existingAccount = await db
-			.selectFrom("account_all")
-			.where("id", "=", accountToSet.id)
-			.where("lixcol_version_id", "=", "global")
-			.selectAll()
-			.executeTakeFirst();
-
-		if (!existingAccount) {
-			// Create the account in global version
-			await db
-				.insertInto("account_all")
-				.values({
-					id: accountToSet.id,
-					name: accountToSet.name,
-					lixcol_version_id: "global",
-				})
-				.execute();
-		}
+	const providedPluginObjects = [...(args.providePlugins ?? [])];
+	const providedPluginStrings = [...(args.providePluginsRaw ?? [])];
 
-		// Use switchAccount to properly set this as the only active account
-		await switchAccount({ lix: { db }, to: [accountToSet] });
-	} else {
-		// Check if there's already an active account
-		const existingActiveAccount = await db
-			.selectFrom("active_account")
-			.selectAll()
-			.executeTakeFirst();
-
-		if (!existingActiveAccount) {
-			// No account provided and no active account exists - create anonymous account
-			const { nanoId } = await import("../deterministic/index.js");
-			const { generateHumanId } = await import(
-				"../deterministic/generate-human-id.js"
-			);
-
-			const activeAccountId = nanoId({
-				lix: { sqlite: database, db: db as any },
-			});
-			const humanName = generateHumanId({
-				lix: { sqlite: database, db: db as any },
-			});
-			const anonymousAccountName = `Anonymous ${humanName}`;
-
-			// Create the anonymous account as untracked
-			await db
-				.insertInto("account_all")
-				.values({
-					id: activeAccountId,
-					name: anonymousAccountName,
-					lixcol_version_id: "global",
-					lixcol_untracked: true,
-				})
-				.execute();
-
-			// Set it as the active account
-			await db
-				.insertInto("active_account")
-				.values({
-					account_id: activeAccountId,
-				})
-				.execute();
-		}
+	for (const plugin of providedPluginObjects) {
+		hostPlugins.push(plugin);
+	}
+	for (const plugin of providedPluginStrings) {
+		hostPlugins.push(await loadPluginFromString(plugin));
 	}
 
-	// Only process keyValues if we're opening an existing blob
-	// (newLixFile already handles keyValues during creation)
-	if (args.blob && args.keyValues && args.keyValues.length > 0) {
-		for (const keyValue of args.keyValues) {
-			// Determine which table to use based on whether version is specified
-			if (keyValue.lixcol_version_id) {
-				// Check if the key already exists in the specified version
-				const existing = await db
-					.selectFrom("key_value_all")
-					.select("key")
-					.where("key", "=", keyValue.key)
-					.where("lixcol_version_id", "=", keyValue.lixcol_version_id)
-					.executeTakeFirst();
+	const environment =
+		args.environment ??
+		(await import("../environment/in-memory.js").then(
+			(m) => new m.InMemoryEnvironment()
+		))!;
 
-				if (existing) {
-					// Update existing key in the specified version
-					await db
-						.updateTable("key_value_all")
-						.set({ value: keyValue.value })
-						.where("key", "=", keyValue.key)
-						.where("lixcol_version_id", "=", keyValue.lixcol_version_id)
-						.execute();
-				} else {
-					// Insert new key into the specified version
-					await db
-						.insertInto("key_value_all")
-						.values({
-							key: keyValue.key,
-							value: keyValue.value,
-							lixcol_version_id: keyValue.lixcol_version_id,
-						})
-						.execute();
-				}
-			} else {
-				// No version specified, use default key_value table (active version)
-				const existing = await db
-					.selectFrom("key_value")
-					.select("key")
-					.where("key", "=", keyValue.key)
-					.executeTakeFirst();
+	// Default behavior: openOrCreate
+	// - If a blob is provided, attempt to create from it (environment may refuse if target exists)
+	// - If no blob is provided, attempt to create a fresh Lix; if environment reports existing DB,
+	//   fall back to open without a blob.
+	const boot = {
+		args: {
+			providePlugins: providedPluginObjects,
+			providePluginsRaw: providedPluginStrings,
+			account: args.account,
+			keyValues: args.keyValues,
+		},
+	} as const;
 
-				if (existing) {
-					// Update existing key
-					await db
-						.updateTable("key_value")
-						.set({ value: keyValue.value })
-						.where("key", "=", keyValue.key)
-						.execute();
-				} else {
-					// Insert new key
-					await db
-						.insertInto("key_value")
-						.values({ key: keyValue.key, value: keyValue.value })
-						.execute();
-				}
-			}
+	if (blob) {
+		await environment.create({ blob: await args.blob!.arrayBuffer() });
+		const res = await environment.open({
+			boot,
+			emit: (ev: EngineEvent) => hooks._emit(ev.type, ev.payload),
+		});
+		engine = res?.engine;
+	} else {
+		const exists = await environment.exists();
+		if (!exists) {
+			const { newLixFile } = await import("./new-lix.js");
+			const seed = await newLixFile({ keyValues: args.keyValues });
+			const seedBytes = await seed.arrayBuffer();
+			await environment.create({ blob: seedBytes });
 		}
+		const res = await environment.open({
+			boot,
+			emit: (ev: EngineEvent) => hooks._emit(ev.type, ev.payload),
+		});
+		engine = res?.engine;
 	}
 
-	const plugins: LixPlugin[] = [];
-	if (args.providePlugins && args.providePlugins.length > 0) {
-		plugins.push(...args.providePlugins);
-	}
-
-	const plugin = {
-		getAll: async () => plugins,
-		getAllSync: () => plugins,
-	};
+	const db = new Kysely<LixDatabaseSchema>({
+		dialect: createEnvironmentDialect({ environment }),
+		plugins: createDefaultPlugins(),
+	});
 
 	const observe = createObserve({ hooks });
 
-	const lix = {
+	await captureOpened({ lix: { db, call: environment.call } });
+
+	const lix: Lix = {
+		// sqlite intentionally not exposed in environment mode
 		db,
-		sqlite: database,
-		plugin,
 		hooks,
 		observe,
+		engine: engine,
+		plugin: {
+			getAll: async () => hostPlugins,
+		},
+		call: async (name: string, args?: unknown): Promise<unknown> =>
+			environment.call(name, args),
 		close: async () => {
-			await storage.close();
+			await environment.close();
 		},
 		toBlob: async () => {
-			commitDeterministicSequenceNumber({ lix: { sqlite: database, db } });
-			commitDeterministicRngState({ lix: { sqlite: database, db } });
-			return storage.export();
+			await environment.call("lix_commit_sequence_number");
+			await environment.call("lix_commit_deterministic_rng_state");
+			return new Blob([await environment.export()]);
 		},
 	};
 
-	// MUST BE AWAITED
-	// The databse queries must run. Otherwise, we get super unexpected behavior
-	// where something is tested and then the async queue kicks in and leads
-	// to unexpected results. For example, cache population!
-	await captureOpened({ lix });
-
-	// Apply file and account schemas now that we have the full lix object with plugins
-	applyFileDatabaseSchema(lix);
-
-	// Connect storage to Lix if the adapter supports it
-	if (storage.connect) {
-		storage.connect({ lix });
-	}
-
 	return lix;
 }
 
-async function captureOpened(args: { lix: Lix }) {
+async function captureOpened(args: { lix: Pick<Lix, "db" | "call"> }) {
 	try {
 		const telemetry = await args.lix.db
 			.selectFrom("key_value")
@@ -342,7 +300,8 @@ async function captureOpened(args: { lix: Lix }) {
 			.executeTakeFirstOrThrow();
 
 		const fileExtensions = await usedFileExtensions(args.lix.db);
-		if (random({ lix: args.lix }) > 0.1) {
+		const sample = await random({ lix: args.lix });
+		if (sample > 0.1) {
 			// Not awaiting to avoid boot up time and knowing that
 			// no database query is performed here. we dont care if the
 			// server responds with an error or not.

package/src/log/create-lix-own-log.test.ts

@@ -1,8 +1,8 @@
 import { expect, test } from "vitest";
 import { openLix } from "../lix/open-lix.js";
 import type { Lix } from "../lix/open-lix.js";
-import { createLixOwnLog } from "./create-lix-own-log.js";
-import type { LixLog } from "./schema.js";
+import type { LixLog } from "./schema-definition.js";
+import { createLixOwnLogSync } from "./create-lix-own-log.js";
 
 test("should insert logs default log levels when lix_log_levels is not set)", async () => {
 	const lix = await openLix({
@@ -19,12 +19,18 @@ test("should insert logs default log levels when lix_log_levels is not set)", as
 	expect(logs.find((log) => log.level === "info")?.message).toBe(
 		"info message"
 	);
+	expect(logs.find((log) => log.level === "info")?.payload).toBeNull();
 	expect(logs.find((log) => log.level === "warn")?.message).toBe(
 		"warn message"
 	);
+	expect(logs.find((log) => log.level === "warn")?.payload).toEqual({
+		reason: "detected",
+		path: "/tmp/example.txt",
+	});
 	expect(logs.find((log) => log.level === "error")?.message).toBe(
 		"error message"
 	);
+	expect(logs.find((log) => log.level === "error")?.payload).toBeNull();
 });
 
 test("should insert only specified levels when lix_log_levels=['warn', 'error']", async () => {
@@ -42,9 +48,14 @@ test("should insert only specified levels when lix_log_levels=['warn', 'error']"
 	expect(logs.find((log) => log.level === "warn")?.message).toBe(
 		"warn message"
 	);
+	expect(logs.find((log) => log.level === "warn")?.payload).toEqual({
+		reason: "detected",
+		path: "/tmp/example.txt",
+	});
 	expect(logs.find((log) => log.level === "error")?.message).toBe(
 		"error message"
 	);
+	expect(logs.find((log) => log.level === "error")?.payload).toBeNull();
 });
 
 test("should insert only specified levels when lix_log_levels=['debug']", async () => {
@@ -56,9 +67,11 @@ test("should insert only specified levels when lix_log_levels=['debug']", async
 
 	const logs = await getLogs(lix);
 	expect(logs).toHaveLength(1); // debug
-	expect(logs.find((log) => log.level === "debug")?.message).toBe(
-		"debug message"
-	);
+	expect(logs.find((log) => log.level === "debug")?.message).toBeNull();
+	expect(logs.find((log) => log.level === "debug")?.payload).toEqual({
+		reason: "verbose",
+		details: { retries: 0 },
+	});
 	expect(logs.find((log) => log.level === "info")).toBeUndefined();
 	expect(logs.find((log) => log.level === "warn")).toBeUndefined();
 	expect(logs.find((log) => log.level === "error")).toBeUndefined();
@@ -73,44 +86,55 @@ test("should insert all levels contain wildcard '*'", async () => {
 
 	const logs = await getLogs(lix);
 	expect(logs).toHaveLength(4); // debug, info, warn, error
-	expect(logs.find((log) => log.level === "debug")?.message).toBe(
-		"debug message"
-	);
+	expect(logs.find((log) => log.level === "debug")?.message).toBeNull();
+	expect(logs.find((log) => log.level === "debug")?.payload).toEqual({
+		reason: "verbose",
+		details: { retries: 0 },
+	});
 	expect(logs.find((log) => log.level === "info")?.message).toBe(
 		"info message"
 	);
+	expect(logs.find((log) => log.level === "info")?.payload).toBeNull();
 	expect(logs.find((log) => log.level === "warn")?.message).toBe(
 		"warn message"
 	);
+	expect(logs.find((log) => log.level === "warn")?.payload).toEqual({
+		reason: "detected",
+		path: "/tmp/example.txt",
+	});
 	expect(logs.find((log) => log.level === "error")?.message).toBe(
 		"error message"
 	);
+	expect(logs.find((log) => log.level === "error")?.payload).toBeNull();
 });
 
 async function createLogs(lix: Lix) {
-	await createLixOwnLog({
-		lix,
+	createLixOwnLogSync({
+		engine: lix.engine!,
 		key: "log_test_debug",
 		level: "debug",
-		message: "debug message",
+		payload: { reason: "verbose", details: { retries: 0 } },
 	});
-	await createLixOwnLog({
-		lix,
+	createLixOwnLogSync({
+		engine: lix.engine!,
 		key: "log_test_info",
 		level: "info",
 		message: "info message",
+		payload: null,
 	});
-	await createLixOwnLog({
-		lix,
+	createLixOwnLogSync({
+		engine: lix.engine!,
 		key: "log_test_warn",
 		level: "warn",
 		message: "warn message",
+		payload: { reason: "detected", path: "/tmp/example.txt" },
 	});
-	await createLixOwnLog({
-		lix,
+	createLixOwnLogSync({
+		engine: lix.engine!,
 		key: "log_test_error",
 		level: "error",
 		message: "error message",
+		payload: null,
 	});
 }
 

package/src/log/create-lix-own-log.ts

@@ -1,5 +1,6 @@
-import { executeSync } from "../database/execute-sync.js";
-import type { Lix } from "../lix/open-lix.js";
+import type { LixEngine } from "../engine/boot.js";
+import { internalQueryBuilder } from "../engine/internal-query-builder.js";
+import type { JSONType } from "../schema-definition/json-type.js";
 
 const DEFAULT_LOG_LEVELS = ["info", "warn", "error"];
 
@@ -34,24 +35,27 @@ export function shouldLog(
  * `["info", "warn", "error"]`. The log entry is only created if the provided
  * `level` is included in the determined list of allowed levels or if the list is '["*"]'.
  *
- * It is recommended to use dot notation for log keys (e.g., 'app.module.component').
+ * Use `snake_case` for log keys (e.g., `app_module_component`) to keep filters predictable.
+ * Provide either a `message`, a structured `payload`, or both depending on your needs.
  *
  * @example
  * // Basic info log (will be logged if 'info' is allowed by lix_log_levels)
- * createLixOwnLog({
- *   lix,
- *   key: 'app.init',
+ *
+ * await createLixOwnLog({
+ *   engine,
+ *   key: 'app_init',
  *   level: 'info',
  *   message: 'Application started.'
  * });
  *
  * @example
- * // Log a warning (will be logged if 'warn' is allowed by lix_log_levels)
- * createLixOwnLog({
- *   lix,
- *   key: 'user.login.failed',
- *   level: 'warn',
- *   message: `Login failed for user ${userId}`
+ * // Log structured data without a message
+ *
+ * await createLixOwnLog({
+ *   engine,
+ *   key: 'user_login_attempt',
+ *   level: 'info',
+ *   payload: { userId, success: false }
  * });
  *
  * @example
@@ -74,18 +78,19 @@ export function shouldLog(
  * @returns A promise that resolves with the created log entry, or undefined if filtered out.
  */
 export function createLixOwnLogSync(args: {
-	lix: Pick<Lix, "sqlite" | "db">;
-	message: string;
+	engine: Pick<LixEngine, "executeSync">;
+	message?: string | null;
+	payload?: JSONType;
 	level: string;
 	key: string;
 }): void {
-	const logLevels = executeSync({
-		lix: args.lix,
-		query: args.lix.db
+	const [logLevels] = args.engine.executeSync(
+		internalQueryBuilder
 			.selectFrom("key_value")
 			.select("value")
-			.where("key", "=", "lix_log_levels"),
-	})[0];
+			.where("key", "=", "lix_log_levels")
+			.compile()
+	).rows;
 
 	// Check if the level is allowed
 	if (!shouldLog(logLevels?.value, args.level)) {
@@ -93,21 +98,15 @@ export function createLixOwnLogSync(args: {
 	}
 
 	// Insert the log
-	executeSync({
-		lix: args.lix,
-		query: args.lix.db.insertInto("log").values({
-			key: args.key,
-			message: args.message,
-			level: args.level,
-		}),
-	});
-}
-
-export async function createLixOwnLog(args: {
-	lix: Pick<Lix, "sqlite" | "db">;
-	message: string;
-	level: string;
-	key: string;
-}): Promise<void> {
-	return createLixOwnLogSync(args);
+	args.engine.executeSync(
+		internalQueryBuilder
+			.insertInto("log")
+			.values({
+				key: args.key,
+				message: args.message ?? null,
+				payload: args.payload ?? null,
+				level: args.level,
+			})
+			.compile()
+	);
 }