@lix-js/sdk 0.5.0-preview.1 → 0.5.1

package/src/lix/new-lix.ts

@@ -1,26 +1,42 @@
 import {
 	createInMemoryDatabase,
 	contentFromDatabase,
-} from "sqlite-wasm-kysely";
-import { initDb } from "../database/init-db.js";
-import { LixVersionSchema, type LixVersion } from "../version/schema.js";
+} from "../database/sqlite/index.js";
+import { prepareEngineDatabase } from "../database/init-db.js";
+import {
+	LixActiveVersionSchema,
+	LixVersionDescriptorSchema,
+	LixVersionTipSchema,
+	type LixVersionDescriptor,
+	type LixVersionTip,
+} from "../version/schema-definition.js";
 import {
 	LixChangeSetSchema,
 	LixChangeSetElementSchema,
 	type LixChangeSet,
 	type LixChangeSetElement,
-} from "../change-set/schema.js";
-import { LixLabelSchema, type LixLabel } from "../label/schema.js";
-import { LixKeyValueSchema, type LixKeyValue } from "../key-value/schema.js";
-import { LixSchemaViewMap } from "../database/schema.js";
-import type { LixChange } from "../change/schema.js";
-import type { LixStoredSchema } from "../stored-schema/schema.js";
+} from "../change-set/schema-definition.js";
+import { LixLabelSchema, type LixLabel } from "../label/schema-definition.js";
+import {
+	LixKeyValueSchema,
+	type LixKeyValue,
+} from "../key-value/schema-definition.js";
+import type { LixChange } from "../change/schema-definition.js";
+import type { LixStoredSchema } from "../stored-schema/schema-definition.js";
 import { createHooks } from "../hooks/create-hooks.js";
-import { humanId } from "human-id";
-import type { NewStateAll } from "../entity-views/types.js";
+import type { NewStateByVersion } from "../engine/entity-views/types.js";
 import { updateUntrackedState } from "../state/untracked/update-untracked-state.js";
+import { populateStateCache } from "../state/cache/populate-state-cache.js";
+import { markStateCacheAsFresh } from "../state/cache/mark-state-cache-as-stale.js";
 import { v7 } from "uuid";
 import { randomNanoId } from "../database/nano-id.js";
+import { LixSchemaViewMap } from "../database/schema-view-map.js";
+import { createExecuteSync } from "../engine/execute-sync.js";
+import type { LixEngine } from "../engine/boot.js";
+import { setDeterministicBoot } from "../engine/deterministic-mode/is-deterministic-mode.js";
+import { getTimestampSync } from "../engine/functions/timestamp.js";
+import { randomHumanIdWord } from "../engine/functions/generate-human-id.js";
+import { createPreprocessor } from "../engine/preprocessor/create-preprocessor.js";
 
 /**
  * A Blob with an attached `._lix` property for easy access to some lix properties.
@@ -101,18 +117,41 @@ export async function newLixFile(args?: {
 	 *    { key: "my_custom_key", value: "my_custom_value", lixcol_version_id: "global" },
 	 *  ]
 	 */
-	keyValues?: NewStateAll<LixKeyValue>[];
+	keyValues?: NewStateByVersion<LixKeyValue>[];
 }): Promise<NewLixBlob> {
 	const sqlite = await createInMemoryDatabase({
 		readOnly: false,
 	});
+	// Configure persisted page size before schema creation so fresh Lix files use 8 KiB pages.
+	sqlite.exec({ sql: "PRAGMA page_size = 8192;" });
 
 	const hooks = createHooks();
+	const runtimeCacheRef = {};
 
-	// applying the schema etc.
-	const db = initDb({ sqlite, hooks });
+	const engine: LixEngine = {
+		sqlite: sqlite,
+		hooks,
+		getAllPluginsSync: () => {
+			throw new Error("preprocessQuery() is not yet initialised");
+		},
+		runtimeCacheRef,
+		preprocessQuery: () => {
+			throw new Error("preprocessQuery() is not yet initialised");
+		},
+		executeSync: () => {
+			throw new Error("executeSync() is not yet initialised");
+		},
+		call: async () => {
+			throw new Error("call() is not yet initialised");
+		},
+		registerFunction: () => {
+			throw new Error("registerFunction() is not yet initialised");
+		},
+		listFunctions: () => {
+			throw new Error("listFunctions() is not yet initialised");
+		},
+	};
 
-	// Check if deterministic mode is enabled
 	const deterministicModeConfig = args?.keyValues?.find(
 		(kv) => kv.key === "lix_deterministic_mode" && typeof kv.value === "object"
 	);
@@ -131,6 +170,16 @@ export async function newLixFile(args?: {
 		useRandomLixId = config.randomLixId === true;
 	}
 
+	if (isDeterministicBootstrap) {
+		setDeterministicBoot({ runtimeCacheRef, value: true });
+	}
+
+	engine.preprocessQuery = createPreprocessor({ engine });
+
+	engine.executeSync = createExecuteSync({ engine });
+
+	prepareEngineDatabase({ engine });
+
 	// Counter for deterministic IDs
 	let deterministicIdCounter = 0;
 
@@ -153,10 +202,9 @@ export async function newLixFile(args?: {
 		return randomNanoId();
 	};
 
-	// Hardcode timestamp to epoch 0 for deterministic bootstrap
 	const created_at = isDeterministicBootstrap
 		? new Date(0).toISOString()
-		: new Date().toISOString();
+		: getTimestampSync({ engine });
 
 	// Create bootstrap changes for initial data
 	const bootstrapChanges = createBootstrapChanges({
@@ -166,6 +214,7 @@ export async function newLixFile(args?: {
 		generateNanoid,
 		isDeterministicBootstrap,
 		useRandomLixId,
+		engineForRandom: engine,
 	});
 
 	// Extract the lix_id from bootstrap changes
@@ -183,7 +232,7 @@ export async function newLixFile(args?: {
 	for (const change of bootstrapChanges) {
 		if (change.snapshot_content) {
 			sqlite.exec({
-				sql: `INSERT INTO internal_snapshot (id, content) VALUES (?, jsonb(?))`,
+				sql: `INSERT INTO lix_internal_snapshot (id, content) VALUES (?, jsonb(?))`,
 				bind: [change.snapshot_id, JSON.stringify(change.snapshot_content)],
 				returnValue: "resultRows",
 			});
@@ -191,8 +240,8 @@ export async function newLixFile(args?: {
 
 		// Insert the change record
 		sqlite.exec({
-			sql: `INSERT INTO internal_change (id, entity_id, schema_key, schema_version, file_id, plugin_key, snapshot_id, created_at)
-				   VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+			sql: `INSERT INTO lix_internal_change (id, entity_id, schema_key, schema_version, file_id, plugin_key, snapshot_id, created_at)
+					VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
 			bind: [
 				change.id,
 				change.entity_id,
@@ -208,27 +257,33 @@ export async function newLixFile(args?: {
 
 	// The initial version ID will be set by createBootstrapChanges
 	const initialVersionId = bootstrapChanges.find(
-		(c) => c.schema_key === "lix_version" && c.snapshot_content?.name === "main"
+		(c) =>
+			c.schema_key === "lix_version_descriptor" &&
+			c.snapshot_content?.name === "main"
 	)?.entity_id;
 
 	// Set active version using updateUntrackedState for proper inheritance handling
+	const activeVersionEntityId = generateNanoid();
+
 	updateUntrackedState({
-		lix: { sqlite, db },
-		change: {
-			entity_id: "active",
-			schema_key: "lix_active_version",
-			file_id: "lix",
-			plugin_key: "lix_own_entity",
-			snapshot_content: JSON.stringify({ version_id: initialVersionId }),
-			schema_version: "1.0",
-			created_at: created_at,
-		},
-		version_id: "global",
+		engine,
+		changes: [
+			{
+				entity_id: activeVersionEntityId,
+				schema_key: "lix_active_version",
+				file_id: "lix",
+				plugin_key: "lix_sdk",
+				snapshot_content: JSON.stringify({
+					id: activeVersionEntityId,
+					version_id: initialVersionId,
+				}),
+				schema_version: "1.0",
+				created_at: created_at,
+				lixcol_version_id: "global",
+			},
+		],
 	});
 
-	// Anonymous account creation has been moved to openLix
-	// No need to set up any accounts here
-
 	// Handle other untracked key values
 	const untrackedKeyValues = args?.keyValues?.filter(
 		(kv) => kv.lixcol_untracked === true
@@ -237,26 +292,37 @@ export async function newLixFile(args?: {
 		for (const kv of untrackedKeyValues) {
 			const versionId = kv.lixcol_version_id ?? "global";
 			updateUntrackedState({
-				lix: { sqlite, db },
-				change: {
-					entity_id: kv.key,
-					schema_key: "lix_key_value",
-					file_id: "lix",
-					plugin_key: "lix_own_entity",
-					snapshot_content: JSON.stringify({
-						key: kv.key,
-						value: kv.value,
-					}),
-					schema_version: LixKeyValueSchema["x-lix-version"],
-					created_at: created_at,
-				},
-				version_id: versionId,
+				engine,
+				changes: [
+					{
+						entity_id: kv.key,
+						schema_key: "lix_key_value",
+						file_id: "lix",
+						plugin_key: "lix_sdk",
+						snapshot_content: JSON.stringify({
+							key: kv.key,
+							value: kv.value,
+						}),
+						schema_version: LixKeyValueSchema["x-lix-version"],
+						created_at: created_at,
+						lixcol_version_id: versionId,
+					},
+				],
 			});
 		}
 	}
 
+	// Initialize the cache stale flag so synchronous reads see a warm cache immediately.
+	markStateCacheAsFresh({
+		engine,
+		timestamp: created_at,
+	});
+	populateStateCache({ engine });
+
 	try {
-		const blob = new Blob([contentFromDatabase(sqlite)]);
+		const blob = new Blob([
+			contentFromDatabase(sqlite) as Uint8Array<ArrayBuffer>,
+		]);
 		// Create a LixBlob by extending the blob with the lix property
 		const lixBlob = Object.assign(blob, {
 			_lix: {
@@ -268,13 +334,15 @@ export async function newLixFile(args?: {
 	} catch (e) {
 		throw new Error(`Failed to create new Lix file: ${e}`, { cause: e });
 	} finally {
-		await db.destroy();
+		setDeterministicBoot({ runtimeCacheRef, value: false });
+		sqlite.close();
 	}
 }
 
-type BootstrapChange = Omit<LixChange, "snapshot_id"> & {
+type BootstrapChange = Omit<LixChange, "snapshot_id" | "metadata"> & {
 	snapshot_content: any;
 	snapshot_id: string;
+	metadata?: Record<string, any> | null;
 };
 
 /**
@@ -282,12 +350,13 @@ type BootstrapChange = Omit<LixChange, "snapshot_id"> & {
  * All entities are created in a single change set to avoid dependency ordering issues.
  */
 function createBootstrapChanges(args: {
-	providedKeyValues?: NewStateAll<LixKeyValue>[];
+	providedKeyValues?: NewStateByVersion<LixKeyValue>[];
 	created_at: string;
 	generateUuid: () => string;
 	generateNanoid: () => string;
 	isDeterministicBootstrap: boolean;
 	useRandomLixId: boolean;
+	engineForRandom: Pick<LixEngine, "executeSync" | "hooks" | "runtimeCacheRef">;
 }): BootstrapChange[] {
 	const changes: BootstrapChange[] = [];
 
@@ -325,7 +394,7 @@ function createBootstrapChanges(args: {
 			schema_key: "lix_change_set",
 			schema_version: LixChangeSetSchema["x-lix-version"],
 			file_id: "lix",
-			plugin_key: "lix_own_entity",
+			plugin_key: "lix_sdk",
 			snapshot_id: args.generateUuid(),
 			snapshot_content: changeSet,
 			created_at: args.created_at,
@@ -337,104 +406,137 @@ function createBootstrapChanges(args: {
 	const mainCommitId = args.generateUuid();
 
 	// Create commit for global version
-	changes.push({
+	const globalCommitChange = {
 		id: args.generateUuid(),
 		entity_id: globalCommitId,
 		schema_key: "lix_commit",
 		schema_version: "1.0",
 		file_id: "lix",
-		plugin_key: "lix_own_entity",
+		plugin_key: "lix_sdk",
 		snapshot_id: args.generateUuid(),
 		snapshot_content: {
 			id: globalCommitId,
 			change_set_id: initialGlobalVersionChangeSetId,
 		},
 		created_at: args.created_at,
-	});
+	} as BootstrapChange;
+	changes.push(globalCommitChange);
 
 	// Create commit for main version
-	changes.push({
+	const mainCommitChange = {
 		id: args.generateUuid(),
 		entity_id: mainCommitId,
 		schema_key: "lix_commit",
 		schema_version: "1.0",
 		file_id: "lix",
-		plugin_key: "lix_own_entity",
+		plugin_key: "lix_sdk",
 		snapshot_id: args.generateUuid(),
 		snapshot_content: {
 			id: mainCommitId,
 			change_set_id: initialChangeSetId,
 		},
 		created_at: args.created_at,
-	});
+	} as BootstrapChange;
+	changes.push(mainCommitChange);
 
 	// Create working commits for both versions
-	changes.push({
+	const globalWorkingCommitChange = {
 		id: args.generateUuid(),
 		entity_id: globalWorkingCommitId,
 		schema_key: "lix_commit",
 		schema_version: "1.0",
 		file_id: "lix",
-		plugin_key: "lix_own_entity",
+		plugin_key: "lix_sdk",
 		snapshot_id: args.generateUuid(),
 		snapshot_content: {
 			id: globalWorkingCommitId,
 			change_set_id: initialGlobalVersionWorkingChangeSetId,
 		},
 		created_at: args.created_at,
-	});
+	} as BootstrapChange;
+	changes.push(globalWorkingCommitChange);
 
-	changes.push({
+	const mainWorkingCommitChange = {
 		id: args.generateUuid(),
 		entity_id: mainWorkingCommitId,
 		schema_key: "lix_commit",
 		schema_version: "1.0",
 		file_id: "lix",
-		plugin_key: "lix_own_entity",
+		plugin_key: "lix_sdk",
 		snapshot_id: args.generateUuid(),
 		snapshot_content: {
 			id: mainWorkingCommitId,
 			change_set_id: initialWorkingChangeSetId,
 		},
 		created_at: args.created_at,
-	});
+	} as BootstrapChange;
+	changes.push(mainWorkingCommitChange);
 
-	// Create global version
+	// Create global version (split only: descriptor + tip)
 	changes.push({
 		id: args.generateUuid(),
 		entity_id: "global",
-		schema_key: "lix_version",
-		schema_version: LixVersionSchema["x-lix-version"],
+		schema_key: LixVersionDescriptorSchema["x-lix-key"],
+		schema_version: LixVersionDescriptorSchema["x-lix-version"],
 		file_id: "lix",
-		plugin_key: "lix_own_entity",
+		plugin_key: "lix_sdk",
 		snapshot_id: args.generateUuid(),
 		snapshot_content: {
 			id: "global",
 			name: "global",
+			inherits_from_version_id: null,
+			hidden: true,
+		} satisfies LixVersionDescriptor,
+		created_at: args.created_at,
+	});
+
+	changes.push({
+		id: args.generateUuid(),
+		entity_id: "global",
+		schema_key: LixVersionTipSchema["x-lix-key"],
+		schema_version: LixVersionTipSchema["x-lix-version"],
+		file_id: "lix",
+		plugin_key: "lix_sdk",
+		snapshot_id: args.generateUuid(),
+		snapshot_content: {
+			id: "global",
 			commit_id: globalCommitId,
 			working_commit_id: globalWorkingCommitId,
-			hidden: true,
-		} satisfies LixVersion,
+		} satisfies LixVersionTip,
 		created_at: args.created_at,
 	});
 
-	// Create main version
+	// Create main version (split only: descriptor + tip)
 	changes.push({
 		id: args.generateUuid(),
 		entity_id: initialVersionId,
-		schema_key: "lix_version",
-		schema_version: LixVersionSchema["x-lix-version"],
+		schema_key: LixVersionDescriptorSchema["x-lix-key"],
+		schema_version: LixVersionDescriptorSchema["x-lix-version"],
 		file_id: "lix",
-		plugin_key: "lix_own_entity",
+		plugin_key: "lix_sdk",
 		snapshot_id: args.generateUuid(),
 		snapshot_content: {
 			id: initialVersionId,
 			name: "main",
-			commit_id: mainCommitId,
-			working_commit_id: mainWorkingCommitId,
 			inherits_from_version_id: "global",
 			hidden: false,
-		} satisfies LixVersion,
+		} satisfies LixVersionDescriptor,
+		created_at: args.created_at,
+	});
+
+	changes.push({
+		id: args.generateUuid(),
+		entity_id: initialVersionId,
+		schema_key: LixVersionTipSchema["x-lix-key"],
+		schema_version: LixVersionTipSchema["x-lix-version"],
+		file_id: "lix",
+		plugin_key: "lix_sdk",
+		snapshot_id: args.generateUuid(),
+		snapshot_content: {
+			id: initialVersionId,
+			commit_id: mainCommitId,
+			working_commit_id: mainWorkingCommitId,
+		} satisfies LixVersionTip,
 		created_at: args.created_at,
 	});
 
@@ -446,7 +548,7 @@ function createBootstrapChanges(args: {
 		schema_key: "lix_label",
 		schema_version: LixLabelSchema["x-lix-version"],
 		file_id: "lix",
-		plugin_key: "lix_own_entity",
+		plugin_key: "lix_sdk",
 		snapshot_id: args.generateUuid(),
 		snapshot_content: {
 			id: checkpointLabelId,
@@ -480,7 +582,7 @@ function createBootstrapChanges(args: {
 		schema_key: "lix_key_value",
 		schema_version: LixKeyValueSchema["x-lix-version"],
 		file_id: "lix",
-		plugin_key: "lix_own_entity",
+		plugin_key: "lix_sdk",
 		snapshot_id: args.generateUuid(),
 		snapshot_content: {
 			key: "lix_id",
@@ -499,7 +601,7 @@ function createBootstrapChanges(args: {
 		schema_key: "lix_key_value",
 		schema_version: LixKeyValueSchema["x-lix-version"],
 		file_id: "lix",
-		plugin_key: "lix_own_entity",
+		plugin_key: "lix_sdk",
 		snapshot_id: args.generateUuid(),
 		snapshot_content: {
 			key: "lix_name",
@@ -507,7 +609,11 @@ function createBootstrapChanges(args: {
 				lixName ??
 				(args.isDeterministicBootstrap
 					? "deterministic-lix-name"
-					: humanId({ separator: "-", capitalize: false })),
+					: randomHumanIdWord({
+							capitalize: false,
+							engine: args.engineForRandom,
+							separator: "-",
+						})),
 		} satisfies LixKeyValue,
 		created_at: args.created_at,
 	});
@@ -533,7 +639,7 @@ function createBootstrapChanges(args: {
 				schema_key: "lix_key_value",
 				schema_version: LixKeyValueSchema["x-lix-version"],
 				file_id: "lix",
-				plugin_key: "lix_own_entity",
+				plugin_key: "lix_sdk",
 				snapshot_id: args.generateUuid(),
 				snapshot_content: {
 					key: kv.key,
@@ -552,19 +658,35 @@ function createBootstrapChanges(args: {
 			schema_key: "lix_stored_schema",
 			schema_version: "1.0",
 			file_id: "lix",
-			plugin_key: "lix_own_entity",
+			plugin_key: "lix_sdk",
 			snapshot_id: args.generateUuid(),
 			snapshot_content: {
-				key: schema["x-lix-key"],
-				version: schema["x-lix-version"],
-				value: JSON.stringify(schema),
+				value: schema,
 			} satisfies LixStoredSchema,
 			created_at: args.created_at,
 		});
 	}
 
-	// Anonymous account creation has been moved to openLix
-	// to only create when no account is provided and no active account exists
+	// Also store version schemas (descriptor, tip, active_version) for validation
+	for (const schema of [
+		LixVersionDescriptorSchema,
+		LixVersionTipSchema,
+		LixActiveVersionSchema,
+	]) {
+		changes.push({
+			id: args.generateUuid(),
+			entity_id: schema["x-lix-key"],
+			schema_key: "lix_stored_schema",
+			schema_version: "1.0",
+			file_id: "lix",
+			plugin_key: "lix_sdk",
+			snapshot_id: args.generateUuid(),
+			snapshot_content: {
+				value: schema,
+			} satisfies LixStoredSchema,
+			created_at: args.created_at,
+		});
+	}
 
 	// Create change set elements linking all changes to the global change set
 	const originalChanges = [...changes]; // snapshot of original changes
@@ -592,7 +714,7 @@ function createBootstrapChanges(args: {
 			schema_key: "lix_change_set_element",
 			schema_version: LixChangeSetElementSchema["x-lix-version"],
 			file_id: "lix",
-			plugin_key: "lix_own_entity",
+			plugin_key: "lix_sdk",
 			snapshot_id: args.generateUuid(),
 			snapshot_content: {
 				change_set_id: targetChangeSetId,
@@ -607,6 +729,31 @@ function createBootstrapChanges(args: {
 		changeSetElementChanges.push(changeSetElementChange);
 	}
 
+	// Map change_set_id -> list of change_ids so commits reference their members
+	const changeIdsByChangeSet = new Map<string, string[]>();
+	for (const cse of changeSetElementChanges) {
+		const snap = cse.snapshot_content as any;
+		const cs = snap.change_set_id as string;
+		const cid = snap.change_id as string;
+		if (!cs || !cid) continue;
+		const list = changeIdsByChangeSet.get(cs) ?? [];
+		list.push(cid);
+		changeIdsByChangeSet.set(cs, list);
+	}
+
+	// Assign change_ids (and empty parent_commit_ids) to initial commits
+	const setCommitMembers = (commitChange: BootstrapChange) => {
+		const csId = (commitChange.snapshot_content as any).change_set_id as string;
+		const ids = changeIdsByChangeSet.get(csId) ?? [];
+		(commitChange.snapshot_content as any).change_ids = ids;
+		(commitChange.snapshot_content as any).parent_commit_ids = [];
+	};
+
+	setCommitMembers(globalCommitChange);
+	setCommitMembers(mainCommitChange);
+	setCommitMembers(globalWorkingCommitChange);
+	setCommitMembers(mainWorkingCommitChange);
+
 	// TODO evaluate if we can come up with a better concept to avoid meta changes (if even desired)
 	//
 	// this is the cost of change set (elements). we need to track what's in a
@@ -623,7 +770,7 @@ function createBootstrapChanges(args: {
 			schema_key: "lix_change_set_element",
 			schema_version: LixChangeSetElementSchema["x-lix-version"],
 			file_id: "lix",
-			plugin_key: "lix_own_entity",
+			plugin_key: "lix_sdk",
 			snapshot_id: args.generateUuid(),
 			snapshot_content: {
 				change_set_id: initialGlobalVersionChangeSetId,

package/src/lix/open-lix.bench.ts

@@ -0,0 +1,23 @@
+import { bench, describe } from "vitest";
+import { openLix } from "./open-lix.js";
+
+describe("openLix baselines", () => {
+	bench("openLix (empty, in-memory)", async () => {
+		const lix = await openLix({});
+		await lix.close();
+	});
+
+	describe("with key values", () => {
+		bench("openLix with 1 keyValue", async () => {
+			const lix = await openLix({
+				keyValues: [
+					{
+						key: "lix_deterministic_mode",
+						value: { enabled: true },
+					},
+				],
+			});
+			await lix.close();
+		});
+	});
+});