@lix-js/sdk 0.5.0-preview.0 → 0.5.0

package/src/engine/functions/random.ts

@@ -0,0 +1,267 @@
+import { sql } from "kysely";
+import {
+	LixKeyValueSchema,
+	type LixKeyValue,
+} from "../../key-value/schema-definition.js";
+import type { Lix } from "../../lix/open-lix.js";
+import type { LixEngine } from "../boot.js";
+import { isDeterministicModeSync } from "../deterministic-mode/is-deterministic-mode.js";
+import { getTimestampSync } from "./timestamp.js";
+import { updateUntrackedState } from "../../state/untracked/update-untracked-state.js";
+import { internalQueryBuilder } from "../internal-query-builder.js";
+import { withRuntimeCache } from "../with-runtime-cache.js";
+
+/** State kept per SQLite connection - 16 bytes for xorshift128+ */
+type RngState = {
+	hi: bigint;
+	lo: bigint;
+	dirty: boolean;
+};
+
+const rngCache = new WeakMap<object, RngState>();
+
+/**
+ * Non-deterministic random number generator using crypto.getRandomValues().
+ * Returns a 53-bit precision float in [0, 1).
+ */
+function randomUnstable(): number {
+	const buf = new Uint32Array(2);
+	if (typeof crypto !== "undefined" && crypto.getRandomValues) {
+		crypto.getRandomValues(buf);
+	} else {
+		// Fallback for environments without crypto API
+		// This is not cryptographically secure but better than failing
+		buf[0] = Math.floor(Math.random() * 0xffffffff);
+		buf[1] = Math.floor(Math.random() * 0xffffffff);
+	}
+	// 53-bit float in [0,1) - standard technique for full precision
+	const high = buf[0]! >>> 5; // 27 bits
+	const low = buf[1]! >>> 6; // 26 bits
+	// Combine to get 53 bits total, then divide by 2^53
+	return (high * 0x4000000 + low) / Math.pow(2, 53);
+}
+
+/**
+ * Sync variant of {@link random}. See {@link random} for behavior and examples.
+ *
+ * @remarks
+ * - Accepts `{ engine }` (or `{ lix }`) and runs next to SQLite.
+ * - Intended for engine/router and UDFs; app code should use {@link random}.
+ *
+ * @see random
+ */
+export function randomSync(args: {
+	engine: Pick<LixEngine, "executeSync" | "hooks" | "runtimeCacheRef">;
+}): number {
+	const engine = args.engine;
+	// Non-deterministic mode: use crypto.getRandomValues()
+	if (!isDeterministicModeSync({ engine })) {
+		return randomUnstable();
+	}
+
+	// Deterministic mode: use xorshift128+
+	let state = rngCache.get(engine.runtimeCacheRef);
+
+	/* First use on this connection → initialize from seed */
+	if (!state) {
+		// Check if we have persisted RNG state
+		const [stateRow] = engine.executeSync(
+			internalQueryBuilder
+				.selectFrom("key_value_by_version")
+				.where("key", "=", "lix_deterministic_rng_state")
+				.where("lixcol_version_id", "=", "global")
+				.select("value")
+				.compile()
+		).rows;
+
+		if (stateRow && stateRow.value) {
+			// Restore persisted state
+			// The value is stored as a JSON string in key_value_by_version view
+			const stateData =
+				typeof stateRow.value === "string"
+					? JSON.parse(stateRow.value)
+					: stateRow.value;
+
+			// Handle both string and direct values
+			state = {
+				hi: BigInt(stateData.hi),
+				lo: BigInt(stateData.lo),
+				dirty: false,
+			};
+		} else {
+			// Initialize from seed
+			const seed = getRngSeed({ engine });
+			state = seedXorshift128Plus(seed);
+		}
+
+		rngCache.set(engine.runtimeCacheRef, state);
+	}
+
+	// Generate next random value using xorshift128+
+	const result = nextXorshift128Plus(state);
+	state.dirty = true;
+	return result;
+}
+
+/**
+ * Get the RNG seed - either from deterministic mode config or derive from lix_id
+ */
+function getRngSeed(args: {
+	engine: Pick<LixEngine, "executeSync" | "hooks" | "runtimeCacheRef">;
+}): string {
+	// Check for seed in the deterministic mode config
+	const [configRow] = withRuntimeCache(
+		args.engine,
+		internalQueryBuilder
+			.selectFrom("lix_internal_state_vtable")
+			.where("entity_id", "=", "lix_deterministic_mode")
+			.where("schema_key", "=", "lix_key_value")
+			.where("snapshot_content", "is not", null)
+			.select(
+				sql`json_extract(snapshot_content, '$.value.random_seed')`.as(
+					"random_seed"
+				)
+			)
+			.compile()
+	).rows;
+
+	if (configRow && configRow.random_seed) {
+		return configRow.random_seed as string;
+	}
+
+	// Derive default seed from lix_id
+	const [idRow] = withRuntimeCache(
+		args.engine,
+		internalQueryBuilder
+			.selectFrom("key_value")
+			.where("key", "=", "lix_id")
+			.select("value")
+			.compile()
+	).rows;
+
+	if (!idRow || !idRow.value) {
+		throw new Error("Could not find lix_id for RNG seed");
+	}
+
+	// Use lix_id as the seed - no need to insert it
+	return idRow.value as string;
+}
+
+/**
+ * Initialize state from a 64-bit hash of the seed string (SplitMix64-style)
+ */
+function seedXorshift128Plus(seed: string): RngState {
+	// FNV-1a hash to get initial value
+	let z = 0xcbf29ce484222325n;
+	for (const ch of seed) {
+		z = ((z ^ BigInt(ch.charCodeAt(0))) * 0x100000001b3n) & 0xffffffffffffffffn;
+	}
+
+	// SplitMix64 to expand into two non-zero words
+	const next = (): bigint => {
+		z = (z + 0x9e3779b97f4a7c15n) & 0xffffffffffffffffn;
+		let t = z;
+		t = ((t ^ (t >> 30n)) * 0xbf58476d1ce4e5b9n) & 0xffffffffffffffffn;
+		t = ((t ^ (t >> 27n)) * 0x94d049bb133111ebn) & 0xffffffffffffffffn;
+		return t ^ (t >> 31n);
+	};
+
+	return {
+		hi: next() || 1n,
+		lo: next() || 2n,
+		dirty: true, // New seed always needs to be persisted
+	};
+}
+
+/**
+ * One xorshift128+ step → returns 53-bit float in [0,1)
+ * This is the same algorithm used by V8/Chrome for Math.random()
+ */
+function nextXorshift128Plus(state: RngState): number {
+	let s1 = state.lo;
+	const s0 = state.hi;
+	state.lo = s0;
+	s1 ^= (s1 << 23n) & 0xffffffffffffffffn;
+	state.hi = (s1 ^ s0 ^ (s1 >> 17n) ^ (s0 >> 26n)) & 0xffffffffffffffffn;
+	const sum = (state.hi + state.lo) & 0xffffffffffffffffn;
+	// 53 high bits → IEEE-754 double in [0,1)
+	return Number(sum >> 11n) / Math.pow(2, 53);
+}
+
+/**
+ * @internal
+ * Flush the in-memory RNG state to persistent storage.
+ *
+ * Called automatically by Lix after each committing write and during
+ * `lix.toBlob()` / `lix.close()`. **Not part of the public API.**
+ */
+export function commitDeterministicRngState(args: {
+	engine: Pick<LixEngine, "executeSync" | "hooks" | "runtimeCacheRef">;
+	timestamp?: string;
+}): void {
+	const engine = args.engine;
+	const state = rngCache.get(engine.runtimeCacheRef);
+	if (!state || !state.dirty) return; // nothing to do
+
+	state.dirty = false; // mark clean _before_ we try to write
+
+	const newValue = JSON.stringify({
+		key: "lix_deterministic_rng_state",
+		value: {
+			hi: state.hi.toString(),
+			lo: state.lo.toString(),
+		},
+	} satisfies LixKeyValue);
+
+	const now = args.timestamp ?? getTimestampSync({ engine });
+	updateUntrackedState({
+		engine: engine,
+		changes: [
+			{
+				entity_id: "lix_deterministic_rng_state",
+				schema_key: LixKeyValueSchema["x-lix-key"],
+				file_id: "lix",
+				plugin_key: "lix_sdk",
+				snapshot_content: newValue,
+				schema_version: LixKeyValueSchema["x-lix-version"],
+				created_at: now,
+				lixcol_version_id: "global",
+			},
+		],
+	});
+}
+
+/**
+ * Returns a random float between 0 (inclusive) and 1 (exclusive).
+ *
+ * In deterministic mode, generates reproducible values using xorshift128+ PRNG
+ * (the same algorithm used by V8/Chrome for Math.random()). In normal mode, uses crypto.getRandomValues().
+ *
+ * - Normal mode: cryptographic quality via crypto.getRandomValues().
+ * - Deterministic mode: xorshift128+ PRNG (same as V8/Chrome's Math.random()).
+ * - Default seed: `lix_id` unless `lix_deterministic_rng_seed` is set.
+ * - State persisted via `lix_deterministic_rng_state`.
+ * - Returns 53-bit precision floats in both modes.
+ * - For ID generation, consider {@link uuidV7} or {@link nanoId} instead.
+ *
+ * @example Normal mode – cryptographically secure randomness
+ * ```ts
+ * const lix = await openLix();
+ * const r1 = await random({ lix }) // 0.823...
+ * const r2 = await random({ lix }) // 0.156...
+ * ```
+ *
+ * @example Deterministic mode – reproducible randomness
+ * ```ts
+ * const lix = await openLix({ keyValues: [{ key: "lix_deterministic_mode", value: { enabled: true } }] });
+ * await random({ lix }) // 0.318...
+ * await random({ lix }) // 0.937...
+ * await random({ lix }) // 0.543...
+ * ```
+ */
+export async function random(args: {
+	lix: Pick<Lix, "call">;
+}): Promise<number> {
+	const res = await args.lix.call("lix_random");
+	return Number(res);
+}

package/src/engine/functions/register-builtins.ts

@@ -0,0 +1,154 @@
+import type { RegisteredFunctionDefinition } from "./function-registry.js";
+import { uuidV7Sync } from "./uuid-v7.js";
+import { nanoIdSync } from "./nano-id.js";
+import { getTimestampSync } from "./timestamp.js";
+import { humanIdSync } from "./generate-human-id.js";
+import { randomSync, commitDeterministicRngState } from "./random.js";
+import {
+	commitSequenceNumberSync,
+	nextSequenceNumberSync,
+} from "./sequence.js";
+import { updateStateCache } from "../../state/cache/update-state-cache.js";
+import { markStateCacheAsFresh } from "../../state/cache/mark-state-cache-as-stale.js";
+import {
+	refreshWorkingChangeSet,
+	type WorkingChange,
+} from "../../state/working-change-set/refresh-working-change-set.js";
+import type { LixEngine } from "../boot.js";
+import { createExplainQuery } from "../explain-query.js";
+
+type RegisterBuiltinArgs = {
+	register: (def: RegisteredFunctionDefinition) => void;
+	engine: Pick<
+		LixEngine,
+		| "sqlite"
+		| "hooks"
+		| "executeSync"
+		| "runtimeCacheRef"
+		| "call"
+		| "preprocessQuery"
+	>;
+};
+
+export function registerBuiltinFunctions({
+	register,
+	engine,
+}: RegisterBuiltinArgs): void {
+	const explain = createExplainQuery({ engine });
+
+	register({
+		name: "lix_uuid_v7",
+		handler: (ctx) => uuidV7Sync({ engine: ctx.engine }),
+	});
+
+	register({
+		name: "lix_nano_id",
+		handler: (ctx, args) =>
+			nanoIdSync({ engine: ctx.engine, length: args?.length }),
+	});
+
+	register({
+		name: "lix_timestamp",
+		handler: (ctx) => getTimestampSync({ engine: ctx.engine }),
+	});
+
+	register({
+		name: "lix_human_id",
+		handler: (ctx, args) => {
+			return humanIdSync({
+				engine: ctx.engine as any,
+				...args,
+			});
+		},
+	});
+
+	register({
+		name: "lix_random",
+		handler: (ctx) => randomSync({ engine: ctx.engine }),
+	});
+
+	// TODO likely remove from public API
+	register({
+		name: "lix_next_sequence_number",
+		handler: (ctx) => nextSequenceNumberSync({ engine: ctx.engine }),
+	});
+
+	// TODO likely remove from public API
+	register({
+		name: "lix_commit_deterministic_rng_state",
+		handler: (ctx, args) => {
+			commitDeterministicRngState({
+				engine: ctx.engine,
+				...(args as any),
+			});
+			return null;
+		},
+	});
+
+	// TODO likely remove from public API
+	register({
+		name: "lix_commit_sequence_number",
+		handler: (ctx, args) => {
+			commitSequenceNumberSync({
+				engine: ctx.engine,
+				...(args as any),
+			});
+			return null;
+		},
+	});
+
+	// TODO likely remove from public API
+	register({
+		name: "lix_update_state_cache",
+		handler: (ctx, args) => {
+			updateStateCache({
+				engine: ctx.engine,
+				...(args as any),
+			});
+			return null;
+		},
+	});
+
+	// TODO likely remove from public API
+	register({
+		name: "lix_mark_state_cache_as_fresh",
+		handler: (ctx) => {
+			markStateCacheAsFresh({ engine: ctx.engine });
+			return null;
+		},
+	});
+
+	register({
+		name: "lix_refresh_working_change_set",
+		handler: (ctx, args) => {
+			const payload = (args ?? {}) as {
+				versionId: string;
+				commitId: string | null;
+				workingCommitId: string | null;
+				timestamp: string;
+				changes: WorkingChange[];
+			};
+			refreshWorkingChangeSet({
+				engine: ctx.engine,
+				version: {
+					id: payload.versionId,
+					commit_id: payload.commitId ?? null,
+					working_commit_id: payload.workingCommitId ?? null,
+				},
+				timestamp: payload.timestamp,
+				changes: payload.changes ?? [],
+			});
+			return null;
+		},
+	});
+
+	register({
+		name: "lix_execute_sync",
+		handler: (ctx, args) => ctx.engine.executeSync(args as any),
+	});
+
+	register({
+		name: "lix_explain_query",
+		handler: (_ctx, args) => explain(args),
+	});
+}

package/src/engine/functions/sequence.test.ts

@@ -0,0 +1,99 @@
+import { expect, test } from "vitest";
+import { openLix } from "../../lix/open-lix.js";
+import { nextSequenceNumber } from "./sequence.js";
+
+test("nextSequenceNumberSync throws error when deterministic mode is false", async () => {
+	const lix = await openLix({
+		keyValues: [{ key: "lix_deterministic_mode", value: { enabled: false } }],
+	});
+
+	await expect(async () => {
+		// call inside to check throw in async wrapper by awaiting the call
+		// but since nextSequenceNumber throws only in sync variant, we emulate by calling and catching
+		await nextSequenceNumber({ lix });
+	}).rejects.toThrow(
+		"nextDeterministicSequenceNumber() is available only when lix_deterministic_mode = true"
+	);
+});
+
+test("nextDeterministicSequenceNumber throws error when deterministic mode is not set", async () => {
+	const lix = await openLix({});
+
+	await expect(async () => {
+		await nextSequenceNumber({ lix });
+	}).rejects.toThrow(
+		"nextDeterministicSequenceNumber() is available only when lix_deterministic_mode = true"
+	);
+});
+
+test("nextDeterministicSequenceNumber works when deterministic mode is true", async () => {
+	const lix = await openLix({
+		keyValues: [{ key: "lix_deterministic_mode", value: { enabled: true } }],
+	});
+
+	const n1 = await nextSequenceNumber({ lix });
+	const n2 = await nextSequenceNumber({ lix });
+	const n3 = await nextSequenceNumber({ lix });
+
+	// Should be strictly incrementing by 1
+	expect(n2).toBe(n1 + 1);
+	expect(n3).toBe(n2 + 1);
+
+	// Should be non-negative integers
+	expect(n1).toBeGreaterThanOrEqual(0);
+	expect(Number.isInteger(n1)).toBe(true);
+});
+
+test("nextDeterministicSequenceNumber persists state across blob operations", async () => {
+	const lix1 = await openLix({
+		keyValues: [
+			{
+				key: "lix_deterministic_mode",
+				value: { enabled: true },
+			},
+		],
+	});
+
+	// Generate some sequence numbers and record the last one
+	const n1 = await nextSequenceNumber({ lix: lix1 });
+	const n2 = await nextSequenceNumber({ lix: lix1 });
+	const n3 = await nextSequenceNumber({ lix: lix1 });
+
+	// Verify they increment properly
+	expect(n2).toBe(n1 + 1);
+	expect(n3).toBe(n2 + 1);
+
+	// Create a new instance from the blob
+	const blob = await lix1.toBlob();
+	const lix2 = await openLix({ blob });
+
+	// Should continue from where it left off
+	const n4 = await nextSequenceNumber({ lix: lix2 });
+	const n5 = await nextSequenceNumber({ lix: lix2 });
+
+	expect(n4).toBe(n3 + 1);
+	expect(n5).toBe(n4 + 1);
+});
+
+test("independent Lix instances have independent sequences", async () => {
+	const lix1 = await openLix({
+		keyValues: [{ key: "lix_deterministic_mode", value: { enabled: true } }],
+	});
+
+	const lix2 = await openLix({
+		keyValues: [{ key: "lix_deterministic_mode", value: { enabled: true } }],
+	});
+
+	// Get initial values - they should be the same for both since they started identically
+	const lix1_n1 = await nextSequenceNumber({ lix: lix1 });
+	const lix2_n1 = await nextSequenceNumber({ lix: lix2 });
+	expect(lix1_n1).toBe(lix2_n1);
+
+	// Both increment independently but maintain the same sequence
+	const lix1_n2 = await nextSequenceNumber({ lix: lix1 });
+	const lix2_n2 = await nextSequenceNumber({ lix: lix2 });
+
+	expect(lix1_n2).toBe(lix1_n1 + 1);
+	expect(lix2_n2).toBe(lix2_n1 + 1);
+	expect(lix1_n2).toBe(lix2_n2);
+});

package/src/engine/functions/sequence.ts

@@ -0,0 +1,165 @@
+import {
+	LixKeyValueSchema,
+	type LixKeyValue,
+} from "../../key-value/schema-definition.js";
+import type { LixEngine } from "../boot.js";
+import type { Lix } from "../../lix/open-lix.js";
+import { isDeterministicModeSync } from "../deterministic-mode/is-deterministic-mode.js";
+import { getTimestampSync } from "./timestamp.js";
+import { updateUntrackedState } from "../../state/untracked/update-untracked-state.js";
+import { sql } from "kysely";
+import { internalQueryBuilder } from "../internal-query-builder.js";
+
+/** State kept per SQLite connection */
+type CounterState = {
+	next: number; // number that will be handed out _next_
+	highestSeen: number; // highest value actually handed out
+	dirty: boolean; // true ⇢ we have to persist on commit
+};
+
+const counterCache = new WeakMap<object, CounterState>();
+
+/**
+ * Returns the next monotonic sequence number, starting at 0.
+ *
+ * Only available in deterministic mode. Provides a simple counter for cases where
+ * you need sequential integers rather than timestamps or random values.
+ *
+ * - Requires `lix_deterministic_mode = true`
+ * - Increments by exactly 1 per call (no gaps or duplicates)
+ * - State persisted via `lix_deterministic_sequence_number` key value
+ * - Clones continue from where the sequence left off
+ * - Consider using {@link getTimestampSync}, {@link uuidV7}, or {@link nanoId} for most ID generation needs
+ *
+ * @example Basic usage (deterministic mode required)
+ * ```ts
+ * const lix = await openLix({
+ *   keyValues: [{ key: "lix_deterministic_mode", value: { enabled: true } }]
+ * });
+ * const n1 = nextDeterministicSequenceNumber({ lix }); // 0
+ * const n2 = nextDeterministicSequenceNumber({ lix }); // 1
+ * const n3 = nextDeterministicSequenceNumber({ lix }); // 2
+ * ```
+ *
+ * @example Collision-free IDs
+ * ```ts
+ * const n = nextDeterministicSequenceNumber({ lix }); // e.g. 7
+ * const id = `ENTITY_${n}`;              // "ENTITY_7"
+ * ```
+ *
+ * @example Pagination cursors
+ * ```ts
+ * const cursor = nextDeterministicSequenceNumber({ lix });
+ * await lix.db.insertInto("page_view")
+ *   .values({ cursor, page_id: "home" })
+ *   .execute();
+ * ```
+ *
+ * @param args.lix - The Lix instance with sqlite and db connections
+ * @returns The next number in the sequence (starting from 0)
+ * @throws {Error} If `lix_deterministic_mode` is not enabled
+ */
+export function nextSequenceNumberSync(args: {
+	engine: Pick<LixEngine, "executeSync" | "hooks" | "runtimeCacheRef">;
+}): number {
+	const engine = args.engine;
+	// Check if deterministic mode is enabled
+	if (!isDeterministicModeSync({ engine: engine })) {
+		throw new Error(
+			"nextDeterministicSequenceNumber() is available only when lix_deterministic_mode = true"
+		);
+	}
+
+	let state = counterCache.get(engine.runtimeCacheRef);
+
+	/* First use on this connection → pull initial value from DB */
+	if (!state) {
+		const compiled = internalQueryBuilder
+			.selectFrom("lix_internal_state_vtable")
+			.where("entity_id", "=", "lix_deterministic_sequence_number")
+			.where("schema_key", "=", "lix_key_value")
+			.where("version_id", "=", "global")
+			.where("snapshot_content", "is not", null)
+			.select(sql`json_extract(snapshot_content, '$.value')`.as("value"))
+			.compile();
+		const { rows } = engine.executeSync({
+			sql: compiled.sql,
+			parameters: compiled.parameters,
+		});
+		const row = rows[0];
+
+		// The persisted value is the next counter to use
+		const start = row ? Number(row.value) + 1 : 0;
+		state = { next: start, highestSeen: start - 1, dirty: false };
+		counterCache.set(engine.runtimeCacheRef, state);
+	}
+
+	/* Hand out current, then bump for the next call */
+	const count = state.next++;
+	state.highestSeen = count;
+	state.dirty = true;
+
+	return count;
+}
+
+/**
+ * Returns the next monotonic sequence number, starting at 0.
+ *
+ * Async wrapper around {@link nextSequenceNumberSync} that runs the computation
+ * next to the database engine via the engine router.
+ *
+ * - Available only when `lix_deterministic_mode.enabled = true`.
+ * - Persists state across `toBlob()`/`openLix({ blob })`.
+ * - Consider {@link getTimestampSync}, {@link uuidV7}, or {@link nanoId} for typical IDs.
+ *
+ * @example
+ * const lix = await openLix({
+ *   keyValues: [{ key: "lix_deterministic_mode", value: { enabled: true } }]
+ * })
+ * const n1 = await nextSequenceNumber({ lix }) // 0
+ * const n2 = await nextSequenceNumber({ lix }) // 1
+ */
+export async function nextSequenceNumber(args: { lix: Lix }): Promise<number> {
+	const res = await args.lix.call("lix_next_sequence_number");
+	return Number(res);
+}
+
+/**
+ * @internal
+ * Flush the in-memory deterministic counter to persistent storage.
+ *
+ * Called automatically by Lix after each committing write and during
+ * `lix.toBlob()` / `lix.close()`.  **Not part of the public API.**
+ */
+export function commitSequenceNumberSync(args: {
+	engine: Pick<LixEngine, "executeSync" | "hooks" | "runtimeCacheRef">;
+	timestamp?: string;
+}): void {
+	const engine = args.engine;
+	const state = counterCache.get(engine.runtimeCacheRef);
+	if (!state || !state.dirty) return; // nothing to do
+
+	state.dirty = false; // mark clean _before_ we try to write
+
+	const newValue = JSON.stringify({
+		key: "lix_deterministic_sequence_number",
+		value: state.highestSeen,
+	} satisfies LixKeyValue);
+
+	const now = args.timestamp ?? getTimestampSync({ engine });
+	updateUntrackedState({
+		engine: engine,
+		changes: [
+			{
+				entity_id: "lix_deterministic_sequence_number",
+				schema_key: LixKeyValueSchema["x-lix-key"],
+				file_id: "lix",
+				plugin_key: "lix_sdk",
+				snapshot_content: newValue,
+				schema_version: LixKeyValueSchema["x-lix-version"],
+				created_at: now,
+				lixcol_version_id: "global",
+			},
+		],
+	});
+}