@lix-js/sdk 0.4.7 → 0.5.0-preview.1

package/src/observe/create-observe.ts

@@ -0,0 +1,184 @@
+import type { SelectQueryBuilder } from "kysely";
+import type { Lix } from "../lix/open-lix.js";
+import { LixObservable } from "./lix-observable.js";
+import { determineSchemaKeys } from "./determine-schema-keys.js";
+import type { Change } from "../change/index.js";
+
+/**
+ * Options for the observe method.
+ */
+export interface ObserveOptions {
+	mode?: "array" | "first" | "firstOrThrow";
+}
+
+/**
+ * Map to track active observables and their cleanup functions.
+ */
+interface ActiveObservable {
+	unsubscribeFromStateCommit: () => void;
+}
+
+/**
+ * Deep equality check for query results.
+ * Compares arrays element by element using JSON.stringify for simplicity.
+ */
+function areResultsEqual<T>(a: T[], b: T[]): boolean {
+	if (a.length !== b.length) return false;
+
+	// Simple deep equality using JSON.stringify
+	// This works well for database results which are typically JSON-serializable
+	return JSON.stringify(a) === JSON.stringify(b);
+}
+
+/**
+ * Creates the observe function for a Lix instance.
+ *
+ * @param lix - The Lix instance to add observables to
+ * @returns The observe function
+ */
+export function createObserve(lix: Pick<Lix, "hooks">) {
+	const activeObservables = new WeakMap<LixObservable<any>, ActiveObservable>();
+
+	/**
+	 * Converts a **Kysely read-query** into a **LixObservable** – an implementation of
+	 * the TC-39 Observable protocol that emits a fresh result-set every time the
+	 * underlying state mutates.
+	 *
+	 * @example **Full table stream**
+	 * ```ts
+	 * observe(
+	 *   lix.db.selectFrom('key_value').selectAll()
+	 * ).subscribe({
+	 *   next: rows => console.table(rows)
+	 * })
+	 * ```
+	 *
+	 * @example **Watch latest change-set (first row)**
+	 * ```ts
+	 * observe(
+	 *   lix.db.selectFrom('change_set_all')
+	 *         .selectAll()
+	 *         .orderBy('created_at desc')
+	 * ).subscribeTakeFirst({
+	 *   next: cs => console.log('head →', cs)
+	 * })
+	 * ```
+	 */
+	return function observe<T>(
+		query: SelectQueryBuilder<any, any, T>,
+		options: ObserveOptions = { mode: "array" }
+	): LixObservable<T> {
+		// Create the observable
+		const observable = new LixObservable<T>((observer) => {
+			let isActive = true;
+			let previousResult: T[] | undefined;
+
+			// Helper to execute the query
+			const executeQuery = async () => {
+				if (!isActive) return;
+
+				try {
+					// Optimize query for first/firstOrThrow modes
+					let optimizedQuery = query;
+					if (options.mode === "first" || options.mode === "firstOrThrow") {
+						// Add limit(1) if we only need the first row
+						optimizedQuery = query.limit(1) as any;
+					}
+
+					// Execute the query
+					const result = await optimizedQuery.execute();
+
+					if (!isActive) return;
+
+					// Check if results have changed
+					const hasChanged =
+						!previousResult || !areResultsEqual(previousResult, result as T[]);
+
+					if (hasChanged) {
+						previousResult = result as T[];
+
+						// Handle different modes
+						if (options.mode === "first") {
+							observer.next?.(result as T[]);
+						} else if (options.mode === "firstOrThrow") {
+							observer.next?.(result as T[]);
+						} else {
+							// Default: array mode
+							observer.next?.(result as T[]);
+						}
+					}
+				} catch (error) {
+					if (!isActive) return;
+					observer.error?.(error);
+				}
+			};
+
+			// Execute initial query
+			void executeQuery();
+
+			const shouldReexecute = (data: { changes: Change[] }) => {
+				// Extract changes from the data structure
+				const changes = data.changes || [];
+
+				// If no changes provided, always re-execute for safety
+				if (!changes || changes.length === 0) {
+					return true;
+				}
+
+				// Get schema keys that this query depends on
+				const schemaKeys = determineSchemaKeys(query.compile());
+				// If no schema keys extracted, always re-execute for safety
+				if (!schemaKeys.length) {
+					return true;
+				}
+
+				// Check if any of the changed entities match our query's schema keys
+				return changes.some((change: any) => {
+					// changesToRealize is an array of arrays: [change_id, entity_id, schema_key, ...]
+					const schemaKey = change[2] || change.schema_key;
+
+					// Special case: queries with 'change' or 'state_all' schema should always re-execute
+					if (
+						schemaKeys.includes("change") ||
+						schemaKeys.includes("state_all")
+					) {
+						return true;
+					}
+
+					// Check if this specific schema key affects our query
+					return schemaKeys.includes(schemaKey);
+				});
+			};
+
+			// Subscribe to state commits for updates
+			const unsubscribeFromStateCommit = lix.hooks.onStateCommit((data) => {
+				if (shouldReexecute(data)) {
+					void executeQuery();
+				}
+			});
+
+			// Store the cleanup function
+			activeObservables.set(observable, { unsubscribeFromStateCommit });
+
+			// Return cleanup function
+			return () => {
+				isActive = false;
+				unsubscribeFromStateCommit();
+				activeObservables.delete(observable);
+			};
+		});
+
+		// Handle mode-specific behavior by returning wrapped observables
+		if (options.mode === "first") {
+			// For compatibility, we still return the base observable
+			// The subscribeTakeFirst method handles the transformation
+			return observable;
+		} else if (options.mode === "firstOrThrow") {
+			// For compatibility, we still return the base observable
+			// The subscribeTakeFirstOrThrow method handles the transformation
+			return observable;
+		}
+
+		return observable;
+	};
+}

package/src/observe/determine-schema-keys.test.ts

@@ -0,0 +1,371 @@
+import { test, expect } from "vitest";
+import { openLix } from "../lix/open-lix.js";
+import { determineSchemaKeys } from "./determine-schema-keys.js";
+
+/**
+ * Tests for the determineSchemaKeys function using real queries from md-app.
+ *
+ * This test suite verifies that the schema key extraction logic correctly
+ * identifies the tables involved in queries and maps them to their schema keys.
+ */
+
+test("should extract schema keys from simple table queries", async () => {
+	const lix = await openLix({});
+
+	// Test selectActiveVersion query from md-app
+	const activeVersionQuery = lix.db
+		.selectFrom("active_version")
+		.innerJoin("version", "active_version.version_id", "version.id")
+		.selectAll("version");
+
+	const schemaKeys = determineSchemaKeys(activeVersionQuery.compile());
+
+	// Should include schema keys for both active_version and version tables
+	expect(schemaKeys).toContain("lix_version");
+	// Note: active_version might not have a direct schema key mapping
+
+	await lix.close();
+});
+
+test("should extract schema keys from key_value queries", async () => {
+	const lix = await openLix({});
+
+	// Test selectCurrentLixName query from md-app
+	const currentLixNameQuery = lix.db
+		.selectFrom("key_value")
+		.where("key", "=", "lix_name")
+		.select("value");
+
+	const schemaKeys = determineSchemaKeys(currentLixNameQuery.compile());
+
+	expect(schemaKeys).toContain("lix_key_value");
+
+	await lix.close();
+});
+
+test("should extract schema keys from state table queries", async () => {
+	const lix = await openLix({});
+
+	// Test selectMdAstRoot query from md-app (this uses "state" table)
+	const mdAstRootQuery = lix.db
+		.selectFrom("state")
+		.where("schema_key", "=", "markdown_root_v1")
+		.where("file_id", "=", "test_file_id")
+		.selectAll();
+
+	const schemaKeys = determineSchemaKeys(mdAstRootQuery.compile());
+
+	// "state" is a virtual table that should map to appropriate schema keys
+	expect(schemaKeys).toContain("state");
+
+	await lix.close();
+});
+
+test("should extract schema keys from state_all table queries", async () => {
+	const lix = await openLix({});
+
+	// Test query using "state_all" table (includes all versions)
+	const stateAllQuery = lix.db
+		.selectFrom("state_all")
+		.where("schema_key", "=", "lix_version")
+		.where("version_id", "=", "test_version_id")
+		.selectAll();
+
+	const schemaKeys = determineSchemaKeys(stateAllQuery.compile());
+
+	// "state_all" should be detected as a special schema key
+	expect(schemaKeys).toContain("state_all");
+
+	await lix.close();
+});
+
+test("should extract schema keys from complex join queries", async () => {
+	const lix = await openLix({});
+
+	// Test selectCheckpoints query from md-app (complex with multiple joins)
+	const checkpointsQuery = lix.db
+		.selectFrom("change_set")
+		.leftJoin(
+			"change_set_element",
+			"change_set.id",
+			"change_set_element.change_set_id"
+		)
+		.leftJoin(
+			"change_set_label",
+			"change_set.id",
+			"change_set_label.change_set_id"
+		)
+		.leftJoin("label", "label.id", "change_set_label.label_id")
+		.where("label.name", "=", "checkpoint")
+		.selectAll("change_set");
+
+	const schemaKeys = determineSchemaKeys(checkpointsQuery.compile());
+
+	// Should include schema keys for all joined tables
+	expect(schemaKeys).toContain("lix_change_set");
+	expect(schemaKeys).toContain("lix_change_set_element");
+	expect(schemaKeys).toContain("lix_change_set_label");
+	expect(schemaKeys).toContain("lix_label");
+
+	await lix.close();
+});
+
+test("should extract schema keys from subquery queries", async () => {
+	const lix = await openLix({});
+
+	// Test selectActiveFile query from md-app (has subquery)
+	const activeFileQuery = lix.db
+		.selectFrom("file")
+		.where(
+			"id",
+			"=",
+			lix.db
+				.selectFrom("key_value")
+				.where("key", "=", "flashtype_active_file")
+				.select("value")
+		)
+		.select(["id", "path", "metadata"]);
+
+	const schemaKeys = determineSchemaKeys(activeFileQuery.compile());
+
+	// Should include schema keys for both main query and subquery tables
+	expect(schemaKeys).toContain("lix_file_descriptor");
+	expect(schemaKeys).toContain("lix_key_value");
+
+	await lix.close();
+});
+
+test("should handle queries with aliases", async () => {
+	const lix = await openLix({});
+
+	// Test query with table aliases using key_value (has schema key lix_key_value)
+	const aliasQuery = lix.db
+		.selectFrom("key_value as kv1")
+		.innerJoin("key_value as kv2", "kv1.key", "kv2.value")
+		.selectAll("kv1");
+
+	const schemaKeys = determineSchemaKeys(aliasQuery.compile());
+
+	// Should recognize the underlying table despite aliases
+	// Since both aliases point to key_value, should contain lix_key_value
+	expect(schemaKeys).toContain("lix_key_value");
+
+	await lix.close();
+});
+
+test("should handle empty results gracefully", async () => {
+	const lix = await openLix({});
+
+	// Test with a table that doesn't exist in our mapping
+	const unknownTableQuery = lix.db
+		.selectFrom("unknown_table" as any)
+		.selectAll();
+
+	const schemaKeys = determineSchemaKeys(unknownTableQuery.compile());
+
+	// Should return empty array for unknown tables
+	expect(Array.isArray(schemaKeys)).toBe(true);
+	expect(schemaKeys.length).toBe(0);
+
+	await lix.close();
+});
+
+test("should handle complex working changes query", async () => {
+	const lix = await openLix({});
+
+	// Test selectWorkingChanges query from md-app (very complex with multiple subqueries)
+	const workingChangesQuery = lix.db
+		.selectFrom("change")
+		.innerJoin(
+			"change_set_element",
+			"change_set_element.change_id",
+			"change.id"
+		)
+		.where(
+			"change_set_element.change_set_id",
+			"=",
+			lix.db
+				.selectFrom("active_version")
+				.innerJoin("version", "active_version.version_id", "version.id")
+				.select("version.working_commit_id")
+		)
+		.where(
+			"change.file_id",
+			"=",
+			lix.db
+				.selectFrom("key_value")
+				.where("key", "=", "flashtype_active_file")
+				.select("value")
+		)
+		.select([
+			"change.id",
+			"change.entity_id",
+			"change.file_id",
+			"change.plugin_key",
+			"change.schema_key",
+			"change.created_at",
+		]);
+
+	const schemaKeys = determineSchemaKeys(workingChangesQuery.compile());
+
+	// Should include schema keys for all tables involved
+	expect(schemaKeys).toContain("change"); // change table
+	expect(schemaKeys).toContain("lix_change_set_element");
+	expect(schemaKeys).toContain("lix_version");
+	expect(schemaKeys).toContain("lix_key_value");
+
+	await lix.close();
+});
+
+test("should handle thread queries with json aggregation", async () => {
+	const lix = await openLix({});
+
+	// Test selectThreads query updated to use entity_thread for universal commenting
+	const threadsQuery = lix.db
+		.selectFrom("thread")
+		.leftJoin("entity_thread", "thread.id", "entity_thread.thread_id")
+		.leftJoin("thread_comment", "thread_comment.thread_id", "thread.id")
+		.leftJoin(
+			"change_author",
+			"thread_comment.lixcol_change_id",
+			"change_author.change_id"
+		)
+		.leftJoin("account", "account.id", "change_author.account_id")
+		.where("entity_thread.entity_id", "=", "test_changeset")
+		.where("entity_thread.schema_key", "=", "lix_change_set")
+		.selectAll("thread");
+
+	const schemaKeys = determineSchemaKeys(threadsQuery.compile());
+
+	// Should include all joined tables
+	expect(schemaKeys).toContain("lix_thread");
+	expect(schemaKeys).toContain("lix_entity_thread");
+	expect(schemaKeys).toContain("lix_thread_comment");
+	expect(schemaKeys).toContain("lix_change_author");
+	expect(schemaKeys).toContain("lix_account");
+
+	await lix.close();
+});
+
+test("should correctly extract schema keys from real md-app queries", async () => {
+	const lix = await openLix({});
+
+	// Test cases based on actual queries from md-app
+	const testCases = [
+		{
+			name: "selectActiveVersion",
+			query: lix.db
+				.selectFrom("active_version")
+				.innerJoin("version", "active_version.version_id", "version.id")
+				.selectAll("version"),
+			expectedKeys: ["lix_version"],
+		},
+		{
+			name: "selectCurrentLixName",
+			query: lix.db
+				.selectFrom("key_value")
+				.where("key", "=", "lix_name")
+				.select("value"),
+			expectedKeys: ["lix_key_value"],
+		},
+		{
+			name: "selectMdAstRoot",
+			query: lix.db
+				.selectFrom("state")
+				.where("schema_key", "=", "markdown_root_v1")
+				.selectAll(),
+			expectedKeys: ["state"],
+		},
+		{
+			name: "selectActiveFile (with subquery)",
+			query: lix.db
+				.selectFrom("file")
+				.where(
+					"id",
+					"=",
+					lix.db
+						.selectFrom("key_value")
+						.where("key", "=", "flashtype_active_file")
+						.select("value")
+				)
+				.select(["id", "path"]),
+			expectedKeys: ["lix_file_descriptor", "lix_key_value"],
+		},
+	];
+
+	for (const testCase of testCases) {
+		const schemaKeys = determineSchemaKeys(testCase.query.compile());
+
+		for (const expectedKey of testCase.expectedKeys) {
+			expect(
+				schemaKeys,
+				`${testCase.name} should include ${expectedKey}`
+			).toContain(expectedKey);
+		}
+	}
+
+	await lix.close();
+});
+
+test("should debug log compiled query structure", async () => {
+	const lix = await openLix({});
+
+	// Test a simple query to see its compiled structure
+	const simpleQuery = lix.db
+		.selectFrom("key_value")
+		.where("key", "=", "test")
+		.select("value");
+
+	const compiled = simpleQuery.compile();
+
+	const schemaKeys = determineSchemaKeys(compiled);
+
+	expect(schemaKeys).toContain("lix_key_value");
+
+	await lix.close();
+});
+
+test("should observe key value and handle untracked insertions", async () => {
+	const lix = await openLix({});
+	const values: any[] = [];
+
+	// Listen to a specific key value
+	const subscription = lix
+		.observe(
+			lix.db
+				.selectFrom("key_value")
+				.selectAll()
+				.where("key", "=", "untracked_test_key")
+		)
+		.subscribe({ next: (rows) => values.push(rows) });
+
+	// Wait for initial emission
+	await new Promise((resolve) => setTimeout(resolve, 10));
+	expect(values).toHaveLength(1);
+	expect(values[0]).toEqual([]);
+
+	// Insert a key value with lixcol_untracked: true
+	await lix.db
+		.insertInto("key_value")
+		.values({
+			key: "untracked_test_key",
+			value: "untracked_value",
+			lixcol_untracked: true,
+		})
+		.execute();
+
+	// Wait for update
+	await new Promise((resolve) => setTimeout(resolve, 10));
+
+	// Should have received an update with the untracked value
+	expect(values).toHaveLength(2);
+	expect(values[1]).toHaveLength(1);
+	expect(values[1][0]).toMatchObject({
+		key: "untracked_test_key",
+		value: "untracked_value",
+		lixcol_untracked: 1, // SQLite stores boolean as integer
+	});
+
+	subscription.unsubscribe();
+	await lix.close();
+});