@lix-js/sdk 0.5.0-preview.0 → 0.5.0

package/src/diff/select-working-diff.test.ts

@@ -0,0 +1,275 @@
+import { test, expect } from "vitest";
+import { openLix } from "../lix/open-lix.js";
+import { createCheckpoint } from "../state/create-checkpoint.js";
+import { selectWorkingDiff } from "./select-working-diff.js";
+import { LixKeyValueSchema } from "../key-value/schema-definition.js";
+
+const FILE_ID = "lix"; // key_value is stored under hardcoded file_id 'lix'
+const KV_SCHEMA = LixKeyValueSchema["x-lix-key"];
+
+// Manually select base diff columns from the aliased subquery "diff"
+const selectBaseDiffCols = [
+	"entity_id",
+	"schema_key",
+	"file_id",
+	"before_version_id",
+	"before_change_id",
+	"before_commit_id",
+	"after_version_id",
+	"after_change_id",
+	"after_commit_id",
+	"status",
+] as const;
+
+async function setKeyValue(lix: any, key: string, value: any) {
+	// Safe upsert for vtable-backed entity: check existence first
+	const existing = await lix.db
+		.selectFrom("key_value")
+		.where("key", "=", key)
+		.select("key")
+		.executeTakeFirst();
+	if (existing) {
+		await lix.db
+			.updateTable("key_value")
+			.set({ value })
+			.where("key", "=", key)
+			.execute();
+	} else {
+		await lix.db.insertInto("key_value").values({ key, value }).execute();
+	}
+}
+
+async function delKeyValue(lix: any, key: string) {
+	await lix.db.deleteFrom("key_value").where("key", "=", key).execute();
+}
+
+async function workingCommitId(lix: any): Promise<string> {
+	const row = await lix.db
+		.selectFrom("active_version")
+		.innerJoin("version", "active_version.version_id", "version.id")
+		.select("version.working_commit_id")
+		.executeTakeFirstOrThrow();
+	return row.working_commit_id;
+}
+
+test("added: new key after checkpoint", async () => {
+	const lix = await openLix({});
+	await createCheckpoint({ lix });
+
+	await setKeyValue(lix, "kv1", "A");
+
+	const wcid = await workingCommitId(lix);
+
+	const rows = await selectWorkingDiff({ lix })
+		.where("file_id", "=", FILE_ID)
+		.where("schema_key", "=", KV_SCHEMA)
+		.select(selectBaseDiffCols as any)
+		.execute();
+
+	expect(rows.length).toBeGreaterThan(0);
+	const created = rows.find(
+		(r) => r.status === "added" && r.entity_id === "kv1"
+	);
+	expect(created).toBeTruthy();
+	expect(created?.before_commit_id).toBeNull();
+	expect(created?.after_commit_id).toBe(wcid);
+	// Assert the value in the after change snapshot is 'A'
+	expect(created?.after_change_id).toBeTruthy();
+	const afterCreated = await lix.db
+		.selectFrom("change")
+		.where("id", "=", created!.after_change_id as string)
+		.selectAll()
+		.executeTakeFirstOrThrow();
+	expect(afterCreated.snapshot_content?.value).toBe("A");
+});
+
+test("modified: key changed since last checkpoint", async () => {
+	const lix = await openLix({});
+	await createCheckpoint({ lix });
+
+	await setKeyValue(lix, "kv2", "A");
+	await createCheckpoint({ lix });
+
+	await setKeyValue(lix, "kv2", "B");
+
+	const rows = await selectWorkingDiff({ lix })
+		.where("file_id", "=", FILE_ID)
+		.where("schema_key", "=", KV_SCHEMA)
+		.select(selectBaseDiffCols as any)
+		.execute();
+
+	const updated = rows.find(
+		(r) => r.status === "modified" && r.entity_id === "kv2"
+	);
+	expect(updated).toBeTruthy();
+	expect(updated?.before_commit_id).toBeTruthy();
+	expect(updated?.after_commit_id).toBeTruthy();
+	expect(updated?.before_change_id).toBeTruthy();
+	expect(updated?.after_change_id).toBeTruthy();
+	// change ids must differ
+	expect(updated!.before_change_id).not.toBe(updated!.after_change_id);
+	// Assert before/after values are 'A' -> 'B'
+	const beforeUpdated = await lix.db
+		.selectFrom("change")
+		.where("id", "=", updated!.before_change_id as string)
+		.selectAll()
+		.executeTakeFirstOrThrow();
+	const afterUpdated = await lix.db
+		.selectFrom("change")
+		.where("id", "=", updated!.after_change_id as string)
+		.selectAll()
+		.executeTakeFirstOrThrow();
+	expect(beforeUpdated.snapshot_content?.value).toBe("A");
+	expect(afterUpdated.snapshot_content?.value).toBe("B");
+});
+
+test("removed: key removed since last checkpoint", async () => {
+	const lix = await openLix({});
+	await createCheckpoint({ lix });
+
+	await setKeyValue(lix, "kv3", "X");
+
+	await createCheckpoint({ lix });
+
+	await delKeyValue(lix, "kv3");
+
+	const rows = await selectWorkingDiff({ lix })
+		.where("file_id", "=", FILE_ID)
+		.where("schema_key", "=", KV_SCHEMA)
+		.select(selectBaseDiffCols)
+		.execute();
+
+	const deleted = rows.find(
+		(r) => r.status === "removed" && r.entity_id === "kv3"
+	);
+
+	expect(deleted).toBeTruthy();
+	expect(deleted?.before_commit_id).toBeTruthy();
+	expect(deleted?.after_commit_id).toBeTruthy();
+	// Assert before had 'X' and after is a tombstone (null)
+	expect(deleted?.before_change_id).toBeTruthy();
+	expect(deleted?.after_change_id).toBeTruthy();
+	const beforeDeleted = await lix.db
+		.selectFrom("change")
+		.where("id", "=", deleted!.before_change_id)
+		.selectAll()
+		.executeTakeFirstOrThrow();
+	const afterDeleted = await lix.db
+		.selectFrom("change")
+		.where("id", "=", deleted!.after_change_id)
+		.selectAll()
+		.executeTakeFirstOrThrow();
+	expect(beforeDeleted.snapshot_content?.value).toBe("X");
+	expect(afterDeleted.snapshot_content).toBeNull();
+});
+
+test("unchanged: no working changes returns no added/modified/removed", async () => {
+	const lix = await openLix({});
+	await createCheckpoint({ lix });
+
+	const rows = await selectWorkingDiff({ lix })
+		.where("file_id", "=", FILE_ID)
+		.where("schema_key", "=", KV_SCHEMA)
+		.select(selectBaseDiffCols)
+		.execute();
+
+	expect(rows.length).toBe(0);
+});
+
+test("latest checkpoint chosen as before", async () => {
+	const lix = await openLix({});
+	await createCheckpoint({ lix });
+
+	// A: create key t='A' and checkpoint
+	await setKeyValue(lix, "t", "A");
+	const A = await createCheckpoint({ lix });
+
+	// B: change t so it's included in the latest checkpoint
+	await setKeyValue(lix, "t", "A1");
+	await createCheckpoint({ lix });
+
+	// Working change: update t to new value
+	await setKeyValue(lix, "t", "B2");
+
+	const rows = await selectWorkingDiff({ lix })
+		.where("file_id", "=", FILE_ID)
+		.where("schema_key", "=", KV_SCHEMA)
+		.where("entity_id", "=", "t")
+		.select(selectBaseDiffCols)
+		.execute();
+
+	const row = rows[0];
+	expect(row).toBeTruthy();
+	expect(row?.status).toBe("modified");
+	expect(row?.before_commit_id).toBeTruthy();
+	expect(row?.before_commit_id).not.toBe(A.id);
+});
+
+test("after_commit_id equals active working commit id", async () => {
+	const lix = await openLix({});
+	await createCheckpoint({ lix });
+	await setKeyValue(lix, "wk1", "Z");
+	const wcid = await workingCommitId(lix);
+
+	const rows = await selectWorkingDiff({ lix })
+		.where("entity_id", "=", "wk1")
+		.select(selectBaseDiffCols)
+		.execute();
+	expect(rows.length).toBe(1);
+	expect(rows[0]!.after_commit_id).toBe(wcid);
+});
+
+test("before_* columns are null when no prior checkpoint", async () => {
+	const lix = await openLix({});
+	// NOTE: no checkpoint before first change
+	await setKeyValue(lix, "ncp", "A");
+
+	const rows = await selectWorkingDiff({ lix })
+		.where("entity_id", "=", "ncp")
+		.select(selectBaseDiffCols)
+		.execute();
+	expect(rows.length).toBe(1);
+	const r = rows[0]!;
+	expect(r.status).toBe("added");
+	expect(r.before_commit_id).toBeNull();
+	expect(r.before_change_id).toBeNull();
+	expect(r.before_version_id).toBeNull();
+});
+
+test("stable sorting by entity_id", async () => {
+	const lix = await openLix({});
+	await createCheckpoint({ lix });
+	await setKeyValue(lix, "b", 1);
+	await setKeyValue(lix, "a", 1);
+	await setKeyValue(lix, "c", 1);
+
+	const rows = await selectWorkingDiff({ lix })
+		.orderBy("entity_id", "asc")
+		.select(selectBaseDiffCols)
+		.execute();
+	const ids = rows.map((r) => r.entity_id);
+	expect(ids).toEqual([...ids].sort());
+});
+
+test("returns unchanged rows for non-inherited state when no working edits", async () => {
+	const lix = await openLix({});
+	// Create a non-inherited tracked key in the active version and checkpoint it
+	await setKeyValue(lix, "kvU", "X");
+	await createCheckpoint({ lix });
+
+	// No further edits; working diff should include 'unchanged' for kvU
+	const rows = await selectWorkingDiff({ lix })
+		.where("file_id", "=", FILE_ID)
+		.where("schema_key", "=", KV_SCHEMA)
+		.where("entity_id", "=", "kvU")
+		.select(selectBaseDiffCols)
+		.execute();
+
+	expect(rows.length).toBe(1);
+	const r = rows[0]!;
+	expect(r.status).toBe("unchanged");
+	// Optional: when unchanged, before/after change ids should match if present
+	if (r.before_change_id && r.after_change_id) {
+		expect(r.before_change_id).toBe(r.after_change_id);
+	}
+});

package/src/diff/select-working-diff.ts

@@ -0,0 +1,233 @@
+import type { SelectQueryBuilder } from "kysely";
+import type { Lix } from "../lix/open-lix.js";
+import type { DiffRow } from "../version/select-version-diff.js";
+import { sql } from "kysely";
+import type { LixDatabaseSchema } from "../database/schema.js";
+
+// Note: snapshots are not projected here; join `change` by id if needed.
+
+/**
+ * Returns the diff between the working state and the last checkpoint.
+ *
+ * The working state contains all changes made since the last checkpoint. This function
+ * compares these changes against the checkpoint to show what has been created, updated,
+ * or deleted. Optimized for real-time UI updates during active editing.
+ *
+ * Unlike {@link selectVersionDiff} which compares two complete versions, this operates
+ * directly on change sets for better performance.
+ *
+ * ## Diff Status Meanings
+ *
+ * | Status      | Description                                  | Before | After  |
+ * |-------------|----------------------------------------------|--------|--------|
+ * | `created`   | Entity added since checkpoint                | null   | exists |
+ * | `updated`   | Entity modified since checkpoint             | exists | exists |
+ * | `deleted`   | Entity removed since checkpoint (tombstone)  | exists | null   |
+ * | `unchanged` | Entity not modified in working state         | same   | same   |
+ *
+ * ## Performance
+ *
+ * - Queries only two change sets (working + latest checkpoint)
+ * - No graph traversal or recursion
+ * - Returns same shape as `selectVersionDiff` for compatibility
+ *
+ * @example
+ * // Get all changes since the last checkpoint
+ * const changes = await selectWorkingDiff({ lix })
+ *   .where('status', '!=', 'unchanged')
+ *   .execute();
+ *
+ * console.log(`${changes.length} changes since checkpoint`);
+ *
+ * @example
+ * // Monitor changes to a specific file
+ * const fileChanges = await selectWorkingDiff({ lix })
+ *   .where('file_id', '=', 'config.json')
+ *   .where('status', '!=', 'unchanged')
+ *   .orderBy('entity_id')
+ *   .execute();
+ *
+ * @example
+ * // Check if specific entities have changed since checkpoint
+ * const entityIds = ['user-1', 'user-2', 'user-3'];
+ * const entityChanges = await selectWorkingDiff({ lix })
+ *   .where('entity_id', 'in', entityIds)
+ *   .where('status', '!=', 'unchanged')
+ *   .execute();
+ *
+ * @example
+ * // Count changes by status
+ * const allChanges = await selectWorkingDiff({ lix })
+ *   .where('status', '!=', 'unchanged')
+ *   .execute();
+ *
+ * const stats = allChanges.reduce((acc, change) => {
+ *   acc[change.status] = (acc[change.status] || 0) + 1;
+ *   return acc;
+ * }, {} as Record<string, number>);
+ *
+ * console.log('Changes:', stats);
+ * // Output: { created: 5, updated: 3, deleted: 1 }
+ *
+ * @param args - Configuration object
+ * @param args.lix - Lix instance with database connection
+ *
+ * @returns A Kysely query builder for `DiffRow` results that can be further filtered,
+ * sorted, or executed.
+ *
+ * @see createCheckpoint - Convert working state into a checkpoint
+ * @see selectVersionDiff - Compare two complete version states
+ *
+ * @example
+ * // Fetch before/after snapshots by joining the `change` table
+ * // Tip: apply filters (file_id, schema_key, status) before joining to keep it fast.
+ * const rows = await selectWorkingDiff({ lix })
+ *   .where('file_id', '=', 'lix')
+ *   .where('schema_key', '=', 'lix_key_value')
+ *   .where('status', '!=', 'unchanged')
+ *   .leftJoin('change as before', 'before.id', 'before_change_id')
+ *   .leftJoin('change as after', 'after.id', 'after_change_id')
+ *   .select([
+ *     'entity_id',
+ *     'schema_key',
+ *     'file_id',
+ *     'status',
+ *     'before.snapshot_content',
+ *     'after.snapshot_content',
+ *   ])
+ *   .execute();
+ *
+ * // rows[i].before_snapshot_content / after_snapshot_content are JSON objects (or null)
+ */
+type DiffDB = { diff: DiffRow };
+
+export function selectWorkingDiff(args: {
+	lix: Lix;
+}): SelectQueryBuilder<LixDatabaseSchema & DiffDB, "diff", DiffRow> {
+	const db = args.lix.db;
+
+	// Active version details
+	const workingCommitIdQ = db
+		.selectFrom("active_version")
+		.innerJoin("version", "active_version.version_id", "version.id")
+		.select((eb) => eb.ref("version.working_commit_id").as("id"));
+
+	// Optimize: previous checkpoint is the parent of the current working commit
+	const latestCheckpointCommitIdQ = db
+		.selectFrom("commit_edge as ce")
+		.innerJoin("commit as p", "p.id", "ce.parent_id")
+		.innerJoin("entity_label as el", (join) =>
+			join
+				.onRef("el.entity_id", "=", "p.id")
+				.on("el.schema_key", "=", "lix_commit")
+		)
+		.innerJoin("label as l", "l.id", "el.label_id")
+		.whereRef("ce.child_id", "=", workingCommitIdQ)
+		.where("l.name", "=", "checkpoint")
+		.select("p.id")
+		.limit(1);
+
+	// Base SELECT: rows from the working change set
+	const base = db
+		.selectFrom("change as ch")
+		.innerJoin("change_set_element as cse", "cse.change_id", "ch.id")
+		.where(
+			"cse.change_set_id",
+			"=",
+			sql`(select change_set_id from wcs)` as any
+		)
+		.where("ch.file_id", "!=", "lix_sdk")
+		.leftJoin("change_set_element as bcse", (join) =>
+			join
+				.onRef("bcse.entity_id", "=", "ch.entity_id")
+				.onRef("bcse.schema_key", "=", "ch.schema_key")
+				.onRef("bcse.file_id", "=", "ch.file_id")
+				.on("bcse.change_set_id", "=", sql`(select change_set_id from ccs)`)
+		)
+		// no join to `change bc` here — project ids only
+		.select((eb) => [
+			eb.ref("ch.entity_id").as("entity_id"),
+			eb.ref("ch.schema_key").as("schema_key"),
+			eb.ref("ch.file_id").as("file_id"),
+
+			sql`NULL`.as("before_version_id"),
+			eb.ref("bcse.change_id").as("before_change_id"),
+			sql`(select id from cc)`.as("before_commit_id"),
+
+			sql`NULL`.as("after_version_id"),
+			eb.ref("ch.id").as("after_change_id"),
+			sql`(select id from wc)`.as("after_commit_id"),
+
+			sql<"added" | "modified" | "removed" | "unchanged">`CASE
+        WHEN bcse.change_id IS NOT NULL AND json(ch.snapshot_content) IS NULL THEN 'removed'
+        WHEN bcse.change_id IS NULL AND json(ch.snapshot_content) IS NOT NULL THEN 'added'
+        WHEN bcse.change_id IS NOT NULL AND json(ch.snapshot_content) IS NOT NULL AND bcse.change_id != ch.id THEN 'modified'
+        ELSE 'unchanged'
+      END`.as("status"),
+		]);
+
+	// Unchanged rows = non-inherited, tracked state rows without a working CSE entry
+	const unchanged = db
+		.selectFrom("state_by_version as sa")
+		.whereRef(
+			"sa.version_id",
+			"=",
+			db.selectFrom("active_version").select("active_version.version_id")
+		)
+		.where("sa.inherited_from_version_id", "is", null)
+		.where("sa.untracked", "=", false)
+		// Anti-join to ensure no working edit exists for this triple
+		.leftJoin("change_set_element as wcse", (join) =>
+			join
+				.onRef("wcse.entity_id", "=", "sa.entity_id")
+				.onRef("wcse.schema_key", "=", "sa.schema_key")
+				.onRef("wcse.file_id", "=", "sa.file_id")
+				.on("wcse.change_set_id", "=", sql`(select change_set_id from wcs)`)
+		)
+		.where("wcse.change_id", "is", null)
+		// For unchanged rows, before == after; reuse sa.* and avoid checkpoint joins
+		.select((eb) => [
+			eb.ref("sa.entity_id").as("entity_id"),
+			eb.ref("sa.schema_key").as("schema_key"),
+			eb.ref("sa.file_id").as("file_id"),
+			sql`NULL`.as("before_version_id"),
+			eb.ref("sa.change_id").as("before_change_id"),
+			sql`(select id from cc)`.as("before_commit_id"),
+			sql`NULL`.as("after_version_id"),
+			eb.ref("sa.change_id").as("after_change_id"),
+			sql`(select id from wc)`.as("after_commit_id"),
+			sql<"unchanged">`'unchanged'`.as("status"),
+		]);
+
+	// CTEs for constants used across both UNION arms
+	const union = sql<DiffRow>`(
+        select * from (${base}) as w
+        union all
+        select * from (${unchanged}) as u
+    )`.as("diff");
+
+	return db
+		.with("wc", () => workingCommitIdQ as any)
+		.with("cc", () => latestCheckpointCommitIdQ as any)
+		.with(
+			"wcs",
+			() =>
+				db
+					.selectFrom("commit")
+					.where("id", "=", sql`(select id from wc)` as any)
+					.select(["change_set_id"]) as any
+		)
+		.with(
+			"ccs",
+			() =>
+				db
+					.selectFrom("commit")
+					.where("id", "=", sql`(select id from cc)` as any)
+					.select(["change_set_id"]) as any
+		)
+		.selectFrom(union) as unknown as SelectQueryBuilder<
+		LixDatabaseSchema & DiffDB,
+		"diff",
+		DiffRow
+	>;
+}

package/src/engine/boot.test.ts

@@ -0,0 +1,88 @@
+import { expect, test } from "vitest";
+import {
+	createInMemoryDatabase,
+	importDatabase,
+} from "../database/sqlite/index.js";
+import { newLixFile } from "../lix/new-lix.js";
+import { boot } from "./boot.js";
+
+test("boot installs engine and triggers plugin on file insert", async () => {
+	const sqlite = await createInMemoryDatabase({ readOnly: false });
+	// Seed with a fresh Lix snapshot
+	const blob = await newLixFile();
+	const buf = new Uint8Array(await blob.arrayBuffer());
+	importDatabase({ db: sqlite, content: buf });
+
+	// Minimal plugin that matches *.md and emits one entity
+	const pluginCode = `
+      export const plugin = {
+        key: 'test_plugin',
+        detectChangesGlob: '*.md',
+        detectChanges: ({ after }) => {
+          return [{
+            entity_id: 'e1',
+            schema: {
+              "x-lix-key": "test_item",
+              "x-lix-version": "1.0",
+              "x-lix-primary-key": ["/id"],
+              "type": "object",
+              "properties": { "id": {"type": "string"}, "title": {"type": "string"} },
+              "required": ["id", "title"],
+              "additionalProperties": false
+            },
+            snapshot_content: { id: 'e1', title: 'Hello' }
+          }];
+        }
+      };
+    `;
+
+	const events: any[] = [];
+	const engine = await boot({
+		sqlite,
+		emit: (ev) => events.push(ev),
+		args: { providePluginsRaw: [pluginCode] },
+	});
+
+	// Sanity check: execSync should return rows as mapped objects including column names.
+	const execResult = engine.executeSync({ sql: "select 1 as value" });
+	expect(execResult.rows).toEqual([{ value: 1 }]);
+
+	// Insert a markdown file; plugin should detect a change
+	const data = new Uint8Array([1, 2, 3]);
+	sqlite.exec({
+		sql: `INSERT INTO file (id, path, data, metadata, hidden) VALUES (?, ?, ?, json(?), 0)`,
+		bind: ["f1", "/doc.md", data, JSON.stringify({})],
+		returnValue: "resultRows",
+	});
+
+	// Verify the plugin-produced entity exists
+	const rows = sqlite.exec({
+		sql: `SELECT COUNT(*) FROM state WHERE schema_key = 'test_item'`,
+		returnValue: "resultRows",
+	}) as any[];
+	const count = Number(rows?.[0]?.[0] ?? 0);
+	expect(count).toBeGreaterThan(0);
+
+	// Verify a state_commit event was bridged
+	expect(events.find((e) => e?.type === "state_commit")).toBeTruthy();
+	sqlite.close();
+});
+
+test("execSync.rows returns a mapped object", async () => {
+	const sqlite = await createInMemoryDatabase({ readOnly: false });
+	const blob = await newLixFile();
+	const buf = new Uint8Array(await blob.arrayBuffer());
+	importDatabase({ db: sqlite, content: buf });
+
+	const engine = await boot({
+		sqlite,
+		emit: () => {},
+		args: {},
+	});
+
+	const result = engine.executeSync({
+		sql: "select 42 as answer, 'meaning' as label",
+	});
+
+	expect(result.rows).toEqual([{ answer: 42, label: "meaning" }]);
+});