@lix-js/sdk 0.5.0-preview.0 → 0.5.0-preview.1

package/src/state/materialize-state.ts

@@ -0,0 +1,235 @@
+import type { SqliteWasmDatabase } from "sqlite-wasm-kysely";
+
+export function applyMaterializeStateSchema(sqlite: SqliteWasmDatabase): void {
+	// View 1: Version tips - one row per version with its current tip commit
+	// Rule: "if a version entity exists, the version is active. even if other versions 'build' on this version by branching away from the commit"
+	// A commit C is the tip for version V iff:
+	// • C appears in at least one lix_version change row for V AND
+	// • there is no commit that is both a child of C in lix_commit_edge table AND referenced by any lix_version row of the same version V
+	sqlite.exec(`
+		CREATE VIEW IF NOT EXISTS internal_materialization_version_tips AS
+		WITH
+		-- 1. every (version, commit) ever recorded
+		version_commits(version_id, commit_id) AS (
+			SELECT
+				v.entity_id,
+				json_extract(v.snapshot_content,'$.commit_id')
+			FROM change v
+			WHERE v.schema_key = 'lix_version'
+		),
+		-- 2. mark (version, commit) pairs that still have a child commit referenced by the same version
+		non_tips AS (
+			SELECT DISTINCT
+				vc.version_id,
+				vc.commit_id
+			FROM version_commits vc
+			JOIN change e -- commit-edge
+				ON e.schema_key = 'lix_commit_edge'
+				AND json_extract(e.snapshot_content,'$.parent_id') = vc.commit_id
+			JOIN version_commits vc_child
+				ON vc_child.commit_id = json_extract(e.snapshot_content,'$.child_id')
+				AND vc_child.version_id = vc.version_id -- same version!
+		)
+		-- 3. tips = version commits that are NOT in the non_tip set
+		SELECT
+			vc.version_id,
+			vc.commit_id AS tip_commit_id
+		FROM version_commits vc
+		WHERE NOT EXISTS (
+			SELECT 1
+			FROM non_tips nt
+			WHERE nt.version_id = vc.version_id
+			AND nt.commit_id = vc.commit_id
+		);
+	`);
+
+	// View 2: Commit graph - lineage with depth (combines old views 3 & 4)
+	sqlite.exec(`
+		CREATE VIEW IF NOT EXISTS internal_materialization_commit_graph AS
+		WITH RECURSIVE commit_paths(commit_id, version_id, depth, path) AS (
+			-- Start from version tips at depth 0
+			SELECT 
+				tip_commit_id, 
+				version_id, 
+				0,
+				',' || tip_commit_id || ',' -- Path for cycle detection
+			FROM internal_materialization_version_tips
+			
+			UNION ALL
+			
+			-- Walk up parent chain, incrementing depth
+			SELECT 
+				json_extract(edge.snapshot_content,'$.parent_id'),
+				g.version_id,
+				g.depth + 1,
+				g.path || json_extract(edge.snapshot_content,'$.parent_id') || ','
+			FROM change edge
+			JOIN commit_paths g ON json_extract(edge.snapshot_content,'$.child_id') = g.commit_id
+			WHERE edge.schema_key = 'lix_commit_edge'
+			  AND json_extract(edge.snapshot_content,'$.parent_id') IS NOT NULL
+			  -- Cycle detection: stop if parent already in path
+			  AND INSTR(g.path, ',' || json_extract(edge.snapshot_content,'$.parent_id') || ',') = 0
+		)
+		-- Group by commit_id and version_id, taking MIN depth for merge commits
+		SELECT 
+			commit_id,
+			version_id,
+			MIN(depth) as depth
+		FROM commit_paths
+		GROUP BY commit_id, version_id;
+	`);
+
+	// View 3: Latest visible state - first seen wins, with proper timestamps
+	sqlite.exec(`
+		CREATE VIEW IF NOT EXISTS internal_materialization_latest_visible_state AS
+		WITH commit_changes AS (
+			SELECT 
+				cg.version_id,
+				cg.commit_id,
+				cg.depth,
+				c.id as change_id,
+				c.entity_id,
+				c.schema_key,
+				c.file_id,
+				c.plugin_key,
+				c.snapshot_content,
+				c.schema_version,
+				c.created_at,
+				ROW_NUMBER() OVER (
+					PARTITION BY cg.version_id, c.entity_id, c.schema_key, c.file_id
+					ORDER BY cg.depth ASC
+				) as first_seen,
+				-- Get timestamps based on commit graph depth, not wall-clock time
+				-- Deepest ancestor (highest depth) → created_at
+				FIRST_VALUE(c.created_at) OVER (
+					PARTITION BY cg.version_id, c.entity_id, c.schema_key, c.file_id
+					ORDER BY cg.depth DESC
+					ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
+				) as entity_created_at,
+				-- Current tip (depth 0) → updated_at  
+				FIRST_VALUE(c.created_at) OVER (
+					PARTITION BY cg.version_id, c.entity_id, c.schema_key, c.file_id
+					ORDER BY cg.depth ASC
+					ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
+				) as entity_updated_at
+			FROM internal_materialization_commit_graph cg
+			-- Get commit's change_set_id
+			JOIN change cmt ON cmt.entity_id = cg.commit_id 
+				AND cmt.schema_key = 'lix_commit'
+			-- Get changes in this change_set  
+			JOIN change cse ON cse.schema_key = 'lix_change_set_element'
+				AND json_extract(cse.snapshot_content,'$.change_set_id') = json_extract(cmt.snapshot_content,'$.change_set_id')
+			-- Get the actual change
+			JOIN change c ON c.id = json_extract(cse.snapshot_content,'$.change_id')
+		)
+		SELECT 
+			version_id,
+			commit_id,
+			depth,
+			change_id,
+			entity_id,
+			schema_key,
+			file_id,
+			plugin_key,
+			snapshot_content,
+			schema_version,
+			entity_created_at as created_at,
+			entity_updated_at as updated_at
+		FROM commit_changes 
+		WHERE first_seen = 1;
+		-- Note: We do NOT filter out NULL snapshots here to allow deletion tracking
+	`);
+
+	// View 4: Version ancestry - computes the complete ancestral lineage for each version
+	// This view recursively follows inherits_from_version_id relationships to build
+	// the full ancestry tree. Each version sees:
+	// - Itself as an ancestor at depth 0
+	// - Its direct parent at depth 1 (if it inherits)
+	// - Its grandparent at depth 2
+	// - And so on up the inheritance chain
+	// Used by the state materializer to implement multi-level inheritance where
+	// child versions can see state from all ancestors unless overridden
+	sqlite.exec(`
+		CREATE VIEW IF NOT EXISTS internal_materialization_version_ancestry AS
+		WITH RECURSIVE version_ancestry(version_id, ancestor_version_id, inheritance_depth, path) AS (
+			-- Each version is its own ancestor at depth 0
+			SELECT 
+				entity_id as version_id, 
+				entity_id as ancestor_version_id, 
+				0 as inheritance_depth,
+				',' || entity_id || ',' -- Path for cycle detection
+			FROM change 
+			WHERE schema_key = 'lix_version'
+			
+			UNION ALL
+			
+			-- Recursively find all ancestors
+			SELECT
+				va.version_id,
+				json_extract(parent_v.snapshot_content,'$.inherits_from_version_id'),
+				va.inheritance_depth + 1,
+				va.path || json_extract(parent_v.snapshot_content,'$.inherits_from_version_id') || ','
+			FROM version_ancestry va
+			JOIN change parent_v 
+				ON parent_v.entity_id = va.ancestor_version_id 
+				AND parent_v.schema_key = 'lix_version'
+			WHERE json_extract(parent_v.snapshot_content,'$.inherits_from_version_id') IS NOT NULL
+			  -- Cycle detection: stop if ancestor already in path
+			  AND INSTR(va.path, ',' || json_extract(parent_v.snapshot_content,'$.inherits_from_version_id') || ',') = 0
+		)
+		SELECT version_id, ancestor_version_id, inheritance_depth 
+		FROM version_ancestry;
+	`);
+
+	// View 5: Final state materializer with multi-level inheritance
+	sqlite.exec(`
+		CREATE VIEW IF NOT EXISTS internal_state_materializer AS
+		WITH all_possible_states AS (
+			SELECT
+				va.version_id,
+				s.entity_id,
+				s.schema_key,
+				s.file_id,
+				s.plugin_key,
+				s.snapshot_content,
+				s.schema_version,
+				s.created_at,
+				s.updated_at,
+				s.change_id,
+				s.commit_id,
+				va.ancestor_version_id,
+				va.inheritance_depth,
+				-- Rank by inheritance depth - closer ancestors win
+				ROW_NUMBER() OVER (
+					PARTITION BY va.version_id, s.entity_id, s.schema_key, s.file_id
+					ORDER BY va.inheritance_depth ASC
+				) as inheritance_rank
+			FROM internal_materialization_version_ancestry va
+			JOIN internal_materialization_latest_visible_state s
+				ON s.version_id = va.ancestor_version_id
+		)
+		SELECT 
+			entity_id,
+			schema_key,
+			file_id,
+			version_id,
+			plugin_key,
+			snapshot_content,
+			schema_version,
+			created_at,
+			updated_at,
+			CASE 
+				WHEN inheritance_depth > 0 THEN ancestor_version_id 
+				ELSE NULL 
+			END as inherited_from_version_id,
+			0 as untracked,
+			change_id,
+			commit_id
+		FROM all_possible_states
+		-- inheritance_rank = 1 means this is the closest ancestor that has state for this entity
+		-- (rank 1 = direct state, rank 2 = parent state, rank 3 = grandparent state, etc.)
+		-- We only want the most specific/closest state, not all inherited variants
+		WHERE inheritance_rank = 1;
+		-- Note: Now includes tombstones (null snapshot_content) for proper deletion handling
+	`);
+}

package/src/state/primary-key.test.ts

@@ -0,0 +1,158 @@
+import { test, expect } from "vitest";
+import {
+	serializeStatePk,
+	parseStatePk,
+	type StatePkTag,
+} from "./primary-key.js";
+
+test("makePk creates correct composite key for simple values", () => {
+	const pk = serializeStatePk("U", "file1", "entity1", "version1");
+	// No special characters, so no encoding needed
+	expect(pk).toBe("U~file1~entity1~version1");
+});
+
+test("makePk handles special characters in fields", () => {
+	const pk = serializeStatePk(
+		"C",
+		"file|with|pipes",
+		"entity-123",
+		"ver/sion:1"
+	);
+	// Special characters should be percent-encoded
+	expect(pk).toBe("C~file%7Cwith%7Cpipes~entity-123~ver%2Fsion%3A1");
+});
+
+test("parsePk correctly parses simple composite key", () => {
+	const result = parseStatePk("UI~file1~entity1~version1");
+	expect(result).toEqual({
+		tag: "UI",
+		fileId: "file1",
+		entityId: "entity1",
+		versionId: "version1",
+	});
+});
+
+test("parsePk handles fields with special characters", () => {
+	// Test with encoded values
+	const result = parseStatePk(
+		"CI~file%7Cwith%7Cpipes~entity-123~ver%2Fsion%3A1"
+	);
+	expect(result).toEqual({
+		tag: "CI",
+		fileId: "file|with|pipes",
+		entityId: "entity-123",
+		versionId: "ver/sion:1",
+	});
+});
+
+test("roundtrip encoding and decoding preserves data", () => {
+	const testCases: Array<[StatePkTag, string, string, string]> = [
+		["U", "simple", "test", "v1"],
+		["UI", "file-with-dash", "entity", "version"],
+		["C", "file", "entity_with_underscore", "version"],
+		["CI", "file", "entity", "version.with.dot"],
+		["U", "a|b", "c:d", "e/f"], // various special chars
+		["C", "---", "test", "v1"], // multiple dashes
+		["UI", "", "", ""], // empty strings
+		["CI", "file-", "-entity", "-version-"], // dashes at edges
+	];
+
+	for (const [tag, fileId, entityId, versionId] of testCases) {
+		const pk = serializeStatePk(tag, fileId, entityId, versionId);
+		const parsed = parseStatePk(pk);
+
+		expect(parsed).toEqual({
+			tag,
+			fileId,
+			entityId,
+			versionId,
+		});
+	}
+});
+
+test("parsePk handles edge cases", () => {
+	// Empty strings are valid values
+	const result1 = parseStatePk("U~~~");
+	expect(result1).toEqual({
+		tag: "U",
+		fileId: "",
+		entityId: "",
+		versionId: "",
+	});
+
+	// Complex field values
+	const result2 = parseStatePk(
+		"C~file/path/to/resource~com.example.entity~v1.2.3"
+	);
+	expect(result2).toEqual({
+		tag: "C",
+		fileId: "file/path/to/resource",
+		entityId: "com.example.entity",
+		versionId: "v1.2.3",
+	});
+});
+
+test("all tag types are handled", () => {
+	const tags: StatePkTag[] = ["U", "UI", "C", "CI"];
+
+	for (const tag of tags) {
+		const pk = serializeStatePk(tag, "file", "entity", "version");
+		const parsed = parseStatePk(pk);
+		expect(parsed.tag).toBe(tag);
+	}
+});
+
+test("makePk and parsePk handle Unicode correctly", () => {
+	const pk = serializeStatePk(
+		"U",
+		"文件-file",
+		"エンティティ-entity",
+		"版本-version"
+	);
+	const parsed = parseStatePk(pk);
+
+	expect(parsed).toEqual({
+		tag: "U",
+		fileId: "文件-file",
+		entityId: "エンティティ-entity",
+		versionId: "版本-version",
+	});
+});
+
+test("handles entity IDs with tildes", () => {
+	// This is the critical test - entity ID contains a tilde
+	const pk = serializeStatePk("C", "test_file", "cat1~id1", "version1");
+	// The tilde in entity ID should be encoded as %7E
+	expect(pk).toBe("C~test_file~cat1%7Eid1~version1");
+
+	// And it should parse back correctly
+	const parsed = parseStatePk(pk);
+	expect(parsed).toEqual({
+		tag: "C",
+		fileId: "test_file",
+		entityId: "cat1~id1",
+		versionId: "version1",
+	});
+});
+
+test("parsePk throws on malformed input", () => {
+	// Missing parts
+	expect(() => parseStatePk("U")).toThrow(
+		"Invalid composite key: U - expected 4 parts"
+	);
+
+	// Only tag and one field
+	expect(() => parseStatePk("C~file")).toThrow(
+		"Invalid composite key: C~file - expected 4 parts"
+	);
+
+	// Two fields
+	expect(() => parseStatePk("UI~file~entity")).toThrow(
+		"Invalid composite key: UI~file~entity - expected 4 parts"
+	);
+
+	// Too many fields
+	expect(() => parseStatePk("CI~file~entity~version~extra")).toThrow(
+		"Invalid composite key: CI~file~entity~version~extra - expected 4 parts"
+	);
+});

package/src/state/primary-key.ts

@@ -0,0 +1,80 @@
+/**
+ * Utilities for encoding/decoding composite primary keys for the resolved state virtual table.
+ *
+ * The composite key format is: tag~file_id~entity_id~version_id
+ * Using ~ as delimiter since it's URL-safe and unlikely to appear in identifiers
+ *
+ * Tag meanings:
+ *
+ * - U: Untracked direct (from internal_state_all_untracked in child version)
+ * - UI: Untracked inherited (from internal_state_all_untracked in parent version)
+ * - C: Cache-tracked direct (from internal_state_cache in child version)
+ * - CI: Cache-tracked inherited (from internal_state_cache in parent version)
+ */
+
+export type StatePkTag = "U" | "UI" | "C" | "CI";
+
+const DELIM = "~"; // still our global separator
+const DELIM_ENC = "%7E"; // RFC 3986 percent-encoding for '~'
+
+/** Encode one segment so it cannot contain the delimiter or raw '%' */
+export function encodeStatePkPart(part: string): string {
+	// encodeURIComponent protects '%' and other reserved chars,
+	// then we additionally escape the delimiter itself.
+	return encodeURIComponent(part).replace(/~/g, DELIM_ENC);
+}
+
+/** Reverse of encodeStatePkPart */
+function decodeStatePkPart(encoded: string): string {
+	// decodeURIComponent happily turns %7E → ~ and %25 → %
+	return decodeURIComponent(encoded);
+}
+
+/**
+ * Build primary key string from components.
+ *
+ * @param tag - The tag indicating the source and ownership
+ * @param fileId - The file ID
+ * @param entityId - The entity ID
+ * @param versionId - The version ID
+ * @returns The encoded composite key
+ */
+export function serializeStatePk(
+	tag: StatePkTag,
+	fileId: string,
+	entityId: string,
+	versionId: string
+): string {
+	return [
+		tag,
+		encodeStatePkPart(fileId),
+		encodeStatePkPart(entityId),
+		encodeStatePkPart(versionId),
+	].join(DELIM);
+}
+
+/**
+ * Parse primary key back into parts.
+ *
+ * @param pk - The encoded composite key
+ * @returns The decoded components
+ */
+export function parseStatePk(pk: string): {
+	tag: StatePkTag;
+	fileId: string;
+	entityId: string;
+	versionId: string;
+} {
+	const parts = pk.split(DELIM);
+
+	if (parts.length !== 4) {
+		throw new Error(`Invalid composite key: ${pk} - expected 4 parts`);
+	}
+
+	return {
+		tag: parts[0] as StatePkTag,
+		fileId: decodeStatePkPart(parts[1] ?? ""),
+		entityId: decodeStatePkPart(parts[2] ?? ""),
+		versionId: decodeStatePkPart(parts[3] ?? ""),
+	};
+}