@lix-js/sdk 0.5.0-preview.0 → 0.5.0-preview.1

package/src/schema-definition/definition.ts

@@ -1,5 +1,38 @@
 import type { FromSchema, JSONSchema } from "json-schema-to-ts";
 
+/**
+ * Foreign key constraint definition
+ */
+export type LixForeignKey = {
+	/**
+	 * Local JSON-schema property names that participate in the FK.
+	 * Must have at least one property.
+	 */
+	properties: readonly string[] | string[];
+
+	/**
+	 * Where they point to.
+	 */
+	references: {
+		/**
+		 * The x-lix-key of the referenced schema
+		 */
+		schemaKey: string;
+		/**
+		 * Remote property names (must have same length as local properties)
+		 */
+		properties: readonly string[] | string[];
+		/**
+		 * Optional version of the referenced schema
+		 */
+		schemaVersion?: string;
+	};
+
+	// Future features - not implemented yet
+	// onDelete?: "cascade" | "restrict" | "set null";
+	// onUpdate?: "cascade" | "restrict";
+};
+
 export const LixSchemaDefinition = {
 	$schema: "http://json-schema.org/draft-07/schema#",
 	title: "Lix Change Schema",
@@ -28,25 +61,51 @@ export const LixSchemaDefinition = {
 					},
 				},
 				"x-lix-foreign-keys": {
-					type: "object",
-					additionalProperties: {
+					type: "array",
+					items: {
 						type: "object",
+						required: ["properties", "references"],
 						properties: {
-							schemaKey: {
-								type: "string",
-								description: "The x-lix-key of the referenced schema",
-							},
-							property: {
-								type: "string",
-								description: "The property name in the referenced schema",
+							properties: {
+								type: "array",
+								minItems: 1,
+								items: { type: "string" },
+								description:
+									"Local JSON-schema property names that participate in the FK",
 							},
-							schemaVersion: {
-								type: "string",
-								pattern: "^\\d+\\.\\d+$",
-								description: "Optional version of the referenced schema",
+							references: {
+								type: "object",
+								required: ["schemaKey", "properties"],
+								properties: {
+									schemaKey: {
+										type: "string",
+										description: "The x-lix-key of the referenced schema",
+									},
+									properties: {
+										type: "array",
+										minItems: 1,
+										items: { type: "string" },
+										description:
+											"Remote property names (same length as local properties)",
+									},
+									schemaVersion: {
+										type: "string",
+										pattern: "^\\d+\\.\\d+$",
+										description: "Optional version of the referenced schema",
+									},
+								},
 							},
+							// onDelete: {
+							//   type: "string",
+							//   enum: ["cascade", "restrict", "set null"],
+							//   description: "Action to take when referenced entity is deleted (future feature)"
+							// },
+							// onUpdate: {
+							//   type: "string",
+							//   enum: ["cascade", "restrict"],
+							//   description: "Action to take when referenced entity is updated (future feature)"
+							// },
 						},
-						required: ["schemaKey", "property"],
 					},
 				},
 				"x-lix-key": {
@@ -149,26 +208,24 @@ export type LixSchemaDefinition = JSONSchema & {
 	 * Foreign key constraints referencing other schemas.
 	 *
 	 * @example
-	 *   {
-	 *     "x-lix-foreign-keys": {
-	 *       "author_id": {
+	 *   [
+	 *     {
+	 *       "properties": ["author_id"],
+	 *       "references": {
 	 *         "schemaKey": "user_profile",
-	 *         "property": "id"
-	 *       },
-	 *       "category_id": {
-	 *         "schemaKey": "post_category",
-	 *         "property": "id"
+	 *         "properties": ["id"]
+	 *       }
+	 *     },
+	 *     {
+	 *       "properties": ["entity_id", "schema_key", "file_id"],
+	 *       "references": {
+	 *         "schemaKey": "state",
+	 *         "properties": ["entity_id", "schema_key", "file_id"]
 	 *       }
 	 *     }
-	 *   }
+	 *   ]
 	 */
-	"x-lix-foreign-keys"?: {
-		[localProperty: string]: {
-			schemaKey: string;
-			property: string;
-			schemaVersion?: string;
-		};
-	};
+	"x-lix-foreign-keys"?: LixForeignKey[] | readonly LixForeignKey[];
 	type: "object";
 	properties?: {
 		[key: string]: LixPropertySchema;

package/src/schema-definition/index.ts

@@ -10,4 +10,4 @@ export { JSONTypeSchema, type JSONType } from "./json-type.js";
 export {
 	validateLixSchema,
 	validateLixSchemaDefinition,
-} from "./validate-lix-schema.js";
+} from "./validate-lix-schema.js";

package/src/schema-definition/validate-lix-schema.test.ts

@@ -1,5 +1,8 @@
 import { test, expect } from "vitest";
-import { validateLixSchema, validateLixSchemaDefinition } from "./validate-lix-schema.js";
+import {
+	validateLixSchema,
+	validateLixSchemaDefinition,
+} from "./validate-lix-schema.js";
 
 test("validateLixSchemaDefinition passes for valid Lix schema", () => {
 	const validSchema = {
@@ -25,7 +28,9 @@ test("validateLixSchemaDefinition throws for invalid schema", () => {
 		// Missing required x-lix-key
 	};
 
-	expect(() => validateLixSchemaDefinition(invalidSchema)).toThrow("Invalid Lix schema definition");
+	expect(() => validateLixSchemaDefinition(invalidSchema)).toThrow(
+		"Invalid Lix schema definition"
+	);
 });
 
 test("validateLixSchema validates both schema and data successfully", () => {
@@ -61,7 +66,9 @@ test("validateLixSchema throws when schema is invalid", () => {
 
 	const data = { id: "123" };
 
-	expect(() => validateLixSchema(invalidSchema, data)).toThrow("Invalid Lix schema definition");
+	expect(() => validateLixSchema(invalidSchema, data)).toThrow(
+		"Invalid Lix schema definition"
+	);
 });
 
 test("validateLixSchema throws when data doesn't match schema", () => {
@@ -81,5 +88,7 @@ test("validateLixSchema throws when data doesn't match schema", () => {
 		// Missing required 'name' field
 	};
 
-	expect(() => validateLixSchema(schema, invalidData)).toThrow("Data validation failed");
-});
+	expect(() => validateLixSchema(schema, invalidData)).toThrow(
+		"Data validation failed"
+	);
+});

package/src/schema-definition/validate-lix-schema.ts

@@ -13,12 +13,12 @@ const _validateLixSchemaDefinition = ajv.compile(LixSchemaDefinition);
 /**
  * Validates that a schema conforms to the LixSchemaDefinition format
  * and then validates data against that schema.
- * 
+ *
  * @param schema - The Lix schema definition to validate against
  * @param data - The data to validate
  * @returns true if both the schema and data are valid
  * @throws Error with validation details if either validation fails
- * 
+ *
  * @example
  * ```typescript
  * const userSchema = {
@@ -32,13 +32,13 @@ const _validateLixSchemaDefinition = ajv.compile(LixSchemaDefinition);
  *   },
  *   required: ["id", "name"]
  * } as const satisfies LixSchemaDefinition;
- * 
+ *
  * const userData = {
  *   id: "123",
  *   name: "John Doe",
  *   age: 30
  * };
- * 
+ *
  * try {
  *   validateLixSchema(userSchema, userData); // returns true
  * } catch (error) {
@@ -46,10 +46,7 @@ const _validateLixSchemaDefinition = ajv.compile(LixSchemaDefinition);
  * }
  * ```
  */
-export function validateLixSchema(
-	schema: unknown,
-	data: unknown
-): boolean {
+export function validateLixSchema(schema: unknown, data: unknown): boolean {
 	// First validate that the schema itself is valid
 	const schemaValid = _validateLixSchemaDefinition(schema);
 	if (!schemaValid) {
@@ -73,22 +70,22 @@ export function validateLixSchema(
 /**
  * Validates only the Lix schema definition format.
  * Use this when you only need to check if a schema is valid.
- * 
+ *
  * @param schema - The schema to validate
  * @returns true if the schema is valid
  * @throws Error with validation details if validation fails
- * 
+ *
  * @example
  * ```typescript
  * const schema = {
- *   "x-lix-key": "product", 
+ *   "x-lix-key": "product",
  *   "x-lix-version": "1.0",
  *   type: "object",
  *   properties: {
  *     id: { type: "string" }
  *   }
  * };
- * 
+ *
  * try {
  *   validateLixSchemaDefinition(schema); // returns true
  * } catch (error) {
@@ -105,4 +102,3 @@ export function validateLixSchemaDefinition(schema: unknown): boolean {
 	}
 	return valid;
 }
-

package/src/server-protocol-handler/environment/create-in-memory-environment.test.ts

@@ -14,7 +14,7 @@ test.skip("opening a lix works", async () => {
 		.executeTakeFirstOrThrow()) as { value: string };
 
 	// initialize the env with the lix file
-	environment.setLix({ id: lixId, blob: await mockLix.toBlob() });
+	await environment.setLix({ id: lixId, blob: await mockLix.toBlob() });
 
 	const open0 = await environment.openLix({ id: lixId });
 
@@ -61,7 +61,7 @@ test.todo("it handles concurrent connections", async () => {
 		.executeTakeFirstOrThrow()) as { value: string };
 
 	// initialize the env with the lix file
-	environment.setLix({ id: lixId, blob: await mockLix.toBlob() });
+	await environment.setLix({ id: lixId, blob: await mockLix.toBlob() });
 
 	const open0 = await environment.openLix({ id: lixId });
 	const open1 = await environment.openLix({ id: lixId });

package/src/server-protocol-handler/routes/push-v1.test.ts

@@ -130,7 +130,7 @@ test.skip("it should return 400 for a failed insert operation", async () => {
 
 	const environment = createLspInMemoryEnvironment();
 
-	environment.setLix({ id, blob: await lix.toBlob() });
+	await environment.setLix({ id, blob: await lix.toBlob() });
 
 	const lsa = await createServerProtocolHandler({ environment });
 

package/src/snapshot/schema.test.ts

@@ -130,6 +130,119 @@ test("can insert null content explicitly", async () => {
 	expect(snapshot.content).toBeNull();
 });
 
+test("snapshot content validation - accepts JSONB", async () => {
+	const lix = await openLix({});
+
+	// This should succeed - content is properly formatted as JSONB
+	await (lix.db as unknown as Kysely<LixInternalDatabaseSchema>)
+		.insertInto("internal_snapshot")
+		.values({
+			id: "valid-jsonb",
+			content: sql`jsonb(${JSON.stringify({ test: "data", value: 123 })})`,
+		})
+		.execute();
+
+	const snapshot = await (
+		lix.db as unknown as Kysely<LixInternalDatabaseSchema>
+	)
+		.selectFrom("internal_snapshot")
+		.where("id", "=", "valid-jsonb")
+		.select(["id", (eb) => eb.fn("json", [`content`]).as("content")])
+		.executeTakeFirstOrThrow();
+
+	expect(snapshot.content).toEqual({ test: "data", value: 123 });
+});
+
+test("snapshot content validation - rejects text JSON", async () => {
+	const lix = await openLix({});
+
+	// This should fail - content is text JSON, not JSONB
+	// SQLite's STRICT tables enforce that BLOB columns can't store TEXT
+	await expect(
+		(lix.db as unknown as Kysely<LixInternalDatabaseSchema>)
+			.insertInto("internal_snapshot")
+			.values({
+				id: "invalid-text-json",
+				content: sql`${JSON.stringify({ test: "data" })}`,
+			})
+			.execute()
+	).rejects.toThrow(
+		/SQLITE_CONSTRAINT_DATATYPE.*cannot store TEXT value in BLOB column/
+	);
+});
+
+test("snapshot content validation - rejects arbitrary binary", async () => {
+	const lix = await openLix({});
+
+	// This should fail - content is arbitrary binary data, not valid JSONB
+	// Using raw SQL to insert arbitrary binary that's not JSONB format
+	const encoder = new TextEncoder();
+	const arbitraryBytes = encoder.encode("arbitrary binary data");
+
+	await expect(async () => {
+		lix.sqlite.exec({
+			sql: `INSERT INTO internal_snapshot (id, content) VALUES (?, ?)`,
+			bind: ["invalid-binary", arbitraryBytes],
+		});
+	}).rejects.toThrow(/CHECK constraint failed.*json_valid/);
+});
+
+test("snapshot content validation - prevents double-stringified JSON storage", async () => {
+	const lix = await openLix({});
+
+	// This test demonstrates the double-stringified JSON issue we were investigating
+	const originalData = { id: "test", name: "value" };
+	const onceStringified = JSON.stringify(originalData);
+	const doubleStringified = JSON.stringify(onceStringified);
+
+	// First, verify that plain text JSON is rejected by STRICT table
+	await expect(
+		(lix.db as unknown as Kysely<LixInternalDatabaseSchema>)
+			.insertInto("internal_snapshot")
+			.values({
+				id: "double-stringified",
+				content: sql`${doubleStringified}`,
+			})
+			.execute()
+	).rejects.toThrow(
+		/SQLITE_CONSTRAINT_DATATYPE.*cannot store TEXT value in BLOB column/
+	);
+
+	// The real issue: jsonb() on double-stringified JSON creates valid JSONB
+	// because a JSON string is valid JSON! But with our new json_type check,
+	// this should now be rejected since it's storing a string, not an object
+	expect(() => {
+		lix.sqlite.exec({
+			sql: `INSERT INTO internal_snapshot (id, content) VALUES (?, jsonb(?))`,
+			bind: ["double-stringified-jsonb", doubleStringified],
+		});
+	}).toThrow(/CHECK constraint failed/);
+
+	// Also verify that storing arrays is rejected (we only want objects)
+	expect(() => {
+		lix.sqlite.exec({
+			sql: `INSERT INTO internal_snapshot (id, content) VALUES (?, jsonb(?))`,
+			bind: ["array-content", JSON.stringify([1, 2, 3])],
+		});
+	}).toThrow(/CHECK constraint failed/);
+
+	// Verify the correct way: jsonb() on properly stringified JSON
+	lix.sqlite.exec({
+		sql: `INSERT INTO internal_snapshot (id, content) VALUES (?, jsonb(?))`,
+		bind: ["correct-jsonb", onceStringified],
+	});
+
+	// Verify it retrieves correctly
+	const correctResult: any = lix.sqlite.exec({
+		sql: `SELECT json(content) as json_content FROM internal_snapshot WHERE id = ?`,
+		bind: ["correct-jsonb"],
+		returnValue: "resultRows",
+	});
+
+	expect(correctResult[0]?.[0]).toBe(onceStringified);
+	expect(JSON.parse(correctResult[0]![0]!)).toEqual(originalData);
+});
+
 test("snapshot ids must be unique", async () => {
 	const lix = await openLix({});
 

package/src/snapshot/schema.ts

@@ -6,8 +6,16 @@ export function applySnapshotDatabaseSchema(
 ): SqliteWasmDatabase {
 	return sqlite.exec(`
   CREATE TABLE IF NOT EXISTS internal_snapshot (
-    id TEXT PRIMARY KEY DEFAULT (uuid_v7()),
-    content BLOB -- jsonb or binary file
+    id TEXT PRIMARY KEY DEFAULT (lix_uuid_v7()),
+    content BLOB -- jsonb,
+
+    -- 8 = strictly JSONB
+    -- https://www.sqlite.org/json1.html#jvalid
+    CHECK (json_valid(content, 8)),
+    
+    -- Ensure content is either NULL or a JSON object (not string, array, etc)
+    -- This prevents double-stringified JSON from being stored
+    CHECK (content IS NULL OR json_type(content) = 'object')
   ) STRICT;
 
   INSERT OR IGNORE INTO internal_snapshot (id, content)

package/src/state/README.md

@@ -0,0 +1,95 @@
+# State Architecture
+
+## Overview
+
+The Lix state system manages all entity changes through a [SQLite virtual table](https://www.sqlite.org/vtab.html). This virtual table serves as the single entry point for all mutations, ensuring consistent change tracking, changeset creation, and state persistence.
+
+## State Mutation Flow
+
+All state mutations flow through a SQLite virtual table that orchestrates the three-stage lifecycle:
+
+```
+     User SQL Operations
+            │
+            v
+    ┌─────────────────────┐
+    │ State Virtual Table │  <── Entry point for ALL mutations
+    └───────┬─────────────┘
+            │
+            v
+┌─────────────────┐     ┌──────────────────┐     ┌─────────────────┐
+│    MUTATION     │ --> │   TRANSACTION    │ --> │     COMMIT      │
+└─────────────────┘     └──────────────────┘     └─────────────────┘
+        │                        │                         │
+        v                        v                         v
+   Validate & Track        Group Changes           Persist to
+      Changes              in Changesets          SQLite Tables
+```
+
+The virtual table:
+
+- Intercepts INSERT, UPDATE, and DELETE operations
+- Tracks every change for audit and history
+- Groups related changes into changesets during transactions
+- Delegates to regular SQLite tables for actual storage
+
+### Stage Characteristics
+
+#### 1. MUTATION Stage
+
+- **Purpose**: Handle individual state changes
+- **Scope**: Single entity operation
+- **Storage**: None (passes through)
+- **Rollback**: Not applicable
+
+#### 2. TRANSACTION Stage
+
+- **Purpose**: Accumulate multiple mutations
+- **Scope**: Multiple entities, single version
+- **Storage**: Temporary (`internal_change_in_transaction`)
+- **Rollback**: Automatic on error or explicit rollback
+
+#### 3. COMMIT Stage
+
+- **Purpose**: Persist changes permanently
+- **Scope**: All pending changes across versions
+- **Storage**: Permanent (`internal_change`, `internal_snapshot`)
+- **Rollback**: Not possible after commit
+
+## Virtual Table Architecture
+
+The virtual table acts as a facade over the actual storage mechanism:
+
+```
+┌─────────────────────────────────────────────────┐
+│              User SQL Queries                   │
+│  (SELECT, INSERT, UPDATE, DELETE on entities)   │
+└─────────────────────┬───────────────────────────┘
+                      │
+                      v
+┌─────────────────────────────────────────────────┐
+│           State Virtual Table                   │
+│                                                 │
+│  • Intercepts all operations                    │
+│  • Maintains change history                     │
+│  • Ensures transactional consistency            │
+│  • Manages cache coherency                      │
+└─────────────────────┬───────────────────────────┘
+                      │
+                      v
+┌─────────────────────────────────────────────────┐
+│         Regular SQLite Tables                   │
+│                                                 │
+│  • internal_change_in_transaction               │
+│  • internal_state_cache                         │
+│  • internal_change                              │
+│  • internal_snapshot                            │
+└─────────────────────────────────────────────────┘
+```
+
+The virtual table provides:
+
+- **Transparent change tracking** - Users work with entities naturally while all changes are tracked
+- **ACID guarantees** - Leverages SQLite's transactional properties
+- **Performance optimization** - Maintains caches and indexes automatically
+- **Version isolation** - Each version sees its own consistent view of the data

package/src/state/cache/clear-state-cache.test.ts

@@ -0,0 +1,43 @@
+import { test, expect } from "vitest";
+import { clearStateCache } from "./clear-state-cache.js";
+import { isStaleStateCache } from "./is-stale-state-cache.js";
+import { openLix } from "../../lix/open-lix.js";
+import type { LixInternalDatabaseSchema } from "../../database/schema.js";
+import type { Kysely } from "kysely";
+
+test("clearStateCache deletes all cache entries", async () => {
+	const lix = await openLix({});
+	const internalDb = lix.db as unknown as Kysely<LixInternalDatabaseSchema>;
+
+	// Insert some data to populate cache
+	await lix.db
+		.insertInto("key_value")
+		.values({
+			key: "test_key",
+			value: "test_value",
+		})
+		.execute();
+
+	// Verify cache has entries
+	const cacheBeforeClear = await internalDb
+		.selectFrom("internal_state_cache")
+		.selectAll()
+		.execute();
+
+	expect(cacheBeforeClear.length).toBeGreaterThan(0);
+
+	// Clear the cache
+	clearStateCache({ lix });
+
+	// Verify cache is empty
+	const cacheAfterClear = await internalDb
+		.selectFrom("internal_state_cache")
+		.selectAll()
+		.execute();
+
+	expect(cacheAfterClear.length).toBe(0);
+
+	// Verify the cache is marked as stale
+	const isStale = isStaleStateCache({ lix });
+	expect(isStale).toBe(true);
+});

package/src/state/cache/clear-state-cache.ts

@@ -0,0 +1,16 @@
+import type { Lix } from "../../lix/open-lix.js";
+import { markStateCacheAsStale } from "./mark-state-cache-as-stale.js";
+
+/**
+ * Clears the internal state cache.
+ */
+export function clearStateCache(args: { lix: Lix }): void {
+	// Mark the cache as stale first to prevent repopulation during delete
+	markStateCacheAsStale({ lix: args.lix });
+
+	// Delete all entries from the cache
+	args.lix.sqlite.exec({
+		sql: `DELETE FROM internal_state_cache`,
+		returnValue: "resultRows",
+	});
+}

package/src/state/cache/is-stale-state-cache.test.ts

@@ -0,0 +1,35 @@
+import { test, expect } from "vitest";
+import { isStaleStateCache } from "./is-stale-state-cache.js";
+import { clearStateCache } from "./clear-state-cache.js";
+import { openLix } from "../../lix/open-lix.js";
+import type { LixInternalDatabaseSchema } from "../../database/schema.js";
+import type { Kysely } from "kysely";
+
+test("cache is stale after cache clear", async () => {
+	const lix = await openLix({});
+	const internalDb = lix.db as unknown as Kysely<LixInternalDatabaseSchema>;
+
+	// Insert some data to ensure cache is populated
+	await lix.db
+		.insertInto("key_value")
+		.values({
+			key: "test_key",
+			value: "test_value",
+		})
+		.execute();
+
+	const cache = await internalDb
+		.selectFrom("internal_state_cache")
+		.selectAll()
+		.execute();
+
+	expect(cache.length).toBeGreaterThan(0);
+
+	// Clear the cache
+	clearStateCache({ lix });
+
+	// Cache should be stale after clearing
+	const result = isStaleStateCache({ lix });
+
+	expect(result).toBe(true);
+});

package/src/state/cache/is-stale-state-cache.ts

@@ -0,0 +1,29 @@
+import { sql, type Kysely } from "kysely";
+import { executeSync } from "../../database/execute-sync.js";
+import type { LixInternalDatabaseSchema } from "../../database/schema.js";
+import type { Lix } from "../../lix/open-lix.js";
+
+const CACHE_STALE_KEY = "lix_state_cache_stale";
+
+export function isStaleStateCache(args: {
+	lix: Pick<Lix, "sqlite" | "db">;
+}): boolean {
+	// Query the resolved view directly to avoid recursion on the vtable itself
+	const res = executeSync({
+		lix: args.lix,
+		query: (args.lix.db as unknown as Kysely<LixInternalDatabaseSchema>)
+			.selectFrom("internal_resolved_state_all")
+			.where("entity_id", "=", CACHE_STALE_KEY)
+			.where("schema_key", "=", "lix_key_value")
+			.where("version_id", "=", "global")
+			.select(sql`json_extract(snapshot_content, '$.value')`.as("value")),
+	});
+
+	// If no flag exists, cache is stale by default
+	if (!res || res.length === 0) {
+		return true;
+	}
+
+	// Parse the snapshot content to get the value
+	return res[0]?.value == true;
+}