@durable-streams/state 0.1.0

package/dist/index.js

@@ -0,0 +1,530 @@
+import { createCollection, createOptimisticAction } from "@tanstack/db";
+import { DurableStream } from "@durable-streams/client";
+
+//#region src/types.ts
+/**
+* Type guard to check if an event is a change event
+*/
+function isChangeEvent(event) {
+	return event != null && `operation` in event.headers;
+}
+/**
+* Type guard to check if an event is a control event
+*/
+function isControlEvent(event) {
+	return event != null && `control` in event.headers;
+}
+
+//#endregion
+//#region src/materialized-state.ts
+/**
+* MaterializedState maintains an in-memory view of state from change events.
+*
+* It organizes data by type, where each type contains a map of key -> value.
+* This supports multi-type streams where different entity types can coexist.
+*/
+var MaterializedState = class {
+	data;
+	constructor() {
+		this.data = new Map();
+	}
+	/**
+	* Apply a single change event to update the materialized state
+	*/
+	apply(event) {
+		const { type, key, value, headers } = event;
+		let typeMap = this.data.get(type);
+		if (!typeMap) {
+			typeMap = new Map();
+			this.data.set(type, typeMap);
+		}
+		switch (headers.operation) {
+			case `insert`:
+				typeMap.set(key, value);
+				break;
+			case `update`:
+				typeMap.set(key, value);
+				break;
+			case `upsert`:
+				typeMap.set(key, value);
+				break;
+			case `delete`:
+				typeMap.delete(key);
+				break;
+		}
+	}
+	/**
+	* Apply a batch of change events
+	*/
+	applyBatch(events) {
+		for (const event of events) this.apply(event);
+	}
+	/**
+	* Get a specific value by type and key
+	*/
+	get(type, key) {
+		const typeMap = this.data.get(type);
+		if (!typeMap) return void 0;
+		return typeMap.get(key);
+	}
+	/**
+	* Get all entries for a specific type
+	*/
+	getType(type) {
+		return this.data.get(type) || new Map();
+	}
+	/**
+	* Clear all state
+	*/
+	clear() {
+		this.data.clear();
+	}
+	/**
+	* Get the number of types in the state
+	*/
+	get typeCount() {
+		return this.data.size;
+	}
+	/**
+	* Get all type names
+	*/
+	get types() {
+		return Array.from(this.data.keys());
+	}
+};
+
+//#endregion
+//#region src/stream-db.ts
+/**
+* Internal event dispatcher that routes stream events to collection handlers
+*/
+var EventDispatcher = class {
+	/** Map from event type to collection handler */
+	handlers = new Map();
+	/** Handlers that have pending writes (need commit) */
+	pendingHandlers = new Set();
+	/** Whether we've received the initial up-to-date signal */
+	isUpToDate = false;
+	/** Resolvers and rejecters for preload promises */
+	preloadResolvers = [];
+	preloadRejecters = [];
+	/** Set of all txids that have been seen and committed */
+	seenTxids = new Set();
+	/** Txids collected during current batch (before commit) */
+	pendingTxids = new Set();
+	/** Resolvers waiting for specific txids */
+	txidResolvers = new Map();
+	/** Track existing keys per collection for upsert logic */
+	existingKeys = new Map();
+	/**
+	* Register a handler for a specific event type
+	*/
+	registerHandler(eventType, handler) {
+		this.handlers.set(eventType, handler);
+		if (!this.existingKeys.has(eventType)) this.existingKeys.set(eventType, new Set());
+	}
+	/**
+	* Dispatch a change event to the appropriate collection.
+	* Writes are buffered until commit() is called via markUpToDate().
+	*/
+	dispatchChange(event) {
+		if (!isChangeEvent(event)) return;
+		if (event.headers.txid && typeof event.headers.txid === `string`) this.pendingTxids.add(event.headers.txid);
+		const handler = this.handlers.get(event.type);
+		if (!handler) return;
+		let operation = event.headers.operation;
+		if (operation !== `delete`) {
+			if (typeof event.value !== `object` || event.value === null) throw new Error(`StreamDB collections require object values; got ${typeof event.value} for type=${event.type}, key=${event.key}`);
+		}
+		const originalValue = event.value ?? {};
+		const value = { ...originalValue };
+		value[handler.primaryKey] = event.key;
+		if (!this.pendingHandlers.has(handler)) {
+			handler.begin();
+			this.pendingHandlers.add(handler);
+		}
+		if (operation === `upsert`) {
+			const keys$1 = this.existingKeys.get(event.type);
+			const existing = keys$1?.has(event.key);
+			operation = existing ? `update` : `insert`;
+		}
+		const keys = this.existingKeys.get(event.type);
+		if (operation === `insert` || operation === `update`) keys?.add(event.key);
+		else keys?.delete(event.key);
+		try {
+			handler.write(value, operation);
+		} catch (error) {
+			console.error(`[StreamDB] Error in handler.write():`, error);
+			console.error(`[StreamDB] Event that caused error:`, {
+				type: event.type,
+				key: event.key,
+				operation
+			});
+			throw error;
+		}
+	}
+	/**
+	* Handle control events from the stream JSON items
+	*/
+	dispatchControl(event) {
+		if (!isControlEvent(event)) return;
+		switch (event.headers.control) {
+			case `reset`:
+				for (const handler of this.handlers.values()) handler.truncate();
+				for (const keys of this.existingKeys.values()) keys.clear();
+				this.pendingHandlers.clear();
+				this.isUpToDate = false;
+				break;
+			case `snapshot-start`:
+			case `snapshot-end`: break;
+		}
+	}
+	/**
+	* Commit all pending writes and handle up-to-date signal
+	*/
+	markUpToDate() {
+		for (const handler of this.pendingHandlers) try {
+			handler.commit();
+		} catch (error) {
+			console.error(`[StreamDB] Error in handler.commit():`, error);
+			if (error instanceof Error && error.message.includes(`already exists in the collection`) && error.message.includes(`live-query`)) {
+				console.warn(`[StreamDB] Known TanStack DB groupBy bug detected - continuing despite error`);
+				console.warn(`[StreamDB] Queries with groupBy may show stale data until fixed`);
+				continue;
+			}
+			throw error;
+		}
+		this.pendingHandlers.clear();
+		for (const txid of this.pendingTxids) {
+			this.seenTxids.add(txid);
+			const resolvers = this.txidResolvers.get(txid);
+			if (resolvers) {
+				for (const { resolve, timeoutId } of resolvers) {
+					clearTimeout(timeoutId);
+					resolve();
+				}
+				this.txidResolvers.delete(txid);
+			}
+		}
+		this.pendingTxids.clear();
+		if (!this.isUpToDate) {
+			this.isUpToDate = true;
+			for (const handler of this.handlers.values()) handler.markReady();
+			for (const resolve of this.preloadResolvers) resolve();
+			this.preloadResolvers = [];
+		}
+	}
+	/**
+	* Wait for the stream to reach up-to-date state
+	*/
+	waitForUpToDate() {
+		if (this.isUpToDate) return Promise.resolve();
+		return new Promise((resolve, reject) => {
+			this.preloadResolvers.push(resolve);
+			this.preloadRejecters.push(reject);
+		});
+	}
+	/**
+	* Reject all waiting preload promises with an error
+	*/
+	rejectAll(error) {
+		for (const reject of this.preloadRejecters) reject(error);
+		this.preloadResolvers = [];
+		this.preloadRejecters = [];
+		for (const resolvers of this.txidResolvers.values()) for (const { reject, timeoutId } of resolvers) {
+			clearTimeout(timeoutId);
+			reject(error);
+		}
+		this.txidResolvers.clear();
+	}
+	/**
+	* Check if we've received up-to-date
+	*/
+	get ready() {
+		return this.isUpToDate;
+	}
+	/**
+	* Wait for a specific txid to be seen in the stream
+	*/
+	awaitTxId(txid, timeout = 5e3) {
+		if (this.seenTxids.has(txid)) return Promise.resolve();
+		return new Promise((resolve, reject) => {
+			const timeoutId = setTimeout(() => {
+				const resolvers = this.txidResolvers.get(txid);
+				if (resolvers) {
+					const index = resolvers.findIndex((r) => r.timeoutId === timeoutId);
+					if (index !== -1) resolvers.splice(index, 1);
+					if (resolvers.length === 0) this.txidResolvers.delete(txid);
+				}
+				reject(new Error(`Timeout waiting for txid: ${txid}`));
+			}, timeout);
+			if (!this.txidResolvers.has(txid)) this.txidResolvers.set(txid, []);
+			this.txidResolvers.get(txid).push({
+				resolve,
+				reject,
+				timeoutId
+			});
+		});
+	}
+};
+/**
+* Create a sync config for a stream-backed collection
+*/
+function createStreamSyncConfig(eventType, dispatcher, primaryKey) {
+	return { sync: ({ begin, write, commit, markReady, truncate }) => {
+		dispatcher.registerHandler(eventType, {
+			begin,
+			write: (value, type) => {
+				write({
+					value,
+					type
+				});
+			},
+			commit,
+			markReady,
+			truncate,
+			primaryKey
+		});
+		if (dispatcher.ready) markReady();
+		return () => {};
+	} };
+}
+/**
+* Reserved collection names that would collide with StreamDB properties
+* (collections are now namespaced, but we still prevent internal name collisions)
+*/
+const RESERVED_COLLECTION_NAMES = new Set([
+	`collections`,
+	`preload`,
+	`close`,
+	`utils`,
+	`actions`
+]);
+/**
+* Create helper functions for a collection
+*/
+function createCollectionHelpers(eventType, primaryKey, schema) {
+	return {
+		insert: ({ key, value, headers }) => {
+			const result = schema[`~standard`].validate(value);
+			if (`issues` in result) throw new Error(`Validation failed for ${eventType} insert: ${result.issues?.map((i) => i.message).join(`, `) ?? `Unknown validation error`}`);
+			const derived = value[primaryKey];
+			const finalKey = key ?? (derived != null && derived !== `` ? String(derived) : void 0);
+			if (finalKey == null || finalKey === ``) throw new Error(`Cannot create ${eventType} insert event: must provide either 'key' or a value with a non-empty '${primaryKey}' field`);
+			return {
+				type: eventType,
+				key: finalKey,
+				value,
+				headers: {
+					...headers,
+					operation: `insert`
+				}
+			};
+		},
+		update: ({ key, value, oldValue, headers }) => {
+			const result = schema[`~standard`].validate(value);
+			if (`issues` in result) throw new Error(`Validation failed for ${eventType} update: ${result.issues?.map((i) => i.message).join(`, `) ?? `Unknown validation error`}`);
+			if (oldValue !== void 0) {
+				const oldResult = schema[`~standard`].validate(oldValue);
+				if (`issues` in oldResult) throw new Error(`Validation failed for ${eventType} update (oldValue): ${oldResult.issues?.map((i) => i.message).join(`, `) ?? `Unknown validation error`}`);
+			}
+			const derived = value[primaryKey];
+			const finalKey = key ?? (derived != null && derived !== `` ? String(derived) : void 0);
+			if (finalKey == null || finalKey === ``) throw new Error(`Cannot create ${eventType} update event: must provide either 'key' or a value with a non-empty '${primaryKey}' field`);
+			return {
+				type: eventType,
+				key: finalKey,
+				value,
+				old_value: oldValue,
+				headers: {
+					...headers,
+					operation: `update`
+				}
+			};
+		},
+		delete: ({ key, oldValue, headers }) => {
+			if (oldValue !== void 0) {
+				const result = schema[`~standard`].validate(oldValue);
+				if (`issues` in result) throw new Error(`Validation failed for ${eventType} delete (oldValue): ${result.issues?.map((i) => i.message).join(`, `) ?? `Unknown validation error`}`);
+			}
+			const finalKey = key ?? (oldValue ? String(oldValue[primaryKey]) : void 0);
+			if (!finalKey) throw new Error(`Cannot create ${eventType} delete event: must provide either 'key' or 'oldValue' with a ${primaryKey} field`);
+			return {
+				type: eventType,
+				key: finalKey,
+				old_value: oldValue,
+				headers: {
+					...headers,
+					operation: `delete`
+				}
+			};
+		},
+		upsert: ({ key, value, headers }) => {
+			const result = schema[`~standard`].validate(value);
+			if (`issues` in result) throw new Error(`Validation failed for ${eventType} upsert: ${result.issues?.map((i) => i.message).join(`, `) ?? `Unknown validation error`}`);
+			const derived = value[primaryKey];
+			const finalKey = key ?? (derived != null && derived !== `` ? String(derived) : void 0);
+			if (finalKey == null || finalKey === ``) throw new Error(`Cannot create ${eventType} upsert event: must provide either 'key' or a value with a non-empty '${primaryKey}' field`);
+			return {
+				type: eventType,
+				key: finalKey,
+				value,
+				headers: {
+					...headers,
+					operation: `upsert`
+				}
+			};
+		}
+	};
+}
+/**
+* Create a state schema definition with typed collections and event helpers
+*/
+function createStateSchema(collections) {
+	for (const name of Object.keys(collections)) if (RESERVED_COLLECTION_NAMES.has(name)) throw new Error(`Reserved collection name "${name}" - this would collide with StreamDB properties (${Array.from(RESERVED_COLLECTION_NAMES).join(`, `)})`);
+	const typeToCollection = new Map();
+	for (const [collectionName, def] of Object.entries(collections)) {
+		const existing = typeToCollection.get(def.type);
+		if (existing) throw new Error(`Duplicate event type "${def.type}" - used by both "${existing}" and "${collectionName}" collections`);
+		typeToCollection.set(def.type, collectionName);
+	}
+	const enhancedCollections = {};
+	for (const [name, collectionDef] of Object.entries(collections)) enhancedCollections[name] = {
+		...collectionDef,
+		...createCollectionHelpers(collectionDef.type, collectionDef.primaryKey, collectionDef.schema)
+	};
+	return enhancedCollections;
+}
+/**
+* Create a stream-backed database with TanStack DB collections
+*
+* This function is synchronous - it creates the stream handle and collections
+* but does not start the stream connection. Call `db.preload()` to connect
+* and sync initial data.
+*
+* @example
+* ```typescript
+* const stateSchema = createStateSchema({
+*   users: { schema: userSchema, type: "user", primaryKey: "id" },
+*   messages: { schema: messageSchema, type: "message", primaryKey: "id" },
+* })
+*
+* // Create a stream DB (synchronous - stream is created lazily on preload)
+* const db = createStreamDB({
+*   streamOptions: {
+*     url: "https://api.example.com/streams/my-stream",
+*     contentType: "application/json",
+*   },
+*   state: stateSchema,
+* })
+*
+* // preload() creates the stream and loads initial data
+* await db.preload()
+* const user = await db.collections.users.get("123")
+* ```
+*/
+function createStreamDB(options) {
+	const { streamOptions, state, actions: actionsFactory } = options;
+	const stream = new DurableStream(streamOptions);
+	const dispatcher = new EventDispatcher();
+	const collectionInstances = {};
+	for (const [name, definition] of Object.entries(state)) {
+		const collection = createCollection({
+			id: `stream-db:${name}`,
+			schema: definition.schema,
+			getKey: (item) => String(item[definition.primaryKey]),
+			sync: createStreamSyncConfig(definition.type, dispatcher, definition.primaryKey),
+			startSync: true,
+			gcTime: 0
+		});
+		console.log(`[StreamDB] Created collection "${name}":`, {
+			type: typeof collection,
+			constructor: collection.constructor.name,
+			isCollection: collection instanceof Object,
+			hasSize: `size` in collection
+		});
+		collectionInstances[name] = collection;
+	}
+	let streamResponse = null;
+	const abortController = new AbortController();
+	let consumerStarted = false;
+	/**
+	* Start the stream consumer (called lazily on first preload)
+	*/
+	const startConsumer = async () => {
+		if (consumerStarted) return;
+		consumerStarted = true;
+		streamResponse = await stream.stream({
+			live: `auto`,
+			signal: abortController.signal
+		});
+		let batchCount = 0;
+		let lastBatchTime = Date.now();
+		streamResponse.subscribeJson(async (batch) => {
+			try {
+				batchCount++;
+				lastBatchTime = Date.now();
+				if (batch.items.length > 0) console.log(`[StreamDB] Processing batch #${batchCount}: ${batch.items.length} items, upToDate=${batch.upToDate}`);
+				for (const event of batch.items) if (isChangeEvent(event)) dispatcher.dispatchChange(event);
+				else if (isControlEvent(event)) dispatcher.dispatchControl(event);
+				if (batch.upToDate) {
+					console.log(`[StreamDB] Marking up-to-date after batch #${batchCount}`);
+					dispatcher.markUpToDate();
+					console.log(`[StreamDB] Successfully marked up-to-date`);
+				}
+				if (batch.items.length > 0) console.log(`[StreamDB] Successfully processed batch #${batchCount}`);
+			} catch (error) {
+				console.error(`[StreamDB] Error processing batch:`, error);
+				console.error(`[StreamDB] Failed batch:`, batch);
+				dispatcher.rejectAll(error);
+				abortController.abort();
+			}
+		});
+		const healthCheck = setInterval(() => {
+			const timeSinceLastBatch = Date.now() - lastBatchTime;
+			console.log(`[StreamDB] Health: ${batchCount} batches processed, last batch ${(timeSinceLastBatch / 1e3).toFixed(1)}s ago`);
+		}, 15e3);
+		abortController.signal.addEventListener(`abort`, () => {
+			clearInterval(healthCheck);
+			console.log(`[StreamDB] Aborted - cleaning up health check`);
+		});
+	};
+	const dbMethods = {
+		stream,
+		preload: async () => {
+			await startConsumer();
+			await dispatcher.waitForUpToDate();
+		},
+		close: () => {
+			dispatcher.rejectAll(new Error(`StreamDB closed`));
+			abortController.abort();
+		},
+		utils: { awaitTxId: (txid, timeout) => dispatcher.awaitTxId(txid, timeout) }
+	};
+	console.log(`[StreamDB] Creating db object with collections:`, Object.keys(collectionInstances));
+	const db = {
+		collections: collectionInstances,
+		...dbMethods
+	};
+	console.log(`[StreamDB] db.collections:`, Object.keys(db.collections));
+	console.log(`[StreamDB] db.collections.events:`, db.collections.events);
+	if (actionsFactory) {
+		const actionDefs = actionsFactory({
+			db,
+			stream
+		});
+		const wrappedActions = {};
+		for (const [name, def] of Object.entries(actionDefs)) wrappedActions[name] = createOptimisticAction({
+			onMutate: def.onMutate,
+			mutationFn: def.mutationFn
+		});
+		return {
+			...db,
+			actions: wrappedActions
+		};
+	}
+	return db;
+}
+
+//#endregion
+export { MaterializedState, createStateSchema, createStreamDB, isChangeEvent, isControlEvent };

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@durable-streams/state",
+  "description": "State change event protocol for Durable Streams",
+  "version": "0.1.0",
+  "author": "Durable Stream contributors",
+  "license": "Apache-2.0",
+  "type": "module",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    }
+  },
+  "files": [
+    "dist",
+    "src",
+    "state-protocol.schema.json",
+    "STATE-PROTOCOL.md"
+  ],
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "sideEffects": false,
+  "dependencies": {
+    "@durable-streams/client": "workspace:*",
+    "@standard-schema/spec": "^1.0.0",
+    "@tanstack/db": "latest"
+  },
+  "devDependencies": {
+    "@durable-streams/server": "workspace:*",
+    "tsdown": "^0.9.0"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "dev": "tsdown --watch",
+    "typecheck": "tsc --noEmit"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}

package/src/index.ts

@@ -0,0 +1,33 @@
+// Types
+export type {
+  Operation,
+  Value,
+  Row,
+  ChangeHeaders,
+  ChangeEvent,
+  ControlEvent,
+  StateEvent,
+} from "./types"
+
+export { isChangeEvent, isControlEvent } from "./types"
+
+// Classes
+export { MaterializedState } from "./materialized-state"
+
+// Stream DB
+export { createStreamDB, createStateSchema } from "./stream-db"
+export type {
+  CollectionDefinition,
+  CollectionEventHelpers,
+  CollectionWithHelpers,
+  StreamStateDefinition,
+  StateSchema,
+  CreateStreamDBOptions,
+  StreamDB,
+  StreamDBMethods,
+  StreamDBUtils,
+  StreamDBWithActions,
+  ActionFactory,
+  ActionMap,
+  ActionDefinition,
+} from "./stream-db"

package/src/materialized-state.ts

@@ -0,0 +1,93 @@
+import type { ChangeEvent } from "./types"
+
+/**
+ * MaterializedState maintains an in-memory view of state from change events.
+ *
+ * It organizes data by type, where each type contains a map of key -> value.
+ * This supports multi-type streams where different entity types can coexist.
+ */
+export class MaterializedState {
+  private data: Map<string, Map<string, unknown>>
+
+  constructor() {
+    this.data = new Map()
+  }
+
+  /**
+   * Apply a single change event to update the materialized state
+   */
+  apply(event: ChangeEvent): void {
+    const { type, key, value, headers } = event
+
+    // Get or create the type map
+    let typeMap = this.data.get(type)
+    if (!typeMap) {
+      typeMap = new Map()
+      this.data.set(type, typeMap)
+    }
+
+    // Apply the operation
+    switch (headers.operation) {
+      case `insert`:
+        typeMap.set(key, value)
+        break
+      case `update`:
+        typeMap.set(key, value)
+        break
+      case `upsert`:
+        typeMap.set(key, value)
+        break
+      case `delete`:
+        typeMap.delete(key)
+        break
+    }
+  }
+
+  /**
+   * Apply a batch of change events
+   */
+  applyBatch(events: Array<ChangeEvent>): void {
+    for (const event of events) {
+      this.apply(event)
+    }
+  }
+
+  /**
+   * Get a specific value by type and key
+   */
+  get<T = unknown>(type: string, key: string): T | undefined {
+    const typeMap = this.data.get(type)
+    if (!typeMap) {
+      return undefined
+    }
+    return typeMap.get(key) as T | undefined
+  }
+
+  /**
+   * Get all entries for a specific type
+   */
+  getType(type: string): Map<string, unknown> {
+    return this.data.get(type) || new Map()
+  }
+
+  /**
+   * Clear all state
+   */
+  clear(): void {
+    this.data.clear()
+  }
+
+  /**
+   * Get the number of types in the state
+   */
+  get typeCount(): number {
+    return this.data.size
+  }
+
+  /**
+   * Get all type names
+   */
+  get types(): Array<string> {
+    return Array.from(this.data.keys())
+  }
+}