@firtoz/db-helpers 0.1.0 → 2.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@firtoz/db-helpers",
-	"version": "0.1.0",
+	"version": "2.0.0",
 	"description": "TanStack DB helpers and utilities",
 	"main": "./src/index.ts",
 	"module": "./src/index.ts",
@@ -19,6 +19,7 @@
 	],
 	"scripts": {
 		"typecheck": "tsc --noEmit -p ./tsconfig.json",
+		"test": "bun test",
 		"lint": "biome check --write src",
 		"lint:ci": "biome ci src",
 		"format": "biome format src --write"
@@ -48,10 +49,15 @@
 	},
 	"peerDependencies": {
 		"@standard-schema/spec": ">=1.1.0",
-		"@tanstack/db": ">=0.5.25"
+		"@tanstack/db": ">=0.5.33"
 	},
 	"devDependencies": {
 		"@standard-schema/spec": "^1.1.0",
-		"@tanstack/db": "^0.5.25"
+		"@tanstack/db": "^0.5.33",
+		"bun-types": "^1.3.10",
+		"zod": "^4.3.6"
+	},
+	"dependencies": {
+		"@firtoz/maybe-error": "^1.5.2"
 	}
 }

package/src/generic-sync.ts

@@ -0,0 +1,357 @@
+import type { CollectionUtils, SyncMessage } from "./sync-types";
+import { exhaustiveGuard } from "@firtoz/maybe-error";
+import type { StandardSchemaV1 } from "@standard-schema/spec";
+import type {
+	CollectionConfig,
+	InferSchemaOutput,
+	SyncConfig,
+	SyncConfigRes,
+	SyncMode,
+	LoadSubsetOptions,
+} from "@tanstack/db";
+import { DeduplicatedLoadSubset } from "@tanstack/db";
+
+// WORKAROUND: DeduplicatedLoadSubset has a bug where toggling queries (e.g., isNull/isNotNull)
+// creates invalid expressions like not(or(isNull(...), not(isNull(...))))
+// See: https://github.com/TanStack/db/issues/828
+// TODO: Re-enable once the bug is fixed
+export const USE_DEDUPE = false as boolean;
+
+/**
+ * Base configuration for sync lifecycle management (generic, no Drizzle dependency).
+ */
+export interface GenericBaseSyncConfig {
+	readyPromise: Promise<void>;
+	syncMode?: SyncMode;
+	debug?: boolean;
+}
+
+/**
+ * Backend-specific implementations required for sync (generic, no Drizzle dependency).
+ */
+export interface GenericSyncBackend<TItem extends object> {
+	initialLoad: () => Promise<Array<TItem>>;
+	loadSubset: (options: LoadSubsetOptions) => Promise<Array<TItem>>;
+	handleInsert: (items: Array<TItem>) => Promise<Array<TItem>>;
+	handleUpdate: (
+		mutations: Array<{
+			key: string;
+			changes: Partial<TItem>;
+			original: TItem;
+		}>,
+	) => Promise<Array<TItem>>;
+	handleDelete: (
+		mutations: Array<{
+			key: string;
+			modified: TItem;
+			original: TItem;
+		}>,
+	) => Promise<void>;
+	handleTruncate?: () => Promise<void>;
+}
+
+/**
+ * Return type for createGenericSyncFunction.
+ */
+export type GenericSyncFunctionResult<TItem extends object> = {
+	sync: SyncConfig<TItem, string>["sync"];
+	onInsert: CollectionConfig<
+		TItem,
+		string,
+		// biome-ignore lint/suspicious/noExplicitAny: Schema type parameter needs to be flexible
+		any
+	>["onInsert"];
+	onUpdate: CollectionConfig<
+		TItem,
+		string,
+		// biome-ignore lint/suspicious/noExplicitAny: Schema type parameter needs to be flexible
+		any
+	>["onUpdate"];
+	onDelete: CollectionConfig<
+		TItem,
+		string,
+		// biome-ignore lint/suspicious/noExplicitAny: Schema type parameter needs to be flexible
+		any
+	>["onDelete"];
+	utils: CollectionUtils<TItem>;
+};
+
+/**
+ * Creates the sync function with common lifecycle management.
+ * Generic version -- no Drizzle dependency.
+ */
+export function createGenericSyncFunction<TItem extends object>(
+	config: GenericBaseSyncConfig,
+	backend: GenericSyncBackend<TItem>,
+): GenericSyncFunctionResult<TItem> {
+	type CollectionType = CollectionConfig<
+		TItem,
+		string,
+		// biome-ignore lint/suspicious/noExplicitAny: Schema type parameter needs to be flexible
+		any
+	>;
+
+	let insertListener: CollectionType["onInsert"];
+	let updateListener: CollectionType["onUpdate"];
+	let deleteListener: CollectionType["onDelete"];
+
+	let syncBegin: (() => void) | null = null;
+	let syncWrite:
+		| ((op: { type: "insert" | "update" | "delete"; value: TItem }) => void)
+		| null = null;
+	let syncCommit: (() => void) | null = null;
+	let syncTruncate: (() => void) | null = null;
+
+	const syncFn: SyncConfig<TItem, string>["sync"] = (params) => {
+		const { begin, write, commit, markReady, truncate } = params;
+
+		syncBegin = begin;
+		syncWrite = write;
+		syncCommit = commit;
+		syncTruncate = truncate;
+
+		const initialSync = async () => {
+			await config.readyPromise;
+
+			try {
+				const items = await backend.initialLoad();
+
+				begin();
+
+				for (const item of items) {
+					write({
+						type: "insert",
+						value: item,
+					});
+				}
+
+				commit();
+			} finally {
+				markReady();
+			}
+		};
+
+		if (config.syncMode === "eager" || !config.syncMode) {
+			initialSync();
+		} else {
+			markReady();
+		}
+
+		insertListener = async (params) => {
+			const results = await backend.handleInsert(
+				params.transaction.mutations.map((m) => m.modified),
+			);
+
+			begin();
+			for (const result of results) {
+				write({
+					type: "insert",
+					value: result,
+				});
+			}
+			commit();
+		};
+
+		updateListener = async (params) => {
+			const results = await backend.handleUpdate(params.transaction.mutations);
+
+			begin();
+			for (const result of results) {
+				write({
+					type: "update",
+					value: result,
+				});
+			}
+			commit();
+		};
+
+		deleteListener = async (params) => {
+			await backend.handleDelete(params.transaction.mutations);
+
+			begin();
+			for (const item of params.transaction.mutations) {
+				write({
+					type: "delete",
+					value: item.modified,
+				});
+			}
+			commit();
+		};
+
+		const loadSubset = async (options: LoadSubsetOptions) => {
+			await config.readyPromise;
+
+			const items = await backend.loadSubset(options);
+
+			begin();
+
+			for (const item of items) {
+				write({
+					type: "insert",
+					value: item,
+				});
+			}
+
+			commit();
+		};
+
+		let loadSubsetDedupe: DeduplicatedLoadSubset | null = null;
+		if (USE_DEDUPE) {
+			loadSubsetDedupe = new DeduplicatedLoadSubset({
+				loadSubset,
+			});
+		}
+
+		return {
+			cleanup: () => {
+				insertListener = undefined;
+				updateListener = undefined;
+				deleteListener = undefined;
+				loadSubsetDedupe?.reset();
+			},
+			loadSubset: loadSubsetDedupe?.loadSubset ?? loadSubset,
+		} satisfies SyncConfigRes;
+	};
+
+	const receiveSync = async (messages: SyncMessage<TItem>[]) => {
+		if (messages.length === 0) return;
+		if (!syncBegin || !syncWrite || !syncCommit || !syncTruncate) {
+			if (config.debug) {
+				console.warn(
+					"[receiveSync] Sync functions not initialized yet - messages will be dropped",
+					messages.length,
+				);
+			}
+			return;
+		}
+		syncBegin();
+		for (const msg of messages) {
+			switch (msg.type) {
+				case "insert":
+					syncWrite({ type: "insert", value: msg.value });
+					break;
+				case "update":
+					syncWrite({ type: "update", value: msg.value });
+					break;
+				case "delete":
+					syncWrite({
+						type: "delete",
+						value: { id: msg.key } as TItem,
+					});
+					break;
+				case "truncate":
+					syncTruncate();
+					break;
+				default:
+					exhaustiveGuard(msg);
+			}
+		}
+		syncCommit();
+	};
+
+	const utils: CollectionUtils<TItem> = {
+		truncate: async () => {
+			if (!backend.handleTruncate) {
+				throw new Error("Truncate not supported by this backend");
+			}
+			if (!syncBegin || !syncTruncate || !syncCommit) {
+				throw new Error(
+					"Sync functions not initialized - sync function may not have been called yet",
+				);
+			}
+			await backend.handleTruncate();
+			syncBegin();
+			syncTruncate();
+			syncCommit();
+		},
+		receiveSync,
+	};
+
+	return {
+		sync: syncFn,
+		onInsert: async (params) => {
+			if (!insertListener) {
+				throw new Error(
+					"insertListener not initialized - sync function may not have been called yet",
+				);
+			}
+			return insertListener(params);
+		},
+		onUpdate: async (params) => {
+			if (!updateListener) {
+				throw new Error(
+					"updateListener not initialized - sync function may not have been called yet",
+				);
+			}
+			return updateListener(params);
+		},
+		onDelete: async (params) => {
+			if (!deleteListener) {
+				throw new Error(
+					"deleteListener not initialized - sync function may not have been called yet",
+				);
+			}
+			return deleteListener(params);
+		},
+		utils,
+	};
+}
+
+/**
+ * Generic collection config factory.
+ * Combines schema, sync, and event handlers into a collection config.
+ * No Drizzle dependency -- uses StandardSchemaV1 directly.
+ */
+export function createGenericCollectionConfig<
+	TItem extends object,
+	TSchema extends StandardSchemaV1,
+>(config: {
+	schema: TSchema;
+	getKey: (item: TItem) => string;
+	syncResult: GenericSyncFunctionResult<TItem>;
+	onInsert?: CollectionConfig<
+		TItem,
+		string,
+		// biome-ignore lint/suspicious/noExplicitAny: Schema type parameter needs to be flexible
+		any
+	>["onInsert"];
+	onUpdate?: CollectionConfig<
+		TItem,
+		string,
+		// biome-ignore lint/suspicious/noExplicitAny: Schema type parameter needs to be flexible
+		any
+	>["onUpdate"];
+	onDelete?: CollectionConfig<
+		TItem,
+		string,
+		// biome-ignore lint/suspicious/noExplicitAny: Schema type parameter needs to be flexible
+		any
+	>["onDelete"];
+	syncMode?: SyncMode;
+}): Omit<
+	CollectionConfig<
+		TItem,
+		string,
+		// biome-ignore lint/suspicious/noExplicitAny: Schema type parameter needs to be flexible
+		any
+	>,
+	"utils"
+> & {
+	schema: TSchema;
+	utils: CollectionUtils<InferSchemaOutput<TSchema>>;
+} {
+	return {
+		schema: config.schema,
+		getKey: config.getKey,
+		sync: {
+			sync: config.syncResult.sync,
+		},
+		onInsert: config.onInsert ?? config.syncResult.onInsert,
+		onUpdate: config.onUpdate ?? config.syncResult.onUpdate,
+		onDelete: config.onDelete ?? config.syncResult.onDelete,
+		syncMode: config.syncMode,
+		utils: config.syncResult.utils as CollectionUtils<
+			InferSchemaOutput<TSchema>
+		>,
+	};
+}

package/src/index.ts

@@ -1,5 +1,22 @@
+export type {
+	CollectionUtils,
+	ExternalSyncEvent,
+	ExternalSyncHandler,
+	SyncMessage,
+} from "./sync-types";
 export {
 	createMemoryCollection,
 	memoryCollectionOptions,
 	type MemoryCollection,
 } from "./memoryCollection";
+
+export { evaluateExpression, getExpressionValue } from "./ir-evaluator";
+
+export {
+	USE_DEDUPE,
+	createGenericSyncFunction,
+	createGenericCollectionConfig,
+	type GenericBaseSyncConfig,
+	type GenericSyncBackend,
+	type GenericSyncFunctionResult,
+} from "./generic-sync";

package/src/ir-evaluator.ts

@@ -0,0 +1,136 @@
+import type { IR } from "@tanstack/db";
+
+/**
+ * Evaluates a TanStack DB IR expression against a plain object item.
+ * @internal Exported for testing and reuse by collection backends
+ */
+export function evaluateExpression(
+	expression: IR.BasicExpression,
+	item: Record<string, unknown>,
+): boolean {
+	switch (expression.type) {
+		case "ref": {
+			const propRef = expression;
+			const columnName = propRef.path[propRef.path.length - 1];
+			return item[columnName as string] !== undefined;
+		}
+		case "val": {
+			const value = expression;
+			return !!value.value;
+		}
+		case "func": {
+			const func = expression;
+
+			switch (func.name) {
+				case "eq": {
+					const left = getExpressionValue(func.args[0], item);
+					const right = getExpressionValue(func.args[1], item);
+					return left === right;
+				}
+				case "ne": {
+					const left = getExpressionValue(func.args[0], item);
+					const right = getExpressionValue(func.args[1], item);
+					return left !== right;
+				}
+				case "gt": {
+					const left = getExpressionValue(func.args[0], item);
+					const right = getExpressionValue(func.args[1], item);
+					return left > right;
+				}
+				case "gte": {
+					const left = getExpressionValue(func.args[0], item);
+					const right = getExpressionValue(func.args[1], item);
+					return left >= right;
+				}
+				case "lt": {
+					const left = getExpressionValue(func.args[0], item);
+					const right = getExpressionValue(func.args[1], item);
+					return left < right;
+				}
+				case "lte": {
+					const left = getExpressionValue(func.args[0], item);
+					const right = getExpressionValue(func.args[1], item);
+					return left <= right;
+				}
+				case "and": {
+					return func.args.every((arg) => evaluateExpression(arg, item));
+				}
+				case "or": {
+					return func.args.some((arg) => evaluateExpression(arg, item));
+				}
+				case "not": {
+					return !evaluateExpression(func.args[0], item);
+				}
+				case "isNull": {
+					const value = getExpressionValue(func.args[0], item);
+					return value === null || value === undefined;
+				}
+				case "isNotNull": {
+					const value = getExpressionValue(func.args[0], item);
+					return value !== null && value !== undefined;
+				}
+				case "like": {
+					const left = String(getExpressionValue(func.args[0], item));
+					const right = String(getExpressionValue(func.args[1], item));
+					const pattern = right.replace(/%/g, ".*").replace(/_/g, ".");
+					return new RegExp(`^${pattern}$`).test(left);
+				}
+				case "ilike": {
+					const left = String(getExpressionValue(func.args[0], item));
+					const right = String(getExpressionValue(func.args[1], item));
+					const pattern = right.replace(/%/g, ".*").replace(/_/g, ".");
+					return new RegExp(`^${pattern}$`, "i").test(left);
+				}
+				case "in": {
+					const left = getExpressionValue(func.args[0], item);
+					const right = getExpressionValue(func.args[1], item);
+					return Array.isArray(right) && right.includes(left);
+				}
+				case "isUndefined": {
+					const value = getExpressionValue(func.args[0], item);
+					return value === null || value === undefined;
+				}
+				default:
+					throw new Error(`Unsupported function: ${func.name}`);
+			}
+		}
+		default: {
+			const _ex: never = expression;
+			void _ex;
+			throw new Error(
+				`Unsupported expression type: ${(expression as { type: string }).type}`,
+			);
+		}
+	}
+}
+
+/**
+ * Gets the value from an IR expression by resolving refs and vals.
+ * @internal Exported for testing and reuse by collection backends
+ */
+export function getExpressionValue(
+	expression: IR.BasicExpression,
+	item: Record<string, unknown>,
+	// biome-ignore lint/suspicious/noExplicitAny: We need any here for dynamic values
+): any {
+	switch (expression.type) {
+		case "ref": {
+			const propRef = expression;
+			const columnName = propRef.path[propRef.path.length - 1];
+			return item[columnName as string];
+		}
+		case "val": {
+			const value = expression;
+			return value.value;
+		}
+		case "func":
+			throw new Error("Cannot get value from func expression");
+		default: {
+			const _ex: never = expression;
+			void _ex;
+			throw new Error(
+				`Cannot get value from expression type: ${(expression as { type: string }).type}`,
+			);
+		}
+	}
+}

package/src/memoryCollection.ts

@@ -1,3 +1,5 @@
+import type { SyncMessage } from "./sync-types";
+import { exhaustiveGuard } from "@firtoz/maybe-error";
 import type { StandardSchemaV1 } from "@standard-schema/spec";
 import {
 	type Collection,
@@ -16,72 +18,100 @@ type MemoryCollectionConfig<TSchema extends StandardSchemaV1> = Omit<
 	"onInsert" | "onUpdate" | "onDelete" | "sync" | "schema"
 > & {
 	schema: TSchema;
+	/** Called when a local mutation is written to the sync layer; use to broadcast to other peers. */
+	onBroadcast?: (
+		changes: SyncMessage<InferSchemaOutput<TSchema>, string | number>[],
+	) => void;
 };
 
-type MemoryUtils = {
+type MemoryUtils<
+	TItem = unknown,
+	TKey extends string | number = string | number,
+> = {
 	truncate: () => Promise<void>;
+	/**
+	 * Apply incoming sync messages without triggering onInsert/onUpdate/onDelete.
+	 * Uses the sync layer (begin/write/commit) so state updates and subscribeChanges fire,
+	 * but no rebroadcast occurs.
+	 */
+	receiveSync: (messages: SyncMessage<TItem, TKey>[]) => Promise<void>;
 };
 
 export function memoryCollectionOptions<TSchema extends StandardSchemaV1>(
 	config: MemoryCollectionConfig<TSchema>,
 ): CollectionConfig<InferSchemaOutput<TSchema>, string | number, TSchema> & {
-	utils: MemoryUtils;
+	utils: MemoryUtils<InferSchemaOutput<TSchema>, string | number>;
 	schema: TSchema;
 } {
 	type TItem = InferSchemaOutput<TSchema>;
+	type TKey = string | number;
 	let syncParams: Parameters<SyncConfig<TItem>["sync"]>[0] | null = null;
 
 	const sync: SyncConfig<TItem>["sync"] = (params) => {
 		syncParams = params;
-
 		params.markReady();
-
-		// Return cleanup function
 		return () => {};
 	};
 
-	// All mutation handlers use the same transaction sender
-	const onInsert = async (params: InsertMutationFnParams<TItem>) => {
+	const writeChanges = (writes: SyncMessage<TItem, TKey>[]) => {
 		if (!syncParams) {
 			throw new Error("Sync parameters not initialized");
 		}
 		syncParams.begin();
-		for (const mutation of params.transaction.mutations) {
-			syncParams.write({
-				type: "insert",
-				value: mutation.modified,
-			});
+		for (const msg of writes) {
+			switch (msg.type) {
+				case "insert":
+					syncParams.write({ type: "insert", value: msg.value });
+					break;
+				case "update":
+					syncParams.write({
+						type: "update",
+						value: msg.value,
+						previousValue: msg.previousValue,
+					});
+					break;
+				case "delete":
+					syncParams.write({ type: "delete", key: msg.key });
+					break;
+				case "truncate":
+					syncParams.truncate();
+					break;
+				default:
+					exhaustiveGuard(msg);
+			}
 		}
 		syncParams.commit();
 	};
 
-	const onUpdate = async (params: UpdateMutationFnParams<TItem>) => {
-		if (!syncParams) {
-			throw new Error("Sync parameters not initialized");
+	const onInsert = async (params: InsertMutationFnParams<TItem>) => {
+		const writes: SyncMessage<TItem, TKey>[] = [];
+		for (const mutation of params.transaction.mutations) {
+			writes.push({ type: "insert", value: mutation.modified });
 		}
-		syncParams.begin();
+		writeChanges(writes);
+		config.onBroadcast?.(writes);
+	};
+
+	const onUpdate = async (params: UpdateMutationFnParams<TItem>) => {
+		const writes: SyncMessage<TItem, TKey>[] = [];
 		for (const mutation of params.transaction.mutations) {
-			syncParams.write({
+			writes.push({
 				type: "update",
 				value: mutation.modified,
 				previousValue: mutation.original,
 			});
 		}
-		syncParams.commit();
+		writeChanges(writes);
+		config.onBroadcast?.(writes);
 	};
 
 	const onDelete = async (params: DeleteMutationFnParams<TItem>) => {
-		if (!syncParams) {
-			throw new Error("Sync parameters not initialized");
-		}
-		syncParams.begin();
+		const writes: SyncMessage<TItem, TKey>[] = [];
 		for (const mutation of params.transaction.mutations) {
-			syncParams.write({
-				type: "delete",
-				key: mutation.key,
-			});
+			writes.push({ type: "delete", key: mutation.key as TKey });
 		}
-		syncParams?.commit();
+		writeChanges(writes);
+		config.onBroadcast?.(writes);
 	};
 
 	const truncate = async () => {
@@ -93,6 +123,11 @@ export function memoryCollectionOptions<TSchema extends StandardSchemaV1>(
 		syncParams.commit();
 	};
 
+	const receiveSync = async (messages: SyncMessage<TItem, TKey>[]) => {
+		if (messages.length === 0) return;
+		writeChanges(messages);
+	};
+
 	return {
 		id: config.id,
 		schema: config.schema,
@@ -103,6 +138,7 @@ export function memoryCollectionOptions<TSchema extends StandardSchemaV1>(
 		onDelete,
 		utils: {
 			truncate,
+			receiveSync,
 		},
 	};
 }
@@ -110,7 +146,7 @@ export function memoryCollectionOptions<TSchema extends StandardSchemaV1>(
 export type MemoryCollection<TSchema extends StandardSchemaV1> = Collection<
 	InferSchemaOutput<TSchema>,
 	string | number,
-	MemoryUtils,
+	MemoryUtils<InferSchemaOutput<TSchema>, string | number>,
 	TSchema,
 	InferSchemaInput<TSchema>
 >;

package/src/sync-types.ts

@@ -0,0 +1,46 @@
+/**
+ * Generic collection sync types. Used by memory, IndexedDB, SQLite, and other
+ * collection backends for a unified sync protocol.
+ */
+
+/**
+ * Canonical per-mutation sync message. Use for broadcast and receive across all collection types.
+ */
+export type SyncMessage<
+	T = unknown,
+	TKey extends string | number = string | number,
+> =
+	| { type: "insert"; value: T }
+	| { type: "update"; value: T; previousValue: T }
+	| { type: "delete"; key: TKey }
+	| { type: "truncate" };
+
+/**
+ * External sync event (batched). Used internally by the sync layer.
+ */
+export type ExternalSyncEvent<T> =
+	| { type: "insert"; items: T[] }
+	| { type: "update"; items: T[] }
+	| { type: "delete"; items: T[] }
+	| { type: "truncate" };
+
+/**
+ * Handler for external sync events (internal use).
+ */
+export type ExternalSyncHandler<T> = (event: ExternalSyncEvent<T>) => void;
+
+/**
+ * Collection utils: truncate and receiveSync (canonical sync protocol).
+ */
+export interface CollectionUtils<T = unknown> {
+	/**
+	 * Clear all data from the store (truncate).
+	 * This clears the backend store and updates the local reactive store.
+	 */
+	truncate: () => Promise<void>;
+	/**
+	 * Apply incoming sync messages without triggering mutation handlers.
+	 * Use the same SyncMessage[] shape for memory and backend collections.
+	 */
+	receiveSync: (messages: SyncMessage<T>[]) => Promise<void>;
+}