@firtoz/collection-sync 1.0.0

package/package.json

@@ -0,0 +1,82 @@
+{
+	"name": "@firtoz/collection-sync",
+	"version": "1.0.0",
+	"description": "WebSocket sync protocol and bridges for TanStack DB collections",
+	"main": "./src/index.ts",
+	"module": "./src/index.ts",
+	"types": "./src/index.ts",
+	"type": "module",
+	"exports": {
+		".": {
+			"types": "./src/index.ts",
+			"import": "./src/index.ts",
+			"require": "./src/index.ts"
+		},
+		"./react": {
+			"types": "./src/react/index.ts",
+			"import": "./src/react/index.ts",
+			"require": "./src/react/index.ts"
+		},
+		"./*": {
+			"types": "./src/*.ts",
+			"import": "./src/*.ts",
+			"require": "./src/*.ts"
+		}
+	},
+	"files": [
+		"src/**/*.ts",
+		"!src/**/*.test.ts",
+		"README.md"
+	],
+	"scripts": {
+		"typecheck": "tsgo --noEmit -p ./tsconfig.json",
+		"lint": "biome check --write src",
+		"lint:ci": "biome ci src",
+		"format": "biome format src --write",
+		"test": "bun test src"
+	},
+	"keywords": [
+		"typescript",
+		"websocket",
+		"sync",
+		"tanstack-db"
+	],
+	"author": "Firtina Ozbalikchi <firtoz@github.com>",
+	"license": "MIT",
+	"homepage": "https://github.com/firtoz/fullstack-toolkit#readme",
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/firtoz/fullstack-toolkit.git",
+		"directory": "packages/collection-sync"
+	},
+	"bugs": {
+		"url": "https://github.com/firtoz/fullstack-toolkit/issues"
+	},
+	"engines": {
+		"node": ">=18.0.0"
+	},
+	"publishConfig": {
+		"access": "public"
+	},
+	"dependencies": {
+		"@firtoz/db-helpers": "^2.1.0",
+		"@firtoz/maybe-error": "^1.5.2",
+		"zod": "^4.3.6"
+	},
+	"peerDependencies": {
+		"@firtoz/websocket-do": "^7.1.0",
+		"@standard-schema/spec": ">=1.1.0",
+		"@tanstack/db": ">=0.6.1",
+		"@tanstack/react-db": ">=0.1.79",
+		"react": ">=19.2.4"
+	},
+	"devDependencies": {
+		"@firtoz/websocket-do": "^7.1.0",
+		"@standard-schema/spec": "^1.1.0",
+		"@tanstack/db": "^0.6.1",
+		"@tanstack/react-db": "0.1.79",
+		"@types/react": "^19.2.14",
+		"react": "^19.2.4",
+		"typescript": "^6.0.2"
+	}
+}

package/src/cache-manager.ts

@@ -0,0 +1,208 @@
+export interface CacheStorageEstimate {
+	usageBytes: number;
+	quotaBytes: number;
+	utilizationRatio: number;
+}
+
+export type CacheEntry = {
+	key: string | number;
+	lastAccessedAt: number;
+	fetchedAt: number;
+	sortPositions: Map<string, unknown>;
+	estimatedSizeBytes: number;
+};
+
+import type { PartialSyncRowRef } from "./partial-sync-row-key";
+import { partialSyncRowKey } from "./partial-sync-row-key";
+
+export interface CacheManagerOptions<TItem extends PartialSyncRowRef> {
+	evictionThreshold?: number;
+	evictionTarget?: number;
+	estimateRowSize?: (row: TItem) => number;
+	getStorageEstimate: () => Promise<CacheStorageEstimate>;
+	deleteRows: (keys: Array<string | number>) => Promise<void>;
+}
+
+export type CacheViewport = {
+	sortColumn: string;
+	sortDirection: "asc" | "desc";
+	fromValue: unknown;
+	toValue: unknown;
+};
+
+export class CacheManager<TItem extends PartialSyncRowRef> {
+	#entries = new Map<string | number, CacheEntry>();
+	readonly evictionThreshold: number;
+	readonly evictionTarget: number;
+
+	constructor(private readonly options: CacheManagerOptions<TItem>) {
+		this.evictionThreshold = options.evictionThreshold ?? 0.85;
+		this.evictionTarget = options.evictionTarget ?? 0.7;
+	}
+
+	get entryCount(): number {
+		return this.#entries.size;
+	}
+
+	/**
+	 * Recompute `sortPositions` for rows already tracked (e.g. after a local edit changes sort keys).
+	 * Does not add new entries; skips keys missing from `getRow`.
+	 */
+	resyncSortPositionsForTrackedRows(
+		getRow: (key: string | number) => TItem | undefined,
+		getSortPositions: (row: TItem) => Record<string, unknown>,
+	): void {
+		for (const key of this.#entries.keys()) {
+			const row = getRow(key);
+			if (row === undefined) continue;
+			const entry = this.#entries.get(key);
+			if (entry === undefined) continue;
+			const sortPositionsObject = getSortPositions(row);
+			entry.sortPositions = new Map<string, unknown>(
+				Object.entries(sortPositionsObject),
+			);
+		}
+	}
+
+	recordFetchedRows(
+		rows: TItem[],
+		getSortPositions: (row: TItem) => Record<string, unknown>,
+	): void {
+		const now = Date.now();
+		for (const row of rows) {
+			const rowKey = partialSyncRowKey(row.id);
+			const existing = this.#entries.get(rowKey);
+			const sortPositionsObject = getSortPositions(row);
+			const sortPositions = new Map<string, unknown>(
+				Object.entries(sortPositionsObject),
+			);
+			const estimatedSizeBytes = this.options.estimateRowSize?.(row) ?? 256;
+			this.#entries.set(rowKey, {
+				key: rowKey,
+				lastAccessedAt: existing?.lastAccessedAt ?? now,
+				fetchedAt: now,
+				sortPositions,
+				estimatedSizeBytes,
+			});
+		}
+	}
+
+	markAccessed(keys: Array<string | number>): void {
+		const now = Date.now();
+		for (const key of keys) {
+			const entry = this.#entries.get(key);
+			if (!entry) continue;
+			entry.lastAccessedAt = now;
+		}
+	}
+
+	removeEntries(keys: Array<string | number>): void {
+		for (const key of keys) this.#entries.delete(key);
+	}
+
+	clear(): void {
+		this.#entries.clear();
+	}
+
+	async estimateStoragePressure(): Promise<CacheStorageEstimate> {
+		const estimate = await this.options.getStorageEstimate();
+		return {
+			usageBytes: Math.max(0, estimate.usageBytes),
+			quotaBytes: Math.max(1, estimate.quotaBytes),
+			utilizationRatio: estimate.usageBytes / Math.max(1, estimate.quotaBytes),
+		};
+	}
+
+	async evictIfNeeded(viewport: CacheViewport): Promise<{
+		evictedKeys: Array<string | number>;
+		estimate: CacheStorageEstimate;
+	}> {
+		const estimate = await this.estimateStoragePressure();
+		if (estimate.utilizationRatio < this.evictionThreshold) {
+			return { evictedKeys: [], estimate };
+		}
+
+		const candidates = Array.from(this.#entries.values())
+			.filter((entry) => !this.#isEntryProtectedByViewport(entry, viewport))
+			.sort(
+				(a, b) => this.#scoreEntry(b, viewport) - this.#scoreEntry(a, viewport),
+			);
+
+		const evictedKeys: Array<string | number> = [];
+		let currentEstimate = estimate;
+		for (const candidate of candidates) {
+			if (currentEstimate.utilizationRatio <= this.evictionTarget) break;
+			evictedKeys.push(candidate.key);
+			this.#entries.delete(candidate.key);
+			currentEstimate = {
+				...currentEstimate,
+				usageBytes: Math.max(
+					0,
+					currentEstimate.usageBytes - candidate.estimatedSizeBytes,
+				),
+				utilizationRatio:
+					Math.max(
+						0,
+						currentEstimate.usageBytes - candidate.estimatedSizeBytes,
+					) / currentEstimate.quotaBytes,
+			};
+		}
+
+		if (evictedKeys.length > 0) {
+			await this.options.deleteRows(evictedKeys);
+		}
+		return { evictedKeys, estimate: currentEstimate };
+	}
+
+	#scoreEntry(entry: CacheEntry, viewport: CacheViewport): number {
+		const now = Date.now();
+		const timeSinceAccessMs = Math.max(0, now - entry.lastAccessedAt);
+		const distance = this.#distanceFromViewport(entry, viewport);
+		// Weighted score: heavily prefer evicting far-away rows that have not been used recently.
+		return timeSinceAccessMs * 0.001 + distance * 1000;
+	}
+
+	#distanceFromViewport(entry: CacheEntry, viewport: CacheViewport): number {
+		const value = entry.sortPositions.get(viewport.sortColumn);
+		if (value === undefined || value === null) return 1;
+		// `fromValue` / `toValue` come from first/last *visible* rows. After a local sort-key edit,
+		// the first row can temporarily sort *after* the last row, so from > to. Treat the band as
+		// [min, max] so rows between them stay protected; otherwise every entry looks "outside" and
+		// eviction can wipe the visible window (empty list, no throw).
+		let low = viewport.fromValue;
+		let high = viewport.toValue;
+		if (this.#compareValues(low, high) > 0) {
+			const t = low;
+			low = high;
+			high = t;
+		}
+		const lowerCompare = this.#compareValues(value, low);
+		const upperCompare = this.#compareValues(value, high);
+		if (lowerCompare >= 0 && upperCompare <= 0) return 0;
+		const lowerDistance = Math.abs(lowerCompare);
+		const upperDistance = Math.abs(upperCompare);
+		return Math.min(lowerDistance, upperDistance);
+	}
+
+	#isEntryProtectedByViewport(
+		entry: CacheEntry,
+		viewport: CacheViewport,
+	): boolean {
+		return this.#distanceFromViewport(entry, viewport) === 0;
+	}
+
+	#compareValues(left: unknown, right: unknown): number {
+		const normalizedLeft = left instanceof Date ? left.getTime() : left;
+		const normalizedRight = right instanceof Date ? right.getTime() : right;
+		if (
+			typeof normalizedLeft === "number" &&
+			typeof normalizedRight === "number"
+		) {
+			if (normalizedLeft === normalizedRight) return 0;
+			return normalizedLeft < normalizedRight ? -1 : 1;
+		}
+		const leftStr = String(normalizedLeft);
+		const rightStr = String(normalizedRight);
+		return leftStr.localeCompare(rightStr);
+	}
+}

package/src/connect-partial-sync.ts

@@ -0,0 +1,349 @@
+import { exhaustiveGuard } from "@firtoz/maybe-error";
+import type { SyncMessage } from "@firtoz/db-helpers";
+import { ZodWebSocketClient } from "@firtoz/websocket-do/zod-client";
+import type { PartialSyncClientBridge } from "./partial-sync-client-bridge";
+import type { SyncClientBridge } from "./sync-client-bridge";
+import {
+	partialSyncRowKey,
+	type PartialSyncRowShape,
+} from "./partial-sync-row-key";
+import {
+	createClientMessageSchema,
+	createServerMessageSchema,
+	DEFAULT_SYNC_COLLECTION_ID,
+	type SyncClientMessage,
+	type SyncServerMessage,
+} from "./sync-protocol";
+
+/**
+ * Optional browser `document` for Page Visibility — accessed via `globalThis` so this module typechecks
+ * under configs that omit the DOM global (e.g. Worker-only `lib`).
+ */
+type PageVisibilityDocument = {
+	readonly visibilityState: string;
+	addEventListener(type: "visibilitychange", listener: () => void): void;
+	removeEventListener(type: "visibilitychange", listener: () => void): void;
+};
+
+function pageVisibilityDocument(): PageVisibilityDocument | undefined {
+	return (
+		globalThis as typeof globalThis & { document?: PageVisibilityDocument }
+	).document;
+}
+
+export type ConnectPartialSyncTransport = "json" | "msgpack";
+
+export type ConnectPartialSyncOptions<
+	TItem extends PartialSyncRowShape = PartialSyncRowShape,
+> = {
+	url: string;
+	transport?: ConnectPartialSyncTransport;
+	/** Prefer a module-level function or `useCallback`; a new inline function each render can churn effects. */
+	serializeJson?: (value: unknown) => string;
+	/** Prefer a module-level function or `useCallback`; a new inline function each render can churn effects. */
+	deserializeJson?: (raw: string) => unknown;
+	setTransportSend: (send: (msg: SyncClientMessage) => void) => void;
+	onServerMessage?: (msg: SyncServerMessage<TItem>) => void;
+	/**
+	 * When set, inbound messages are split: range traffic → `bridge`, ack/reject/syncBackfill → mutation bridge;
+	 * `syncBatch` applies via mutation bridge then updates partial cache ids (no double `receiveSync`).
+	 */
+	mutationBridge?: SyncClientBridge<TItem>;
+};
+
+/**
+ * @internal Exported for unit tests; prefer {@link connectPartialSync} in apps.
+ * After a `rangePatch`, call `partialBridge.flushPendingCoalescedInboundUpdates()` unless you use
+ * `connectPartialSync` (the pump flushes coalesced updates after each inbound job).
+ */
+export async function dispatchPartialSyncServerMessage<
+	TItem extends PartialSyncRowShape,
+>(
+	msg: SyncServerMessage<TItem>,
+	partialBridge: PartialSyncClientBridge<TItem>,
+	mutationBridge: SyncClientBridge<TItem> | undefined,
+): Promise<void> {
+	const mid = msg.collectionId ?? DEFAULT_SYNC_COLLECTION_ID;
+	const forPartial = mid === partialBridge.collectionId;
+	const forMutation =
+		mutationBridge !== undefined && mid === mutationBridge.collectionId;
+
+	if (mutationBridge === undefined) {
+		if (!forPartial) return;
+		await partialBridge.handleServerMessage(msg);
+		return;
+	}
+
+	switch (msg.type) {
+		case "queryRangeChunk":
+		case "rangeUpToDate":
+		case "rangeDelta":
+		case "rangeReconcileResult":
+		case "rangePatch":
+		case "pong":
+			if (!forPartial) return;
+			await partialBridge.handleServerMessage(msg);
+			return;
+		case "ack":
+		case "reject":
+		case "syncBackfill":
+			if (!forMutation) return;
+			await mutationBridge.handleServerMessage(msg);
+			return;
+		case "syncBatch": {
+			if (!forMutation) return;
+			await mutationBridge.handleServerMessage(msg);
+			if (forPartial) {
+				const changes = msg.changes as SyncMessage<TItem>[];
+				partialBridge.syncTrackedIdsFromMessages(changes);
+			}
+			return;
+		}
+		default:
+			exhaustiveGuard(msg);
+	}
+}
+
+export type CoalescedRangePatchMessage<TItem extends PartialSyncRowShape> =
+	Extract<SyncServerMessage<TItem>, { type: "rangePatch" }>;
+
+/**
+ * Last-wins merge of buffered `rangePatch` messages (same row id keeps the latest patch).
+ * @internal Exported for unit tests.
+ */
+export function mergeCoalescedRangePatches<TItem extends PartialSyncRowShape>(
+	patches: CoalescedRangePatchMessage<TItem>[],
+): CoalescedRangePatchMessage<TItem>[] {
+	let truncateWinner: CoalescedRangePatchMessage<TItem> | undefined;
+	const byRow = new Map<string | number, CoalescedRangePatchMessage<TItem>>();
+	for (const p of patches) {
+		const ch = p.change;
+		switch (ch.type) {
+			case "truncate":
+				truncateWinner = p;
+				byRow.clear();
+				break;
+			case "insert":
+			case "update": {
+				const k = partialSyncRowKey(ch.value.id);
+				byRow.set(k, p);
+				break;
+			}
+			case "delete":
+				byRow.set(ch.key, p);
+				break;
+			default:
+				exhaustiveGuard(ch);
+		}
+	}
+	if (truncateWinner !== undefined) return [truncateWinner];
+	return [...byRow.values()];
+}
+
+export function connectPartialSync<
+	TItem extends PartialSyncRowShape = PartialSyncRowShape,
+>(
+	bridge: PartialSyncClientBridge<TItem>,
+	options: ConnectPartialSyncOptions<TItem>,
+): () => void {
+	const clientSchema = createClientMessageSchema();
+	const serverSchema = createServerMessageSchema<TItem>();
+	const useMsgpack = options.transport === "msgpack";
+	const mutationBridge = options.mutationBridge;
+	/**
+	 * Serialized inbound handling without chaining `.then` per message (that grows the promise graph
+	 * linearly with traffic → accumulating latency, memory pressure, and eventual tab crashes).
+	 */
+	type InboundJob = () => Promise<void>;
+	const inboundWorkQueue: InboundJob[] = [];
+	let inboundPumpRunning = false;
+	let connectDisposed = false;
+
+	/**
+	 * Single pump: drain all queued jobs, then **await** `flushPendingCoalescedInboundUpdates` so
+	 * coalesced `rangePatch` rows are applied before the next drain pass (prevents duplicate-insert
+	 * from `ack`/`syncBatch`). If new jobs arrived during the flush, loop.
+	 *
+	 * Only one pump runs at a time (`inboundPumpRunning`). Unlike the earlier version that left
+	 * the flag set while awaiting a potentially-stalled flush, the flag is reset in a `finally`
+	 * that runs **after** both the drain and the flush, and errors are caught per-job so one
+	 * failure cannot block subsequent work.
+	 */
+	const runInboundPump = async (): Promise<void> => {
+		if (inboundPumpRunning) return;
+		inboundPumpRunning = true;
+		try {
+			do {
+				while (inboundWorkQueue.length > 0) {
+					const job = inboundWorkQueue.shift();
+					if (job === undefined) continue;
+					try {
+						await job();
+					} catch (err) {
+						console.error("[connectPartialSync] inbound job error", err);
+					}
+					if (connectDisposed) return;
+				}
+				try {
+					await bridge.flushPendingCoalescedInboundUpdates();
+				} catch (err) {
+					console.error("[connectPartialSync] coalesced flush error", err);
+				}
+			} while (!connectDisposed && inboundWorkQueue.length > 0);
+		} finally {
+			inboundPumpRunning = false;
+			if (!connectDisposed && inboundWorkQueue.length > 0) {
+				void runInboundPump();
+			}
+		}
+	};
+
+	const enqueueInbound = (job: InboundJob): void => {
+		inboundWorkQueue.push(job);
+		void runInboundPump();
+	};
+
+	let rangePatchCoalesceBuffer: CoalescedRangePatchMessage<TItem>[] = [];
+	/**
+	 * `requestAnimationFrame` can run between WebSocket deliveries, flushing a single `rangePatch`
+	 * while more patches are still queued on the inbound chain — causing duplicate `receiveSync`
+	 * and stepped replay. A deduped microtask flush runs after the current sync work so bursts
+	 * in the same turn coalesce before paint.
+	 */
+	let rangePatchFlushMicrotaskQueued = false;
+
+	const drainRangePatchCoalesceBuffer = async (): Promise<void> => {
+		if (rangePatchCoalesceBuffer.length === 0) return;
+		const merged = mergeCoalescedRangePatches(rangePatchCoalesceBuffer);
+		rangePatchCoalesceBuffer = [];
+		for (const m of merged) {
+			await dispatchPartialSyncServerMessage(m, bridge, mutationBridge);
+		}
+	};
+
+	const scheduleCoalescedRangePatchFlushMicrotask = (): void => {
+		if (rangePatchFlushMicrotaskQueued) return;
+		rangePatchFlushMicrotaskQueued = true;
+		queueMicrotask(() => {
+			rangePatchFlushMicrotaskQueued = false;
+			enqueueInbound(async () => {
+				await drainRangePatchCoalesceBuffer();
+			});
+		});
+	};
+
+	const cancelCoalescedRangePatchDeferredFlush = (): void => {
+		rangePatchFlushMicrotaskQueued = false;
+	};
+
+	const flushCoalescedRangePatchesInline = async (): Promise<void> => {
+		cancelCoalescedRangePatchDeferredFlush();
+		await drainRangePatchCoalesceBuffer();
+	};
+
+	const zodClient = new ZodWebSocketClient({
+		url: options.url,
+		clientSchema,
+		serverSchema,
+		enableBufferMessages: useMsgpack,
+		...(useMsgpack
+			? {}
+			: {
+					serializeJson: options.serializeJson ?? JSON.stringify,
+					deserializeJson: options.deserializeJson ?? JSON.parse,
+				}),
+		onMessage: (msg) => {
+			enqueueInbound(async () => {
+				options.onServerMessage?.(msg);
+				if (mutationBridge === undefined) {
+					await dispatchPartialSyncServerMessage(msg, bridge, mutationBridge);
+					return;
+				}
+				const mid = msg.collectionId ?? DEFAULT_SYNC_COLLECTION_ID;
+				const forPartial = mid === bridge.collectionId;
+				if (msg.type === "rangePatch" && forPartial) {
+					rangePatchCoalesceBuffer.push(msg);
+					scheduleCoalescedRangePatchFlushMicrotask();
+					return;
+				}
+				await flushCoalescedRangePatchesInline();
+				await dispatchPartialSyncServerMessage(msg, bridge, mutationBridge);
+			});
+		},
+	});
+
+	/** DO not call `WebSocket.send` until `OPEN`; queue outbound messages until then. */
+	const pendingOutbound: SyncClientMessage[] = [];
+
+	const flushPendingOutbound = () => {
+		while (
+			pendingOutbound.length > 0 &&
+			zodClient.socket.readyState === WebSocket.OPEN
+		) {
+			const message = pendingOutbound.shift();
+			if (message !== undefined) {
+				zodClient.send(message);
+			}
+		}
+	};
+
+	options.setTransportSend((message) => {
+		if (zodClient.socket.readyState === WebSocket.OPEN) {
+			zodClient.send(message);
+		} else {
+			pendingOutbound.push(message);
+		}
+	});
+
+	bridge.setConnecting();
+
+	const onOpen = () => {
+		flushPendingOutbound();
+		bridge.setConnected(true);
+		mutationBridge?.setConnected(true);
+	};
+	const onClose = () => {
+		pendingOutbound.length = 0;
+		bridge.setConnected(false);
+		mutationBridge?.setConnected(false);
+	};
+
+	zodClient.socket.addEventListener("open", onOpen);
+	zodClient.socket.addEventListener("close", onClose);
+
+	if (zodClient.socket.readyState === WebSocket.OPEN) {
+		onOpen();
+	}
+
+	const onVisibilityFlush = (): void => {
+		const doc = pageVisibilityDocument();
+		if (doc === undefined) return;
+		if (doc.visibilityState !== "visible") return;
+		enqueueInbound(async () => {
+			await flushCoalescedRangePatchesInline();
+		});
+	};
+
+	const visibilityDoc = pageVisibilityDocument();
+	if (visibilityDoc !== undefined) {
+		visibilityDoc.addEventListener("visibilitychange", onVisibilityFlush);
+	}
+
+	return () => {
+		connectDisposed = true;
+		const docCleanup = pageVisibilityDocument();
+		if (docCleanup !== undefined) {
+			docCleanup.removeEventListener("visibilitychange", onVisibilityFlush);
+		}
+		cancelCoalescedRangePatchDeferredFlush();
+		inboundWorkQueue.length = 0;
+		rangePatchCoalesceBuffer = [];
+		zodClient.socket.removeEventListener("open", onOpen);
+		zodClient.socket.removeEventListener("close", onClose);
+		pendingOutbound.length = 0;
+		bridge.setConnected(false);
+		mutationBridge?.setConnected(false);
+		options.setTransportSend(() => {});
+		zodClient.close();
+	};
+}