@firtoz/collection-sync 1.0.0

package/src/sync-protocol.ts

@@ -0,0 +1,362 @@
+import type { SyncMessage } from "@firtoz/db-helpers";
+import { exhaustiveGuard } from "@firtoz/maybe-error";
+import { z } from "zod";
+
+/** Default {@link SyncClientMessage} / {@link SyncServerMessage} `collectionId` when omitted on the wire. */
+export const DEFAULT_SYNC_COLLECTION_ID = "default" as const;
+
+const collectionIdSchema = z
+	.string()
+	.min(1)
+	.default(DEFAULT_SYNC_COLLECTION_ID);
+
+const mutationTypeSchema = z.enum(["insert", "update", "delete", "truncate"]);
+
+export const mutationIntentSchema = z.object({
+	clientMutationId: z.string().min(1),
+	type: mutationTypeSchema,
+	value: z.record(z.string(), z.unknown()).optional(),
+	previousValue: z.record(z.string(), z.unknown()).optional(),
+	key: z.union([z.string(), z.number()]).optional(),
+});
+
+export type MutationIntent = z.infer<typeof mutationIntentSchema>;
+
+export type SyncSortDirection = "asc" | "desc";
+export type SyncRangeSort = {
+	column: string;
+	direction: SyncSortDirection;
+};
+
+/** Fixed operator set for predicate-based range queries (e.g. spatial filters). */
+export type RangeConditionOp =
+	| "gt"
+	| "gte"
+	| "lt"
+	| "lte"
+	| "eq"
+	| "neq"
+	| "between";
+
+export type RangeCondition = {
+	column: string;
+	op: RangeConditionOp;
+	value: unknown;
+	/** Required when `op` is `"between"`. */
+	valueTo?: unknown;
+};
+
+export type IndexRangeCursor = {
+	kind: "index";
+	mode: "cursor";
+	sort: SyncRangeSort;
+	limit: number;
+	afterCursor: unknown | null;
+};
+
+export type IndexRangeOffset = {
+	kind: "index";
+	mode: "offset";
+	sort: SyncRangeSort;
+	limit: number;
+	offset: number;
+};
+
+export type PredicateRange = {
+	kind: "predicate";
+	conditions: RangeCondition[];
+	sort?: SyncRangeSort;
+	limit?: number;
+};
+
+export type SyncRange = IndexRangeCursor | IndexRangeOffset | PredicateRange;
+
+/** Client watermark for reconciliation (max row version in range + count). */
+export type RangeFingerprint = {
+	version: number;
+	count: number;
+};
+
+const syncRangeSortSchema = z.object({
+	column: z.string().min(1),
+	direction: z.enum(["asc", "desc"]),
+});
+
+const rangeConditionOpSchema = z.enum([
+	"gt",
+	"gte",
+	"lt",
+	"lte",
+	"eq",
+	"neq",
+	"between",
+]);
+
+const rangeConditionSchema = z.object({
+	column: z.string().min(1),
+	op: rangeConditionOpSchema,
+	value: z.unknown(),
+	valueTo: z.unknown().optional(),
+});
+
+const indexRangeCursorSchema = z.object({
+	kind: z.literal("index"),
+	mode: z.literal("cursor"),
+	sort: syncRangeSortSchema,
+	limit: z.number().int().positive(),
+	afterCursor: z.unknown().nullable(),
+});
+
+const indexRangeOffsetSchema = z.object({
+	kind: z.literal("index"),
+	mode: z.literal("offset"),
+	sort: syncRangeSortSchema,
+	limit: z.number().int().positive(),
+	offset: z.number().int().nonnegative(),
+});
+
+const predicateRangeSchema = z.object({
+	kind: z.literal("predicate"),
+	conditions: z.array(rangeConditionSchema).min(1),
+	sort: syncRangeSortSchema.optional(),
+	limit: z.number().int().positive().optional(),
+});
+
+export const syncRangeSchema: z.ZodType<SyncRange> = z.union([
+	indexRangeCursorSchema,
+	indexRangeOffsetSchema,
+	predicateRangeSchema,
+]);
+
+const rangeFingerprintSchema = z.object({
+	version: z.number().int().nonnegative(),
+	count: z.number().int().nonnegative(),
+});
+
+export function createClientMessageSchema() {
+	return z.discriminatedUnion("type", [
+		z.object({
+			type: z.literal("mutateBatch"),
+			collectionId: collectionIdSchema,
+			clientId: z.string().min(1),
+			mutations: z.array(mutationIntentSchema).min(1),
+		}),
+		z.object({
+			type: z.literal("syncHello"),
+			collectionId: collectionIdSchema,
+			clientId: z.string().min(1),
+			lastAckedServerVersion: z.number().int().nonnegative(),
+		}),
+		z.object({
+			type: z.literal("ping"),
+			collectionId: collectionIdSchema,
+			clientId: z.string().min(1),
+			timestamp: z.number(),
+		}),
+		z.object({
+			type: z.literal("queryRange"),
+			collectionId: collectionIdSchema,
+			clientId: z.string().min(1),
+			requestId: z.string().min(1),
+			sort: z.object({
+				column: z.string().min(1),
+				direction: z.enum(["asc", "desc"]),
+			}),
+			limit: z.number().int().positive(),
+			afterCursor: z.unknown().nullable(),
+		}),
+		z.object({
+			type: z.literal("queryByOffset"),
+			collectionId: collectionIdSchema,
+			clientId: z.string().min(1),
+			requestId: z.string().min(1),
+			sort: z.object({
+				column: z.string().min(1),
+				direction: z.enum(["asc", "desc"]),
+			}),
+			limit: z.number().int().positive(),
+			offset: z.number().int().nonnegative(),
+		}),
+		z.object({
+			type: z.literal("rangeQuery"),
+			collectionId: collectionIdSchema,
+			clientId: z.string().min(1),
+			requestId: z.string().min(1),
+			range: syncRangeSchema,
+			fingerprint: rangeFingerprintSchema.optional(),
+		}),
+		z.object({
+			type: z.literal("rangeReconcile"),
+			collectionId: collectionIdSchema,
+			clientId: z.string().min(1),
+			requestId: z.string().min(1),
+			range: syncRangeSchema,
+			manifest: z.array(
+				z.object({
+					id: z.union([z.string(), z.number()]),
+					version: z.number().int().nonnegative(),
+				}),
+			),
+		}),
+	]);
+}
+
+export type SyncClientMessage = z.infer<
+	ReturnType<typeof createClientMessageSchema>
+>;
+
+export const clientMessageSchema = createClientMessageSchema();
+
+/** How to apply {@link SyncServerMessage} `syncBackfill` changes on the client. */
+export type SyncBackfillMode = "snapshot" | "delta";
+
+type DistributiveOmit<T, K extends PropertyKey> = T extends unknown
+	? Omit<T, K>
+	: never;
+
+/** Client payload before `collectionId` is attached (bridge outbox helpers). */
+export type SyncClientMessageBody = DistributiveOmit<
+	SyncClientMessage,
+	"collectionId"
+>;
+
+export function createServerMessageSchema<
+	TItem = unknown,
+	TKey extends string | number = string | number,
+>() {
+	const syncMessageSchema = z.custom<SyncMessage<TItem, TKey>>();
+	return z.discriminatedUnion("type", [
+		z.object({
+			type: z.literal("ack"),
+			collectionId: collectionIdSchema,
+			clientId: z.string().min(1),
+			clientMutationIds: z.array(z.string().min(1)),
+			serverVersion: z.number().int().nonnegative(),
+			changes: z.array(syncMessageSchema),
+		}),
+		z.object({
+			type: z.literal("syncBatch"),
+			collectionId: collectionIdSchema,
+			serverVersion: z.number().int().nonnegative(),
+			changes: z.array(syncMessageSchema),
+		}),
+		z.object({
+			type: z.literal("syncBackfill"),
+			collectionId: collectionIdSchema,
+			mode: z.enum(["snapshot", "delta"]),
+			serverVersion: z.number().int().nonnegative(),
+			changes: z.array(syncMessageSchema),
+			chunkIndex: z.number().int().nonnegative().optional(),
+			totalChunks: z.number().int().positive().optional(),
+		}),
+		z.object({
+			type: z.literal("reject"),
+			collectionId: collectionIdSchema,
+			clientId: z.string().min(1),
+			clientMutationId: z.string().min(1),
+			reason: z.string().min(1),
+			correctiveChanges: z.array(syncMessageSchema).default([]),
+		}),
+		z.object({
+			type: z.literal("pong"),
+			collectionId: collectionIdSchema,
+			timestamp: z.number(),
+		}),
+		z.object({
+			type: z.literal("queryRangeChunk"),
+			collectionId: collectionIdSchema,
+			requestId: z.string().min(1),
+			rows: z.array(z.custom<TItem>()),
+			totalCount: z.number().int().nonnegative(),
+			lastCursor: z.unknown().nullable(),
+			hasMore: z.boolean(),
+			chunkIndex: z.number().int().nonnegative(),
+			done: z.boolean(),
+		}),
+		z.object({
+			type: z.literal("rangePatch"),
+			collectionId: collectionIdSchema,
+			change: syncMessageSchema,
+			viewTransition: z.enum(["enterView", "exitView"]).optional(),
+		}),
+		z.object({
+			type: z.literal("rangeUpToDate"),
+			collectionId: collectionIdSchema,
+			requestId: z.string().min(1),
+			totalCount: z.number().int().nonnegative(),
+		}),
+		z.object({
+			type: z.literal("rangeDelta"),
+			collectionId: collectionIdSchema,
+			requestId: z.string().min(1),
+			totalCount: z.number().int().nonnegative(),
+			changes: z.array(syncMessageSchema),
+			lastCursor: z.unknown().optional(),
+		}),
+		z.object({
+			type: z.literal("rangeReconcileResult"),
+			collectionId: collectionIdSchema,
+			requestId: z.string().min(1),
+			added: z.array(syncMessageSchema),
+			updated: z.array(syncMessageSchema),
+			stale: z.array(z.union([z.string(), z.number()])),
+			movedHints: z.array(
+				z.object({
+					id: z.union([z.string(), z.number()]),
+					hint: z.record(z.string(), z.unknown()),
+				}),
+			),
+			totalCount: z.number().int().nonnegative(),
+		}),
+	]);
+}
+
+export type SyncServerMessage<
+	TItem = unknown,
+	TKey extends string | number = string | number,
+> = z.infer<ReturnType<typeof createServerMessageSchema<TItem, TKey>>>;
+
+/** Server payload before `collectionId` is attached (bridge outbox helpers). */
+export type SyncServerMessageBody<
+	TItem = unknown,
+	TKey extends string | number = string | number,
+> = DistributiveOmit<SyncServerMessage<TItem, TKey>, "collectionId">;
+
+/** Attach `collectionId` to an outbound server message (single-collection servers). */
+export function withServerCollectionId<TItem, TKey extends string | number>(
+	collectionId: string,
+	message: SyncServerMessage<TItem, TKey>,
+): SyncServerMessage<TItem, TKey> {
+	return { ...message, collectionId };
+}
+
+export const serverMessageSchema = createServerMessageSchema();
+
+export function toSyncMessage(intent: MutationIntent): SyncMessage {
+	switch (intent.type) {
+		case "insert":
+			if (!intent.value) throw new Error("Insert intent requires value");
+			return { type: "insert", value: intent.value };
+		case "update":
+			if (!intent.value || !intent.previousValue) {
+				throw new Error("Update intent requires value and previousValue");
+			}
+			return {
+				type: "update",
+				value: intent.value,
+				previousValue: intent.previousValue,
+			};
+		case "delete":
+			if (intent.key === undefined)
+				throw new Error("Delete intent requires key");
+			return { type: "delete", key: intent.key };
+		case "truncate":
+			return { type: "truncate" };
+		default:
+			exhaustiveGuard(intent.type);
+	}
+}
+
+export function createClientMutationId(prefix = "m"): string {
+	return `${prefix}_${crypto.randomUUID()}`;
+}

package/src/sync-server-bridge.ts

@@ -0,0 +1,267 @@
+import type { SyncMessage } from "@firtoz/db-helpers";
+import { exhaustiveGuard } from "@firtoz/maybe-error";
+import type {
+	MutationIntent,
+	SyncBackfillMode,
+	SyncClientMessage,
+	SyncServerMessage,
+	SyncServerMessageBody,
+} from "./sync-protocol";
+import { DEFAULT_SYNC_COLLECTION_ID } from "./sync-protocol";
+import { toSyncMessage } from "./sync-protocol";
+import {
+	partialSyncRowKey,
+	type PartialSyncRowShape,
+} from "./partial-sync-row-key";
+
+export interface SyncServerBridgeStore<TItem> {
+	applySyncMessages: (messages: SyncMessage<TItem>[]) => Promise<void>;
+	getSnapshotMessages: () => Promise<SyncMessage<TItem>[]>;
+	getRow: (key: string | number) => Promise<TItem | undefined>;
+}
+
+export interface SyncServerBridgeOptions<TItem> {
+	store: SyncServerBridgeStore<TItem>;
+	sendToClient: (clientId: string, message: SyncServerMessage<TItem>) => void;
+	broadcastExcept: (
+		excludeClientId: string,
+		message: SyncServerMessage<TItem>,
+	) => void | Promise<void>;
+	/** Deliver a server message to every connected client (e.g. {@link SyncServerBridge.pushServerChanges}). */
+	broadcastAll?: (message: SyncServerMessage<TItem>) => void;
+	/** Maximum number of changes per `syncBackfill` frame. Defaults to 500. */
+	backfillChunkSize?: number;
+	/** Multiplex key for sync messages. Default {@link DEFAULT_SYNC_COLLECTION_ID}. */
+	collectionId?: string;
+}
+
+export class SyncServerBridge<TItem extends PartialSyncRowShape> {
+	#serverVersion = 0;
+	#changeLog: Array<{ serverVersion: number; changes: SyncMessage<TItem>[] }> =
+		[];
+	readonly #cid: string;
+
+	constructor(private readonly options: SyncServerBridgeOptions<TItem>) {
+		this.#cid = options.collectionId ?? DEFAULT_SYNC_COLLECTION_ID;
+	}
+
+	get collectionId(): string {
+		return this.#cid;
+	}
+
+	#emit(clientId: string, body: SyncServerMessageBody<TItem>): void {
+		this.options.sendToClient(clientId, {
+			...body,
+			collectionId: this.#cid,
+		} as SyncServerMessage<TItem>);
+	}
+
+	#broadcastExcept(
+		excludeClientId: string,
+		body: SyncServerMessageBody<TItem>,
+	): void | Promise<void> {
+		return this.options.broadcastExcept(excludeClientId, {
+			...body,
+			collectionId: this.#cid,
+		} as SyncServerMessage<TItem>);
+	}
+
+	#broadcastAll(body: SyncServerMessageBody<TItem>): void {
+		this.options.broadcastAll?.({
+			...body,
+			collectionId: this.#cid,
+		} as SyncServerMessage<TItem>);
+	}
+
+	get serverVersion(): number {
+		return this.#serverVersion;
+	}
+
+	async handleClientMessage(message: SyncClientMessage): Promise<void> {
+		const mid = message.collectionId ?? DEFAULT_SYNC_COLLECTION_ID;
+		if (mid !== this.#cid) return;
+		switch (message.type) {
+			case "ping":
+				this.#emit(message.clientId, {
+					type: "pong",
+					timestamp: message.timestamp,
+				});
+				return;
+			case "syncHello": {
+				const { mode, changes } = await this.#resolveBackfill(
+					message.lastAckedServerVersion,
+				);
+				const chunks = this.#chunkBackfillChanges(changes);
+				const totalChunks = chunks.length;
+				for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex += 1) {
+					this.#emit(message.clientId, {
+						type: "syncBackfill",
+						mode,
+						serverVersion: this.#serverVersion,
+						changes: chunks[chunkIndex],
+						chunkIndex,
+						totalChunks,
+					});
+				}
+				return;
+			}
+			case "mutateBatch":
+				await this.#handleMutateBatch(message);
+				return;
+			case "queryRange":
+			case "queryByOffset":
+			case "rangeQuery":
+			case "rangeReconcile":
+				// Not supported by the full-sync bridge; partial sync uses PartialSyncServerBridge.
+				return;
+			default:
+				exhaustiveGuard(message);
+		}
+	}
+
+	async #handleMutateBatch(
+		message: Extract<SyncClientMessage, { type: "mutateBatch" }>,
+	): Promise<void> {
+		const resolvedChanges: SyncMessage<TItem>[] = [];
+		const acceptedMutationIds: string[] = [];
+
+		for (const mutation of message.mutations) {
+			try {
+				const change = await this.#intentToMessageLww(mutation);
+				if (!change) {
+					continue;
+				}
+				resolvedChanges.push(change);
+				acceptedMutationIds.push(mutation.clientMutationId);
+			} catch (error) {
+				this.#emit(message.clientId, {
+					type: "reject",
+					clientId: message.clientId,
+					clientMutationId: mutation.clientMutationId,
+					reason: error instanceof Error ? error.message : String(error),
+					correctiveChanges: [],
+				});
+			}
+		}
+
+		if (resolvedChanges.length === 0) return;
+
+		await this.options.store.applySyncMessages(resolvedChanges);
+		this.#serverVersion += 1;
+		this.#changeLog.push({
+			serverVersion: this.#serverVersion,
+			changes: resolvedChanges,
+		});
+
+		this.#emit(message.clientId, {
+			type: "ack",
+			clientId: message.clientId,
+			clientMutationIds: acceptedMutationIds,
+			serverVersion: this.#serverVersion,
+			changes: resolvedChanges,
+		});
+
+		await Promise.resolve(
+			this.#broadcastExcept(message.clientId, {
+				type: "syncBatch",
+				serverVersion: this.#serverVersion,
+				changes: resolvedChanges,
+			}),
+		);
+	}
+
+	async #intentToMessageLww(
+		intent: MutationIntent,
+	): Promise<SyncMessage<TItem> | null> {
+		const base = toSyncMessage(intent) as SyncMessage<TItem>;
+		if (base.type !== "update") {
+			return base;
+		}
+		const key = partialSyncRowKey(intent.key ?? base.value.id);
+		const existing = await this.options.store.getRow(key);
+		if (!existing) {
+			return base;
+		}
+		const incomingUpdatedAt = this.#toMillis(base.value.updatedAt);
+		const existingUpdatedAt = this.#toMillis(existing.updatedAt);
+		if (incomingUpdatedAt <= existingUpdatedAt) {
+			return {
+				type: "update",
+				value: existing,
+				previousValue: base.previousValue,
+			};
+		}
+		return base;
+	}
+
+	#toMillis(value: number | Date | null | undefined): number {
+		if (typeof value === "number") return value;
+		if (value instanceof Date) return value.getTime();
+		return 0;
+	}
+
+	async #resolveBackfill(lastAckedServerVersion: number): Promise<{
+		mode: SyncBackfillMode;
+		changes: SyncMessage<TItem>[];
+	}> {
+		if (lastAckedServerVersion <= 0) {
+			return {
+				mode: "snapshot",
+				changes: await this.options.store.getSnapshotMessages(),
+			};
+		}
+		// Client can be "ahead" after server restart/reset; force a full snapshot.
+		if (lastAckedServerVersion > this.#serverVersion) {
+			return {
+				mode: "snapshot",
+				changes: await this.options.store.getSnapshotMessages(),
+			};
+		}
+		const oldestLogged = this.#changeLog[0]?.serverVersion;
+		if (
+			oldestLogged !== undefined &&
+			lastAckedServerVersion < oldestLogged - 1
+		) {
+			return {
+				mode: "snapshot",
+				changes: await this.options.store.getSnapshotMessages(),
+			};
+		}
+
+		const changes: SyncMessage<TItem>[] = [];
+		for (const entry of this.#changeLog) {
+			if (entry.serverVersion > lastAckedServerVersion) {
+				changes.push(...entry.changes);
+			}
+		}
+		return { mode: "delta", changes };
+	}
+
+	#chunkBackfillChanges(changes: SyncMessage<TItem>[]): SyncMessage<TItem>[][] {
+		const chunkSize = Math.max(1, this.options.backfillChunkSize ?? 500);
+		if (changes.length === 0) return [[]];
+		const chunks: SyncMessage<TItem>[][] = [];
+		for (let i = 0; i < changes.length; i += chunkSize) {
+			chunks.push(changes.slice(i, i + chunkSize));
+		}
+		return chunks;
+	}
+
+	/**
+	 * Apply mutations that originated on the server (cron, admin API, etc.) and fan out to all clients.
+	 */
+	async pushServerChanges(changes: SyncMessage<TItem>[]): Promise<void> {
+		if (changes.length === 0) return;
+		await this.options.store.applySyncMessages(changes);
+		this.#serverVersion += 1;
+		this.#changeLog.push({
+			serverVersion: this.#serverVersion,
+			changes,
+		});
+		this.#broadcastAll({
+			type: "syncBatch",
+			serverVersion: this.#serverVersion,
+			changes,
+		});
+	}
+}