@livestore/sync-cf 0.4.0-dev.8 → 0.4.0

package/src/client/transport/http-rpc-client.ts

@@ -1,5 +1,6 @@
-import { InvalidPullError, InvalidPushError, SyncBackend, UnexpectedError } from '@livestore/common'
+import { SyncBackend, UnknownError } from '@livestore/common'
 import type { EventSequenceNumber } from '@livestore/common/schema'
+import { splitChunkBySize } from '@livestore/common/sync'
 import { omit } from '@livestore/utils'
 import {
   Chunk,
@@ -18,6 +19,8 @@ import {
   SubscriptionRef,
   UrlParams,
 } from '@livestore/utils/effect'
+
+import { MAX_HTTP_REQUEST_BYTES, MAX_PUSH_EVENTS_PER_REQUEST } from '../../common/constants.ts'
 import { SyncHttpRpc } from '../../common/http-rpc-schema.ts'
 import { SearchParamsSchema } from '../../common/mod.ts'
 import type { SyncMetadata } from '../../common/sync-message-types.ts'
@@ -74,7 +77,7 @@ export const makeHttpSync =
         storeId,
         payload,
         transport: 'http',
-      }).pipe(UnexpectedError.mapToUnexpectedError)
+      }).pipe(UnknownError.mapToUnknownError)
 
       const urlParams = UrlParams.fromInput(urlParamsData)
 
@@ -101,7 +104,7 @@ export const makeHttpSync =
 
         yield* SubscriptionRef.set(isConnected, true)
       }).pipe(
-        UnexpectedError.mapToUnexpectedError,
+        UnknownError.mapToUnknownError,
         Effect.timeout(pingTimeout),
         Effect.catchTag('TimeoutException', () => SubscriptionRef.set(isConnected, false)),
       )
@@ -114,14 +117,14 @@ export const makeHttpSync =
       }
 
       // Helps already establish a TCP connection to the server
-      const connect = ping.pipe(UnexpectedError.mapToUnexpectedError)
+      const connect = ping.pipe(UnknownError.mapToUnknownError)
 
       const backendIdHelper = yield* SyncBackend.makeBackendIdHelper
 
       const mapCursor = (cursor: Option.Option<{ eventSequenceNumber: number }>) =>
         cursor.pipe(
           Option.map((a) => ({
-            eventSequenceNumber: a.eventSequenceNumber as EventSequenceNumber.GlobalEventSequenceNumber,
+            eventSequenceNumber: a.eventSequenceNumber as EventSequenceNumber.Global.Type,
             backendId: backendIdHelper.get().pipe(Option.getOrThrow),
           })),
         )
@@ -132,7 +135,7 @@ export const makeHttpSync =
           payload,
           cursor: mapCursor(cursor),
         }).pipe(
-          options?.live
+          options?.live === true
             ? // Phase 2: Simulate `live` pull by polling for new events
               Stream.concatWithLastElement((lastElement) => {
                 const initialPhase2Cursor = lastElement.pipe(
@@ -164,26 +167,49 @@ export const makeHttpSync =
             : identity,
           Stream.tap((res) => backendIdHelper.lazySet(res.backendId)),
           Stream.map((res) => omit(res, ['backendId'])),
-          Stream.mapError((cause) => (cause._tag === 'InvalidPullError' ? cause : InvalidPullError.make({ cause }))),
+          Stream.mapError((cause) =>
+            cause._tag === 'UnknownError' || cause._tag === 'BackendIdMismatchError'
+              ? cause
+              : new UnknownError({ cause }),
+          ),
           Stream.withSpan('http-sync-client:pull'),
         )
 
       const pushSemaphore = yield* Effect.makeSemaphore(1)
 
-      const push: SyncBackend.SyncBackend<SyncMetadata>['push'] = (batch) =>
-        Effect.gen(function* () {
+      const push: SyncBackend.SyncBackend<SyncMetadata>['push'] = Effect.fn('http-sync-client:push')(
+        function* (batch) {
           if (batch.length === 0) {
             return
           }
 
-          yield* rpcClient.SyncHttpRpc.Push({ storeId, payload, batch, backendId: backendIdHelper.get() })
-        }).pipe(
-          pushSemaphore.withPermits(1),
-          Effect.mapError((cause) =>
-            cause._tag === 'InvalidPushError' ? cause : new InvalidPushError({ cause: new UnexpectedError({ cause }) }),
-          ),
-          Effect.withSpan('http-sync-client:push'),
-        )
+          const backendId = backendIdHelper.get()
+          const batchChunks = yield* Chunk.fromIterable(batch).pipe(
+            splitChunkBySize({
+              maxItems: MAX_PUSH_EVENTS_PER_REQUEST,
+              maxBytes: MAX_HTTP_REQUEST_BYTES,
+              encode: (items) => ({
+                batch: items,
+                storeId,
+                payload,
+                backendId,
+              }),
+            }),
+            Effect.mapError((cause) => new UnknownError({ cause })),
+          )
+
+          for (const chunk of Chunk.toReadonlyArray(batchChunks)) {
+            const chunkArray = Chunk.toReadonlyArray(chunk)
+            yield* rpcClient.SyncHttpRpc.Push({ storeId, payload, batch: chunkArray, backendId })
+          }
+        },
+        pushSemaphore.withPermits(1),
+        Effect.mapError((cause) =>
+          cause._tag === 'UnknownError' || cause._tag === 'ServerAheadError' || cause._tag === 'BackendIdMismatchError'
+            ? cause
+            : new UnknownError({ cause }),
+        ),
+      )
 
       return SyncBackend.of({
         connect,

package/src/client/transport/ws-rpc-client.ts

@@ -1,6 +1,9 @@
-import { InvalidPullError, InvalidPushError, IsOfflineError, SyncBackend, UnexpectedError } from '@livestore/common'
+import { IsOfflineError, SyncBackend, UnknownError } from '@livestore/common'
+import type { LiveStoreEvent } from '@livestore/common/schema'
+import { splitChunkBySize } from '@livestore/common/sync'
 import { omit } from '@livestore/utils'
 import {
+  Chunk,
   type Duration,
   Effect,
   Layer,
@@ -14,8 +17,10 @@ import {
   Stream,
   SubscriptionRef,
   UrlParams,
-  type WebSocket,
 } from '@livestore/utils/effect'
+import type { WebSocket } from '@livestore/utils/effect/browser'
+
+import { MAX_PUSH_EVENTS_PER_REQUEST, MAX_WS_MESSAGE_BYTES } from '../../common/constants.ts'
 import { SearchParamsSchema } from '../../common/mod.ts'
 import type { SyncMetadata } from '../../common/sync-message-types.ts'
 import { SyncWsRpc } from '../../common/ws-rpc-schema.ts'
@@ -69,7 +74,7 @@ export const makeWsSync =
         storeId,
         payload,
         transport: 'ws',
-      }).pipe(UnexpectedError.mapToUnexpectedError)
+      }).pipe(UnknownError.mapToUnknownError)
 
       const urlParams = UrlParams.fromInput(urlParamsData)
       const wsUrl = `${options.url}?${UrlParams.toString(urlParams)}`
@@ -91,7 +96,10 @@ export const makeWsSync =
 
       const ProtocolLive = RpcClient.layerProtocolSocketWithIsConnected({
         isConnected,
-        retryTransientErrors: Schedule.fixed(1000),
+        retryTransientErrors: Schedule.exponential('1 seconds').pipe(
+          Schedule.union(Schedule.fixed('30 seconds')),
+          Schedule.jittered,
+        ),
         pingSchedule: Schedule.once.pipe(Schedule.andThen(Schedule.fixed(pingInterval))),
         url: wsUrl,
       }).pipe(
@@ -115,7 +123,7 @@ export const makeWsSync =
       }).pipe(
         Effect.timeout(pingTimeout),
         Effect.catchTag('TimeoutException', () => SubscriptionRef.set(isConnected, false)),
-        UnexpectedError.mapToUnexpectedError,
+        UnknownError.mapToUnknownError,
         Effect.withSpan('ping'),
       )
 
@@ -134,39 +142,54 @@ export const makeWsSync =
                 backendId: backendIdHelper.get().pipe(Option.getOrThrow),
               })),
             ),
-            live: options?.live ?? false,
+            live: options?.live === true,
           }).pipe(
             Stream.tap((res) => backendIdHelper.lazySet(res.backendId)),
             Stream.map((res) => omit(res, ['backendId'])),
             Stream.mapError((cause) =>
-              cause._tag === 'RpcClientError' && Socket.isSocketError(cause.cause)
+              cause._tag === 'RpcClientError' && Socket.isSocketError(cause.cause) === true
                 ? new IsOfflineError({ cause: cause.cause })
-                : cause._tag === 'InvalidPullError'
+                : cause._tag === 'UnknownError' || cause._tag === 'BackendIdMismatchError'
                   ? cause
-                  : InvalidPullError.make({ cause }),
+                  : new UnknownError({ cause }),
             ),
             Stream.withSpan('pull'),
           ),
 
-        push: (batch) =>
-          Effect.gen(function* () {
-            if (batch.length === 0) {
-              return
-            }
+        push: Effect.fn('push')(function* (batch) {
+          if (batch.length === 0) return
+
+          const encodePayload = (batch: ReadonlyArray<LiveStoreEvent.Global.Encoded>) => ({
+            storeId,
+            payload,
+            batch,
+            backendId: backendIdHelper.get(),
+          })
+
+          const chunksChunk = yield* Chunk.fromIterable(batch).pipe(
+            splitChunkBySize({
+              maxItems: MAX_PUSH_EVENTS_PER_REQUEST,
+              maxBytes: MAX_WS_MESSAGE_BYTES,
+              encode: encodePayload,
+            }),
+            Effect.mapError((cause) => new UnknownError({ cause })),
+          )
 
-            return yield* rpcClient.SyncWsRpc.Push({
+          for (const sub of chunksChunk) {
+            yield* rpcClient.SyncWsRpc.Push({
               storeId,
               payload,
-              batch,
+              batch: Chunk.toReadonlyArray(sub),
               backendId: backendIdHelper.get(),
             }).pipe(
               Effect.mapError((cause) =>
-                cause._tag === 'InvalidPushError'
+                cause._tag === 'UnknownError' || cause._tag === 'ServerAheadError' || cause._tag === 'BackendIdMismatchError'
                   ? cause
-                  : new InvalidPushError({ cause: new UnexpectedError({ cause }) }),
+                  : new UnknownError({ cause }),
               ),
             )
-          }).pipe(Effect.withSpan('push')),
+          }
+        }),
         ping,
         metadata: {
           name: '@livestore/cf-sync',

package/src/common/constants.ts

@@ -0,0 +1,18 @@
+// Shared transport limits for Cloudflare sync provider
+// Keep payloads comfortably below ~1MB frame caps across Cloudflare transports.
+// References:
+// - Durable Objects WebSockets + hibernation best practices:
+//   https://developers.cloudflare.com/durable-objects/best-practices/websockets/
+// - Workers platform limits (general context):
+//   https://developers.cloudflare.com/workers/platform/limits/
+// Empirically, frames just below 1MB can fail on hibernated DO WebSockets; we use 900_000 bytes to keep a safety margin.
+export const MAX_TRANSPORT_PAYLOAD_BYTES = 900_000
+
+export const MAX_WS_MESSAGE_BYTES = MAX_TRANSPORT_PAYLOAD_BYTES
+export const MAX_DO_RPC_REQUEST_BYTES = MAX_TRANSPORT_PAYLOAD_BYTES
+export const MAX_HTTP_REQUEST_BYTES = MAX_TRANSPORT_PAYLOAD_BYTES
+
+// Upper bound for items per message/request. Mirrors server broadcast chunking.
+// Not Cloudflare-enforced; chosen to balance payload size and latency.
+export const MAX_PULL_EVENTS_PER_MESSAGE = 100
+export const MAX_PUSH_EVENTS_PER_REQUEST = 100

package/src/common/do-rpc-schema.ts

@@ -1,10 +1,11 @@
-import { InvalidPullError, InvalidPushError } from '@livestore/common'
+import { BackendIdMismatchError, ServerAheadError, UnknownError } from '@livestore/common'
 import { Rpc, RpcGroup, Schema } from '@livestore/utils/effect'
+
 import * as SyncMessage from './sync-message-types.ts'
 
 const commonPayloadFields = {
   /**
-   * While the storeId is already implied by the durable object, we still need the explicit storeId
+   * While the storeId is already implied by the Durable Object, we still need the explicit storeId
    * since a DO doesn't know its own id.name value. 🫠
    * https://community.cloudflare.com/t/how-can-i-get-the-name-of-a-durable-object-from-itself/505961/8
    */
@@ -34,7 +35,7 @@ export class SyncDoRpc extends RpcGroup.make(
       rpcRequestId: Schema.String,
       ...SyncMessage.PullResponse.fields,
     }),
-    error: InvalidPullError,
+    error: Schema.Union(UnknownError, BackendIdMismatchError),
     stream: true,
   }),
   Rpc.make('SyncDoRpc.Push', {
@@ -43,7 +44,7 @@ export class SyncDoRpc extends RpcGroup.make(
       ...commonPayloadFields,
     },
     success: SyncMessage.PushAck,
-    error: InvalidPushError,
+    error: Schema.Union(UnknownError, ServerAheadError, BackendIdMismatchError),
   }),
   Rpc.make('SyncDoRpc.Ping', {
     payload: {

package/src/common/http-rpc-schema.ts

@@ -1,5 +1,6 @@
-import { InvalidPullError, InvalidPushError, UnexpectedError } from '@livestore/common'
+import { BackendIdMismatchError, ServerAheadError, UnknownError } from '@livestore/common'
 import { Rpc, RpcGroup, Schema } from '@livestore/utils/effect'
+
 import * as SyncMessage from './sync-message-types.ts'
 
 /**
@@ -17,7 +18,7 @@ export class SyncHttpRpc extends RpcGroup.make(
       ...SyncMessage.PullRequest.fields,
     }),
     success: SyncMessage.PullResponse,
-    error: InvalidPullError,
+    error: Schema.Union(UnknownError, BackendIdMismatchError),
     stream: true,
   }),
   Rpc.make('SyncHttpRpc.Push', {
@@ -27,7 +28,7 @@ export class SyncHttpRpc extends RpcGroup.make(
       ...SyncMessage.PushRequest.fields,
     }),
     success: SyncMessage.PushAck,
-    error: InvalidPushError,
+    error: Schema.Union(UnknownError, ServerAheadError, BackendIdMismatchError),
   }),
   Rpc.make('SyncHttpRpc.Ping', {
     payload: Schema.Struct({
@@ -35,6 +36,6 @@ export class SyncHttpRpc extends RpcGroup.make(
       payload: Schema.optional(Schema.JsonValue),
     }),
     success: SyncMessage.Pong,
-    error: UnexpectedError,
+    error: UnknownError,
   }),
 ) {}

package/src/common/mod.ts

@@ -1,15 +1,17 @@
+import { OversizeChunkItemError, splitChunkBySize } from '@livestore/common/sync'
 import { Schema } from '@livestore/utils/effect'
 
 export type { CfTypes } from '@livestore/common-cf'
-
+export * from './constants.ts'
 export { SyncHttpRpc } from './http-rpc-schema.ts'
 export * as SyncMessage from './sync-message-types.ts'
+export { OversizeChunkItemError, splitChunkBySize }
 
 export const SearchParamsSchema = Schema.Struct({
   storeId: Schema.String,
   payload: Schema.compose(Schema.StringFromUriComponent, Schema.parseJson(Schema.JsonValue)).pipe(Schema.UndefinedOr),
   // NOTE `do-rpc` is handled differently
-  transport: Schema.Union(Schema.Literal('http'), Schema.Literal('ws')),
+  transport: Schema.Literal('http', 'ws'),
 })
 
 export type SearchParams = typeof SearchParamsSchema.Type

package/src/common/sync-message-types.ts

@@ -20,7 +20,7 @@ export const PullRequest = Schema.Struct({
   cursor: Schema.Option(
     Schema.Struct({
       backendId: BackendId,
-      eventSequenceNumber: EventSequenceNumber.GlobalEventSequenceNumber,
+      eventSequenceNumber: EventSequenceNumber.Global.Schema,
     }),
   ),
 }).annotations({ title: '@livestore/sync-cf:PullRequest' })
@@ -30,7 +30,7 @@ export type PullRequest = typeof PullRequest.Type
 export const PullResponse = Schema.Struct({
   batch: Schema.Array(
     Schema.Struct({
-      eventEncoded: LiveStoreEvent.AnyEncodedGlobal,
+      eventEncoded: LiveStoreEvent.Global.Encoded,
       metadata: Schema.Option(SyncMetadata),
     }),
   ),
@@ -48,7 +48,7 @@ export const emptyPullResponse = (backendId: string) =>
 export type PullResponse = typeof PullResponse.Type
 
 export const PushRequest = Schema.Struct({
-  batch: Schema.Array(LiveStoreEvent.AnyEncodedGlobal),
+  batch: Schema.Array(LiveStoreEvent.Global.Encoded),
   backendId: Schema.Option(BackendId),
 }).annotations({ title: '@livestore/sync-cf:PushRequest' })
 

package/src/common/ws-rpc-schema.ts

@@ -1,5 +1,6 @@
-import { InvalidPullError, InvalidPushError } from '@livestore/common'
+import { BackendIdMismatchError, ServerAheadError, UnknownError } from '@livestore/common'
 import { Rpc, RpcGroup, Schema } from '@livestore/utils/effect'
+
 import * as SyncMessage from './sync-message-types.ts'
 
 /**
@@ -19,7 +20,7 @@ export class SyncWsRpc extends RpcGroup.make(
       ...SyncMessage.PullRequest.fields,
     }),
     success: SyncMessage.PullResponse,
-    error: InvalidPullError,
+    error: Schema.Union(UnknownError, BackendIdMismatchError),
     stream: true,
   }),
   Rpc.make('SyncWsRpc.Push', {
@@ -29,7 +30,7 @@ export class SyncWsRpc extends RpcGroup.make(
       ...SyncMessage.PushRequest.fields,
     }),
     success: SyncMessage.PushAck,
-    error: InvalidPushError,
+    error: Schema.Union(UnknownError, ServerAheadError, BackendIdMismatchError),
   }),
   // Ping <> Pong is handled by DO WS auto-response
   // TODO add admin RPCs