@livestore/sync-cf 0.4.0 → 0.5.0-dev.0

package/src/client/transport/http-rpc-client.ts

@@ -1,9 +1,7 @@
 import { SyncBackend, UnknownError } from '@livestore/common'
-import type { EventSequenceNumber } from '@livestore/common/schema'
-import { splitChunkBySize } from '@livestore/common/sync'
-import { omit } from '@livestore/utils'
+import type { EventSequenceNumber, LiveStoreEvent } from '@livestore/common/schema'
+import { splitArrayBySize } from '@livestore/common/sync'
 import {
-  Chunk,
   type Duration,
   Effect,
   HttpClient,
@@ -11,11 +9,14 @@ import {
   identity,
   Layer,
   Option,
+  ReadonlyArray as EffectArray,
   RpcClient,
   RpcSerialization,
   Schedule,
   Schema,
+  Semaphore,
   Stream,
+  Struct,
   SubscriptionRef,
   UrlParams,
 } from '@livestore/utils/effect'
@@ -25,6 +26,8 @@ import { SyncHttpRpc } from '../../common/http-rpc-schema.ts'
 import { SearchParamsSchema } from '../../common/mod.ts'
 import type { SyncMetadata } from '../../common/sync-message-types.ts'
 
+type PushBatchItem = LiveStoreEvent.Global.Encoded
+
 export interface HttpSyncOptions {
   /**
    * URL of the sync backend
@@ -41,7 +44,7 @@ export interface HttpSyncOptions {
      * How often to poll for new events
      * @default 5 seconds
      */
-    pollInterval?: Duration.DurationInput
+    pollInterval?: Duration.Input
   }
   ping?: {
     /**
@@ -52,12 +55,12 @@ export interface HttpSyncOptions {
      * How long to wait for a ping response before timing out
      * @default 10 seconds
      */
-    requestTimeout?: Duration.DurationInput
+    requestTimeout?: Duration.Input
     /**
      * How often to send ping requests
      * @default 10 seconds
      */
-    requestInterval?: Duration.DurationInput
+    requestInterval?: Duration.Input
   }
 }
 
@@ -73,7 +76,7 @@ export const makeHttpSync =
 
       const livePullInterval = options.livePull?.pollInterval ?? 5_000
 
-      const urlParamsData = yield* Schema.encode(SearchParamsSchema)({
+      const urlParamsData = yield* Schema.encodeEffect(SearchParamsSchema)({
         storeId,
         payload,
         transport: 'http',
@@ -83,9 +86,10 @@ export const makeHttpSync =
 
       // Setup HTTP RPC Protocol
       const HttpProtocolLive = RpcClient.layerProtocolHttp({
-        url: `${options.url}?${UrlParams.toString(urlParams)}`,
+        url: options.url,
         transformClient: HttpClient.mapRequest((request) =>
           request.pipe(
+            HttpClientRequest.appendUrlParams(urlParams),
             HttpClientRequest.setHeaders({
               ...options.headers,
               // Used in CF Worker to identify the store (additionally to storeId embedded in the RPC requests)
@@ -100,13 +104,13 @@ export const makeHttpSync =
       const pingTimeout = options.ping?.requestTimeout ?? 10_000
 
       const ping: SyncBackend.SyncBackend<SyncMetadata>['ping'] = Effect.gen(function* () {
-        yield* rpcClient.SyncHttpRpc.Ping({ storeId, payload })
+        yield* rpcClient['SyncHttpRpc.Ping']({ storeId, payload })
 
         yield* SubscriptionRef.set(isConnected, true)
       }).pipe(
         UnknownError.mapToUnknownError,
         Effect.timeout(pingTimeout),
-        Effect.catchTag('TimeoutException', () => SubscriptionRef.set(isConnected, false)),
+        Effect.catchTag('TimeoutError', () => SubscriptionRef.set(isConnected, false)),
       )
 
       const pingInterval = options.ping?.requestInterval ?? 10_000
@@ -130,7 +134,7 @@ export const makeHttpSync =
         )
 
       const pull: SyncBackend.SyncBackend<SyncMetadata>['pull'] = (cursor, options) =>
-        rpcClient.SyncHttpRpc.Pull({
+        rpcClient['SyncHttpRpc.Pull']({
           storeId,
           payload,
           cursor: mapCursor(cursor),
@@ -139,34 +143,40 @@ export const makeHttpSync =
             ? // Phase 2: Simulate `live` pull by polling for new events
               Stream.concatWithLastElement((lastElement) => {
                 const initialPhase2Cursor = lastElement.pipe(
-                  Option.flatMap((_) => Option.fromNullable(_.batch.at(-1)?.eventEncoded.seqNum)),
+                  Option.flatMap((_) => Option.fromNullishOr(_.batch.at(-1)?.eventEncoded.seqNum)),
                   Option.map((eventSequenceNumber) => ({ eventSequenceNumber })),
                   Option.orElse(() => cursor),
                   mapCursor,
                 )
 
-                return Stream.unfoldChunkEffect(initialPhase2Cursor, (currentCursor) =>
+                return Stream.paginate(initialPhase2Cursor, (currentCursor) =>
                   Effect.gen(function* () {
                     yield* Effect.sleep(livePullInterval)
 
-                    const items = yield* rpcClient.SyncHttpRpc.Pull({ storeId, payload, cursor: currentCursor }).pipe(
-                      Stream.runCollect,
-                    )
+                    const items = yield* rpcClient['SyncHttpRpc.Pull']({
+                      storeId,
+                      payload,
+                      cursor: currentCursor,
+                    }).pipe(Stream.runCollect)
 
-                    const nextCursor = Chunk.last(items).pipe(
-                      Option.flatMap((item) => Option.fromNullable(item.batch.at(-1)?.eventEncoded.seqNum)),
+                    const nextCursor = EffectArray.last(items).pipe(
+                      Option.flatMap((item) => Option.fromNullishOr(item.batch.at(-1)?.eventEncoded.seqNum)),
                       Option.map((eventSequenceNumber) => ({ eventSequenceNumber })),
                       Option.orElse(() => currentCursor),
                       mapCursor,
                     )
 
-                    return Option.some([items, nextCursor])
+                    const page: readonly [typeof items, Option.Option<typeof currentCursor>] = [
+                      items,
+                      Option.some(nextCursor),
+                    ]
+                    return page
                   }),
                 )
               })
             : identity,
           Stream.tap((res) => backendIdHelper.lazySet(res.backendId)),
-          Stream.map((res) => omit(res, ['backendId'])),
+          Stream.map((res) => Struct.omit(res, ['backendId'])),
           Stream.mapError((cause) =>
             cause._tag === 'UnknownError' || cause._tag === 'BackendIdMismatchError'
               ? cause
@@ -175,7 +185,7 @@ export const makeHttpSync =
           Stream.withSpan('http-sync-client:pull'),
         )
 
-      const pushSemaphore = yield* Effect.makeSemaphore(1)
+      const pushSemaphore = yield* Semaphore.make(1)
 
       const push: SyncBackend.SyncBackend<SyncMetadata>['push'] = Effect.fn('http-sync-client:push')(
         function* (batch) {
@@ -184,23 +194,23 @@ export const makeHttpSync =
           }
 
           const backendId = backendIdHelper.get()
-          const batchChunks = yield* Chunk.fromIterable(batch).pipe(
-            splitChunkBySize({
-              maxItems: MAX_PUSH_EVENTS_PER_REQUEST,
-              maxBytes: MAX_HTTP_REQUEST_BYTES,
-              encode: (items) => ({
-                batch: items,
-                storeId,
-                payload,
-                backendId,
-              }),
+          if (EffectArray.isReadonlyArrayNonEmpty(batch) === false) {
+            return
+          }
+
+          const batchChunks = yield* splitArrayBySize({
+            maxItems: MAX_PUSH_EVENTS_PER_REQUEST,
+            maxBytes: MAX_HTTP_REQUEST_BYTES,
+            encode: (items: ReadonlyArray<PushBatchItem>) => ({
+              batch: items,
+              storeId,
+              payload,
+              backendId,
             }),
-            Effect.mapError((cause) => new UnknownError({ cause })),
-          )
+          })(batch).pipe(Effect.mapError((cause) => new UnknownError({ cause })))
 
-          for (const chunk of Chunk.toReadonlyArray(batchChunks)) {
-            const chunkArray = Chunk.toReadonlyArray(chunk)
-            yield* rpcClient.SyncHttpRpc.Push({ storeId, payload, batch: chunkArray, backendId })
+          for (const batchChunk of batchChunks) {
+            yield* rpcClient['SyncHttpRpc.Push']({ storeId, payload, batch: batchChunk, backendId })
           }
         },
         pushSemaphore.withPermits(1),

package/src/client/transport/ws-rpc-client.ts

@@ -1,13 +1,12 @@
 import { IsOfflineError, SyncBackend, UnknownError } from '@livestore/common'
 import type { LiveStoreEvent } from '@livestore/common/schema'
-import { splitChunkBySize } from '@livestore/common/sync'
-import { omit } from '@livestore/utils'
+import { splitArrayBySize } from '@livestore/common/sync'
 import {
-  Chunk,
-  type Duration,
+  Duration,
   Effect,
   Layer,
   Option,
+  ReadonlyArray as EffectArray,
   RpcClient,
   RpcSerialization,
   Schedule,
@@ -15,6 +14,7 @@ import {
   type Scope,
   Socket,
   Stream,
+  Struct,
   SubscriptionRef,
   UrlParams,
 } from '@livestore/utils/effect'
@@ -48,12 +48,12 @@ export interface WsSyncOptions {
      * How long to wait for a ping response before timing out
      * @default 10 seconds
      */
-    requestTimeout?: Duration.DurationInput
+    requestTimeout?: Duration.Input
     /**
      * How often to send ping requests
      * @default 10 seconds
      */
-    requestInterval?: Duration.DurationInput
+    requestInterval?: Duration.Input
   }
 }
 
@@ -70,7 +70,7 @@ export const makeWsSync =
   (options: WsSyncOptions): SyncBackend.SyncBackendConstructor<SyncMetadata> =>
   ({ storeId, payload }) =>
     Effect.gen(function* () {
-      const urlParamsData = yield* Schema.encode(SearchParamsSchema)({
+      const urlParamsData = yield* Schema.encodeEffect(SearchParamsSchema)({
         storeId,
         payload,
         transport: 'ws',
@@ -89,7 +89,7 @@ export const makeWsSync =
       // }
       // TODO bring this back in a cross-platform way
       // if (navigator.onLine === false) {
-      //   yield* Effect.async((cb) => self.addEventListener('online', () => cb(Effect.void)))
+      //   yield* Effect.callback((cb) => self.addEventListener('online', () => cb(Effect.void)))
       // }
 
       const pingInterval = options.ping?.requestInterval ?? 10_000
@@ -97,10 +97,10 @@ export const makeWsSync =
       const ProtocolLive = RpcClient.layerProtocolSocketWithIsConnected({
         isConnected,
         retryTransientErrors: Schedule.exponential('1 seconds').pipe(
-          Schedule.union(Schedule.fixed('30 seconds')),
+          Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(30)))),
           Schedule.jittered,
         ),
-        pingSchedule: Schedule.once.pipe(Schedule.andThen(Schedule.fixed(pingInterval))),
+        pingSchedule: Schedule.recurs(1).pipe(Schedule.concat(Schedule.fixed(pingInterval))),
         url: wsUrl,
       }).pipe(
         Layer.provide(Socket.layerWebSocket(wsUrl)),
@@ -122,7 +122,7 @@ export const makeWsSync =
         yield* SubscriptionRef.set(isConnected, true)
       }).pipe(
         Effect.timeout(pingTimeout),
-        Effect.catchTag('TimeoutException', () => SubscriptionRef.set(isConnected, false)),
+        Effect.catchTag('TimeoutError', () => SubscriptionRef.set(isConnected, false)),
         UnknownError.mapToUnknownError,
         Effect.withSpan('ping'),
       )
@@ -133,7 +133,7 @@ export const makeWsSync =
         isConnected,
         connect: ping,
         pull: (cursor, options) =>
-          rpcClient.SyncWsRpc.Pull({
+          rpcClient['SyncWsRpc.Pull']({
             storeId,
             payload,
             cursor: cursor.pipe(
@@ -145,7 +145,7 @@ export const makeWsSync =
             live: options?.live === true,
           }).pipe(
             Stream.tap((res) => backendIdHelper.lazySet(res.backendId)),
-            Stream.map((res) => omit(res, ['backendId'])),
+            Stream.map((res) => Struct.omit(res, ['backendId'])),
             Stream.mapError((cause) =>
               cause._tag === 'RpcClientError' && Socket.isSocketError(cause.cause) === true
                 ? new IsOfflineError({ cause: cause.cause })
@@ -166,24 +166,27 @@ export const makeWsSync =
             backendId: backendIdHelper.get(),
           })
 
-          const chunksChunk = yield* Chunk.fromIterable(batch).pipe(
-            splitChunkBySize({
-              maxItems: MAX_PUSH_EVENTS_PER_REQUEST,
-              maxBytes: MAX_WS_MESSAGE_BYTES,
-              encode: encodePayload,
-            }),
-            Effect.mapError((cause) => new UnknownError({ cause })),
-          )
-
-          for (const sub of chunksChunk) {
-            yield* rpcClient.SyncWsRpc.Push({
+          if (EffectArray.isReadonlyArrayNonEmpty(batch) === false) {
+            return
+          }
+
+          const batchChunks = yield* splitArrayBySize({
+            maxItems: MAX_PUSH_EVENTS_PER_REQUEST,
+            maxBytes: MAX_WS_MESSAGE_BYTES,
+            encode: encodePayload,
+          })(batch).pipe(Effect.mapError((cause) => new UnknownError({ cause })))
+
+          for (const batchChunk of batchChunks) {
+            yield* rpcClient['SyncWsRpc.Push']({
               storeId,
               payload,
-              batch: Chunk.toReadonlyArray(sub),
+              batch: batchChunk,
               backendId: backendIdHelper.get(),
             }).pipe(
               Effect.mapError((cause) =>
-                cause._tag === 'UnknownError' || cause._tag === 'ServerAheadError' || cause._tag === 'BackendIdMismatchError'
+                cause._tag === 'UnknownError' ||
+                cause._tag === 'ServerAheadError' ||
+                cause._tag === 'BackendIdMismatchError'
                   ? cause
                   : new UnknownError({ cause }),
               ),

package/src/common/do-rpc-schema.ts

@@ -11,7 +11,7 @@ const commonPayloadFields = {
    */
   storeId: Schema.String,
   /** Needed for various reasons (e.g. auth) */
-  payload: Schema.optional(Schema.JsonValue),
+  payload: Schema.optional(Schema.Json),
 }
 
 export class SyncDoRpc extends RpcGroup.make(
@@ -35,7 +35,7 @@ export class SyncDoRpc extends RpcGroup.make(
       rpcRequestId: Schema.String,
       ...SyncMessage.PullResponse.fields,
     }),
-    error: Schema.Union(UnknownError, BackendIdMismatchError),
+    error: Schema.Union([UnknownError, BackendIdMismatchError]),
     stream: true,
   }),
   Rpc.make('SyncDoRpc.Push', {
@@ -44,7 +44,7 @@ export class SyncDoRpc extends RpcGroup.make(
       ...commonPayloadFields,
     },
     success: SyncMessage.PushAck,
-    error: Schema.Union(UnknownError, ServerAheadError, BackendIdMismatchError),
+    error: Schema.Union([UnknownError, ServerAheadError, BackendIdMismatchError]),
   }),
   Rpc.make('SyncDoRpc.Ping', {
     payload: {

package/src/common/http-rpc-schema.ts

@@ -14,26 +14,26 @@ export class SyncHttpRpc extends RpcGroup.make(
   Rpc.make('SyncHttpRpc.Pull', {
     payload: Schema.Struct({
       storeId: Schema.String,
-      payload: Schema.optional(Schema.JsonValue),
+      payload: Schema.optional(Schema.Json),
       ...SyncMessage.PullRequest.fields,
     }),
     success: SyncMessage.PullResponse,
-    error: Schema.Union(UnknownError, BackendIdMismatchError),
+    error: Schema.Union([UnknownError, BackendIdMismatchError]),
     stream: true,
   }),
   Rpc.make('SyncHttpRpc.Push', {
     payload: Schema.Struct({
       storeId: Schema.String,
-      payload: Schema.optional(Schema.JsonValue),
+      payload: Schema.optional(Schema.Json),
       ...SyncMessage.PushRequest.fields,
     }),
     success: SyncMessage.PushAck,
-    error: Schema.Union(UnknownError, ServerAheadError, BackendIdMismatchError),
+    error: Schema.Union([UnknownError, ServerAheadError, BackendIdMismatchError]),
   }),
   Rpc.make('SyncHttpRpc.Ping', {
     payload: Schema.Struct({
       storeId: Schema.String,
-      payload: Schema.optional(Schema.JsonValue),
+      payload: Schema.optional(Schema.Json),
     }),
     success: SyncMessage.Pong,
     error: UnknownError,

package/src/common/mod.ts

@@ -1,17 +1,16 @@
-import { OversizeChunkItemError, splitChunkBySize } from '@livestore/common/sync'
+export { OversizeChunkItemError, splitArrayBySize } from '@livestore/common/sync'
 import { Schema } from '@livestore/utils/effect'
 
 export type { CfTypes } from '@livestore/common-cf'
 export * from './constants.ts'
 export { SyncHttpRpc } from './http-rpc-schema.ts'
 export * as SyncMessage from './sync-message-types.ts'
-export { OversizeChunkItemError, splitChunkBySize }
 
 export const SearchParamsSchema = Schema.Struct({
   storeId: Schema.String,
-  payload: Schema.compose(Schema.StringFromUriComponent, Schema.parseJson(Schema.JsonValue)).pipe(Schema.UndefinedOr),
+  payload: Schema.optional(Schema.StringFromUriComponent.pipe(Schema.decodeTo(Schema.fromJsonString(Schema.Json)))),
   // NOTE `do-rpc` is handled differently
-  transport: Schema.Literal('http', 'ws'),
+  transport: Schema.Literals(['http', 'ws']),
 })
 
 export type SearchParams = typeof SearchParamsSchema.Type

package/src/common/sync-message-types.ts

@@ -11,7 +11,7 @@ import { Schema } from '@livestore/utils/effect'
 export const SyncMetadata = Schema.TaggedStruct('SyncMessage.SyncMetadata', {
   /** ISO date format */
   createdAt: Schema.String,
-}).annotations({ title: '@livestore/sync-cf:SyncMetadata' })
+}).annotate({ title: '@livestore/sync-cf:SyncMetadata' })
 
 export type SyncMetadata = typeof SyncMetadata.Type
 
@@ -23,7 +23,7 @@ export const PullRequest = Schema.Struct({
       eventSequenceNumber: EventSequenceNumber.Global.Schema,
     }),
   ),
-}).annotations({ title: '@livestore/sync-cf:PullRequest' })
+}).annotate({ title: '@livestore/sync-cf:PullRequest' })
 
 export type PullRequest = typeof PullRequest.Type
 
@@ -36,7 +36,7 @@ export const PullResponse = Schema.Struct({
   ),
   pageInfo: SyncBackend.PullResPageInfo,
   backendId: BackendId,
-}).annotations({ title: '@livestore/sync-cf:PullResponse' })
+}).annotate({ title: '@livestore/sync-cf:PullResponse' })
 
 export const emptyPullResponse = (backendId: string) =>
   PullResponse.make({
@@ -50,32 +50,32 @@ export type PullResponse = typeof PullResponse.Type
 export const PushRequest = Schema.Struct({
   batch: Schema.Array(LiveStoreEvent.Global.Encoded),
   backendId: Schema.Option(BackendId),
-}).annotations({ title: '@livestore/sync-cf:PushRequest' })
+}).annotate({ title: '@livestore/sync-cf:PushRequest' })
 
 export type PushRequest = typeof PushRequest.Type
 
-export const PushAck = Schema.Struct({}).annotations({
+export const PushAck = Schema.Struct({}).annotate({
   title: '@livestore/sync-cf:PushAck',
 })
 
 export type PushAck = typeof PushAck.Type
 
-export const Ping = Schema.TaggedStruct('SyncMessage.Ping', {}).annotations({ title: '@livestore/sync-cf:Ping' })
+export const Ping = Schema.TaggedStruct('SyncMessage.Ping', {}).annotate({ title: '@livestore/sync-cf:Ping' })
 
 export type Ping = typeof Ping.Type
 
-export const Pong = Schema.TaggedStruct('SyncMessage.Pong', {}).annotations({ title: '@livestore/sync-cf:Pong' })
+export const Pong = Schema.TaggedStruct('SyncMessage.Pong', {}).annotate({ title: '@livestore/sync-cf:Pong' })
 
 export type Pong = typeof Pong.Type
 
 // Admin operations
 export const AdminResetRoomRequest = Schema.TaggedStruct('SyncMessage.AdminResetRoomRequest', {
   adminSecret: Schema.String,
-}).annotations({ title: '@livestore/sync-cf:AdminResetRoomRequest' })
+}).annotate({ title: '@livestore/sync-cf:AdminResetRoomRequest' })
 
 export type AdminResetRoomRequest = typeof AdminResetRoomRequest.Type
 
-export const AdminResetRoomResponse = Schema.TaggedStruct('SyncMessage.AdminResetRoomResponse', {}).annotations({
+export const AdminResetRoomResponse = Schema.TaggedStruct('SyncMessage.AdminResetRoomResponse', {}).annotate({
   title: '@livestore/sync-cf:AdminResetRoomResponse',
 })
 
@@ -83,7 +83,7 @@ export type AdminResetRoomResponse = typeof AdminResetRoomResponse.Type
 
 export const AdminInfoRequest = Schema.TaggedStruct('SyncMessage.AdminInfoRequest', {
   adminSecret: Schema.String,
-}).annotations({ title: '@livestore/sync-cf:AdminInfoRequest' })
+}).annotate({ title: '@livestore/sync-cf:AdminInfoRequest' })
 
 export type AdminInfoRequest = typeof AdminInfoRequest.Type
 
@@ -91,27 +91,27 @@ export const AdminInfoResponse = Schema.TaggedStruct('SyncMessage.AdminInfoRespo
   info: Schema.Struct({
     durableObjectId: Schema.String,
   }),
-}).annotations({ title: '@livestore/sync-cf:AdminInfoResponse' })
+}).annotate({ title: '@livestore/sync-cf:AdminInfoResponse' })
 
 export type AdminInfoResponse = typeof AdminInfoResponse.Type
 
-export const BackendToClientMessage = Schema.Union(
+export const BackendToClientMessage = Schema.Union([
   PullResponse,
   PushAck,
   Pong,
   AdminResetRoomResponse,
   AdminInfoResponse,
-)
+])
 export type BackendToClientMessage = typeof BackendToClientMessage.Type
 
-export const ClientToBackendMessage = Schema.Union(
+export const ClientToBackendMessage = Schema.Union([
   PullRequest,
   PushRequest,
   Ping,
   AdminResetRoomRequest,
   AdminInfoRequest,
-)
+])
 export type ClientToBackendMessage = typeof ClientToBackendMessage.Type
 
-export const Message = Schema.Union(BackendToClientMessage, ClientToBackendMessage)
+export const Message = Schema.Union([BackendToClientMessage, ClientToBackendMessage])
 export type Message = typeof Message.Type

package/src/common/ws-rpc-schema.ts

@@ -14,23 +14,23 @@ export class SyncWsRpc extends RpcGroup.make(
   Rpc.make('SyncWsRpc.Pull', {
     payload: Schema.Struct({
       storeId: Schema.String,
-      payload: Schema.optional(Schema.JsonValue),
+      payload: Schema.optional(Schema.Json),
       /** Whether to keep the pull stream alive and wait for more events */
       live: Schema.Boolean,
       ...SyncMessage.PullRequest.fields,
     }),
     success: SyncMessage.PullResponse,
-    error: Schema.Union(UnknownError, BackendIdMismatchError),
+    error: Schema.Union([UnknownError, BackendIdMismatchError]),
     stream: true,
   }),
   Rpc.make('SyncWsRpc.Push', {
     payload: Schema.Struct({
       storeId: Schema.String,
-      payload: Schema.optional(Schema.JsonValue),
+      payload: Schema.optional(Schema.Json),
       ...SyncMessage.PushRequest.fields,
     }),
     success: SyncMessage.PushAck,
-    error: Schema.Union(UnknownError, ServerAheadError, BackendIdMismatchError),
+    error: Schema.Union([UnknownError, ServerAheadError, BackendIdMismatchError]),
   }),
   // Ping <> Pong is handled by DO WS auto-response
   // TODO add admin RPCs