@livestore/sync-cf 0.4.0-dev.2 → 0.4.0-dev.20

package/src/cf-worker/worker.ts

@@ -1,92 +1,70 @@
-import type * as CfWorker from '@cloudflare/workers-types'
-import { UnexpectedError } from '@livestore/common'
-import type { Schema } from '@livestore/utils/effect'
-import { Effect, UrlParams } from '@livestore/utils/effect'
-
-import { SearchParamsSchema } from '../common/mod.ts'
-
-import type { Env } from './durable-object.ts'
+import { env as importedEnv } from 'cloudflare:workers'
+import { UnknownError } from '@livestore/common'
+import type { HelperTypes } from '@livestore/common-cf'
+import { Effect, Schema } from '@livestore/utils/effect'
+import type { CfTypes, SearchParams } from '../common/mod.ts'
+import type { CfDeclare } from './mod.ts'
+import { type Env, matchSyncRequest } from './shared.ts'
 
 // NOTE We need to redeclare runtime types here to avoid type conflicts with the lib.dom Response type.
-declare class Response extends CfWorker.Response {}
-
-export namespace HelperTypes {
-  type AnyDON = CfWorker.DurableObjectNamespace<undefined>
-
-  type DOKeys<T> = {
-    [K in keyof T]-?: T[K] extends AnyDON ? K : never
-  }[keyof T]
-
-  type NonBuiltins<T> = Omit<T, keyof Env>
-
-  /**
-   * Helper type to extract DurableObject keys from Env to give consumer type safety.
-   *
-   * @example
-   * ```ts
-   *  type PlatformEnv = {
-   *    DB: D1Database
-   *    ADMIN_TOKEN: string
-   *    WEBSOCKET_SERVER: DurableObjectNamespace<WebSocketServer>
-   * }
-   *  export default makeWorker<PlatformEnv>({
-   *    durableObject: { name: "WEBSOCKET_SERVER" },
-   *    // ^ (property) name?: "WEBSOCKET_SERVER" | undefined
-   *  });
-   */
-  export type ExtractDurableObjectKeys<TEnv = Env> = DOKeys<NonBuiltins<TEnv>> extends never
-    ? string
-    : DOKeys<NonBuiltins<TEnv>>
-}
+declare class Response extends CfDeclare.Response {}
 
 // HINT: If we ever extend user's custom worker RPC, type T can help here with expected return type safety. Currently unused.
-export type CFWorker<TEnv extends Env = Env, _T extends CfWorker.Rpc.DurableObjectBranded | undefined = undefined> = {
+export type CFWorker<TEnv extends Env = Env, _T extends CfTypes.Rpc.DurableObjectBranded | undefined = undefined> = {
   fetch: <CFHostMetada = unknown>(
-    request: CfWorker.Request<CFHostMetada>,
+    request: CfTypes.Request<CFHostMetada>,
     env: TEnv,
-    ctx: CfWorker.ExecutionContext,
-  ) => Promise<CfWorker.Response>
+    ctx: CfTypes.ExecutionContext,
+  ) => Promise<CfTypes.Response>
 }
 
-export type MakeWorkerOptions<TEnv extends Env = Env> = {
+/**
+ * Options accepted by {@link makeWorker}. The Durable Object binding has to be
+ * supplied explicitly so we never fall back to deprecated defaults when Cloudflare config changes.
+ */
+export type MakeWorkerOptions<TEnv extends Env = Env, TSyncPayload = Schema.JsonValue> = {
+  /**
+   * Binding name of the sync Durable Object declared in wrangler config.
+   */
+  syncBackendBinding: HelperTypes.ExtractDurableObjectKeys<TEnv>
   /**
    * Validates the payload during WebSocket connection establishment.
    * Note: This runs only at connection time, not for individual push events.
    * For push event validation, use the `onPush` callback in the durable object.
    */
-  validatePayload?: (payload: Schema.JsonValue | undefined, context: { storeId: string }) => void | Promise<void>
+  /**
+   * Optionally pass a schema to decode the client-provided payload into a typed object
+   * before calling {@link validatePayload}. If omitted, the raw JSON value is forwarded.
+   */
+  syncPayloadSchema?: Schema.Schema<TSyncPayload>
+  /**
+   * Validates the (optionally decoded) payload during WebSocket connection establishment.
+   * If {@link syncPayloadSchema} is provided, `payload` will be of the schema’s inferred type.
+   */
+  validatePayload?: (payload: TSyncPayload, context: { storeId: string }) => void | Promise<void>
   /** @default false */
   enableCORS?: boolean
-  durableObject?: {
-    /**
-     * Needs to match the binding name from the wrangler config
-     *
-     * @default 'WEBSOCKET_SERVER'
-     */
-    name?: HelperTypes.ExtractDurableObjectKeys<TEnv>
-  }
 }
 
+/**
+ * Produces a Cloudflare Worker `fetch` handler that delegates sync traffic to the
+ * Durable Object identified by `syncBackendBinding`.
+ *
+ * For more complex setups prefer implementing a custom `fetch` and call {@link handleSyncRequest}
+ * from the branch that handles LiveStore sync requests.
+ */
 export const makeWorker = <
   TEnv extends Env = Env,
-  TDurableObjectRpc extends CfWorker.Rpc.DurableObjectBranded | undefined = undefined,
+  TDurableObjectRpc extends CfTypes.Rpc.DurableObjectBranded | undefined = undefined,
+  TSyncPayload = Schema.JsonValue,
 >(
-  options: MakeWorkerOptions<TEnv> = {},
+  options: MakeWorkerOptions<TEnv, TSyncPayload>,
 ): CFWorker<TEnv, TDurableObjectRpc> => {
   return {
     fetch: async (request, env, _ctx) => {
       const url = new URL(request.url)
 
-      await new Promise((resolve) => setTimeout(resolve, 500))
-
-      if (request.method === 'GET' && url.pathname === '/') {
-        return new Response('Info: WebSocket sync backend endpoint for @livestore/sync-cf.', {
-          status: 200,
-          headers: { 'Content-Type': 'text/plain' },
-        })
-      }
-
-      const corsHeaders: CfWorker.HeadersInit = options.enableCORS
+      const corsHeaders: CfTypes.HeadersInit = options.enableCORS
         ? {
             'Access-Control-Allow-Origin': '*',
             'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
@@ -101,11 +79,27 @@ export const makeWorker = <
         })
       }
 
-      if (url.pathname.endsWith('/websocket')) {
-        return handleWebSocket<TEnv, TDurableObjectRpc>(request, env, _ctx, {
+      const searchParams = matchSyncRequest(request)
+
+      // Check if this is a sync request first, before showing info message
+      if (searchParams !== undefined) {
+        return handleSyncRequest<TEnv, TDurableObjectRpc, unknown, TSyncPayload>({
+          request,
+          searchParams,
+          env,
+          ctx: _ctx,
+          syncBackendBinding: options.syncBackendBinding,
           headers: corsHeaders,
           validatePayload: options.validatePayload,
-          durableObject: options.durableObject,
+          syncPayloadSchema: options.syncPayloadSchema,
+        })
+      }
+
+      // Only show info message for GET requests to / without sync parameters
+      if (request.method === 'GET' && url.pathname === '/') {
+        return new Response('Info: Sync backend endpoint for @livestore/sync-cf.', {
+          status: 200,
+          headers: { 'Content-Type': 'text/plain' },
         })
       }
 
@@ -124,7 +118,7 @@ export const makeWorker = <
 }
 
 /**
- * Handles `/websocket` endpoint.
+ * Handles LiveStore sync requests (e.g. with search params `?storeId=...&transport=...`).
  *
  * @example
  * ```ts
@@ -137,85 +131,113 @@ export const makeWorker = <
  *
  * export default {
  *   fetch: async (request, env, ctx) => {
- *     if (request.url.endsWith('/websocket')) {
- *       return handleWebSocket(request, env, ctx, { headers: {}, validatePayload })
+ *     const searchParams = matchSyncRequest(request)
+ *
+ *     // Is LiveStore sync request
+ *     if (searchParams !== undefined) {
+ *       return handleSyncRequest({
+ *         request,
+ *         searchParams,
+ *         env,
+ *         ctx,
+ *         syncBackendBinding: 'SYNC_BACKEND_DO',
+ *         headers: {},
+ *         validatePayload,
+ *       })
  *     }
  *
  *     return new Response('Invalid path', { status: 400 })
- *     return new Response('Invalid path', { status: 400 })
  *   }
  * }
  * ```
  *
- * @throws {UnexpectedError} If the payload is invalid
+ * @throws {UnknownError} If the payload is invalid
  */
-export const handleWebSocket = <
+export const handleSyncRequest = <
   TEnv extends Env = Env,
-  TDurableObjectRpc extends CfWorker.Rpc.DurableObjectBranded | undefined = undefined,
+  TDurableObjectRpc extends CfTypes.Rpc.DurableObjectBranded | undefined = undefined,
   CFHostMetada = unknown,
->(
-  request: CfWorker.Request<CFHostMetada>,
-  env: TEnv,
-  _ctx: CfWorker.ExecutionContext,
-  options: {
-    headers?: CfWorker.HeadersInit
-    durableObject?: MakeWorkerOptions<TEnv>['durableObject']
-    validatePayload?: (payload: Schema.JsonValue | undefined, context: { storeId: string }) => void | Promise<void>
-  } = {},
-): Promise<CfWorker.Response> =>
+  TSyncPayload = Schema.JsonValue,
+>({
+  request,
+  searchParams: { storeId, payload, transport },
+  env: explicitlyProvidedEnv,
+  syncBackendBinding,
+  headers,
+  validatePayload,
+  syncPayloadSchema,
+}: {
+  request: CfTypes.Request<CFHostMetada>
+  searchParams: SearchParams
+  env?: TEnv | undefined
+  /** Only there for type-level reasons */
+  ctx: CfTypes.ExecutionContext
+  /** Binding name of the sync backend Durable Object */
+  syncBackendBinding: MakeWorkerOptions<TEnv, TSyncPayload>['syncBackendBinding']
+  headers?: CfTypes.HeadersInit | undefined
+  validatePayload?: MakeWorkerOptions<TEnv, TSyncPayload>['validatePayload']
+  syncPayloadSchema?: MakeWorkerOptions<TEnv, TSyncPayload>['syncPayloadSchema']
+}): Promise<CfTypes.Response> =>
   Effect.gen(function* () {
-    const url = new URL(request.url)
-
-    const urlParams = UrlParams.fromInput(url.searchParams)
-    const paramsResult = yield* UrlParams.schemaStruct(SearchParamsSchema)(urlParams).pipe(Effect.either)
-
-    if (paramsResult._tag === 'Left') {
-      return new Response(`Invalid search params: ${paramsResult.left.toString()}`, {
-        status: 500,
-        headers: options?.headers,
-      })
-    }
-
-    const { storeId, payload } = paramsResult.right
-
-    if (options.validatePayload !== undefined) {
-      const result = yield* Effect.promise(async () => options.validatePayload!(payload, { storeId })).pipe(
-        UnexpectedError.mapToUnexpectedError,
-        Effect.either,
-      )
-
-      if (result._tag === 'Left') {
-        console.error('Invalid payload', result.left)
-        return new Response(result.left.toString(), { status: 400, headers: options.headers })
+    if (validatePayload !== undefined) {
+      // Always decode with the supplied schema when present, even if payload is undefined.
+      // This ensures required payloads are enforced by the schema.
+      if (syncPayloadSchema !== undefined) {
+        const decodedEither = Schema.decodeUnknownEither(syncPayloadSchema)(payload)
+        if (decodedEither._tag === 'Left') {
+          const message = decodedEither.left.toString()
+          console.error('Invalid payload (decode failed)', message)
+          return new Response(message, { status: 400, headers })
+        }
+
+        const result = yield* Effect.promise(async () =>
+          validatePayload(decodedEither.right as TSyncPayload, { storeId }),
+        ).pipe(UnknownError.mapToUnknownError, Effect.either)
+
+        if (result._tag === 'Left') {
+          console.error('Invalid payload (validation failed)', result.left)
+          return new Response(result.left.toString(), { status: 400, headers })
+        }
+      } else {
+        const result = yield* Effect.promise(async () => validatePayload(payload as TSyncPayload, { storeId })).pipe(
+          UnknownError.mapToUnknownError,
+          Effect.either,
+        )
+
+        if (result._tag === 'Left') {
+          console.error('Invalid payload (validation failed)', result.left)
+          return new Response(result.left.toString(), { status: 400, headers })
+        }
       }
     }
 
-    const durableObjectName = options.durableObject?.name ?? 'WEBSOCKET_SERVER'
-    if (!(durableObjectName in env)) {
+    const env = explicitlyProvidedEnv ?? (importedEnv as TEnv)
+
+    if (!(syncBackendBinding in env)) {
       return new Response(
-        `Failed dependency: Required Durable Object binding '${durableObjectName as string}' not available`,
+        `Failed dependency: Required Durable Object binding '${syncBackendBinding as string}' not available`,
         {
           status: 424,
-          headers: options.headers,
+          headers,
         },
       )
     }
 
     const durableObjectNamespace = env[
-      durableObjectName as keyof TEnv
-    ] as CfWorker.DurableObjectNamespace<TDurableObjectRpc>
+      syncBackendBinding as keyof TEnv
+    ] as CfTypes.DurableObjectNamespace<TDurableObjectRpc>
 
     const id = durableObjectNamespace.idFromName(storeId)
     const durableObject = durableObjectNamespace.get(id)
 
+    // Handle WebSocket upgrade request
     const upgradeHeader = request.headers.get('Upgrade')
-    if (!upgradeHeader || upgradeHeader !== 'websocket') {
+    if (transport === 'ws' && (upgradeHeader === null || upgradeHeader !== 'websocket')) {
       return new Response('Durable Object expected Upgrade: websocket', {
         status: 426,
-        headers: options?.headers,
+        headers,
       })
     }
 
-    // Cloudflare Durable Object type clashing with lib.dom Response type, which is why we need the casts here.
     return yield* Effect.promise(() => durableObject.fetch(request))
   }).pipe(Effect.tapCauseLogPretty, Effect.runPromise)

package/src/client/mod.ts

@@ -0,0 +1,3 @@
+export * from './transport/do-rpc-client.ts'
+export * from './transport/http-rpc-client.ts'
+export * from './transport/ws-rpc-client.ts'

package/src/client/transport/do-rpc-client.ts

@@ -0,0 +1,189 @@
+import { InvalidPullError, InvalidPushError, SyncBackend, UnknownError } from '@livestore/common'
+import { splitChunkBySize } from '@livestore/common/sync'
+import { type CfTypes, layerProtocolDurableObject } from '@livestore/common-cf'
+import { omit, shouldNeverHappen } from '@livestore/utils'
+import {
+  Chunk,
+  Effect,
+  identity,
+  Layer,
+  Mailbox,
+  Option,
+  RpcClient,
+  RpcSerialization,
+  Schema,
+  Stream,
+  SubscriptionRef,
+} from '@livestore/utils/effect'
+import type { SyncBackendRpcInterface } from '../../cf-worker/shared.ts'
+import { MAX_DO_RPC_REQUEST_BYTES, MAX_PUSH_EVENTS_PER_REQUEST } from '../../common/constants.ts'
+import { SyncDoRpc } from '../../common/do-rpc-schema.ts'
+import { SyncMessage } from '../../common/mod.ts'
+import type { SyncMetadata } from '../../common/sync-message-types.ts'
+
+export interface SyncBackendRpcStub extends CfTypes.DurableObjectStub, SyncBackendRpcInterface {}
+
+// TODO we probably need better scoping for the requestIdMailboxMap (i.e. support multiple stores, ...)
+type EffectRpcRequestId = string // 0, 1, 2, ...
+const requestIdMailboxMap = new Map<EffectRpcRequestId, Mailbox.Mailbox<SyncMessage.PullResponse>>()
+
+export interface DoRpcSyncOptions {
+  /** Durable Object stub that implements the SyncDoRpc interface */
+  syncBackendStub: SyncBackendRpcStub
+  /** Information about this DurableObject instance so the Sync DO instance can call back to this instance */
+  durableObjectContext: {
+    /** See `wrangler.toml` for the binding name */
+    bindingName: string
+    /** `state.id.toString()` in the DO */
+    durableObjectId: string
+  }
+}
+
+/**
+ * Creates a sync backend that uses Durable Object RPC to communicate with the sync backend.
+ *
+ * Used internally by `@livestore/adapter-cf` to connect to the sync backend.
+ */
+export const makeDoRpcSync =
+  ({ syncBackendStub, durableObjectContext }: DoRpcSyncOptions): SyncBackend.SyncBackendConstructor<SyncMetadata> =>
+  ({ storeId, payload }) =>
+    Effect.gen(function* () {
+      const isConnected = yield* SubscriptionRef.make(true)
+
+      const ProtocolLive = layerProtocolDurableObject({
+        callRpc: (payload) => syncBackendStub.rpc(payload),
+        callerContext: durableObjectContext,
+      }).pipe(Layer.provide(RpcSerialization.layerJson))
+
+      const context = yield* Layer.build(ProtocolLive)
+
+      const rpcClient = yield* RpcClient.make(SyncDoRpc).pipe(Effect.provide(context))
+
+      // Nothing to do here
+      const connect = Effect.void
+
+      const backendIdHelper = yield* SyncBackend.makeBackendIdHelper
+
+      const pull: SyncBackend.SyncBackend<SyncMetadata>['pull'] = (cursor, options) =>
+        rpcClient.SyncDoRpc.Pull({
+          cursor: cursor.pipe(
+            Option.map((a) => ({
+              eventSequenceNumber: a.eventSequenceNumber,
+              backendId: backendIdHelper.get().pipe(Option.getOrThrow),
+            })),
+          ),
+          storeId,
+          rpcContext: options?.live ? { callerContext: durableObjectContext } : undefined,
+        }).pipe(
+          options?.live
+            ? Stream.concatWithLastElement((res) =>
+                Effect.gen(function* () {
+                  if (res._tag === 'None')
+                    return shouldNeverHappen('There should at least be a no-more page info response')
+
+                  const mailbox = yield* Mailbox.make<SyncMessage.PullResponse>().pipe(
+                    Effect.acquireRelease((mailbox) => mailbox.shutdown),
+                  )
+
+                  requestIdMailboxMap.set(res.value.rpcRequestId, mailbox)
+
+                  return Mailbox.toStream(mailbox)
+                }).pipe(Stream.unwrapScoped),
+              )
+            : identity,
+          Stream.tap((res) => backendIdHelper.lazySet(res.backendId)),
+          Stream.map((res) => omit(res, ['backendId'])),
+          Stream.mapError((cause) => (cause._tag === 'InvalidPullError' ? cause : InvalidPullError.make({ cause }))),
+          Stream.withSpan('rpc-sync-client:pull'),
+        )
+
+      const push: SyncBackend.SyncBackend<{ createdAt: string }>['push'] = (batch) =>
+        Effect.gen(function* () {
+          if (batch.length === 0) {
+            return
+          }
+
+          const backendId = backendIdHelper.get()
+          const batchChunks = yield* Chunk.fromIterable(batch).pipe(
+            splitChunkBySize({
+              maxItems: MAX_PUSH_EVENTS_PER_REQUEST,
+              maxBytes: MAX_DO_RPC_REQUEST_BYTES,
+              encode: (items) => ({
+                batch: items,
+                storeId,
+                backendId,
+              }),
+            }),
+            Effect.mapError((cause) => new InvalidPushError({ cause: new UnknownError({ cause }) })),
+          )
+
+          for (const chunk of Chunk.toReadonlyArray(batchChunks)) {
+            const chunkArray = Chunk.toReadonlyArray(chunk)
+            yield* rpcClient.SyncDoRpc.Push({ batch: chunkArray, storeId, backendId })
+          }
+        }).pipe(
+          Effect.mapError((cause) =>
+            cause._tag === 'InvalidPushError' ? cause : InvalidPushError.make({ cause: new UnknownError({ cause }) }),
+          ),
+          Effect.withSpan('rpc-sync-client:push'),
+        )
+
+      const ping: SyncBackend.SyncBackend<{ createdAt: string }>['ping'] = rpcClient.SyncDoRpc.Ping({
+        storeId,
+        payload,
+      }).pipe(UnknownError.mapToUnknownError, Effect.withSpan('rpc-sync-client:ping'))
+
+      return SyncBackend.of({
+        connect,
+        isConnected,
+        pull,
+        push,
+        ping,
+        metadata: {
+          name: 'rpc-sync-client',
+          description: 'Cloudflare Durable Object RPC Sync Client',
+          protocol: 'rpc',
+          storeId,
+        },
+        supports: {
+          pullPageInfoKnown: true,
+          pullLive: true,
+        },
+      })
+    }).pipe(Effect.withSpan('rpc-sync-client:makeDoRpcSync'))
+
+/**
+ *
+ * ```ts
+ * import { DurableObject } from 'cloudflare:workers'
+ * import { ClientDoWithRpcCallback } from '@livestore/common-cf'
+ *
+ * export class MyDurableObject extends DurableObject implements ClientDoWithRpcCallback {
+ *   // ...
+ *
+ *   async syncUpdateRpc(payload: RpcMessage.ResponseChunkEncoded) {
+ *     return handleSyncUpdateRpc(payload)
+ *   }
+ * }
+ * ```
+ */
+export const handleSyncUpdateRpc = (payload: unknown) =>
+  Effect.gen(function* () {
+    const decodedPayload = yield* Schema.decodeUnknown(ResponseChunkEncoded)(payload)
+    const decoded = yield* Schema.decodeUnknown(SyncMessage.PullResponse)(decodedPayload.values[0]!)
+
+    const pullStreamMailbox = requestIdMailboxMap.get(decodedPayload.requestId)
+
+    if (pullStreamMailbox === undefined) {
+      // Case: DO was hibernated, so we need to manually update the store
+      yield* Effect.log(`No mailbox found for ${decodedPayload.requestId}`)
+    } else {
+      // Case: DO was still alive, so the existing `pull` will pick up the new events
+      yield* pullStreamMailbox.offer(decoded)
+    }
+  }).pipe(Effect.withSpan('rpc-sync-client:rpcCallback'), Effect.tapCauseLogPretty, Effect.runPromise)
+
+const ResponseChunkEncoded = Schema.Struct({
+  requestId: Schema.String,
+  values: Schema.Array(Schema.Any),
+})