@livestore/sync-cf 0.4.0-dev.7 → 0.4.0-dev.9

package/src/cf-worker/do/push.ts

@@ -6,10 +6,21 @@ import {
   UnexpectedError,
 } from '@livestore/common'
 import { type CfTypes, emitStreamResponse } from '@livestore/common-cf'
-import { Effect, Option, type RpcMessage, Schema } from '@livestore/utils/effect'
+import { Chunk, Effect, Option, type RpcMessage, Schema } from '@livestore/utils/effect'
 import { SyncMessage } from '../../common/mod.ts'
-import { type Env, type MakeDurableObjectClassOptions, type StoreId, WebSocketAttachmentSchema } from '../shared.ts'
+import {
+  type Env,
+  MAX_PULL_EVENTS_PER_MESSAGE,
+  MAX_WS_MESSAGE_BYTES,
+  type MakeDurableObjectClassOptions,
+  type StoreId,
+  WebSocketAttachmentSchema,
+} from '../shared.ts'
 import { DoCtx } from './layer.ts'
+import { splitChunkBySize } from './ws-chunking.ts'
+
+const encodePullResponse = Schema.encodeSync(SyncMessage.PullResponse)
+type PullBatchItem = SyncMessage.PullResponse['batch'][number]
 
 export const makePush =
   ({
@@ -51,6 +62,13 @@ export const makePush =
         // Validate the batch
         const firstEventParent = pushRequest.batch[0]!.parentSeqNum
         if (firstEventParent !== currentHead) {
+          // yield* Effect.logDebug('ServerAheadError: backend head mismatch', {
+          //   expectedHead: currentHead,
+          //   providedHead: firstEventParent,
+          //   batchSize: pushRequest.batch.length,
+          //   backendId,
+          // })
+
           return yield* new ServerAheadError({ minimumExpectedNum: currentHead, providedNum: firstEventParent })
         }
 
@@ -68,40 +86,69 @@ export const makePush =
       yield* Effect.gen(function* () {
         const connectedClients = ctx.getWebSockets()
 
-        // Dual broadcasting: WebSocket + RPC clients
-        const pullRes = SyncMessage.PullResponse.make({
-          batch: pushRequest.batch.map((eventEncoded) => ({
-            eventEncoded,
-            metadata: Option.some(SyncMessage.SyncMetadata.make({ createdAt })),
-          })),
-          pageInfo: SyncBackend.pageInfoNoMore,
-          backendId,
-        })
+        // Preparing chunks of responses to make sure we don't exceed the WS message size limit.
+        const responses = Chunk.fromIterable(pushRequest.batch).pipe(
+          splitChunkBySize({
+            maxItems: MAX_PULL_EVENTS_PER_MESSAGE,
+            maxBytes: MAX_WS_MESSAGE_BYTES,
+            encode: (items) =>
+              encodePullResponse(
+                SyncMessage.PullResponse.make({
+                  batch: items.map(
+                    (eventEncoded): PullBatchItem => ({
+                      eventEncoded,
+                      metadata: Option.some(SyncMessage.SyncMetadata.make({ createdAt })),
+                    }),
+                  ),
+                  pageInfo: SyncBackend.pageInfoNoMore,
+                  backendId,
+                }),
+              ),
+          }),
+          Chunk.map((eventsChunk) => {
+            const batchWithMetadata = Chunk.toReadonlyArray(eventsChunk).map((eventEncoded) => ({
+              eventEncoded,
+              metadata: Option.some(SyncMessage.SyncMetadata.make({ createdAt })),
+            }))
+
+            const response = SyncMessage.PullResponse.make({
+              batch: batchWithMetadata,
+              pageInfo: SyncBackend.pageInfoNoMore,
+              backendId,
+            })
+
+            return {
+              response,
+              encoded: Schema.encodeSync(SyncMessage.PullResponse)(response),
+            }
+          }),
+        )
 
-        const pullResEnc = Schema.encodeSync(SyncMessage.PullResponse)(pullRes)
+        // Dual broadcasting: WebSocket + RPC clients
 
         // Broadcast to WebSocket clients
         if (connectedClients.length > 0) {
-          // Only calling once for now.
-          if (options?.onPullRes) {
-            yield* Effect.tryAll(() => options.onPullRes!(pullRes)).pipe(UnexpectedError.mapToUnexpectedError)
-          }
+          for (const { response, encoded } of responses) {
+            // Only calling once for now.
+            if (options?.onPullRes) {
+              yield* Effect.tryAll(() => options.onPullRes!(response)).pipe(UnexpectedError.mapToUnexpectedError)
+            }
 
-          // NOTE we're also sending the pullRes to the pushing ws client as a confirmation
-          for (const conn of connectedClients) {
-            // conn.send(pullResEnc)
-            const attachment = Schema.decodeSync(WebSocketAttachmentSchema)(conn.deserializeAttachment())
-
-            // We're doing something a bit "advanced" here as we're directly emitting Effect RPC-compatible
-            // response messsages on the Effect RPC-managed websocket connection to the WS client.
-            // For this we need to get the RPC `requestId` from the WebSocket attachment.
-            for (const requestId of attachment.pullRequestIds) {
-              const res: RpcMessage.ResponseChunkEncoded = {
-                _tag: 'Chunk',
-                requestId,
-                values: [pullResEnc],
+            // NOTE we're also sending the pullRes chunk to the pushing ws client as confirmation
+            for (const conn of connectedClients) {
+              const attachment = Schema.decodeSync(WebSocketAttachmentSchema)(conn.deserializeAttachment())
+
+              // We're doing something a bit "advanced" here as we're directly emitting Effect RPC-compatible
+              // response messsages on the Effect RPC-managed websocket connection to the WS client.
+              // For this we need to get the RPC `requestId` from the WebSocket attachment.
+              for (const requestId of attachment.pullRequestIds) {
+                const res: RpcMessage.ResponseChunkEncoded = {
+                  _tag: 'Chunk',
+                  requestId,
+                  values: [encoded],
+                }
+                conn.send(JSON.stringify(res))
               }
-              conn.send(JSON.stringify(res))
             }
           }
 
@@ -110,17 +157,16 @@ export const makePush =
 
         // RPC broadcasting would require reconstructing client stubs from clientIds
         if (rpcSubscriptions.size > 0) {
-          yield* Effect.forEach(
-            rpcSubscriptions.values(),
-            (subscription) =>
-              emitStreamResponse({
+          for (const subscription of rpcSubscriptions.values()) {
+            for (const { encoded } of responses) {
+              yield* emitStreamResponse({
                 callerContext: subscription.callerContext,
                 env,
                 requestId: subscription.requestId,
-                values: [pullResEnc],
-              }).pipe(Effect.tapCauseLogPretty, Effect.exit),
-            { concurrency: 'unbounded' },
-          )
+                values: [encoded],
+              }).pipe(Effect.tapCauseLogPretty, Effect.exit)
+            }
+          }
 
           yield* Effect.logDebug(`Broadcasted to ${rpcSubscriptions.size} RPC clients`)
         }
@@ -144,6 +190,7 @@ export const makePush =
         }),
       ),
       Effect.mapError((cause) => InvalidPushError.make({ cause })),
+      Effect.withSpan('sync-cf:do:push', { attributes: { storeId, batchSize: pushRequest.batch.length } }),
     )
 
 /**

package/src/cf-worker/do/sync-storage.ts

@@ -1,18 +1,21 @@
 import { UnexpectedError } from '@livestore/common'
 import type { LiveStoreEvent } from '@livestore/common/schema'
 import type { CfTypes } from '@livestore/common-cf'
-import { Effect, Option, Schema } from '@livestore/utils/effect'
+import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
 import { SyncMetadata } from '../../common/sync-message-types.ts'
 import { type Env, PERSISTENCE_FORMAT_VERSION, type StoreId } from '../shared.ts'
 import { eventlogTable } from './sqlite.ts'
 
 export type SyncStorage = {
   dbName: string
-  // getHead: Effect.Effect<EventSequenceNumber.GlobalEventSequenceNumber, UnexpectedError>
-  getEvents: (
-    cursor: number | undefined,
-  ) => Effect.Effect<
-    ReadonlyArray<{ eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> }>,
+  getEvents: (cursor: number | undefined) => Effect.Effect<
+    {
+      total: number
+      stream: Stream.Stream<
+        { eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> },
+        UnexpectedError
+      >
+    },
     UnexpectedError
   >
   appendEvents: (
@@ -34,35 +37,111 @@ export const makeStorage = (ctx: CfTypes.DurableObjectState, env: Env, storeId:
       Effect.withSpan('@livestore/sync-cf:durable-object:execDb'),
     )
 
-  // const getHead: Effect.Effect<EventSequenceNumber.GlobalEventSequenceNumber, UnexpectedError> = Effect.gen(
-  //   function* () {
-  //     const result = yield* execDb<{ seqNum: EventSequenceNumber.GlobalEventSequenceNumber }>((db) =>
-  //       db.prepare(`SELECT seqNum FROM ${dbName} ORDER BY seqNum DESC LIMIT 1`).all(),
-  //     )
+  // Cloudflare's D1 HTTP endpoint rejects JSON responses once they exceed ~1MB.
+  // Keep individual SELECT batches comfortably below that threshold so we can
+  // serve large histories without tripping the limit.
+  const D1_MAX_JSON_RESPONSE_BYTES = 1_000_000
+  const D1_RESPONSE_SAFETY_MARGIN_BYTES = 64 * 1024
+  const D1_TARGET_RESPONSE_BYTES = D1_MAX_JSON_RESPONSE_BYTES - D1_RESPONSE_SAFETY_MARGIN_BYTES
+  const D1_INITIAL_PAGE_SIZE = 256
+  const D1_MIN_PAGE_SIZE = 1
+
+  const decodeEventlogRows = Schema.decodeUnknownSync(Schema.Array(eventlogTable.rowSchema))
+  const textEncoder = new TextEncoder()
+
+  const decreaseLimit = (limit: number) => Math.max(D1_MIN_PAGE_SIZE, Math.floor(limit / 2))
+  const increaseLimit = (limit: number) => Math.min(D1_INITIAL_PAGE_SIZE, limit * 2)
 
-  //     return result[0]?.seqNum ?? EventSequenceNumber.ROOT.global
-  //   },
-  // ).pipe(UnexpectedError.mapToUnexpectedError)
+  const computeNextLimit = (limit: number, encodedSize: number) => {
+    if (encodedSize > D1_TARGET_RESPONSE_BYTES && limit > D1_MIN_PAGE_SIZE) {
+      const next = decreaseLimit(limit)
+      return next === limit ? limit : next
+    }
+
+    if (encodedSize < D1_TARGET_RESPONSE_BYTES / 2 && limit < D1_INITIAL_PAGE_SIZE) {
+      const next = increaseLimit(limit)
+      return next === limit ? limit : next
+    }
+
+    return limit
+  }
 
-  // TODO support streaming
   const getEvents = (
     cursor: number | undefined,
   ): Effect.Effect<
-    ReadonlyArray<{ eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> }>,
+    {
+      total: number
+      stream: Stream.Stream<
+        { eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> },
+        UnexpectedError
+      >
+    },
     UnexpectedError
   > =>
     Effect.gen(function* () {
-      const whereClause = cursor === undefined ? '' : `WHERE seqNum > ${cursor}`
-      const sql = `SELECT * FROM ${dbName} ${whereClause} ORDER BY seqNum ASC`
-      // TODO handle case where `cursor` was not found
-      const rawEvents = yield* execDb((db) => db.prepare(sql).all())
-      const events = Schema.decodeUnknownSync(Schema.Array(eventlogTable.rowSchema))(rawEvents).map(
-        ({ createdAt, ...eventEncoded }) => ({
-          eventEncoded,
-          metadata: Option.some(SyncMetadata.make({ createdAt })),
-        }),
-      )
-      return events
+      const countStatement =
+        cursor === undefined
+          ? `SELECT COUNT(*) as total FROM ${dbName}`
+          : `SELECT COUNT(*) as total FROM ${dbName} WHERE seqNum > ?`
+
+      const countRows = yield* execDb<{ total: number }>((db) => {
+        const prepared = db.prepare(countStatement)
+        return cursor === undefined ? prepared.all() : prepared.bind(cursor).all()
+      })
+
+      const total = Number(countRows[0]?.total ?? 0)
+
+      type State = { cursor: number | undefined; limit: number }
+      type EmittedEvent = { eventEncoded: LiveStoreEvent.AnyEncodedGlobal; metadata: Option.Option<SyncMetadata> }
+
+      const initialState: State = { cursor, limit: D1_INITIAL_PAGE_SIZE }
+
+      const fetchPage = (
+        state: State,
+      ): Effect.Effect<Option.Option<readonly [Chunk.Chunk<EmittedEvent>, State]>, UnexpectedError> =>
+        Effect.gen(function* () {
+          const statement =
+            state.cursor === undefined
+              ? `SELECT * FROM ${dbName} ORDER BY seqNum ASC LIMIT ?`
+              : `SELECT * FROM ${dbName} WHERE seqNum > ? ORDER BY seqNum ASC LIMIT ?`
+
+          const rawEvents = yield* execDb((db) => {
+            const prepared = db.prepare(statement)
+            return state.cursor === undefined
+              ? prepared.bind(state.limit).all()
+              : prepared.bind(state.cursor, state.limit).all()
+          })
+
+          if (rawEvents.length === 0) {
+            return Option.none()
+          }
+
+          const encodedSize = textEncoder.encode(JSON.stringify(rawEvents)).byteLength
+
+          if (encodedSize > D1_TARGET_RESPONSE_BYTES && state.limit > D1_MIN_PAGE_SIZE) {
+            const nextLimit = decreaseLimit(state.limit)
+
+            if (nextLimit !== state.limit) {
+              return yield* fetchPage({ cursor: state.cursor, limit: nextLimit })
+            }
+          }
+
+          const decodedRows = Chunk.fromIterable(decodeEventlogRows(rawEvents))
+
+          const eventsChunk = Chunk.map(decodedRows, ({ createdAt, ...eventEncoded }) => ({
+            eventEncoded,
+            metadata: Option.some(SyncMetadata.make({ createdAt })),
+          }))
+
+          const lastSeqNum = Chunk.unsafeLast(decodedRows).seqNum
+          const nextState: State = { cursor: lastSeqNum, limit: computeNextLimit(state.limit, encodedSize) }
+
+          return Option.some([eventsChunk, nextState] as const)
+        })
+
+      const stream = Stream.unfoldChunkEffect(initialState, fetchPage)
+
+      return { total, stream }
     }).pipe(
       UnexpectedError.mapToUnexpectedError,
       Effect.withSpan('@livestore/sync-cf:durable-object:getEvents', { attributes: { dbName, cursor } }),

package/src/cf-worker/do/transport/do-rpc-server.ts

@@ -1,5 +1,5 @@
 import { InvalidPullError, InvalidPushError } from '@livestore/common'
-import { toDurableObjectHandler } from '@livestore/common-cf'
+import { type CfTypes, toDurableObjectHandler } from '@livestore/common-cf'
 import {
   Effect,
   Headers,
@@ -22,7 +22,9 @@ export interface DoRpcHandlerOptions {
   input: Omit<DoCtxInput, 'from'>
 }
 
-export const createDoRpcHandler = (options: DoRpcHandlerOptions) =>
+export const createDoRpcHandler = (
+  options: DoRpcHandlerOptions,
+): Effect.Effect<Uint8Array<ArrayBuffer> | CfTypes.ReadableStream> =>
   Effect.gen(this, function* () {
     const { payload, input } = options
     // const { rpcSubscriptions, backendId, doOptions, ctx, env } = yield* DoCtx

package/src/cf-worker/do/ws-chunking.ts

@@ -0,0 +1,76 @@
+import { Chunk } from '@livestore/utils/effect'
+
+const textEncoder = new TextEncoder()
+
+/**
+ * Configuration describing how to break a chunk into smaller payload-safe chunks.
+ */
+export interface ChunkingOptions<A> {
+  /** Maximum number of items that may appear in any emitted chunk. */
+  readonly maxItems: number
+  /** Maximum encoded byte size allowed for any emitted chunk. */
+  readonly maxBytes: number
+  /**
+   * Callback that produces a JSON-serialisable structure whose byte size should
+   * fit within {@link maxBytes}. This lets callers control framing overhead.
+   */
+  readonly encode: (items: ReadonlyArray<A>) => unknown
+}
+
+/**
+ * Derives a function that splits an input chunk into sub-chunks confined by
+ * both item count and encoded byte size limits. Designed for transports with
+ * strict frame caps (e.g. Cloudflare hibernated WebSockets).
+ */
+export const splitChunkBySize =
+  <A>(options: ChunkingOptions<A>) =>
+  (chunk: Chunk.Chunk<A>): Chunk.Chunk<Chunk.Chunk<A>> => {
+    const maxItems = Math.max(1, options.maxItems)
+    const maxBytes = Math.max(1, options.maxBytes)
+    const encode = options.encode
+
+    const measure = (items: ReadonlyArray<A>) => {
+      const encoded = encode(items)
+      return textEncoder.encode(JSON.stringify(encoded)).byteLength
+    }
+
+    const items = Chunk.toReadonlyArray(chunk)
+    if (items.length === 0) {
+      return Chunk.fromIterable<Chunk.Chunk<A>>([])
+    }
+
+    const result: Array<Chunk.Chunk<A>> = []
+    let current: Array<A> = []
+
+    const flushCurrent = () => {
+      if (current.length > 0) {
+        result.push(Chunk.fromIterable(current))
+        current = []
+      }
+    }
+
+    for (const item of items) {
+      current.push(item)
+      const exceedsLimit = current.length > maxItems || measure(current) > maxBytes
+
+      if (exceedsLimit) {
+        // remove the item we just added and emit the previous chunk if it exists
+        const last = current.pop()!
+        flushCurrent()
+
+        if (last !== undefined) {
+          current = [last]
+          const singleItemTooLarge = measure(current) > maxBytes
+          if (singleItemTooLarge || current.length > maxItems) {
+            // Emit the oversized item on its own; downstream can decide how to handle it.
+            result.push(Chunk.of(last))
+            current = []
+          }
+        }
+      }
+    }
+
+    flushCurrent()
+
+    return Chunk.fromIterable(result)
+  }

package/src/cf-worker/shared.ts

@@ -1,6 +1,7 @@
 import type { InvalidPullError, InvalidPushError } from '@livestore/common'
 import type { CfTypes } from '@livestore/common-cf'
-import { Effect, type Option, Schema, UrlParams } from '@livestore/utils/effect'
+import { Effect, Schema, UrlParams } from '@livestore/utils/effect'
+import type { SearchParams } from '../common/mod.ts'
 import { SearchParamsSchema, SyncMessage } from '../common/mod.ts'
 
 export interface Env {
@@ -48,20 +49,32 @@ export type DurableObjectId = string
  */
 export const PERSISTENCE_FORMAT_VERSION = 7
 
-export const DEFAULT_SYNC_DURABLE_OBJECT_NAME = 'SYNC_BACKEND_DO'
-
 export const encodeOutgoingMessage = Schema.encodeSync(Schema.parseJson(SyncMessage.BackendToClientMessage))
 export const encodeIncomingMessage = Schema.encodeSync(Schema.parseJson(SyncMessage.ClientToBackendMessage))
 
-export const getSyncRequestSearchParams = (request: CfTypes.Request): Option.Option<typeof SearchParamsSchema.Type> => {
+/**
+ * Extracts the LiveStore sync search parameters from a request. Returns
+ * `undefined` when the request does not carry valid sync metadata so callers
+ * can fall back to custom routing.
+ */
+export const matchSyncRequest = (request: CfTypes.Request): SearchParams | undefined => {
   const url = new URL(request.url)
   const urlParams = UrlParams.fromInput(url.searchParams)
   const paramsResult = UrlParams.schemaStruct(SearchParamsSchema)(urlParams).pipe(Effect.option, Effect.runSync)
 
-  return paramsResult
+  if (paramsResult._tag === 'None') {
+    return undefined
+  }
+
+  return paramsResult.value
 }
 
-export const PULL_CHUNK_SIZE = 100
+export const MAX_PULL_EVENTS_PER_MESSAGE = 100
+
+// Cloudflare hibernated WebSocket frames begin failing just below 1MB. Keep our
+// payloads comfortably beneath that ceiling so we don't rely on implementation
+// quirks of local dev servers.
+export const MAX_WS_MESSAGE_BYTES = 900_000
 
 // RPC subscription storage (TODO refactor)
 export type RpcSubscription = {