@livestore/sync-cf 0.4.0-dev.8 → 0.4.0

package/src/cf-worker/do/layer.ts

@@ -1,9 +1,10 @@
-import { UnexpectedError } from '@livestore/common'
-import { EventSequenceNumber, State } from '@livestore/common/schema'
+import { UnknownError } from '@livestore/common'
 import type { CfTypes } from '@livestore/common-cf'
+import { EventSequenceNumber, State } from '@livestore/common/schema'
 import { shouldNeverHappen } from '@livestore/utils'
 import { Effect, Predicate } from '@livestore/utils/effect'
 import { nanoid } from '@livestore/utils/nanoid'
+
 import type { Env, MakeDurableObjectClassOptions, RpcSubscription } from '../shared.ts'
 import { contextTable, eventlogTable } from './sqlite.ts'
 import { makeStorage } from './sync-storage.ts'
@@ -27,7 +28,7 @@ export class DoCtx extends Effect.Service<DoCtx>()('DoCtx', {
       }
 
       const getStoreId = (from: CfTypes.Request | { storeId: string }) => {
-        if (Predicate.hasProperty(from, 'url')) {
+        if (Predicate.hasProperty(from, 'url') === true) {
           const url = new URL(from.url)
           return (
             url.searchParams.get('storeId') ?? shouldNeverHappen(`No storeId provided in request URL search params`)
@@ -37,15 +38,36 @@ export class DoCtx extends Effect.Service<DoCtx>()('DoCtx', {
       }
 
       const storeId = getStoreId(from)
-      const storage = makeStorage(doSelf.ctx, doSelf.env, storeId)
+      // Resolve storage engine
+      const makeEngine = Effect.gen(function* () {
+        const opt = doOptions?.storage
+        if (opt?._tag === 'd1') {
+          const db = (doSelf.env as any)[opt.binding]
+          if (db == null) {
+            return yield* UnknownError.make({ cause: new Error(`D1 binding '${opt.binding}' not found on env`) })
+          }
+          return { _tag: 'd1' as const, db }
+        } else if (opt?._tag === 'do-sqlite' || opt === undefined) {
+          return { _tag: 'do-sqlite' as const }
+        } else return shouldNeverHappen(`Invalid storage engine`, opt)
+      })
+
+      const engine = yield* makeEngine
+
+      const storage = makeStorage(doSelf.ctx, storeId, engine)
 
       // Initialize database tables
       {
         const colSpec = State.SQLite.makeColumnSpec(eventlogTable.sqliteDef.ast)
-        // D1 database is async, so we need to use a promise
-        yield* Effect.promise(() =>
-          doSelf.env.DB.exec(`CREATE TABLE IF NOT EXISTS "${storage.dbName}" (${colSpec}) strict`),
-        )
+        if (engine._tag === 'd1') {
+          // D1 database is async, so we need to use a promise
+          yield* Effect.promise(() =>
+            engine.db.exec(`CREATE TABLE IF NOT EXISTS "${storage.dbName}" (${colSpec}) strict`),
+          )
+        } else {
+          // DO SQLite table lives in Durable Object storage
+          doSelf.ctx.storage.sql.exec(`CREATE TABLE IF NOT EXISTS "${storage.dbName}" (${colSpec}) strict`)
+        }
       }
       {
         const colSpec = State.SQLite.makeColumnSpec(contextTable.sqliteDef.ast)
@@ -56,14 +78,14 @@ export class DoCtx extends Effect.Service<DoCtx>()('DoCtx', {
         .exec(`SELECT * FROM "${contextTable.sqliteDef.name}" WHERE storeId = ?`, storeId)
         .toArray()[0] as typeof contextTable.rowSchema.Type | undefined
 
-      const currentHeadRef = { current: storageRow?.currentHead ?? EventSequenceNumber.ROOT.global }
+      const currentHeadRef = { current: storageRow?.currentHead ?? EventSequenceNumber.Client.ROOT.global }
 
       // TODO do concistency check with eventlog table to make sure the head is consistent
 
-      // Should be the same backendId for lifetime of the durable object
+      // Should be the same backendId for lifetime of the Durable Object
       const backendId = storageRow?.backendId ?? nanoid()
 
-      const updateCurrentHead = (currentHead: EventSequenceNumber.GlobalEventSequenceNumber) => {
+      const updateCurrentHead = (currentHead: EventSequenceNumber.Global.Type) => {
         doSelf.ctx.storage.sql.exec(
           `INSERT OR REPLACE INTO "${contextTable.sqliteDef.name}" (storeId, currentHead, backendId) VALUES (?, ?, ?)`,
           storeId,
@@ -96,12 +118,12 @@ export class DoCtx extends Effect.Service<DoCtx>()('DoCtx', {
 
       // Set initial current head to root
       if (storageRow === undefined) {
-        updateCurrentHead(EventSequenceNumber.ROOT.global)
+        updateCurrentHead(EventSequenceNumber.Client.ROOT.global)
       }
 
       return storageCache
     },
-    UnexpectedError.mapToUnexpectedError,
+    UnknownError.mapToUnknownError,
     Effect.withSpan('@livestore/sync-cf:durable-object:makeDoCtx'),
   ),
 }) {}

package/src/cf-worker/do/pull.ts

@@ -1,9 +1,14 @@
-import { BackendIdMismatchError, InvalidPullError, SyncBackend, UnexpectedError } from '@livestore/common'
-import { Chunk, Effect, Option, type Schema, Stream } from '@livestore/utils/effect'
+import { BackendIdMismatchError, SyncBackend, UnknownError } from '@livestore/common'
+import { splitChunkBySize } from '@livestore/common/sync'
+import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
+
+import { MAX_PULL_EVENTS_PER_MESSAGE, MAX_WS_MESSAGE_BYTES } from '../../common/constants.ts'
 import { SyncMessage } from '../../common/mod.ts'
-import { PULL_CHUNK_SIZE } from '../shared.ts'
+import type { ForwardedHeaders } from '../shared.ts'
 import { DoCtx } from './layer.ts'
 
+const encodePullResponse = Schema.encodeSync(SyncMessage.PullResponse)
+
 // Notes on stream handling:
 // We're intentionally closing the stream once we've read all existing events
 //
@@ -13,16 +18,27 @@ import { DoCtx } from './layer.ts'
 // DO RPC:
 // - Further chunks will be emitted manually in `push.ts`
 // - If the client sends a `Interrupt` RPC message, TODO
-export const makeEndingPullStream = (
-  req: SyncMessage.PullRequest,
-  payload: Schema.JsonValue | undefined,
-): Stream.Stream<SyncMessage.PullResponse, InvalidPullError, DoCtx> =>
+export const makeEndingPullStream = ({
+  req,
+  payload,
+  headers,
+}: {
+  req: SyncMessage.PullRequest
+  payload: Schema.JsonValue | undefined
+  headers: ForwardedHeaders | undefined
+}): Stream.Stream<SyncMessage.PullResponse, UnknownError | BackendIdMismatchError, DoCtx> =>
   Effect.gen(function* () {
     const { doOptions, backendId, storeId, storage } = yield* DoCtx
 
-    if (doOptions?.onPull) {
-      yield* Effect.tryAll(() => doOptions!.onPull!(req, { storeId, payload })).pipe(
-        UnexpectedError.mapToUnexpectedError,
+    if (doOptions?.onPull !== undefined) {
+      yield* Effect.tryAll(() =>
+        doOptions.onPull!(req, {
+          storeId,
+          ...(payload !== undefined ? { payload } : {}),
+          ...(headers !== undefined ? { headers } : {}),
+        }),
+      ).pipe(
+        UnknownError.mapToUnknownError,
       )
     }
 
@@ -35,7 +51,16 @@ export const makeEndingPullStream = (
     )
 
     return storedEvents.pipe(
-      Stream.grouped(PULL_CHUNK_SIZE),
+      Stream.mapChunksEffect(
+        splitChunkBySize({
+          maxItems: MAX_PULL_EVENTS_PER_MESSAGE,
+          maxBytes: MAX_WS_MESSAGE_BYTES,
+          encode: (batch) =>
+            encodePullResponse(
+              SyncMessage.PullResponse.make({ batch, pageInfo: SyncBackend.pageInfoNoMore, backendId }),
+            ),
+        }),
+      ),
       Stream.mapAccum(total, (remaining, chunk) => {
         const asArray = Chunk.toReadonlyArray(chunk)
         const nextRemaining = Math.max(0, remaining - asArray.length)
@@ -51,8 +76,8 @@ export const makeEndingPullStream = (
       }),
       Stream.tap(
         Effect.fn(function* (res) {
-          if (doOptions?.onPullRes) {
-            yield* Effect.tryAll(() => doOptions.onPullRes!(res)).pipe(UnexpectedError.mapToUnexpectedError)
+          if (doOptions?.onPullRes !== undefined) {
+            yield* Effect.tryAll(() => doOptions.onPullRes!(res)).pipe(UnknownError.mapToUnknownError)
           }
         }),
       ),
@@ -60,6 +85,10 @@ export const makeEndingPullStream = (
     )
   }).pipe(
     Stream.unwrap,
-    Stream.mapError((cause) => InvalidPullError.make({ cause })),
+    Stream.mapError((cause) =>
+      cause._tag === 'BackendIdMismatchError' || cause._tag === 'UnknownError'
+        ? cause
+        : new UnknownError({ cause }),
+    ),
     Stream.withSpan('cloudflare-provider:pull'),
   )

package/src/cf-worker/do/push.ts

@@ -1,25 +1,39 @@
 import {
   BackendIdMismatchError,
-  InvalidPushError,
   ServerAheadError,
   SyncBackend,
-  UnexpectedError,
+  UnknownError,
 } from '@livestore/common'
 import { type CfTypes, emitStreamResponse } from '@livestore/common-cf'
-import { Effect, Option, type RpcMessage, Schema } from '@livestore/utils/effect'
+import { splitChunkBySize } from '@livestore/common/sync'
+import { Chunk, Effect, Option, type RpcMessage, Schema } from '@livestore/utils/effect'
+
+import { MAX_PUSH_EVENTS_PER_REQUEST, MAX_WS_MESSAGE_BYTES } from '../../common/constants.ts'
 import { SyncMessage } from '../../common/mod.ts'
-import { type Env, type MakeDurableObjectClassOptions, type StoreId, WebSocketAttachmentSchema } from '../shared.ts'
+import {
+  type Env,
+  type ForwardedHeaders,
+  type MakeDurableObjectClassOptions,
+  type StoreId,
+  WebSocketAttachmentSchema,
+} from '../shared.ts'
 import { DoCtx } from './layer.ts'
 
+const encodePullResponse = Schema.encodeSync(SyncMessage.PullResponse)
+const jsonStringify = Schema.encodeSync(Schema.parseJson())
+type PullBatchItem = SyncMessage.PullResponse['batch'][number]
+
 export const makePush =
   ({
     payload,
+    headers,
     options,
     storeId,
     ctx,
     env,
   }: {
     payload: Schema.JsonValue | undefined
+    headers: ForwardedHeaders | undefined
     options: MakeDurableObjectClassOptions | undefined
     storeId: StoreId
     ctx: CfTypes.DurableObjectState
@@ -34,9 +48,15 @@ export const makePush =
         return SyncMessage.PushAck.make({})
       }
 
-      if (options?.onPush) {
-        yield* Effect.tryAll(() => options.onPush!(pushRequest, { storeId, payload })).pipe(
-          UnexpectedError.mapToUnexpectedError,
+      if (options?.onPush !== undefined) {
+        yield* Effect.tryAll(() =>
+          options.onPush!(pushRequest, {
+            storeId,
+            ...(payload !== undefined ? { payload } : {}),
+            ...(headers !== undefined ? { headers } : {}),
+          }),
+        ).pipe(
+          UnknownError.mapToUnknownError,
         )
       }
 
@@ -51,6 +71,13 @@ export const makePush =
         // Validate the batch
         const firstEventParent = pushRequest.batch[0]!.parentSeqNum
         if (firstEventParent !== currentHead) {
+          // yield* Effect.logDebug('ServerAheadError: backend head mismatch', {
+          //   expectedHead: currentHead,
+          //   providedHead: firstEventParent,
+          //   batchSize: pushRequest.batch.length,
+          //   backendId,
+          // })
+
           return yield* new ServerAheadError({ minimumExpectedNum: currentHead, providedNum: firstEventParent })
         }
 
@@ -68,40 +95,71 @@ export const makePush =
       yield* Effect.gen(function* () {
         const connectedClients = ctx.getWebSockets()
 
-        // Dual broadcasting: WebSocket + RPC clients
-        const pullRes = SyncMessage.PullResponse.make({
-          batch: pushRequest.batch.map((eventEncoded) => ({
-            eventEncoded,
-            metadata: Option.some(SyncMessage.SyncMetadata.make({ createdAt })),
-          })),
-          pageInfo: SyncBackend.pageInfoNoMore,
-          backendId,
-        })
+        // Preparing chunks of responses to make sure we don't exceed the WS message size limit.
+        const responses = yield* Chunk.fromIterable(pushRequest.batch).pipe(
+          splitChunkBySize({
+            maxItems: MAX_PUSH_EVENTS_PER_REQUEST,
+            maxBytes: MAX_WS_MESSAGE_BYTES,
+            encode: (items) =>
+              encodePullResponse(
+                SyncMessage.PullResponse.make({
+                  batch: items.map(
+                    (eventEncoded): PullBatchItem => ({
+                      eventEncoded,
+                      metadata: Option.some(SyncMessage.SyncMetadata.make({ createdAt })),
+                    }),
+                  ),
+                  pageInfo: SyncBackend.pageInfoNoMore,
+                  backendId,
+                }),
+              ),
+          }),
+          Effect.map(
+            Chunk.map((eventsChunk) => {
+              const batchWithMetadata = Chunk.toReadonlyArray(eventsChunk).map((eventEncoded) => ({
+                eventEncoded,
+                metadata: Option.some(SyncMessage.SyncMetadata.make({ createdAt })),
+              }))
+
+              const response = SyncMessage.PullResponse.make({
+                batch: batchWithMetadata,
+                pageInfo: SyncBackend.pageInfoNoMore,
+                backendId,
+              })
+
+              return {
+                response,
+                encoded: Schema.encodeSync(SyncMessage.PullResponse)(response),
+              }
+            }),
+          ),
+        )
 
-        const pullResEnc = Schema.encodeSync(SyncMessage.PullResponse)(pullRes)
+        // Dual broadcasting: WebSocket + RPC clients
 
         // Broadcast to WebSocket clients
         if (connectedClients.length > 0) {
-          // Only calling once for now.
-          if (options?.onPullRes) {
-            yield* Effect.tryAll(() => options.onPullRes!(pullRes)).pipe(UnexpectedError.mapToUnexpectedError)
-          }
+          for (const { response, encoded } of responses) {
+            // Only calling once for now.
+            if (options?.onPullRes !== undefined) {
+              yield* Effect.tryAll(() => options.onPullRes!(response)).pipe(UnknownError.mapToUnknownError)
+            }
 
-          // NOTE we're also sending the pullRes to the pushing ws client as a confirmation
-          for (const conn of connectedClients) {
-            // conn.send(pullResEnc)
-            const attachment = Schema.decodeSync(WebSocketAttachmentSchema)(conn.deserializeAttachment())
-
-            // We're doing something a bit "advanced" here as we're directly emitting Effect RPC-compatible
-            // response messsages on the Effect RPC-managed websocket connection to the WS client.
-            // For this we need to get the RPC `requestId` from the WebSocket attachment.
-            for (const requestId of attachment.pullRequestIds) {
-              const res: RpcMessage.ResponseChunkEncoded = {
-                _tag: 'Chunk',
-                requestId,
-                values: [pullResEnc],
+            // NOTE we're also sending the pullRes chunk to the pushing ws client as confirmation
+            for (const conn of connectedClients) {
+              const attachment = yield* Schema.decode(WebSocketAttachmentSchema)(conn.deserializeAttachment())
+
+              // We're doing something a bit "advanced" here as we're directly emitting Effect RPC-compatible
+              // response messsages on the Effect RPC-managed websocket connection to the WS client.
+              // For this we need to get the RPC `requestId` from the WebSocket attachment.
+              for (const requestId of attachment.pullRequestIds) {
+                const res: RpcMessage.ResponseChunkEncoded = {
+                  _tag: 'Chunk',
+                  requestId,
+                  values: [encoded],
+                }
+                conn.send(jsonStringify(res))
               }
-              conn.send(JSON.stringify(res))
             }
           }
 
@@ -110,17 +168,16 @@ export const makePush =
 
         // RPC broadcasting would require reconstructing client stubs from clientIds
         if (rpcSubscriptions.size > 0) {
-          yield* Effect.forEach(
-            rpcSubscriptions.values(),
-            (subscription) =>
-              emitStreamResponse({
+          for (const subscription of rpcSubscriptions.values()) {
+            for (const { encoded } of responses) {
+              yield* emitStreamResponse({
                 callerContext: subscription.callerContext,
                 env,
                 requestId: subscription.requestId,
-                values: [pullResEnc],
-              }).pipe(Effect.tapCauseLogPretty, Effect.exit),
-            { concurrency: 'unbounded' },
-          )
+                values: [encoded],
+              }).pipe(Effect.tapCauseLogPretty, Effect.exit)
+            }
+          }
 
           yield* Effect.logDebug(`Broadcasted to ${rpcSubscriptions.size} RPC clients`)
         }
@@ -138,12 +195,16 @@ export const makePush =
     }).pipe(
       Effect.tap(
         Effect.fn(function* (message) {
-          if (options?.onPushRes) {
-            yield* Effect.tryAll(() => options.onPushRes!(message)).pipe(UnexpectedError.mapToUnexpectedError)
+          if (options?.onPushRes !== undefined) {
+            yield* Effect.tryAll(() => options.onPushRes!(message)).pipe(UnknownError.mapToUnknownError)
           }
         }),
       ),
-      Effect.mapError((cause) => InvalidPushError.make({ cause })),
+      Effect.mapError((cause) =>
+        cause._tag === 'BackendIdMismatchError' || cause._tag === 'ServerAheadError' || cause._tag === 'UnknownError'
+          ? cause
+          : new UnknownError({ cause }),
+      ),
       Effect.withSpan('sync-cf:do:push', { attributes: { storeId, batchSize: pushRequest.batch.length } }),
     )
 

package/src/cf-worker/do/sqlite.ts

@@ -1,13 +1,19 @@
 import { EventSequenceNumber, State } from '@livestore/common/schema'
 import { Schema } from '@livestore/utils/effect'
+
 import { PERSISTENCE_FORMAT_VERSION } from '../shared.ts'
 
+/**
+ * Main event log table storing all LiveStore events.
+ *
+ * ⚠️  IMPORTANT: Any changes to this schema require bumping PERSISTENCE_FORMAT_VERSION in shared.ts
+ */
 export const eventlogTable = State.SQLite.table({
   // NOTE actual table name is determined at runtime to use proper storeId
   name: `eventlog_${PERSISTENCE_FORMAT_VERSION}_$storeId`,
   columns: {
-    seqNum: State.SQLite.integer({ primaryKey: true, schema: EventSequenceNumber.GlobalEventSequenceNumber }),
-    parentSeqNum: State.SQLite.integer({ schema: EventSequenceNumber.GlobalEventSequenceNumber }),
+    seqNum: State.SQLite.integer({ primaryKey: true, schema: EventSequenceNumber.Global.Schema }),
+    parentSeqNum: State.SQLite.integer({ schema: EventSequenceNumber.Global.Schema }),
     name: State.SQLite.text({}),
     args: State.SQLite.text({ schema: Schema.parseJson(Schema.Any), nullable: true }),
     /** ISO date format. Currently only used for debugging purposes. */
@@ -17,12 +23,16 @@ export const eventlogTable = State.SQLite.table({
   },
 })
 
-/** Will only ever have one row per durable object. */
+/**
+ * Context metadata table - one row per Durable Object.
+ *
+ * ⚠️  IMPORTANT: Any changes to this schema require bumping PERSISTENCE_FORMAT_VERSION in shared.ts
+ */
 export const contextTable = State.SQLite.table({
   name: `context_${PERSISTENCE_FORMAT_VERSION}`,
   columns: {
     storeId: State.SQLite.text({ primaryKey: true }),
-    currentHead: State.SQLite.integer({ schema: EventSequenceNumber.GlobalEventSequenceNumber }),
+    currentHead: State.SQLite.integer({ schema: EventSequenceNumber.Global.Schema }),
     backendId: State.SQLite.text({}),
   },
 })