@livestore/sync-cf 0.4.0 → 0.5.0-dev.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livestore/sync-cf",
-  "version": "0.4.0",
+  "version": "0.5.0-dev.0",
   "license": "Apache-2.0",
   "repository": {
     "type": "git",
@@ -24,51 +24,35 @@
   },
   "dependencies": {
     "@cloudflare/workers-types": "4.20251118.0",
-    "@livestore/common": "^0.4.0",
-    "@livestore/common-cf": "^0.4.0",
-    "@livestore/utils": "^0.4.0"
+    "@livestore/utils": "^0.5.0-dev.0",
+    "@livestore/common": "^0.5.0-dev.0",
+    "@livestore/common-cf": "^0.5.0-dev.0"
   },
   "devDependencies": {
-    "@effect/ai": "0.35.0",
-    "@effect/cli": "0.75.1",
-    "@effect/cluster": "0.58.2",
-    "@effect/experimental": "0.60.0",
-    "@effect/opentelemetry": "0.63.0",
-    "@effect/platform": "0.96.1",
-    "@effect/platform-browser": "0.76.0",
-    "@effect/platform-bun": "0.89.0",
-    "@effect/platform-node": "0.106.0",
-    "@effect/printer": "0.49.0",
-    "@effect/printer-ansi": "0.49.0",
-    "@effect/rpc": "0.75.1",
-    "@effect/sql": "0.51.1",
-    "@effect/typeclass": "0.40.0",
-    "@effect/vitest": "0.29.0",
+    "@effect/opentelemetry": "4.0.0-rc.111",
+    "@effect/platform-browser": "4.0.0-rc.111",
+    "@effect/platform-bun": "4.0.0-rc.111",
+    "@effect/platform-node": "4.0.0-rc.111",
+    "@effect/platform-node-shared": "4.0.0-rc.111",
+    "@effect/vitest": "4.0.0-rc.111",
     "@opentelemetry/api": "1.9.0",
     "@opentelemetry/resources": "2.2.0",
     "@standard-schema/spec": "1.1.0",
-    "effect": "3.21.2"
+    "effect": "4.0.0-rc.111",
+    "vitest": "4.1.9",
+    "@livestore/utils-dev": "^0.5.0-dev.0"
   },
   "peerDependencies": {
-    "@effect/ai": "^0.35.0",
-    "@effect/cli": "^0.75.1",
-    "@effect/cluster": "^0.58.2",
-    "@effect/experimental": "^0.60.0",
-    "@effect/opentelemetry": "^0.63.0",
-    "@effect/platform": "^0.96.1",
-    "@effect/platform-browser": "^0.76.0",
-    "@effect/platform-bun": "^0.89.0",
-    "@effect/platform-node": "^0.106.0",
-    "@effect/printer": "^0.49.0",
-    "@effect/printer-ansi": "^0.49.0",
-    "@effect/rpc": "^0.75.1",
-    "@effect/sql": "^0.51.1",
-    "@effect/typeclass": "^0.40.0",
-    "@effect/vitest": "^0.29.0",
+    "@effect/opentelemetry": "^4.0.0-rc.111",
+    "@effect/platform-browser": "^4.0.0-rc.111",
+    "@effect/platform-bun": "^4.0.0-rc.111",
+    "@effect/platform-node": "^4.0.0-rc.111",
+    "@effect/platform-node-shared": "^4.0.0-rc.111",
+    "@effect/vitest": "^4.0.0-rc.111",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/resources": "^2.2.0",
     "@standard-schema/spec": "^1.1.0",
-    "effect": "^3.21.2"
+    "effect": "^4.0.0-rc.111"
   },
   "$genie": {
     "source": "package.json.genie.ts",
@@ -83,6 +67,6 @@
     ]
   },
   "scripts": {
-    "test": "echo 'No tests yet'"
+    "test": "vitest run"
   }
 }

package/src/cf-worker/do/durable-object.ts

@@ -9,8 +9,8 @@ import {
   FetchHttpClient,
   Layer,
   Logger,
-  LogLevel,
   Otlp,
+  References,
   RpcMessage,
   Schema,
   type Scope,
@@ -24,17 +24,17 @@ import {
   type SyncBackendRpcInterface,
   WebSocketAttachmentSchema,
 } from '../shared.ts'
-import { DoCtx } from './layer.ts'
+import * as DoCtx from './layer.ts'
 import { createDoRpcHandler } from './transport/do-rpc-server.ts'
 import { createHttpRpcHandler } from './transport/http-rpc-server.ts'
 import { makeRpcServer } from './transport/ws-rpc-server.ts'
 
-// NOTE We need to redeclare runtime types here to avoid type conflicts with the lib.dom Response type.
-// TODO get rid of those once CF fixed their type mismatch in the worker types
-declare class Request extends CfDeclare.Request {}
-declare class Response extends CfDeclare.Response {}
-declare class WebSocketPair extends CfDeclare.WebSocketPair {}
-declare class WebSocketRequestResponsePair extends CfDeclare.WebSocketRequestResponsePair {}
+const Response = CfDeclare.Response
+const WebSocketPair = CfDeclare.WebSocketPair
+const WebSocketRequestResponsePair = CfDeclare.WebSocketRequestResponsePair
+
+/** Module-scoped JSON encoder; keeping the sync codec out of Effect generators avoids `schemaSyncInEffect`. */
+const jsonStringify = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown))
 
 const DurableObjectBase = DurableObject as any as new (
   state: CfTypes.DurableObjectState,
@@ -52,14 +52,14 @@ export type MakeDurableObjectClass = (options?: MakeDurableObjectClassOptions) =
 /**
  * Creates a Durable Object class for handling WebSocket-based sync.
  * A sync Durable Object is uniquely scoped to a specific `storeId`.
- * 
+ *
  * The sync DO supports 3 transport modes:
  * - HTTP JSON-RPC
  * - WebSocket
  * - Durable Object RPC calls (only works in combination with `@livestore/adapter-cf`)
  *
  * Example:
- * 
+ *
  * ```ts
  * // In your Cloudflare Worker file
  * import { makeDurableObject } from '@livestore/sync-cf/cf-worker'
@@ -88,17 +88,16 @@ export type MakeDurableObjectClass = (options?: MakeDurableObjectClassOptions) =
 export const makeDurableObject: MakeDurableObjectClass = (options) => {
   const enabledTransports = options?.enabledTransports ?? new Set(['http', 'ws', 'do-rpc'])
 
-  const Logging = Logger.consoleWithThread('SyncDo')
-
-  const Observability: Layer.Layer<never> = options?.otel?.baseUrl !== undefined
-    ? (Otlp.layer({
-        baseUrl: options.otel.baseUrl,
-        tracerExportInterval: 50,
-        resource: {
-          serviceName: options.otel.serviceName ?? 'sync-cf-do',
-        },
-      }).pipe(Layer.provide(FetchHttpClient.layer)) as Layer.Layer<never>)
-    : Layer.empty
+  const Observability: Layer.Layer<never> =
+    options?.otel?.baseUrl !== undefined
+      ? Otlp.layerJson({
+          baseUrl: options.otel.baseUrl,
+          tracerExportInterval: 50,
+          resource: {
+            serviceName: options.otel.serviceName ?? 'sync-cf-do',
+          },
+        }).pipe(Layer.provide(FetchHttpClient.layer))
+      : Layer.empty
 
   return class SyncBackendDOBase extends DurableObjectBase implements SyncBackendRpcInterface {
     __DURABLE_OBJECT_BRAND = 'SyncBackendDOBase' as never
@@ -120,7 +119,7 @@ export const makeDurableObject: MakeDurableObjectClass = (options) => {
             if (request._tag === 'Request' && request.tag === 'SyncWsRpc.Pull') {
               // Is Pull request: add requestId to pullRequestIds
               const attachment = ws.deserializeAttachment()
-              const { pullRequestIds, ...rest } = Schema.decodeSync(WebSocketAttachmentSchema)(attachment)
+              const { pullRequestIds, ...rest } = Schema.decodeUnknownSync(WebSocketAttachmentSchema)(attachment)
               ws.serializeAttachment(
                 Schema.encodeSync(WebSocketAttachmentSchema)({
                   ...rest,
@@ -130,7 +129,7 @@ export const makeDurableObject: MakeDurableObjectClass = (options) => {
             } else if (request._tag === 'Interrupt') {
               // Is Interrupt request: remove requestId from pullRequestIds
               const attachment = ws.deserializeAttachment()
-              const { pullRequestIds, ...rest } = Schema.decodeSync(WebSocketAttachmentSchema)(attachment)
+              const { pullRequestIds, ...rest } = Schema.decodeUnknownSync(WebSocketAttachmentSchema)(attachment)
               ws.serializeAttachment(
                 Schema.encodeSync(WebSocketAttachmentSchema)({
                   ...rest,
@@ -145,8 +144,8 @@ export const makeDurableObject: MakeDurableObjectClass = (options) => {
       }
     }
 
-    override fetch = async (request: Request): Promise<Response> =>
-      Effect.gen(this, function* () {
+    override fetch = async (request: CfTypes.Request): Promise<CfTypes.Response> =>
+      Effect.gen({ self: this }, function* () {
         const searchParams = matchSyncRequest(request)
         if (searchParams === undefined) {
           throw new Error('No search params found in request URL')
@@ -170,9 +169,16 @@ export const makeDurableObject: MakeDurableObjectClass = (options) => {
 
           // Since we're using websocket hibernation, we need to remember the storeId for subsequent `webSocketMessage` calls
           // Also store forwarded headers so they're available after hibernation resume
-          server.serializeAttachment(
-            Schema.encodeSync(WebSocketAttachmentSchema)({ storeId, payload, pullRequestIds: [], headers }),
-          )
+          // Encoding known-valid domain data: an encode failure is an invariant violation (a defect),
+          // so `Effect.orDie` is the correct modeling. serializeAttachment takes the value synchronously,
+          // so resolve the encoded value first, then pass it in.
+          const attachment = yield* Schema.encodeEffect(WebSocketAttachmentSchema)({
+            storeId,
+            payload,
+            pullRequestIds: [],
+            headers,
+          }).pipe(Effect.orDie)
+          server.serializeAttachment(attachment)
 
           // See https://developers.cloudflare.com/durable-objects/examples/websocket-hibernation-server
 
@@ -180,10 +186,7 @@ export const makeDurableObject: MakeDurableObjectClass = (options) => {
 
           // Ping requests are sent by Effect RPC internally
           this.ctx.setWebSocketAutoResponse(
-            new WebSocketRequestResponsePair(
-              JSON.stringify(RpcMessage.constPing),
-              JSON.stringify(RpcMessage.constPong),
-            ),
+            new WebSocketRequestResponsePair(jsonStringify(RpcMessage.constPing), jsonStringify(RpcMessage.constPong)),
           )
 
           return new Response(null, {
@@ -200,11 +203,11 @@ export const makeDurableObject: MakeDurableObjectClass = (options) => {
         })
       }).pipe(
         Effect.tapCauseLogPretty, // Also log errors to console before catching them
-        Effect.catchAllCause((cause) =>
+        Effect.catchCause((cause) =>
           Effect.succeed(new Response('Error', { status: 500, statusText: cause.toString() })),
         ),
         Effect.withSpan('@livestore/sync-cf:durable-object:fetch'),
-        Effect.provide(DoCtx.Default({ doSelf: this, doOptions: options, from: request })),
+        Effect.provide(DoCtx.layer({ doSelf: this, doOptions: options, from: request })),
         this.runEffectAsPromise,
       )
 
@@ -237,8 +240,14 @@ export const makeDurableObject: MakeDurableObjectClass = (options) => {
     private runEffectAsPromise = <T, E = never>(effect: Effect.Effect<T, E, Scope.Scope>): Promise<T> =>
       effect.pipe(
         Effect.tapCauseLogPretty,
-        Logger.withMinimumLogLevel(LogLevel.Debug),
-        Effect.provide(Layer.mergeAll(Observability, Logging)),
+        Effect.annotateLogs({ thread: 'SyncDo' }),
+        Effect.provide(
+          Layer.mergeAll(
+            Observability,
+            Logger.layer([Logger.consoleStructured]),
+            Layer.succeed(References.MinimumLogLevel, 'Debug'),
+          ),
+        ),
         Effect.scoped,
         Effect.runPromise,
       )

package/src/cf-worker/do/layer.ts

@@ -2,16 +2,17 @@ import { UnknownError } from '@livestore/common'
 import type { CfTypes } from '@livestore/common-cf'
 import { EventSequenceNumber, State } from '@livestore/common/schema'
 import { shouldNeverHappen } from '@livestore/utils'
-import { Effect, Predicate } from '@livestore/utils/effect'
+import { Context, Effect, Layer, Predicate, Semaphore } from '@livestore/utils/effect'
 import { nanoid } from '@livestore/utils/nanoid'
 
-import type { Env, MakeDurableObjectClassOptions, RpcSubscription } from '../shared.ts'
+import type { Env, MakeDurableObjectClassOptions } from '../shared.ts'
 import { contextTable, eventlogTable } from './sqlite.ts'
-import { makeStorage } from './sync-storage.ts'
+import { makeStorage, type StorageEngine, type SyncStorage } from './sync-storage.ts'
 
 const CacheSymbol = Symbol('Cache')
+const InitializationSemaphoreSymbol = Symbol('InitializationSemaphore')
 
-export interface DoCtxInput {
+export interface Options {
   doSelf: CfTypes.DurableObject & {
     ctx: CfTypes.DurableObjectState
     env: Env
@@ -20,110 +21,152 @@ export interface DoCtxInput {
   from: CfTypes.Request | { storeId: string }
 }
 
-export class DoCtx extends Effect.Service<DoCtx>()('DoCtx', {
-  effect: Effect.fn(
-    function* ({ doSelf, doOptions, from }: DoCtxInput) {
-      if ((doSelf as any)[CacheSymbol] !== undefined) {
-        return (doSelf as any)[CacheSymbol] as never
-      }
-
-      const getStoreId = (from: CfTypes.Request | { storeId: string }) => {
-        if (Predicate.hasProperty(from, 'url') === true) {
-          const url = new URL(from.url)
-          return (
-            url.searchParams.get('storeId') ?? shouldNeverHappen(`No storeId provided in request URL search params`)
-          )
+export type DoCtxInput = Options
+
+export interface Service {
+  readonly storeId: string
+  readonly backendId: string
+  readonly currentHeadRef: { current: EventSequenceNumber.Global.Type }
+  readonly updateCurrentHead: (currentHead: EventSequenceNumber.Global.Type) => void
+  /** Serializes push admission across concurrent RPC fibers for this Durable Object instance. */
+  readonly pushSemaphore: Semaphore.Semaphore
+  readonly storage: SyncStorage
+  readonly doOptions: MakeDurableObjectClassOptions | undefined
+  readonly env: Env
+  readonly ctx: CfTypes.DurableObjectState
+}
+
+export class DoCtx extends Context.Service<DoCtx, Service>()('@livestore/sync-cf/DoCtx') {}
+
+type DurableObjectCacheHost = Options['doSelf'] & {
+  [CacheSymbol]?: Service
+  [InitializationSemaphoreSymbol]?: Semaphore.Semaphore
+}
+
+export const make = Effect.fn(
+  function* ({ doSelf, doOptions, from }: Options) {
+    const cacheHost = doSelf as DurableObjectCacheHost
+    if (cacheHost[CacheSymbol] !== undefined) {
+      return DoCtx.of(cacheHost[CacheSymbol])
+    }
+
+    // Request layers can be constructed concurrently during the first RPCs to a DO.
+    // Install this lock synchronously so every contender shares the same initializer.
+    const initializationSemaphore = cacheHost[InitializationSemaphoreSymbol] ?? Semaphore.makeUnsafe(1)
+    cacheHost[InitializationSemaphoreSymbol] = initializationSemaphore
+
+    return yield* initializationSemaphore.withPermits(1)(
+      Effect.gen(function* () {
+        if (cacheHost[CacheSymbol] !== undefined) {
+          return DoCtx.of(cacheHost[CacheSymbol])
         }
-        return from.storeId
-      }
-
-      const storeId = getStoreId(from)
-      // Resolve storage engine
-      const makeEngine = Effect.gen(function* () {
-        const opt = doOptions?.storage
-        if (opt?._tag === 'd1') {
-          const db = (doSelf.env as any)[opt.binding]
-          if (db == null) {
-            return yield* UnknownError.make({ cause: new Error(`D1 binding '${opt.binding}' not found on env`) })
+
+        const getStoreId = (from: CfTypes.Request | { storeId: string }) => {
+          if (Predicate.hasProperty(from, 'url') === true) {
+            const url = new URL(from.url)
+            return (
+              url.searchParams.get('storeId') ?? shouldNeverHappen(`No storeId provided in request URL search params`)
+            )
           }
-          return { _tag: 'd1' as const, db }
-        } else if (opt?._tag === 'do-sqlite' || opt === undefined) {
-          return { _tag: 'do-sqlite' as const }
-        } else return shouldNeverHappen(`Invalid storage engine`, opt)
-      })
+          return from.storeId
+        }
 
-      const engine = yield* makeEngine
+        const storeId = getStoreId(from)
+        const engine = yield* resolveStorageEngine(doOptions?.storage, doSelf.env)
+
+        const storage = makeStorage(doSelf.ctx, storeId, engine)
+
+        // Initialize database tables
+        {
+          const colSpec = State.SQLite.makeColumnSpec(eventlogTable.sqliteDef.ast)
+          if (engine._tag === 'd1') {
+            // D1 database is async, so we need to use a promise
+            yield* Effect.promise(() =>
+              engine.db.exec(`CREATE TABLE IF NOT EXISTS "${storage.dbName}" (${colSpec}) strict`),
+            )
+          } else {
+            // DO SQLite table lives in Durable Object storage
+            doSelf.ctx.storage.sql.exec(`CREATE TABLE IF NOT EXISTS "${storage.dbName}" (${colSpec}) strict`)
+          }
+        }
+        {
+          const colSpec = State.SQLite.makeColumnSpec(contextTable.sqliteDef.ast)
+          doSelf.ctx.storage.sql.exec(`CREATE TABLE IF NOT EXISTS "${contextTable.sqliteDef.name}" (${colSpec}) strict`)
+        }
 
-      const storage = makeStorage(doSelf.ctx, storeId, engine)
+        const storageRow = doSelf.ctx.storage.sql
+          .exec(`SELECT * FROM "${contextTable.sqliteDef.name}" WHERE storeId = ?`, storeId)
+          .toArray()[0] as typeof contextTable.rowSchema.Type | undefined
 
-      // Initialize database tables
-      {
-        const colSpec = State.SQLite.makeColumnSpec(eventlogTable.sqliteDef.ast)
-        if (engine._tag === 'd1') {
-          // D1 database is async, so we need to use a promise
-          yield* Effect.promise(() =>
-            engine.db.exec(`CREATE TABLE IF NOT EXISTS "${storage.dbName}" (${colSpec}) strict`),
-          )
-        } else {
-          // DO SQLite table lives in Durable Object storage
-          doSelf.ctx.storage.sql.exec(`CREATE TABLE IF NOT EXISTS "${storage.dbName}" (${colSpec}) strict`)
-        }
-      }
-      {
-        const colSpec = State.SQLite.makeColumnSpec(contextTable.sqliteDef.ast)
-        doSelf.ctx.storage.sql.exec(`CREATE TABLE IF NOT EXISTS "${contextTable.sqliteDef.name}" (${colSpec}) strict`)
-      }
+        const currentHeadRef = { current: storageRow?.currentHead ?? EventSequenceNumber.Client.ROOT.global }
+        const pushSemaphore = yield* Semaphore.make(1)
 
-      const storageRow = doSelf.ctx.storage.sql
-        .exec(`SELECT * FROM "${contextTable.sqliteDef.name}" WHERE storeId = ?`, storeId)
-        .toArray()[0] as typeof contextTable.rowSchema.Type | undefined
+        // TODO: Reconcile this persisted head with the event log's highest sequence number.
+        // D1 event batches and this DO-local context row cannot be updated atomically, so a
+        // failure between those writes can leave push admission using a stale head after restart.
 
-      const currentHeadRef = { current: storageRow?.currentHead ?? EventSequenceNumber.Client.ROOT.global }
+        // Should be the same backendId for lifetime of the Durable Object
+        const backendId = storageRow?.backendId ?? nanoid()
 
-      // TODO do concistency check with eventlog table to make sure the head is consistent
+        const updateCurrentHead = (currentHead: EventSequenceNumber.Global.Type) => {
+          doSelf.ctx.storage.sql.exec(
+            `INSERT OR REPLACE INTO "${contextTable.sqliteDef.name}" (storeId, currentHead, backendId) VALUES (?, ?, ?)`,
+            storeId,
+            currentHead,
+            backendId,
+          )
 
-      // Should be the same backendId for lifetime of the Durable Object
-      const backendId = storageRow?.backendId ?? nanoid()
+          currentHeadRef.current = currentHead
+        }
 
-      const updateCurrentHead = (currentHead: EventSequenceNumber.Global.Type) => {
-        doSelf.ctx.storage.sql.exec(
-          `INSERT OR REPLACE INTO "${contextTable.sqliteDef.name}" (storeId, currentHead, backendId) VALUES (?, ?, ?)`,
+        const storageCache = {
           storeId,
-          currentHead,
           backendId,
-        )
-
-        currentHeadRef.current = currentHead
-
-        // I still don't know why we need to re-assign this ref to the `doSelf` object but somehow this seems to be needed 😵‍💫
-        // @ts-expect-error
-        doSelf[CacheSymbol].currentHeadRef = { current: currentHead }
-      }
-
-      const rpcSubscriptions = new Map<string, RpcSubscription>()
-
-      const storageCache = {
-        storeId,
-        backendId,
-        currentHeadRef,
-        updateCurrentHead,
-        storage,
-        doOptions,
-        env: doSelf.env,
-        ctx: doSelf.ctx,
-        rpcSubscriptions,
-      }
-
-      ;(doSelf as any)[CacheSymbol] = storageCache
-
-      // Set initial current head to root
-      if (storageRow === undefined) {
-        updateCurrentHead(EventSequenceNumber.Client.ROOT.global)
-      }
-
-      return storageCache
-    },
-    UnknownError.mapToUnknownError,
-    Effect.withSpan('@livestore/sync-cf:durable-object:makeDoCtx'),
-  ),
-}) {}
+          currentHeadRef,
+          updateCurrentHead,
+          pushSemaphore,
+          storage,
+          doOptions,
+          env: doSelf.env,
+          ctx: doSelf.ctx,
+        }
+
+        cacheHost[CacheSymbol] = storageCache
+
+        // Set initial current head to root
+        if (storageRow === undefined) {
+          updateCurrentHead(EventSequenceNumber.Client.ROOT.global)
+        }
+
+        return DoCtx.of(storageCache)
+      }),
+    )
+  },
+  UnknownError.mapToUnknownError,
+  Effect.withSpan('@livestore/sync-cf:durable-object:makeDoCtx'),
+)
+
+export const layer = (options: Options) => Layer.effect(DoCtx, make(options))
+
+/** Resolves storage configuration into the concrete database handle used at runtime. */
+const resolveStorageEngine = Effect.fn(function* (
+  storageOption: MakeDurableObjectClassOptions['storage'],
+  env: Env,
+): Effect.fn.Return<StorageEngine, UnknownError> {
+  if (storageOption === undefined || storageOption._tag === 'do-sqlite') {
+    return { _tag: 'do-sqlite' }
+  }
+
+  if (storageOption._tag === 'd1') {
+    const db = (env as Record<string, CfTypes.D1Database | undefined>)[storageOption.binding]
+    if (db === undefined) {
+      return yield* UnknownError.make({
+        cause: new Error(`D1 binding '${storageOption.binding}' not found on env`),
+      })
+    }
+
+    return { _tag: 'd1', db }
+  }
+
+  return shouldNeverHappen(`Invalid storage engine`, storageOption)
+})

package/src/cf-worker/do/pull.ts

@@ -1,11 +1,11 @@
 import { BackendIdMismatchError, SyncBackend, UnknownError } from '@livestore/common'
-import { splitChunkBySize } from '@livestore/common/sync'
-import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
+import { splitArrayBySize } from '@livestore/common/sync'
+import { Effect, Option, Schema, Stream } from '@livestore/utils/effect'
 
 import { MAX_PULL_EVENTS_PER_MESSAGE, MAX_WS_MESSAGE_BYTES } from '../../common/constants.ts'
 import { SyncMessage } from '../../common/mod.ts'
 import type { ForwardedHeaders } from '../shared.ts'
-import { DoCtx } from './layer.ts'
+import * as DoCtx from './layer.ts'
 
 const encodePullResponse = Schema.encodeSync(SyncMessage.PullResponse)
 
@@ -24,22 +24,20 @@ export const makeEndingPullStream = ({
   headers,
 }: {
   req: SyncMessage.PullRequest
-  payload: Schema.JsonValue | undefined
+  payload: Schema.Json | undefined
   headers: ForwardedHeaders | undefined
-}): Stream.Stream<SyncMessage.PullResponse, UnknownError | BackendIdMismatchError, DoCtx> =>
+}): Stream.Stream<SyncMessage.PullResponse, UnknownError | BackendIdMismatchError, DoCtx.DoCtx> =>
   Effect.gen(function* () {
-    const { doOptions, backendId, storeId, storage } = yield* DoCtx
+    const { doOptions, backendId, storeId, storage } = yield* DoCtx.DoCtx
 
     if (doOptions?.onPull !== undefined) {
-      yield* Effect.tryAll(() =>
+      yield* Effect.trySyncOrPromiseOrEffect(() =>
         doOptions.onPull!(req, {
           storeId,
           ...(payload !== undefined ? { payload } : {}),
           ...(headers !== undefined ? { headers } : {}),
         }),
-      ).pipe(
-        UnknownError.mapToUnknownError,
-      )
+      ).pipe(UnknownError.mapToUnknownError)
     }
 
     if (req.cursor._tag === 'Some' && req.cursor.value.backendId !== backendId) {
@@ -51,8 +49,8 @@ export const makeEndingPullStream = ({
     )
 
     return storedEvents.pipe(
-      Stream.mapChunksEffect(
-        splitChunkBySize({
+      Stream.mapArrayEffect(
+        splitArrayBySize({
           maxItems: MAX_PULL_EVENTS_PER_MESSAGE,
           maxBytes: MAX_WS_MESSAGE_BYTES,
           encode: (batch) =>
@@ -61,23 +59,27 @@ export const makeEndingPullStream = ({
             ),
         }),
       ),
-      Stream.mapAccum(total, (remaining, chunk) => {
-        const asArray = Chunk.toReadonlyArray(chunk)
-        const nextRemaining = Math.max(0, remaining - asArray.length)
+      Stream.mapAccum(
+        () => total,
+        (remaining, array) => {
+          const nextRemaining = Math.max(0, remaining - array.length)
 
-        return [
-          nextRemaining,
-          SyncMessage.PullResponse.make({
-            batch: asArray,
-            pageInfo: nextRemaining > 0 ? SyncBackend.pageInfoMoreKnown(nextRemaining) : SyncBackend.pageInfoNoMore,
-            backendId,
-          }),
-        ] as const
-      }),
+          return [
+            nextRemaining,
+            [
+              SyncMessage.PullResponse.make({
+                batch: array,
+                pageInfo: nextRemaining > 0 ? SyncBackend.pageInfoMoreKnown(nextRemaining) : SyncBackend.pageInfoNoMore,
+                backendId,
+              }),
+            ],
+          ] as const
+        },
+      ),
       Stream.tap(
         Effect.fn(function* (res) {
           if (doOptions?.onPullRes !== undefined) {
-            yield* Effect.tryAll(() => doOptions.onPullRes!(res)).pipe(UnknownError.mapToUnknownError)
+            yield* Effect.trySyncOrPromiseOrEffect(() => doOptions.onPullRes!(res)).pipe(UnknownError.mapToUnknownError)
           }
         }),
       ),
@@ -86,9 +88,7 @@ export const makeEndingPullStream = ({
   }).pipe(
     Stream.unwrap,
     Stream.mapError((cause) =>
-      cause._tag === 'BackendIdMismatchError' || cause._tag === 'UnknownError'
-        ? cause
-        : new UnknownError({ cause }),
+      cause._tag === 'BackendIdMismatchError' || cause._tag === 'UnknownError' ? cause : new UnknownError({ cause }),
     ),
     Stream.withSpan('cloudflare-provider:pull'),
   )