@livestore/sync-cf 0.5.0-dev.0 → 0.5.0

package/src/cf-worker/shared.ts

@@ -1,6 +1,6 @@
 import type { UnknownError } from '@livestore/common'
 import type { CfTypes } from '@livestore/common-cf'
-import type { Effect } from '@livestore/utils/effect'
+import type { Effect, Layer } from '@livestore/utils/effect'
 import { Result, Schema } from '@livestore/utils/effect'
 
 import type { SearchParams } from '../common/mod.ts'
@@ -107,12 +107,19 @@ export type MakeDurableObjectClassOptions = {
     responseHeaders?: Record<string, string>
   }
 
-  otel?: {
-    baseUrl?: string
-    serviceName?: string
-  }
+  /** Optional telemetry. Omit to avoid creating an exporter or sending telemetry. */
+  otel?: SyncBackendOtelOptions
 }
 
+export type SyncBackendOtelOptions =
+  /** Application-supplied tracer layer; LiveStore builds and finalizes its resources per sync operation. */
+  | Layer.Layer<never>
+  | {
+      /** Convenience OTLP/HTTP JSON exporter owned by LiveStore for each operation. */
+      baseUrl?: string
+      serviceName?: string
+    }
+
 export type StoreId = string
 export type DurableObjectId = string
 
@@ -146,7 +153,7 @@ const SearchParamsFromUrlSearchParams = Schema.fromURLSearchParams(SearchParamsS
  */
 export const matchSyncRequest = (request: CfTypes.Request): SearchParams | undefined => {
   const url = new URL(request.url)
-  const paramsResult = Schema.decodeUnknownResult(SearchParamsFromUrlSearchParams)(url.searchParams)
+  const paramsResult = Schema.decodeResult(SearchParamsFromUrlSearchParams)(url.searchParams)
 
   if (Result.isFailure(paramsResult) === true) {
     return undefined

package/src/cf-worker/worker.ts

@@ -200,7 +200,7 @@ export const handleSyncRequest = <
       // Always decode with the supplied schema when present, even if payload is undefined.
       // This ensures required payloads are enforced by the schema.
       if (syncPayloadSchema !== undefined) {
-        const decodedResult = Schema.decodeUnknownResult(syncPayloadSchema)(payload)
+        const decodedResult = Schema.decodeResult(syncPayloadSchema)(payload)
         if (Result.isFailure(decodedResult) === true) {
           const message = decodedResult.failure.toString()
           console.error('Invalid payload (decode failed)', message)

package/src/client/transport/do-rpc-client.test.ts

@@ -0,0 +1,411 @@
+import { BackendIdMismatchError, IsOfflineError, ServerAheadError, SyncBackend } from '@livestore/common'
+import { type CfTypes, toDurableObjectHandler } from '@livestore/common-cf'
+import { EventSequenceNumber } from '@livestore/common/schema'
+import { Vitest } from '@livestore/utils-dev/node-vitest'
+import {
+  Cause,
+  Clock,
+  Deferred,
+  Duration,
+  Effect,
+  Exit,
+  FetchHttpClient,
+  Fiber,
+  KeyValueStore,
+  Layer,
+  Option,
+  Stream,
+  TestClock,
+} from '@livestore/utils/effect'
+
+import { SyncDoRpc } from '../../common/do-rpc-schema.ts'
+import { makeDoRpcSync, type SyncBackendRpcStub } from './do-rpc-client.ts'
+
+Vitest.describe('DO-RPC sync client transport failures', () => {
+  Vitest.effect('a retryable rejection fails the pull as offline after a backoff', () =>
+    Effect.gen(function* () {
+      const { backend } = yield* makeBackend([rejectWith(cloudflareError({ retryable: true }))])
+
+      const { exit, delays } = yield* runRecordingBackoff(backend.pull(Option.none()).pipe(Stream.runCollect))
+
+      Vitest.expect(failureOf(exit)).toBeInstanceOf(IsOfflineError)
+      expectBackoff(delays[0]!, 1_000)
+    }),
+  )
+
+  Vitest.effect('a retryable push rejection fails as offline', () =>
+    Effect.gen(function* () {
+      const { backend } = yield* makeBackend([rejectWith(cloudflareError({ retryable: true }))])
+
+      const { exit } = yield* runRecordingBackoff(backend.push([event]))
+
+      Vitest.expect(failureOf(exit)).toBeInstanceOf(IsOfflineError)
+    }),
+  )
+
+  Vitest.effect('a retryable ping rejection fails as offline', () =>
+    Effect.gen(function* () {
+      const { backend } = yield* makeBackend([rejectWith(cloudflareError({ retryable: true }))])
+
+      const { exit } = yield* runRecordingBackoff(backend.ping)
+
+      Vitest.expect(failureOf(exit)).toBeInstanceOf(IsOfflineError)
+    }),
+  )
+
+  Vitest.describe('keeps terminal failures terminal', () => {
+    // Overload wins over `retryable`, and `remote` alone does not mean retryable (infrastructure errors can carry it).
+    const terminal = [
+      { name: 'overloaded and retryable', flags: { overloaded: true, retryable: true } },
+      { name: 'remote only', flags: { remote: true } },
+      { name: 'unflagged', flags: {} },
+    ]
+
+    for (const { name, flags } of terminal) {
+      Vitest.effect(`${name}: pull dies with the original error without waiting`, () =>
+        Effect.gen(function* () {
+          const rejected = cloudflareError(flags)
+          const { backend } = yield* makeBackend([rejectWith(rejected)])
+
+          const { exit, delays } = yield* runRecordingBackoff(backend.pull(Option.none()).pipe(Stream.runCollect))
+
+          Vitest.expect(defectOf(exit)).toBe(rejected)
+          Vitest.expect(delays).toEqual([])
+        }),
+      )
+
+      Vitest.effect(`${name}: push dies with the original error`, () =>
+        Effect.gen(function* () {
+          const rejected = cloudflareError(flags)
+          const { backend } = yield* makeBackend([rejectWith(rejected)])
+
+          const { exit } = yield* runRecordingBackoff(backend.push([event]))
+
+          Vitest.expect(defectOf(exit)).toBe(rejected)
+        }),
+      )
+    }
+  })
+
+  Vitest.describe('passes application errors through unchanged', () => {
+    Vitest.effect('pull: BackendIdMismatchError', () =>
+      Effect.gen(function* () {
+        const { backend } = yield* makeBackend([
+          serve({ pull: () => Stream.fail(new BackendIdMismatchError({ expected: 'a', received: 'b' })) }),
+        ])
+
+        const { exit } = yield* runRecordingBackoff(backend.pull(Option.none()).pipe(Stream.runCollect))
+
+        Vitest.expect(failureOf(exit)).toBeInstanceOf(BackendIdMismatchError)
+      }),
+    )
+
+    Vitest.effect('push: ServerAheadError', () =>
+      Effect.gen(function* () {
+        const { backend } = yield* makeBackend([
+          serve({
+            push: () =>
+              Effect.fail(
+                new ServerAheadError({
+                  minimumExpectedNum: EventSequenceNumber.Global.make(2),
+                  providedNum: EventSequenceNumber.Global.make(1),
+                }),
+              ),
+          }),
+        ])
+
+        const { exit } = yield* runRecordingBackoff(backend.push([event]))
+
+        Vitest.expect(failureOf(exit)).toBeInstanceOf(ServerAheadError)
+      }),
+    )
+  })
+
+  Vitest.describe('backoff', () => {
+    Vitest.effect('grows to the cap across pulls that each deliver a page and then fail', () =>
+      Effect.gen(function* () {
+        const failure = pageThenRejectWith(cloudflareError({ retryable: true }))
+        const expectedDelays = [1_000, 2_000, 4_000, 8_000, 16_000, 30_000, 30_000]
+        const { backend } = yield* makeBackend(expectedDelays.map(() => failure))
+
+        for (const expectedDelay of expectedDelays) {
+          const { exit, delays } = yield* runRecordingBackoff(backend.pull(Option.none()).pipe(Stream.runCollect))
+          Vitest.expect(failureOf(exit)).toBeInstanceOf(IsOfflineError)
+          Vitest.expect(delays).toHaveLength(1)
+          expectBackoff(delays[0]!, expectedDelay)
+        }
+      }),
+    )
+
+    Vitest.effect('interrupts a pull while it is waiting to retry', () =>
+      Effect.gen(function* () {
+        const { backend } = yield* makeBackend([rejectWith(cloudflareError({ retryable: true }))])
+        const clock = yield* Clock.Clock
+        const waiting = yield* Deferred.make<void>()
+        const fiber = yield* backend.pull(Option.none()).pipe(
+          Stream.runCollect,
+          Effect.provideService(Clock.Clock, {
+            ...clock,
+            sleep: (duration) => {
+              if (Duration.toMillis(duration) >= 60_000) return clock.sleep(duration)
+              return Deferred.succeed(waiting, undefined).pipe(Effect.andThen(clock.sleep(duration)))
+            },
+          }),
+          Effect.forkChild,
+        )
+
+        yield* Deferred.await(waiting)
+        Vitest.expect(fiber.pollUnsafe()).toBeUndefined()
+        yield* Fiber.interrupt(fiber)
+        Vitest.expect(Exit.hasInterrupts(yield* Fiber.await(fiber))).toBe(true)
+      }),
+    )
+
+    Vitest.effect('resets once a pull completes its catch-up', () =>
+      Effect.gen(function* () {
+        const failure = rejectWith(cloudflareError({ retryable: true }))
+        const { backend } = yield* makeBackend([
+          failure,
+          failure,
+          serve({ pull: () => Stream.make(pullResponse(SyncBackend.pageInfoNoMore)) }),
+          failure,
+        ])
+        const pull = runRecordingBackoff(backend.pull(Option.none()).pipe(Stream.runCollect))
+
+        yield* pull
+        yield* pull
+        const completed = yield* pull
+        const afterReset = yield* pull
+
+        Vitest.expect(Exit.isSuccess(completed.exit)).toBe(true)
+        expectBackoff(afterReset.delays[0]!, 1_000)
+      }),
+    )
+  })
+
+  Vitest.effect('recovers on a fresh stub after a stub breaks', () =>
+    Effect.gen(function* () {
+      const broken = rejectWith(cloudflareError({ retryable: true }))
+      const healthy = serve({ pull: () => Stream.make(pullResponse(SyncBackend.pageInfoNoMore)) })
+      const { backend } = yield* makeBackend([broken, healthy])
+
+      const first = yield* runRecordingBackoff(backend.pull(Option.none()).pipe(Stream.runCollect))
+      const second = yield* runRecordingBackoff(backend.pull(Option.none()).pipe(Stream.runCollect))
+
+      Vitest.expect(failureOf(first.exit)).toBeInstanceOf(IsOfflineError)
+      Vitest.expect(Exit.isSuccess(second.exit)).toBe(true)
+    }),
+  )
+
+  Vitest.effect('a stalled catch-up releases its producer and recovers on a fresh stub', () =>
+    Effect.gen(function* () {
+      const pageSeen = yield* Deferred.make<void>()
+      const cancelled = yield* Deferred.make<void>()
+      const { backend } = yield* makeBackend([
+        serve({
+          pull: () =>
+            Stream.make(pullResponse(SyncBackend.pageInfoMoreUnknown)).pipe(
+              Stream.concat(Stream.never),
+              Stream.ensuring(Deferred.succeed(cancelled, undefined)),
+            ),
+        }),
+        serve({ pull: () => Stream.make(pullResponse(SyncBackend.pageInfoNoMore)) }),
+      ])
+      const fiber = yield* backend.pull(Option.none()).pipe(
+        Stream.tap(() => Deferred.succeed(pageSeen, undefined)),
+        Stream.runCollect,
+        Effect.exit,
+        Effect.forkChild,
+      )
+      yield* Deferred.await(pageSeen)
+      yield* TestClock.adjust('62 seconds')
+      Vitest.expect(failureOf(yield* Fiber.join(fiber))).toBeInstanceOf(IsOfflineError)
+      yield* Deferred.await(cancelled)
+      const recovered = yield* backend.pull(Option.none()).pipe(Stream.runCollect)
+      Vitest.expect(recovered).toHaveLength(1)
+    }),
+  )
+
+  Vitest.effect('a live pull can remain idle after catch-up without reconnecting', () =>
+    Effect.gen(function* () {
+      const caughtUp = yield* Deferred.make<void>()
+      const healthy = serve({ pull: () => Stream.make(pullResponse(SyncBackend.pageInfoNoMore)) })
+      const { backend } = yield* makeBackend([healthy, healthy]) // second call is unsubscribe on interruption
+      const fiber = yield* backend.pull(Option.none(), { live: true }).pipe(
+        Stream.tap(() => Deferred.succeed(caughtUp, undefined)),
+        Stream.runDrain,
+        Effect.forkChild,
+      )
+      yield* Deferred.await(caughtUp)
+      yield* TestClock.adjust('2 minutes')
+      Vitest.expect(fiber.pollUnsafe()).toBeUndefined()
+      yield* Fiber.interrupt(fiber)
+    }),
+  )
+
+  Vitest.effect('the catch-up deadline restarts for each page', () =>
+    Effect.gen(function* () {
+      const sendFirst = yield* Deferred.make<void>()
+      const sendLast = yield* Deferred.make<void>()
+      const firstSeen = yield* Deferred.make<void>()
+      const { backend } = yield* makeBackend([
+        serve({
+          pull: () =>
+            Stream.fromEffect(Deferred.await(sendFirst)).pipe(
+              Stream.map(() => pullResponse(SyncBackend.pageInfoMoreUnknown)),
+              Stream.concat(
+                Stream.fromEffect(Deferred.await(sendLast)).pipe(
+                  Stream.map(() => pullResponse(SyncBackend.pageInfoNoMore)),
+                ),
+              ),
+            ),
+        }),
+      ])
+      const fiber = yield* backend.pull(Option.none()).pipe(
+        Stream.tap(() => Deferred.succeed(firstSeen, undefined)),
+        Stream.runCollect,
+        Effect.forkChild,
+      )
+      yield* TestClock.adjust('40 seconds')
+      yield* Deferred.succeed(sendFirst, undefined)
+      yield* Deferred.await(firstSeen)
+      yield* TestClock.adjust('40 seconds')
+      yield* Deferred.succeed(sendLast, undefined)
+      Vitest.expect(yield* Fiber.join(fiber)).toHaveLength(2)
+    }),
+  )
+
+  Vitest.effect('overload on a stream read remains terminal', () =>
+    Effect.gen(function* () {
+      const error = cloudflareError({ overloaded: true, retryable: true })
+      const { backend } = yield* makeBackend([pageThenRejectWith(error)])
+      const { exit, delays } = yield* runRecordingBackoff(backend.pull(Option.none()).pipe(Stream.runCollect))
+      Vitest.expect(defectOf(exit)).toBe(error)
+      Vitest.expect(delays).toEqual([])
+    }),
+  )
+})
+
+type CallRpc = (payload: Uint8Array) => Promise<Uint8Array | CfTypes.ReadableStream>
+
+const backendId = 'backend'
+
+const event = {
+  name: 'todoCreated',
+  args: {},
+  seqNum: EventSequenceNumber.Global.make(1),
+  parentSeqNum: EventSequenceNumber.Global.make(0),
+  clientId: 'client',
+  sessionId: 'session',
+}
+
+const pullResponse = (pageInfo: SyncBackend.PullResPageInfo) => ({
+  rpcRequestId: '0',
+  batch: [{ eventEncoded: event, metadata: Option.none() }],
+  pageInfo,
+  backendId,
+})
+
+/** A Cloudflare DO error carries runtime flags as own properties on the thrown `Error`. */
+const cloudflareError = (flags: { retryable?: boolean; overloaded?: boolean; remote?: boolean }) =>
+  Object.assign(new Error('Network connection lost.'), flags)
+
+const rejectWith =
+  (error: Error): CallRpc =>
+  () =>
+    Promise.reject(error)
+
+/** Serves the real handler, so each call carries encoded responses exactly as a backend DO sends them. */
+const serve =
+  (handlers: {
+    pull?: () => Stream.Stream<ReturnType<typeof pullResponse>, BackendIdMismatchError>
+    push?: () => Effect.Effect<{}, ServerAheadError>
+  }): CallRpc =>
+  (payload) =>
+    toDurableObjectHandler(SyncDoRpc, {
+      layer: SyncDoRpc.toLayer({
+        'SyncDoRpc.Pull': () => handlers.pull?.() ?? Stream.empty,
+        'SyncDoRpc.Push': () => handlers.push?.() ?? Effect.succeed({}),
+        'SyncDoRpc.Ping': () => Effect.void,
+        'SyncDoRpc.Unsubscribe': () => Effect.void,
+      }),
+    })(new Uint8Array(payload)).pipe(Effect.runPromise)
+
+/** Delivers one page, then rejects the next stream read with `error`. */
+const pageThenRejectWith =
+  (error: Error): CallRpc =>
+  async (payload) => {
+    const response = await serve({
+      pull: () => Stream.make(pullResponse(SyncBackend.pageInfoMoreUnknown)),
+    })(payload)
+    if (response instanceof Uint8Array) return response
+    const source = response.getReader()
+    // Keep only the page frame; replace the normal RPC exit with a transport failure.
+    const { value } = await source.read()
+    await source.cancel()
+    source.releaseLock()
+    let sentPage = false
+    const stream = new ReadableStream<Uint8Array>({
+      pull(controller) {
+        if (sentPage === true) return controller.error(error)
+        sentPage = true
+        controller.enqueue(value)
+      },
+    })
+    // oxlint-disable-next-line typescript-eslint(no-unsafe-type-assertion) -- bridge platform ReadableStream to the CF type
+    return stream as unknown as CfTypes.ReadableStream
+  }
+
+/** Each supplied RPC implementation belongs to one stub, and stays attached if that stub is reused. */
+const makeBackend = (stubs: ReadonlyArray<CallRpc>) =>
+  Effect.gen(function* () {
+    let nextStub = 0
+    const backend = yield* makeDoRpcSync({
+      getSyncBackendStub: () => {
+        const rpc = stubs[nextStub++]
+        if (rpc === undefined) throw new Error('Unexpected extra stub')
+        // oxlint-disable-next-line typescript-eslint(no-unsafe-type-assertion) -- a test double only needs `rpc`
+        return { rpc } as unknown as SyncBackendRpcStub
+      },
+      // oxlint-disable-next-line typescript-eslint(no-unsafe-type-assertion) -- only live pulls read the DO state
+      durableObjectState: {} as CfTypes.DurableObjectState,
+      durableObjectContext: { bindingName: 'CLIENT_DO', durableObjectId: 'client-do' },
+    })({ storeId: 'store', clientId: 'client', payload: undefined }).pipe(
+      Effect.provide(Layer.mergeAll(FetchHttpClient.layer, KeyValueStore.layerMemory)),
+    )
+
+    return { backend }
+  })
+
+/** Observe the waits requested by production code without polling fibers or waiting on wall-clock timers. */
+const runRecordingBackoff = <A, E>(effect: Effect.Effect<A, E>) =>
+  Effect.gen(function* () {
+    const clock = yield* Clock.Clock
+    const delays: number[] = []
+    const exit = yield* effect.pipe(
+      Effect.provideService(Clock.Clock, {
+        ...clock,
+        sleep: (duration) => {
+          if (Duration.toMillis(duration) >= 60_000) return clock.sleep(duration)
+          return Effect.sync(() => {
+            delays.push(Duration.toMillis(duration))
+          })
+        },
+      }),
+      Effect.exit,
+    )
+    return { exit, delays }
+  })
+
+const expectBackoff = (delay: number, baseMs: number) => {
+  Vitest.expect(delay).toBeGreaterThanOrEqual(baseMs * 0.8)
+  Vitest.expect(delay).toBeLessThanOrEqual(baseMs * 1.2)
+}
+
+const failureOf = (exit: Exit.Exit<unknown, unknown>): unknown =>
+  Exit.isFailure(exit) === true ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined
+
+const defectOf = (exit: Exit.Exit<unknown, unknown>): unknown =>
+  Exit.isFailure(exit) === true && Option.isNone(Cause.findErrorOption(exit.cause)) === true
+    ? Cause.squash(exit.cause)
+    : undefined

package/src/client/transport/do-rpc-client.ts

@@ -1,14 +1,19 @@
-import { SyncBackend, UnknownError } from '@livestore/common'
-import { type CfTypes, layerProtocolDurableObject } from '@livestore/common-cf'
+import { IsOfflineError, SyncBackend, UnknownError } from '@livestore/common'
+import { type CfTypes, DoRpcReadError, layerProtocolDurableObject } from '@livestore/common-cf'
 import { splitArrayBySize } from '@livestore/common/sync'
 import { shouldNeverHappen } from '@livestore/utils'
 import {
+  Duration,
   Effect,
   identity,
   Layer,
+  Match,
   Option,
+  Predicate,
   Queue,
+  Random,
   ReadonlyArray as EffectArray,
+  Ref,
   RpcClient,
   RpcSerialization,
   Schema,
@@ -28,8 +33,11 @@ type PushBatchItem = SyncMessage.PushRequest['batch'][number]
 export interface SyncBackendRpcStub extends CfTypes.DurableObjectStub, SyncBackendRpcInterface {}
 
 export interface DoRpcSyncOptions {
-  /** Durable Object stub that implements the SyncDoRpc interface */
-  syncBackendStub: SyncBackendRpcStub
+  /**
+   * Returns a stub for the sync backend Durable Object. Called for every RPC call, because Cloudflare leaves a stub
+   * broken after many exceptions, so a retry needs a fresh one.
+   */
+  getSyncBackendStub: () => SyncBackendRpcStub
   /**
    * State handle of the client DurableObject running this sync backend. Scopes live-pull routing to
    * this instance so it resets when the DO is reconstructed (see {@link handleSyncUpdateRpc}).
@@ -51,7 +59,7 @@ export interface DoRpcSyncOptions {
  */
 export const makeDoRpcSync =
   ({
-    syncBackendStub,
+    getSyncBackendStub,
     durableObjectState,
     durableObjectContext,
   }: DoRpcSyncOptions): SyncBackend.SyncBackendConstructor<SyncMetadata> =>
@@ -60,14 +68,16 @@ export const makeDoRpcSync =
       const isConnected = yield* SubscriptionRef.make(true)
 
       const ProtocolLive = layerProtocolDurableObject({
-        callRpc: (payload) => syncBackendStub.rpc(payload),
+        callRpc: (payload) => getSyncBackendStub().rpc(payload),
         callerContext: durableObjectContext,
-      }).pipe(Layer.provide(RpcSerialization.layerJson))
+      })
 
       const context = yield* Layer.build(ProtocolLive)
 
       const rpcClient = yield* RpcClient.make(SyncDoRpc).pipe(Effect.provide(context))
 
+      const pullBackoff = yield* makePullBackoff
+
       // Nothing to do here
       const connect = Effect.void
 
@@ -84,6 +94,12 @@ export const makeDoRpcSync =
           storeId,
           rpcContext: options?.live === true ? { callerContext: durableObjectContext } : undefined,
         }).pipe(
+          // Catch-up must make progress. Live callbacks below may legitimately remain idle indefinitely.
+          Stream.timeoutOrElse({
+            duration: '60 seconds',
+            orElse: () =>
+              Stream.fail(new IsOfflineError({ cause: new Error('DO-RPC pull made no progress for 60 seconds') })),
+          }),
           options?.live === true
             ? Stream.concatWithLastElement((res) =>
                 Effect.gen(function* () {
@@ -100,14 +116,28 @@ export const makeDoRpcSync =
 
                   routing.set(requestId, queue)
 
+                  // Graceful shutdown only (eviction runs no finalizers); matched on requestId so a late send can't drop a newer pull's row
+                  yield* Effect.addFinalizer(() =>
+                    rpcClient['SyncDoRpc.Unsubscribe']({
+                      storeId,
+                      durableObjectId: durableObjectContext.durableObjectId,
+                      requestId,
+                    }).pipe(Effect.timeout('5 seconds'), Effect.tapCauseLogPretty, Effect.ignore),
+                  )
+
                   return Stream.fromQueue(queue)
                 }).pipe(Stream.unwrap),
               )
             : identity,
+          Stream.catchReason('RpcClientError', 'RpcClientDefect', (reason) =>
+            Stream.fromEffect(classifyTransportFailure(reason.cause)),
+          ),
+          Stream.tapError((error) => (error._tag === 'IsOfflineError' ? pullBackoff.wait : Effect.void)),
+          Stream.tap((res) => (res.pageInfo._tag === 'NoMore' ? pullBackoff.reset : Effect.void)),
           Stream.tap((res) => backendIdHelper.lazySet(res.backendId)),
           Stream.map((res) => Struct.omit(res, ['backendId'])),
           Stream.mapError((cause) =>
-            cause._tag === 'UnknownError' || cause._tag === 'BackendIdMismatchError'
+            cause._tag === 'UnknownError' || cause._tag === 'BackendIdMismatchError' || cause._tag === 'IsOfflineError'
               ? cause
               : new UnknownError({ cause }),
           ),
@@ -139,8 +169,12 @@ export const makeDoRpcSync =
             yield* rpcClient['SyncDoRpc.Push']({ batch: batchChunk, storeId, backendId })
           }
         },
+        Effect.catchReason('RpcClientError', 'RpcClientDefect', (reason) => classifyTransportFailure(reason.cause)),
         Effect.mapError((cause) =>
-          cause._tag === 'UnknownError' || cause._tag === 'ServerAheadError' || cause._tag === 'BackendIdMismatchError'
+          cause._tag === 'UnknownError' ||
+          cause._tag === 'ServerAheadError' ||
+          cause._tag === 'BackendIdMismatchError' ||
+          cause._tag === 'IsOfflineError'
             ? cause
             : new UnknownError({ cause }),
         ),
@@ -149,7 +183,11 @@ export const makeDoRpcSync =
       const ping: SyncBackend.SyncBackend<{ createdAt: string }>['ping'] = rpcClient['SyncDoRpc.Ping']({
         storeId,
         payload,
-      }).pipe(UnknownError.mapToUnknownError, Effect.withSpan('rpc-sync-client:ping'))
+      }).pipe(
+        Effect.catchReason('RpcClientError', 'RpcClientDefect', (reason) => classifyTransportFailure(reason.cause)),
+        Effect.mapError((cause) => (cause._tag === 'IsOfflineError' ? cause : new UnknownError({ cause }))),
+        Effect.withSpan('rpc-sync-client:ping'),
+      )
 
       return SyncBackend.of({
         connect,
@@ -191,12 +229,12 @@ export const makeDoRpcSync =
  */
 export const handleSyncUpdateRpc = (ctx: CfTypes.DurableObjectState, payload: Uint8Array<ArrayBuffer>) =>
   Effect.gen(function* () {
-    const parser = RpcSerialization.msgPack.makeUnsafe()
-    const decodedMessage = parser.decode(payload)
-    const [response] = Array.isArray(decodedMessage) === true ? decodedMessage.flat(1) : [decodedMessage]
+    const serialization = yield* RpcSerialization.RpcSerialization
+    const parser = serialization.makeUnsafe()
+    const [response] = parser.decode(payload)
     const decodedPayload = yield* Schema.decodeUnknownEffect(ResponseChunkEncoded)(response)
-    const decoded = yield* Schema.decodeUnknownEffect(Schema.toCodecJson(SyncMessage.PullResponse))(
-      decodedPayload.values[0],
+    const decoded = yield* Schema.decodeEffect(serialization.codecFor(Schema.NonEmptyArray(SyncMessage.PullResponse)))(
+      decodedPayload.values,
     )
 
     const pullStreamQueue = pullRoutingFor(ctx).get(decodedPayload.requestId)
@@ -206,13 +244,18 @@ export const handleSyncUpdateRpc = (ctx: CfTypes.DurableObjectState, payload: Ui
       yield* Effect.log(`No pull stream queue found for ${decodedPayload.requestId}`)
     } else {
       // Case: DO was still alive, so the existing `pull` will pick up the new events
-      yield* Queue.offer(pullStreamQueue, decoded)
+      yield* Queue.offerAll(pullStreamQueue, decoded)
     }
-  }).pipe(Effect.withSpan('rpc-sync-client:rpcCallback'), Effect.tapCauseLogPretty, Effect.runPromise)
+  }).pipe(
+    Effect.provide(RpcSerialization.layerSchemaBinary()),
+    Effect.withSpan('rpc-sync-client:rpcCallback'),
+    Effect.tapCauseLogPretty,
+    Effect.runPromise,
+  )
 
 const ResponseChunkEncoded = Schema.Struct({
   requestId: Schema.String,
-  values: Schema.Array(Schema.Any),
+  values: Schema.Uint8Array,
 })
 
 type EffectRpcRequestId = string // 0, 1, 2, ...
@@ -235,3 +278,42 @@ const pullRoutingFor = (ctx: CfTypes.DurableObjectState): PullRouting => {
   pullRoutingByInstance.set(ctx, routing)
   return routing
 }
+
+/**
+ * Calls require Cloudflare's retryable flag. Transferred streams can lose those flags, so unflagged read
+ * failures are also offline (application failures arrive as encoded RPC exits). Explicit overload and
+ * non-retryable remote errors stay terminal.
+ * https://developers.cloudflare.com/durable-objects/best-practices/error-handling/
+ */
+const classifyTransportFailure = (failure: unknown) => {
+  const cause = failure instanceof DoRpcReadError ? failure.cause : failure
+  return Match.value({
+    retryable:
+      hasTrueFlag(cause, 'retryable') || (failure instanceof DoRpcReadError && hasTrueFlag(cause, 'remote') === false),
+    overloaded: hasTrueFlag(cause, 'overloaded'),
+  }).pipe(
+    Match.when({ overloaded: true }, () => Effect.die(cause)),
+    Match.when({ retryable: true }, () => Effect.fail(new IsOfflineError({ cause }))),
+    Match.orElse(() => Effect.die(cause)),
+  )
+}
+
+const hasTrueFlag = (cause: unknown, flag: 'retryable' | 'overloaded' | 'remote') =>
+  Predicate.hasProperty(cause, flag) === true && cause[flag] === true
+
+/**
+ * This transport delays retryable failures before surfacing them, independently of the leader's connectivity gate.
+ * The attempt count spans pull calls and resets only when a pull completes its catch-up, so a pull that delivers a
+ * page and then fails does not reset it.
+ */
+const makePullBackoff = Effect.gen(function* () {
+  const attempt = yield* Ref.make(0)
+  return {
+    wait: Effect.gen(function* () {
+      const n = yield* Ref.getAndUpdate(attempt, (n) => n + 1)
+      const jitter = yield* Random.nextBetween(0.8, 1.2)
+      yield* Effect.sleep(Duration.millis(Math.min(1_000 * 2 ** n, 30_000) * jitter))
+    }),
+    reset: Ref.set(attempt, 0),
+  }
+})