@livestore/common 0.4.0-dev.8 → 0.4.0

package/src/leader-thread/LeaderSyncProcessor.ts

@@ -1,4 +1,4 @@
-import { casesHandled, isNotUndefined, LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
+import { casesHandled, isNotUndefined, LS_DEV, TRACE_VERBOSE } from '@livestore/utils'
 import type { HttpClient, Runtime, Scope, Tracer } from '@livestore/utils/effect'
 import {
   BucketQueue,
@@ -10,42 +10,37 @@ import {
   FiberHandle,
   Layer,
   Option,
-  OtelTracer,
-  pipe,
   Queue,
   ReadonlyArray,
   Schedule,
+  Schema,
   Stream,
   Subscribable,
   SubscriptionRef,
 } from '@livestore/utils/effect'
-import type * as otel from '@opentelemetry/api'
-import {
-  type IntentionalShutdownCause,
-  type MaterializeError,
-  type SqliteDb,
-  UnexpectedError,
-} from '../adapter-types.ts'
+
+import { type MaterializeError, type SqliteDb, UnknownError } from '../adapter-types.ts'
+import { IntentionalShutdownCause } from '../errors.ts'
 import { makeMaterializerHash } from '../materializer-helper.ts'
 import type { LiveStoreSchema } from '../schema/mod.ts'
-import { EventSequenceNumber, getEventDef, LiveStoreEvent, SystemTables } from '../schema/mod.ts'
-import {
-  type InvalidPullError,
-  type InvalidPushError,
-  type IsOfflineError,
-  LeaderAheadError,
-  type SyncBackend,
-} from '../sync/sync.ts'
+import { EventSequenceNumber, LiveStoreEvent, resolveEventDef, SystemTables } from '../schema/mod.ts'
+import { EVENTLOG_META_TABLE, SYNC_STATUS_TABLE } from '../schema/state/sqlite/system-tables/eventlog-tables.ts'
+import type { BackendIdMismatchError, IsOfflineError, SyncBackend } from '../sync/sync.ts'
+import { isRejectedPushError, LeaderAheadError, NonMonotonicBatchError, StaleRebaseGenerationError } from './RejectedPushError.ts'
 import * as SyncState from '../sync/syncstate.ts'
 import { sql } from '../util.ts'
 import * as Eventlog from './eventlog.ts'
 import { rollback } from './materialize-event.ts'
+import type { ShutdownChannel } from './shutdown-channel.ts'
 import type { InitialBlockingSyncContext, LeaderSyncProcessor } from './types.ts'
 import { LeaderThreadCtx } from './types.ts'
 
+/** Serialize value to JSON string for trace attributes */
+const jsonStringify = Schema.encodeSync(Schema.parseJson())
+
 type LocalPushQueueItem = [
-  event: LiveStoreEvent.EncodedWithMeta,
-  deferred: Deferred.Deferred<void, LeaderAheadError> | undefined,
+  event: LiveStoreEvent.Client.EncodedWithMeta,
+  deferred: Deferred.Deferred<void, LeaderAheadError | StaleRebaseGenerationError> | undefined,
 ]
 
 /**
@@ -66,11 +61,11 @@ type LocalPushQueueItem = [
  *   - Maintains events in ascending order.
  *   - Uses `Deferred` objects to resolve/reject events based on application success.
  * - Processes events from the queue, applying events in batches.
- * - Controlled by a `Latch` to manage execution flow.
- * - The latch closes on pull receipt and re-opens post-pull completion.
+ * - Controlled by a mutex (`Semaphore(1)`) to ensure mutual exclusion between local push and backend pull processing.
+ * - The backend pull side acquires the mutex before processing and releases it on post-pull completion.
  * - Processes up to `maxBatchSize` events per cycle.
  *
- * Currently we're advancing the state db and eventlog in lockstep, but we could also decouple this in the future
+ * Currently, we're advancing the state db and eventlog in lockstep, but we could also decouple this in the future
  *
  * Tricky concurrency scenarios:
  * - Queued local push batches becoming invalid due to a prior local push item being rejected.
@@ -84,6 +79,7 @@ export const makeLeaderSyncProcessor = ({
   initialBlockingSyncContext,
   initialSyncState,
   onError,
+  onBackendIdMismatch,
   livePull,
   params,
   testing,
@@ -93,13 +89,60 @@ export const makeLeaderSyncProcessor = ({
   initialBlockingSyncContext: InitialBlockingSyncContext
   /** Initial sync state rehydrated from the persisted eventlog or initial sync state */
   initialSyncState: SyncState.SyncState
+  /**
+   * What to do when a failure (any cause) occurs (except `BackendIdMismatchError`).
+   *
+   * - `'shutdown'`: Send the error to the shutdown channel and terminate the sync processor.
+   * - `'ignore'`: Continue running.
+   */
   onError: 'shutdown' | 'ignore'
+  /**
+   * What to do when the sync backend identity has changed (i.e. the backend was reset).
+   *
+   * - `'reset'`: Clear local databases (eventlog and state) and send an intentional shutdown signal.
+   * - `'shutdown'`: Send a shutdown signal without clearing local storage.
+   * - `'ignore'`: Continue running with stale data.
+   */
+  onBackendIdMismatch: 'reset' | 'shutdown' | 'ignore'
   params: {
     /**
+     * Maximum number of local events to process per batch cycle.
+     *
+     * This controls how many events from client sessions are applied to the local state
+     * in a single iteration before yielding to allow potential backend pulls.
+     *
+     * **Trade-offs:**
+     * - **Lower values (1-5):** More responsive to remote updates since pull processing can
+     *   interleave more frequently. Better for high-conflict scenarios where rebases are common.
+     *   Slightly higher per-event overhead due to more frequent transaction commits.
+     *
+     * - **Higher values (10-50+):** Better throughput for bulk local writes as more events are
+     *   batched into a single transaction. However, may delay remote update processing and
+     *   increase rebase complexity if many local events queue up during a slow pull.
+     *
+     * - **Very high values (100+):** Risk of starvation for pull processing if local pushes
+     *   arrive continuously. May cause larger rollbacks during rebases. Not recommended
+     *   unless you have a write-heavy workload with minimal remote synchronization.
+     *
      * @default 10
      */
     localPushBatchSize?: number
     /**
+     * Maximum number of events to push to the sync backend per batch.
+     *
+     * This controls how many events are sent in a single push request to the remote server.
+     *
+     * **Trade-offs:**
+     * - **Lower values (1-10):** Lower latency for each push operation. Faster feedback on
+     *   push success/failure. Slightly higher network overhead due to more requests.
+     *
+     * - **Higher values (50-100):** Better network efficiency by amortizing request overhead.
+     *   Preferred for high-throughput scenarios. May increase latency to first confirmation.
+     *
+     * - **Very high values (200+):** Risk of hitting server request size limits or timeouts.
+     *   A single failed request loses the entire batch (will be retried). May cause memory
+     *   pressure if events accumulate faster than they can be pushed.
+     *
      * @default 50
      */
     backendPushBatchSize?: number
@@ -114,18 +157,16 @@ export const makeLeaderSyncProcessor = ({
       localPushProcessing?: Effect.Effect<void>
     }
   }
-}): Effect.Effect<LeaderSyncProcessor, UnexpectedError, Scope.Scope> =>
+}): Effect.Effect<LeaderSyncProcessor, never, Scope.Scope> =>
   Effect.gen(function* () {
-    const syncBackendPushQueue = yield* BucketQueue.make<LiveStoreEvent.EncodedWithMeta>()
-    const localPushBatchSize = params.localPushBatchSize ?? 1
-    const backendPushBatchSize = params.backendPushBatchSize ?? 2
+    const syncBackendPushQueue = yield* BucketQueue.make<LiveStoreEvent.Client.EncodedWithMeta>()
+    const localPushBatchSize = params.localPushBatchSize ?? 10
+    const backendPushBatchSize = params.backendPushBatchSize ?? 50
 
     const syncStateSref = yield* SubscriptionRef.make<SyncState.SyncState | undefined>(undefined)
 
-    const isClientEvent = (eventEncoded: LiveStoreEvent.EncodedWithMeta) => {
-      const { eventDef } = getEventDef(schema, eventEncoded.name)
-      return eventDef.options.clientOnly
-    }
+    const isClientEvent = (eventEncoded: LiveStoreEvent.Client.EncodedWithMeta) =>
+      schema.eventsDefsMap.get(eventEncoded.name)?.options.clientOnly ?? false
 
     const connectedClientSessionPullQueues = yield* makePullQueueSet
 
@@ -134,7 +175,6 @@ export const makeLeaderSyncProcessor = ({
       current: undefined as
         | undefined
         | {
-            otelSpan: otel.Span | undefined
             span: Tracer.Span
             devtoolsLatch: Effect.Latch | undefined
             runtime: Runtime.Runtime<LeaderThreadCtx>
@@ -142,8 +182,8 @@ export const makeLeaderSyncProcessor = ({
     }
 
     const localPushesQueue = yield* BucketQueue.make<LocalPushQueueItem>()
-    const localPushesLatch = yield* Effect.makeLatch(true)
-    const pullLatch = yield* Effect.makeLatch(true)
+    // Ensures mutual exclusion between local push and backend pull processing.
+    const localPushBackendPullMutex = yield* Effect.makeSemaphore(1)
 
     /**
      * Additionally to the `syncStateSref` we also need the `pushHeadRef` in order to prevent old/duplicate
@@ -154,9 +194,9 @@ export const makeLeaderSyncProcessor = ({
      *
      * Thus the purpose of the pushHeadRef is the guard the integrity of the local push queue
      */
-    const pushHeadRef = { current: EventSequenceNumber.ROOT }
-    const advancePushHead = (eventNum: EventSequenceNumber.EventSequenceNumber) => {
-      pushHeadRef.current = EventSequenceNumber.max(pushHeadRef.current, eventNum)
+    const pushHeadRef = { current: EventSequenceNumber.Client.ROOT }
+    const advancePushHead = (eventNum: EventSequenceNumber.Client.Composite) => {
+      pushHeadRef.current = EventSequenceNumber.Client.max(pushHeadRef.current, eventNum)
     }
 
     // NOTE: New events are only pushed to sync backend after successful local push processing
@@ -172,8 +212,8 @@ export const makeLeaderSyncProcessor = ({
 
         const waitForProcessing = options?.waitForProcessing ?? false
 
-        if (waitForProcessing) {
-          const deferreds = yield* Effect.forEach(newEvents, () => Deferred.make<void, LeaderAheadError>())
+        if (waitForProcessing === true) {
+          const deferreds = yield* Effect.forEach(newEvents, () => Deferred.make<void, LeaderAheadError | StaleRebaseGenerationError>())
 
           const items = newEvents.map((eventEncoded, i) => [eventEncoded, deferreds[i]] as LocalPushQueueItem)
 
@@ -188,41 +228,62 @@ export const makeLeaderSyncProcessor = ({
         Effect.withSpan('@livestore/common:LeaderSyncProcessor:push', {
           attributes: {
             batchSize: newEvents.length,
-            batch: TRACE_VERBOSE ? newEvents : undefined,
+            batch: TRACE_VERBOSE === true ? newEvents : undefined,
           },
-          links: ctxRef.current?.span ? [{ _tag: 'SpanLink', span: ctxRef.current.span, attributes: {} }] : undefined,
+          links:
+            ctxRef.current?.span !== undefined
+              ? [{ _tag: 'SpanLink', span: ctxRef.current.span, attributes: {} }]
+              : undefined,
         }),
       )
 
     const pushPartial: LeaderSyncProcessor['pushPartial'] = ({ event: { name, args }, clientId, sessionId }) =>
       Effect.gen(function* () {
-        const syncState = yield* syncStateSref
-        if (syncState === undefined) return shouldNeverHappen('Not initialized')
+        const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger)
+
+        const resolution = yield* resolveEventDef(schema, {
+          operation: '@livestore/common:LeaderSyncProcessor:pushPartial',
+          event: {
+            name,
+            args,
+            clientId,
+            sessionId,
+            seqNum: syncState.localHead,
+          },
+        })
 
-        const { eventDef } = getEventDef(schema, name)
+        if (resolution._tag === 'unknown') {
+          // Ignore partial pushes for unrecognised events – they are still
+          // persisted server-side once a schema update ships.
+          return
+        }
 
-        const eventEncoded = new LiveStoreEvent.EncodedWithMeta({
+        const eventEncoded = new LiveStoreEvent.Client.EncodedWithMeta({
           name,
           args,
           clientId,
           sessionId,
-          ...EventSequenceNumber.nextPair({ seqNum: syncState.localHead, isClient: eventDef.options.clientOnly }),
+          ...EventSequenceNumber.Client.nextPair({
+            seqNum: syncState.localHead,
+            isClient: resolution.eventDef.options.clientOnly,
+          }),
         })
 
         yield* push([eventEncoded])
-      }).pipe(Effect.catchTag('LeaderAheadError', Effect.orDie))
+      }).pipe(
+        // pushPartial constructs the event sequence number internally, so these errors should never happen.
+        Effect.catchIf(isRejectedPushError, Effect.die),
+      )
 
     // Starts various background loops
     const boot: LeaderSyncProcessor['boot'] = Effect.gen(function* () {
       const span = yield* Effect.currentSpan.pipe(Effect.orDie)
-      const otelSpan = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)))
       const { devtools, shutdownChannel } = yield* LeaderThreadCtx
       const runtime = yield* Effect.runtime<LeaderThreadCtx>()
 
       ctxRef.current = {
-        otelSpan,
         span,
-        devtoolsLatch: devtools.enabled ? devtools.syncBackendLatch : undefined,
+        devtoolsLatch: devtools.enabled === true ? devtools.syncBackendLatch : undefined,
         runtime,
       }
 
@@ -232,10 +293,10 @@ export const makeLeaderSyncProcessor = ({
       // Rehydrate sync queue
       if (initialSyncState.pending.length > 0) {
         const globalPendingEvents = initialSyncState.pending
-          // Don't sync clientOnly events
+          // Don't sync client-local events
           .filter((eventEncoded) => {
-            const { eventDef } = getEventDef(schema, eventEncoded.name)
-            return eventDef.options.clientOnly === false
+            const eventDef = schema.eventsDefsMap.get(eventEncoded.name)
+            return eventDef === undefined ? true : eventDef.options.clientOnly === false
           })
 
         if (globalPendingEvents.length > 0) {
@@ -243,19 +304,18 @@ export const makeLeaderSyncProcessor = ({
         }
       }
 
+      const handleBackendIdMismatchError = (error: BackendIdMismatchError) =>
+        handleBackendIdMismatch({ error, onBackendIdMismatch, shutdownChannel })
+
       const maybeShutdownOnError = (
         cause: Cause.Cause<
-          | UnexpectedError
-          | IntentionalShutdownCause
-          | IsOfflineError
-          | InvalidPushError
-          | InvalidPullError
+          | UnknownError
           | MaterializeError
         >,
       ) =>
         Effect.gen(function* () {
           if (onError === 'ignore') {
-            if (LS_DEV) {
+            if (LS_DEV === true) {
               yield* Effect.logDebug(
                 `Ignoring sync error (${cause._tag === 'Fail' ? cause.error._tag : cause._tag})`,
                 Cause.pretty(cause),
@@ -264,35 +324,38 @@ export const makeLeaderSyncProcessor = ({
             return
           }
 
-          const errorToSend = Cause.isFailType(cause) ? cause.error : UnexpectedError.make({ cause })
+          const errorToSend = Cause.isFailType(cause) === true ? cause.error : UnknownError.make({ cause })
           yield* shutdownChannel.send(errorToSend).pipe(Effect.orDie)
 
-          return yield* Effect.die(cause)
+          return yield* Effect.failCause(cause).pipe(Effect.orDie)
         })
 
       yield* backgroundApplyLocalPushes({
-        localPushesLatch,
+        localPushBackendPullMutex,
         localPushesQueue,
-        pullLatch,
         syncStateSref,
         syncBackendPushQueue,
         schema,
         isClientEvent,
-        otelSpan,
         connectedClientSessionPullQueues,
         localPushBatchSize,
         testing: {
           delay: testing?.delays?.localPushProcessing,
         },
-      }).pipe(Effect.catchAllCause(maybeShutdownOnError), Effect.forkScoped)
+      }).pipe(
+        Effect.catchAllCause(maybeShutdownOnError),
+        Effect.forkScoped,
+      )
 
       const backendPushingFiberHandle = yield* FiberHandle.make<void, never>()
       const backendPushingEffect = backgroundBackendPushing({
         syncBackendPushQueue,
-        otelSpan,
         devtoolsLatch: ctxRef.current?.devtoolsLatch,
         backendPushBatchSize,
-      }).pipe(Effect.catchAllCause(maybeShutdownOnError))
+      }).pipe(
+        Effect.catchTag('BackendIdMismatchError', handleBackendIdMismatchError),
+        Effect.catchAllCause(maybeShutdownOnError),
+      )
 
       yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect)
 
@@ -311,20 +374,21 @@ export const makeLeaderSyncProcessor = ({
             yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect)
           }),
         syncStateSref,
-        localPushesLatch,
-        pullLatch,
+        localPushBackendPullMutex,
         livePull,
         dbState,
-        otelSpan,
         initialBlockingSyncContext,
         devtoolsLatch: ctxRef.current?.devtoolsLatch,
         connectedClientSessionPullQueues,
         advancePushHead,
       }).pipe(
         Effect.retry({
-          // We want to retry pulling if we've lost connection to the sync backend
-          while: (cause) => cause._tag === 'IsOfflineError',
+          // Retry pulling when we've lost connection to the sync backend
+          // We're using `until` with a refinement instead of `while` to narrow `IsOfflineError` out of the error type.
+          // See https://github.com/Effect-TS/effect/issues/6122
+          until: (error): error is Exclude<typeof error, IsOfflineError> => error._tag !== 'IsOfflineError',
         }),
+        Effect.catchTag('BackendIdMismatchError', handleBackendIdMismatchError),
         Effect.catchAllCause(maybeShutdownOnError),
         // Needed to avoid `Fiber terminated with an unhandled error` logs which seem to happen because of the `Effect.retry` above.
         // This might be a bug in Effect. Only seems to happen in the browser.
@@ -355,17 +419,16 @@ export const makeLeaderSyncProcessor = ({
       - full new state db snapshot in the "rebase" case
         - downside: importing the snapshot is expensive
     */
-    const pullQueue: LeaderSyncProcessor['pullQueue'] = ({ cursor }) => {
-      const runtime = ctxRef.current?.runtime ?? shouldNeverHappen('Not initialized')
-      return connectedClientSessionPullQueues.makeQueue(cursor).pipe(Effect.provide(runtime))
-    }
+    const pullQueue: LeaderSyncProcessor['pullQueue'] = ({ cursor }) =>
+      Effect.fromNullable(ctxRef.current?.runtime).pipe(
+        Effect.orDieDebugger,
+        Effect.flatMap((runtime) =>
+          connectedClientSessionPullQueues.makeQueue(cursor).pipe(Effect.provide(runtime))
+        )
+      )
 
     const syncState = Subscribable.make({
-      get: Effect.gen(function* () {
-        const syncState = yield* syncStateSref
-        if (syncState === undefined) return shouldNeverHappen('Not initialized')
-        return syncState
-      }),
+      get: syncStateSref.pipe(Effect.flatMap(Effect.fromNullable), Effect.orDieDebugger),
       changes: syncStateSref.changes.pipe(Stream.filter(isNotUndefined)),
     })
 
@@ -380,26 +443,22 @@ export const makeLeaderSyncProcessor = ({
   })
 
 const backgroundApplyLocalPushes = ({
-  localPushesLatch,
+  localPushBackendPullMutex,
   localPushesQueue,
-  pullLatch,
   syncStateSref,
   syncBackendPushQueue,
   schema,
   isClientEvent,
-  otelSpan,
   connectedClientSessionPullQueues,
   localPushBatchSize,
   testing,
 }: {
-  pullLatch: Effect.Latch
-  localPushesLatch: Effect.Latch
+  localPushBackendPullMutex: Effect.Semaphore
   localPushesQueue: BucketQueue.BucketQueue<LocalPushQueueItem>
   syncStateSref: SubscriptionRef.SubscriptionRef<SyncState.SyncState | undefined>
-  syncBackendPushQueue: BucketQueue.BucketQueue<LiveStoreEvent.EncodedWithMeta>
+  syncBackendPushQueue: BucketQueue.BucketQueue<LiveStoreEvent.Client.EncodedWithMeta>
   schema: LiveStoreSchema
-  isClientEvent: (eventEncoded: LiveStoreEvent.EncodedWithMeta) => boolean
-  otelSpan: otel.Span | undefined
+  isClientEvent: (eventEncoded: LiveStoreEvent.Client.EncodedWithMeta) => boolean
   connectedClientSessionPullQueues: PullQueueSet
   localPushBatchSize: number
   testing: {
@@ -414,136 +473,149 @@ const backgroundApplyLocalPushes = ({
 
       const batchItems = yield* BucketQueue.takeBetween(localPushesQueue, 1, localPushBatchSize)
 
-      // Wait for the backend pulling to finish
-      yield* localPushesLatch.await
-
-      // Prevent backend pull processing until this local push is finished
-      yield* pullLatch.close
-
-      const syncState = yield* syncStateSref
-      if (syncState === undefined) return shouldNeverHappen('Not initialized')
+      // Applies a batch of local pushes, guarded by the localPushBackendPullMutex to ensure mutual exclusion with backend pulling
+      yield* Effect.gen(function* () {
+        const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger)
 
-      const currentRebaseGeneration = syncState.localHead.rebaseGeneration
+        const currentRebaseGeneration = syncState.localHead.rebaseGeneration
 
-      // Since the rebase generation might have changed since enqueuing, we need to filter out items with older generation
-      // It's important that we filter after we got localPushesLatch, otherwise we might filter with the old generation
-      const [newEvents, deferreds] = pipe(
-        batchItems,
-        ReadonlyArray.filter(([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration === currentRebaseGeneration),
-        ReadonlyArray.unzip,
-      )
-
-      if (newEvents.length === 0) {
-        // console.log('dropping old-gen batch', currentLocalPushGenerationRef.current)
-        // Allow the backend pulling to start
-        yield* pullLatch.open
-        continue
-      }
+        // Since the rebase generation might have changed since enqueuing, we need to filter out items with older generation
+        // It's important that we filter after acquiring the localPushBackendPullMutex, otherwise we might filter with the old generation
+        const [droppedItems, filteredItems] = ReadonlyArray.partition(
+          batchItems,
+          ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= currentRebaseGeneration,
+        )
 
-      const mergeResult = SyncState.merge({
-        syncState,
-        payload: { _tag: 'local-push', newEvents },
-        isClientEvent,
-        isEqualEvent: LiveStoreEvent.isEqualEncoded,
-      })
+        if (droppedItems.length > 0) {
+          yield* Effect.spanEvent(`push:drop-old-generation`, {
+          droppedCount: droppedItems.length,
+          currentRebaseGeneration,
+        })
 
-      switch (mergeResult._tag) {
-        case 'unexpected-error': {
-          otelSpan?.addEvent(`push:unexpected-error`, {
-            batchSize: newEvents.length,
-            newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
-          })
-          return yield* new UnexpectedError({ cause: mergeResult.message })
+        /**
+         * Dropped pushes may still have a deferred awaiting completion.
+         * Fail it so the caller learns the leader advanced and resubmits with the updated generation.
+         */
+        yield* Effect.forEach(
+          droppedItems.filter(
+            (item): item is [LiveStoreEvent.Client.EncodedWithMeta, Deferred.Deferred<void, LeaderAheadError | StaleRebaseGenerationError>] =>
+                item[1] !== undefined,
+            ),
+            ([eventEncoded, deferred]) =>
+              Deferred.fail(
+                deferred,
+                StaleRebaseGenerationError.make({
+                  currentRebaseGeneration,
+                  providedRebaseGeneration: eventEncoded.seqNum.rebaseGeneration,
+                sessionId: eventEncoded.sessionId,
+                }),
+              ),
+          )
         }
-        case 'rebase': {
-          return shouldNeverHappen('The leader thread should never have to rebase due to a local push')
+
+        if (filteredItems.length === 0) {
+          return
         }
-        case 'reject': {
-          otelSpan?.addEvent(`push:reject`, {
-            batchSize: newEvents.length,
-            mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
-          })
 
-          // TODO: how to test this?
-          const nextRebaseGeneration = currentRebaseGeneration + 1
+        const [newEvents, deferreds] = ReadonlyArray.unzip(filteredItems)
 
-          const providedNum = newEvents.at(0)!.seqNum
-          // All subsequent pushes with same generation should be rejected as well
-          // We're also handling the case where the localPushQueue already contains events
-          // from the next generation which we preserve in the queue
-          const remainingEventsMatchingGeneration = yield* BucketQueue.takeSplitWhere(
-            localPushesQueue,
-            ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= nextRebaseGeneration,
-          )
+        yield* Effect.annotateCurrentSpan({
+        'batchSize': newEvents.length,
+        ...(TRACE_VERBOSE === true ? { 'newEvents': jsonStringify(newEvents) } : {}),
+        })
+
+        const mergeResult = yield* SyncState.merge({
+          syncState,
+          payload: { _tag: 'local-push', newEvents },
+          isClientEvent,
+          isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
+        })
 
-          // TODO we still need to better understand and handle this scenario
-          if (LS_DEV && (yield* BucketQueue.size(localPushesQueue)) > 0) {
-            console.log('localPushesQueue is not empty', yield* BucketQueue.size(localPushesQueue))
-            // biome-ignore lint/suspicious/noDebugger: debugging
-            debugger
+        switch (mergeResult._tag) {
+          case 'rebase': {
+            return yield* Effect.dieDebugger('The leader thread should never have to rebase due to a local push')
           }
+          case 'reject': {
+            yield* Effect.spanEvent(`push:reject`, {
+              batchSize: newEvents.length,
+              ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
+            })
 
-          const allDeferredsToReject = [
-            ...deferreds,
-            ...remainingEventsMatchingGeneration.map(([_, deferred]) => deferred),
-          ].filter(isNotUndefined)
+            // TODO: how to test this?
+            const nextRebaseGeneration = currentRebaseGeneration + 1
 
-          yield* Effect.forEach(allDeferredsToReject, (deferred) =>
-            Deferred.fail(
-              deferred,
-              LeaderAheadError.make({ minimumExpectedNum: mergeResult.expectedMinimumId, providedNum }),
-            ),
-          )
+            const providedNum = newEvents.at(0)!.seqNum
+            // All subsequent pushes with same generation should be rejected as well
+            // We're also handling the case where the localPushQueue already contains events
+            // from the next generation which we preserve in the queue
+            const remainingEventsMatchingGeneration = yield* BucketQueue.takeSplitWhere(
+              localPushesQueue,
+              ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= nextRebaseGeneration,
+            )
 
-          // Allow the backend pulling to start
-          yield* pullLatch.open
+            // TODO we still need to better understand and handle this scenario
+            if (LS_DEV === true && (yield* BucketQueue.size(localPushesQueue)) > 0) {
+              console.log('localPushesQueue is not empty', yield* BucketQueue.size(localPushesQueue))
+              // oxlint-disable-next-line eslint(no-debugger) -- intentional breakpoint for unexpected queue state
+              debugger
+            }
 
-          // In this case we're skipping state update and down/upstream processing
-          // We've cleared the local push queue and are now waiting for new local pushes / backend pulls
-          continue
-        }
-        case 'advance': {
-          break
-        }
-        default: {
-          casesHandled(mergeResult)
-        }
-      }
+            const allDeferredsToReject = [
+              ...deferreds,
+              ...remainingEventsMatchingGeneration.map(([_, deferred]) => deferred),
+            ].filter(isNotUndefined)
 
-      yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState)
+            yield* Effect.forEach(allDeferredsToReject, (deferred) =>
+              Deferred.fail(
+                deferred,
+                LeaderAheadError.make({ minimumExpectedNum: mergeResult.expectedMinimumId, providedNum, sessionId: newEvents.at(0)!.sessionId }),
+              ),
+            )
 
-      yield* connectedClientSessionPullQueues.offer({
-        payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
-        leaderHead: mergeResult.newSyncState.localHead,
-      })
+            // In this case we're skipping state update and down/upstream processing
+            // We've cleared the local push queue and are now waiting for new local pushes / backend pulls
+            return
+          }
+          case 'advance': {
+            break
+          }
+          default: {
+            casesHandled(mergeResult)
+          }
+        }
 
-      otelSpan?.addEvent(`push:advance`, {
-        batchSize: newEvents.length,
-        mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
-      })
+        yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState)
 
-      // Don't sync clientOnly events
-      const filteredBatch = mergeResult.newEvents.filter((eventEncoded) => {
-        const { eventDef } = getEventDef(schema, eventEncoded.name)
-        return eventDef.options.clientOnly === false
-      })
+        yield* connectedClientSessionPullQueues.offer({
+          payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
+          leaderHead: mergeResult.newSyncState.localHead,
+        })
 
-      yield* BucketQueue.offerAll(syncBackendPushQueue, filteredBatch)
+        yield* Effect.spanEvent(`push:advance`, {
+          batchSize: newEvents.length,
+        ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
+        })
 
-      yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds })
+        // Don't sync client-local events
+        const filteredBatch = mergeResult.newEvents.filter((eventEncoded) => {
+          const eventDef = schema.eventsDefsMap.get(eventEncoded.name)
+          return eventDef === undefined ? true : eventDef.options.clientOnly === false
+        })
 
-      // Allow the backend pulling to start
-      yield* pullLatch.open
+        yield* BucketQueue.offerAll(syncBackendPushQueue, filteredBatch)
+
+        yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds })
+      }).pipe(localPushBackendPullMutex.withPermits(1))
     }
   })
 
 type MaterializeEventsBatch = (_: {
-  batchItems: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>
+  batchItems: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>
   /**
    * The deferreds are used by the caller to know when the mutation has been processed.
    * Indexes are aligned with `batchItems`
    */
-  deferreds: ReadonlyArray<Deferred.Deferred<void, LeaderAheadError> | undefined> | undefined
+  deferreds: ReadonlyArray<Deferred.Deferred<void, LeaderAheadError | StaleRebaseGenerationError> | undefined> | undefined
 }) => Effect.Effect<void, MaterializeError, LeaderThreadCtx>
 
 // TODO how to handle errors gracefully
@@ -557,7 +629,7 @@ const materializeEventsBatch: MaterializeEventsBatch = ({ batchItems, deferreds
 
     yield* Effect.addFinalizer((exit) =>
       Effect.gen(function* () {
-        if (Exit.isSuccess(exit)) return
+        if (Exit.isSuccess(exit) === true) return
 
         // Rollback in case of an error
         db.execute('ROLLBACK', undefined)
@@ -586,90 +658,99 @@ const materializeEventsBatch: MaterializeEventsBatch = ({ batchItems, deferreds
     Effect.tapCauseLogPretty,
   )
 
-const backgroundBackendPulling = ({
+const backgroundBackendPulling = Effect.fn('@livestore/common:LeaderSyncProcessor:backend-pulling')(function* ({
   isClientEvent,
   restartBackendPushing,
-  otelSpan,
   dbState,
   syncStateSref,
-  localPushesLatch,
+  localPushBackendPullMutex,
   livePull,
-  pullLatch,
   devtoolsLatch,
   initialBlockingSyncContext,
   connectedClientSessionPullQueues,
   advancePushHead,
 }: {
-  isClientEvent: (eventEncoded: LiveStoreEvent.EncodedWithMeta) => boolean
+  isClientEvent: (eventEncoded: LiveStoreEvent.Client.EncodedWithMeta) => boolean
   restartBackendPushing: (
-    filteredRebasedPending: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>,
-  ) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx | HttpClient.HttpClient>
-  otelSpan: otel.Span | undefined
+    filteredRebasedPending: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>,
+  ) => Effect.Effect<void, never, LeaderThreadCtx | HttpClient.HttpClient>
   syncStateSref: SubscriptionRef.SubscriptionRef<SyncState.SyncState | undefined>
   dbState: SqliteDb
-  localPushesLatch: Effect.Latch
-  pullLatch: Effect.Latch
+  localPushBackendPullMutex: Effect.Semaphore
   livePull: boolean
   devtoolsLatch: Effect.Latch | undefined
   initialBlockingSyncContext: InitialBlockingSyncContext
   connectedClientSessionPullQueues: PullQueueSet
-  advancePushHead: (eventNum: EventSequenceNumber.EventSequenceNumber) => void
-}) =>
-  Effect.gen(function* () {
-    const { syncBackend, dbState: db, dbEventlog, schema } = yield* LeaderThreadCtx
+  advancePushHead: (eventNum: EventSequenceNumber.Client.Composite) => void
+}) {
+  const { syncBackend, dbState: db, dbEventlog, schema } = yield* LeaderThreadCtx
 
-    if (syncBackend === undefined) return
+  if (syncBackend === undefined) return
 
-    const onNewPullChunk = (newEvents: LiveStoreEvent.EncodedWithMeta[], pageInfo: SyncBackend.PullResPageInfo) =>
-      Effect.gen(function* () {
-        if (newEvents.length === 0) return
+  let pullMutexHeld = false
 
-        if (devtoolsLatch !== undefined) {
-          yield* devtoolsLatch.await
+  const releasePullMutexIfHeld = Effect.gen(function* () {
+    if (pullMutexHeld === false) return
+    pullMutexHeld = false
+    yield* localPushBackendPullMutex.release(1)
+  })
+
+  const isPullPaginationComplete = (pageInfo: SyncBackend.PullResPageInfo) => pageInfo._tag === 'NoMore'
+
+  const onNewPullChunk = (newEvents: LiveStoreEvent.Client.EncodedWithMeta[], pageInfo: SyncBackend.PullResPageInfo) =>
+    Effect.gen(function* () {
+      if (devtoolsLatch !== undefined) {
+        yield* devtoolsLatch.await
+      }
+
+      if (newEvents.length === 0) {
+        if (isPullPaginationComplete(pageInfo) === true) {
+          yield* releasePullMutexIfHeld
         }
+        return
+      }
 
-        // Prevent more local pushes from being processed until this pull is finished
-        yield* localPushesLatch.close
+      // Prevent more local pushes from being processed until this pull pagination sequence is finished.
+      if (pullMutexHeld === false) {
+        yield* localPushBackendPullMutex.take(1)
+        pullMutexHeld = true
+      }
 
-        // Wait for pending local pushes to finish
-        yield* pullLatch.await
+      const chunkExit = yield* Effect.gen(function* () {
+        const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger)
 
-        const syncState = yield* syncStateSref
-        if (syncState === undefined) return shouldNeverHappen('Not initialized')
+        yield* Effect.annotateCurrentSpan({
+        'merge.newEventsCount': newEvents.length,
+        ...(TRACE_VERBOSE === true ? { 'merge.newEvents': jsonStringify(newEvents) } : {}),
+      })
 
-        const mergeResult = SyncState.merge({
-          syncState,
-          payload: SyncState.PayloadUpstreamAdvance.make({ newEvents }),
-          isClientEvent,
-          isEqualEvent: LiveStoreEvent.isEqualEncoded,
-          ignoreClientEvents: true,
-        })
+      const mergeResult = yield* SyncState.merge({
+        syncState,
+        payload: SyncState.PayloadUpstreamAdvance.make({ newEvents }),
+        isClientEvent,
+        isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
+        ignoreClientEvents: true,
+      })
 
-        if (mergeResult._tag === 'reject') {
-          return shouldNeverHappen('The leader thread should never reject upstream advances')
-        } else if (mergeResult._tag === 'unexpected-error') {
-          otelSpan?.addEvent(`pull:unexpected-error`, {
-            newEventsCount: newEvents.length,
-            newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
-          })
-          return yield* new UnexpectedError({ cause: mergeResult.message })
-        }
+      if (mergeResult._tag === 'reject') {
+        return yield* Effect.dieDebugger('The leader thread should never reject upstream advances')
+      }
 
         const newBackendHead = newEvents.at(-1)!.seqNum
 
         Eventlog.updateBackendHead(dbEventlog, newBackendHead)
 
         if (mergeResult._tag === 'rebase') {
-          otelSpan?.addEvent(`pull:rebase[${mergeResult.newSyncState.localHead.rebaseGeneration}]`, {
-            newEventsCount: newEvents.length,
-            newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
-            rollbackCount: mergeResult.rollbackEvents.length,
-            mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
+          yield* Effect.spanEvent(`pull:rebase[${mergeResult.newSyncState.localHead.rebaseGeneration}]`, {
+          newEventsCount: newEvents.length,
+          ...(TRACE_VERBOSE === true ? { newEvents: jsonStringify(newEvents) } : {}),
+          rollbackCount: mergeResult.rollbackEvents.length,
+          ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
           })
 
           const globalRebasedPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
-            const { eventDef } = getEventDef(schema, event.name)
-            return eventDef.options.clientOnly === false
+            const eventDef = schema.eventsDefsMap.get(event.name)
+            return eventDef === undefined ? true : eventDef.options.clientOnly === false
           })
           yield* restartBackendPushing(globalRebasedPendingEvents)
 
@@ -686,15 +767,15 @@ const backgroundBackendPulling = ({
             leaderHead: mergeResult.newSyncState.localHead,
           })
         } else {
-          otelSpan?.addEvent(`pull:advance`, {
+          yield* Effect.spanEvent(`pull:advance`, {
             newEventsCount: newEvents.length,
-            mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
+          ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
           })
 
           // Ensure push fiber is active after advance by restarting with current pending (non-client) events
           const globalPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
-            const { eventDef } = getEventDef(schema, event.name)
-            return eventDef.options.clientOnly === false
+            const eventDef = schema.eventsDefsMap.get(event.name)
+            return eventDef === undefined ? true : eventDef.options.clientOnly === false
           })
           yield* restartBackendPushing(globalPendingEvents)
 
@@ -708,10 +789,10 @@ const backgroundBackendPulling = ({
             // `newEvents` instead which we filter via `mergeResult.confirmedEvents`
             const confirmedNewEvents = newEvents.filter((event) =>
               mergeResult.confirmedEvents.some((confirmedEvent) =>
-                EventSequenceNumber.isEqual(event.seqNum, confirmedEvent.seqNum),
+                EventSequenceNumber.Client.isEqual(event.seqNum, confirmedEvent.seqNum),
               ),
             )
-            yield* Eventlog.updateSyncMetadata(confirmedNewEvents).pipe(UnexpectedError.mapToUnexpectedError)
+            yield* Eventlog.updateSyncMetadata(confirmedNewEvents).pipe(Effect.orDieDebugger)
           }
         }
 
@@ -723,138 +804,128 @@ const backgroundBackendPulling = ({
         yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds: undefined })
 
         yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState)
+      }).pipe(Effect.exit)
 
-        // Allow local pushes to be processed again
-        if (pageInfo._tag === 'NoMore') {
-          yield* localPushesLatch.open
-        }
-      })
+      if (Exit.isFailure(chunkExit) === true) {
+        yield* releasePullMutexIfHeld
+        return yield* Effect.failCause(chunkExit.cause)
+      }
 
-    const syncState = yield* syncStateSref
-    if (syncState === undefined) return shouldNeverHappen('Not initialized')
-    const cursorInfo = yield* Eventlog.getSyncBackendCursorInfo({ remoteHead: syncState.upstreamHead.global })
+      if (isPullPaginationComplete(pageInfo) === true) {
+        yield* releasePullMutexIfHeld
+      }
+    })
 
-    const hashMaterializerResult = makeMaterializerHash({ schema, dbState })
+  const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger)
+  const cursorInfo = yield* Eventlog.getSyncBackendCursorInfo({ remoteHead: syncState.upstreamHead.global })
 
-    yield* syncBackend.pull(cursorInfo, { live: livePull }).pipe(
-      // TODO only take from queue while connected
-      Stream.tap(({ batch, pageInfo }) =>
-        Effect.gen(function* () {
-          // yield* Effect.spanEvent('batch', {
-          //   attributes: {
-          //     batchSize: batch.length,
-          //     batch: TRACE_VERBOSE ? batch : undefined,
-          //   },
-          // })
-          // NOTE we only want to take process events when the sync backend is connected
-          // (e.g. needed for simulating being offline)
-          // TODO remove when there's a better way to handle this in stream above
-          yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
-          yield* onNewPullChunk(
-            batch.map((_) =>
-              LiveStoreEvent.EncodedWithMeta.fromGlobal(_.eventEncoded, {
-                syncMetadata: _.metadata,
-                // TODO we can't really know the materializer result here yet beyond the first event batch item as we need to materialize it one by one first
-                // This is a bug and needs to be fixed https://github.com/livestorejs/livestore/issues/503#issuecomment-3114533165
-                materializerHashLeader: hashMaterializerResult(LiveStoreEvent.encodedFromGlobal(_.eventEncoded)),
-                materializerHashSession: Option.none(),
-              }),
-            ),
-            pageInfo,
-          )
-          yield* initialBlockingSyncContext.update({ processed: batch.length, pageInfo })
-        }),
-      ),
-      Stream.runDrain,
-      Effect.interruptible,
-    )
+  const hashMaterializerResult = makeMaterializerHash({ schema, dbState })
 
-    // Should only ever happen when livePull is false
-    yield* Effect.logDebug('backend-pulling finished', { livePull })
-  }).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pulling'))
+  yield* syncBackend.pull(cursorInfo, { live: livePull }).pipe(
+    // TODO only take from queue while connected
+    Stream.tap(({ batch, pageInfo }) =>
+      Effect.gen(function* () {
+        // NOTE we only want to take process events when the sync backend is connected
+        // (e.g. needed for simulating being offline)
+        // TODO remove when there's a better way to handle this in stream above
+        yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
+        yield* onNewPullChunk(
+          batch.map((_) =>
+            LiveStoreEvent.Client.EncodedWithMeta.fromGlobal(_.eventEncoded, {
+              syncMetadata: _.metadata,
+              // TODO we can't really know the materializer result here yet beyond the first event batch item as we need to materialize it one by one first
+              // This is a bug and needs to be fixed https://github.com/livestorejs/livestore/issues/503#issuecomment-3114533165
+              materializerHashLeader: hashMaterializerResult(LiveStoreEvent.Global.toClientEncoded(_.eventEncoded)),
+              materializerHashSession: Option.none(),
+            }),
+          ),
+          pageInfo,
+        )
+        yield* initialBlockingSyncContext.update({ processed: batch.length, pageInfo })
+      }),
+    ),
+    Stream.runDrain,
+    Effect.interruptible,
+    Effect.ensuring(releasePullMutexIfHeld),
+  )
+
+  // Should only ever happen when livePull is false
+  yield* Effect.logDebug('backend-pulling finished', { livePull })
+})
 
-const backgroundBackendPushing = ({
+const backgroundBackendPushing = Effect.fn('@livestore/common:LeaderSyncProcessor:backend-pushing')(function* ({
   syncBackendPushQueue,
-  otelSpan,
   devtoolsLatch,
   backendPushBatchSize,
 }: {
-  syncBackendPushQueue: BucketQueue.BucketQueue<LiveStoreEvent.EncodedWithMeta>
-  otelSpan: otel.Span | undefined
+  syncBackendPushQueue: BucketQueue.BucketQueue<LiveStoreEvent.Client.EncodedWithMeta>
   devtoolsLatch: Effect.Latch | undefined
   backendPushBatchSize: number
-}) =>
-  Effect.gen(function* () {
-    const { syncBackend } = yield* LeaderThreadCtx
-    if (syncBackend === undefined) return
+}) {
+  const { syncBackend } = yield* LeaderThreadCtx
+  if (syncBackend === undefined) return
 
-    while (true) {
-      yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
+  while (true) {
+    yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
 
-      const queueItems = yield* BucketQueue.takeBetween(syncBackendPushQueue, 1, backendPushBatchSize)
+    const queueItems = yield* BucketQueue.takeBetween(syncBackendPushQueue, 1, backendPushBatchSize)
 
-      yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
+    yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
 
-      if (devtoolsLatch !== undefined) {
-        yield* devtoolsLatch.await
-      }
+    if (devtoolsLatch !== undefined) {
+      yield* devtoolsLatch.await
+    }
 
-      otelSpan?.addEvent('backend-push', {
-        batchSize: queueItems.length,
-        batch: TRACE_VERBOSE ? JSON.stringify(queueItems) : undefined,
-      })
+    yield* Effect.spanEvent('backend-push', {
+      batchSize: queueItems.length,
+      ...(TRACE_VERBOSE === true ? { batch: jsonStringify(queueItems) } : {}),
+    })
 
-      // Push with declarative retry/backoff using Effect schedules
-      // - Exponential backoff starting at 1s and doubling (1s, 2s, 4s, 8s, 16s, 30s ...)
-      // - Delay clamped at 30s (continues retrying at 30s)
-      // - Resets automatically after successful push
-      // TODO(metrics): expose counters/gauges for retry attempts and queue health via devtools/metrics
-
-      // Only retry for transient UnexpectedError cases
-      const isRetryable = (err: InvalidPushError | IsOfflineError) =>
-        err._tag === 'InvalidPushError' && err.cause._tag === 'LiveStore.UnexpectedError'
-
-      // Input: InvalidPushError | IsOfflineError, Output: Duration
-      const retrySchedule: Schedule.Schedule<Duration.DurationInput, InvalidPushError | IsOfflineError> =
-        Schedule.exponential(Duration.seconds(1)).pipe(
-          Schedule.andThenEither(Schedule.spaced(Duration.seconds(30))), // clamp at 30 second intervals
-          Schedule.compose(Schedule.elapsed),
-          Schedule.whileInput(isRetryable),
-        )
+    // Push with declarative retry/backoff using Effect schedules
+    // - Exponential backoff starting at 1s and doubling (1s, 2s, 4s, 8s, 16s, 30s ...)
+    // - Delay clamped at 30s (continues retrying at 30s)
+    // - Resets automatically after successful push
+    // TODO(metrics): expose counters/gauges for retry attempts and queue health via devtools/metrics
+    yield* Effect.gen(function* () {
+      const iteration = yield* Schedule.CurrentIterationMetadata
 
-      yield* Effect.gen(function* () {
-        const iteration = yield* Schedule.CurrentIterationMetadata
+      const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either)
 
-        const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either)
+      const retries = iteration.recurrence
+      if (retries > 0 && pushResult._tag === 'Right') {
+        yield* Effect.spanEvent('backend-push-retry-success', { retries, batchSize: queueItems.length })
+      }
 
-        const retries = iteration.recurrence
-        if (retries > 0 && pushResult._tag === 'Right') {
-          otelSpan?.addEvent('backend-push-retry-success', { retries, batchSize: queueItems.length })
+      if (pushResult._tag === 'Left') {
+        yield* Effect.spanEvent('backend-push-error', {
+          error: pushResult.left.toString(),
+          retries,
+          batchSize: queueItems.length,
+        })
+        const error = pushResult.left
+        if (error._tag === 'ServerAheadError') {
+          // It's a core part of the sync protocol that the sync backend will emit a new pull chunk alongside the ServerAheadError
+          yield* Effect.logDebug('handled backend-push-error (waiting for interupt caused by pull)', { error })
+          return yield* Effect.never
         }
 
-        if (pushResult._tag === 'Left') {
-          otelSpan?.addEvent('backend-push-error', {
-            error: pushResult.left.toString(),
-            retries,
-            batchSize: queueItems.length,
-          })
-          const error = pushResult.left
-          if (
-            error._tag === 'IsOfflineError' ||
-            (error._tag === 'InvalidPushError' && error.cause._tag === 'ServerAheadError')
-          ) {
-            // It's a core part of the sync protocol that the sync backend will emit a new pull chunk alongside the ServerAheadError
-            yield* Effect.logDebug('handled backend-push-error (waiting for interupt caused by pull)', { error })
-            return yield* Effect.never
-          }
-
-          return yield* error
-        }
-      }).pipe(Effect.retry(retrySchedule))
-    }
-  }).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pushing'))
+        return yield* error
+      }
+    }).pipe(
+      // Retry transient errors
+      Effect.retry({
+        schedule: Schedule.exponential(Duration.seconds(1)).pipe(
+          Schedule.modifyDelay((_, delay) => Duration.min(delay, Duration.seconds(30))) // Cap delay at 30s intervals.
+        ),
+        while: (error) => error._tag === 'IsOfflineError' || error._tag === 'UnknownError',
+      }),
+      // This is needed to narrow the Error type. Our retry policy runs indefinitely, but Effect.retry does not narrow the Error type.
+      Effect.catchIf((error) => error._tag === 'IsOfflineError' || error._tag === 'UnknownError', Effect.die),
+    )
+  }
+}, Effect.interruptible)
 
-const trimChangesetRows = (db: SqliteDb, newHead: EventSequenceNumber.EventSequenceNumber) => {
+const trimChangesetRows = (db: SqliteDb, newHead: EventSequenceNumber.Client.Composite) => {
   // Since we're using the session changeset rows to query for the current head,
   // we're keeping at least one row for the current head, and thus are using `<` instead of `<=`
   db.execute(sql`DELETE FROM ${SystemTables.SESSION_CHANGESET_META_TABLE} WHERE seqNumGlobal < ${newHead.global}`)
@@ -862,16 +933,16 @@ const trimChangesetRows = (db: SqliteDb, newHead: EventSequenceNumber.EventSeque
 
 interface PullQueueSet {
   makeQueue: (
-    cursor: EventSequenceNumber.EventSequenceNumber,
+    cursor: EventSequenceNumber.Client.Composite,
   ) => Effect.Effect<
     Queue.Queue<{ payload: typeof SyncState.PayloadUpstream.Type }>,
-    UnexpectedError,
+    never,
     Scope.Scope | LeaderThreadCtx
   >
   offer: (item: {
     payload: typeof SyncState.PayloadUpstream.Type
-    leaderHead: EventSequenceNumber.EventSequenceNumber
-  }) => Effect.Effect<void, UnexpectedError>
+    leaderHead: EventSequenceNumber.Client.Composite
+  }) => Effect.Effect<void, never>
 }
 
 const makePullQueueSet = Effect.gen(function* () {
@@ -901,17 +972,17 @@ const makePullQueueSet = Effect.gen(function* () {
 
       const payloadsSinceCursor = Array.from(cachedPayloads.entries())
         .flatMap(([seqNumStr, payloads]) =>
-          payloads.map((payload) => ({ payload, seqNum: EventSequenceNumber.fromString(seqNumStr) })),
+          payloads.map((payload) => ({ payload, seqNum: EventSequenceNumber.Client.fromString(seqNumStr) })),
         )
-        .filter(({ seqNum }) => EventSequenceNumber.isGreaterThan(seqNum, cursor))
-        .toSorted((a, b) => EventSequenceNumber.compare(a.seqNum, b.seqNum))
+        .filter(({ seqNum }) => EventSequenceNumber.Client.isGreaterThan(seqNum, cursor))
+        .toSorted((a, b) => EventSequenceNumber.Client.compare(a.seqNum, b.seqNum))
         .map(({ payload }) => {
           if (payload._tag === 'upstream-advance') {
             return {
               payload: {
                 _tag: 'upstream-advance' as const,
                 newEvents: ReadonlyArray.dropWhile(payload.newEvents, (eventEncoded) =>
-                  EventSequenceNumber.isGreaterThanOrEqual(cursor, eventEncoded.seqNum),
+                  EventSequenceNumber.Client.isGreaterThanOrEqual(cursor, eventEncoded.seqNum),
                 ),
               },
             }
@@ -957,8 +1028,8 @@ const makePullQueueSet = Effect.gen(function* () {
 
   const offer: PullQueueSet['offer'] = (item) =>
     Effect.gen(function* () {
-      const seqNumStr = EventSequenceNumber.toString(item.leaderHead)
-      if (cachedPayloads.has(seqNumStr)) {
+      const seqNumStr = EventSequenceNumber.Client.toString(item.leaderHead)
+      if (cachedPayloads.has(seqNumStr) === true) {
         cachedPayloads.get(seqNumStr)!.push(item.payload)
       } else {
         cachedPayloads.set(seqNumStr, [item.payload])
@@ -982,29 +1053,108 @@ const makePullQueueSet = Effect.gen(function* () {
   }
 })
 
+/**
+ * Validate a client-provided batch before it is admitted to the leader queue.
+ * Ensures the numbers form a strictly increasing chain and that the first
+ * event sits ahead of the current push head.
+ */
 const validatePushBatch = (
-  batch: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>,
-  pushHead: EventSequenceNumber.EventSequenceNumber,
+  batch: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>,
+  pushHead: EventSequenceNumber.Client.Composite,
 ) =>
   Effect.gen(function* () {
     if (batch.length === 0) {
       return
     }
 
-    // Make sure batch is monotonically increasing
+    // Defensive check: callers should already provide a strictly increasing sequence
+    // of event numbers.
     for (let i = 1; i < batch.length; i++) {
-      if (EventSequenceNumber.isGreaterThanOrEqual(batch[i - 1]!.seqNum, batch[i]!.seqNum)) {
-        shouldNeverHappen(
-          `Events must be ordered in monotonically ascending order by eventNum. Received: [${batch.map((e) => EventSequenceNumber.toString(e.seqNum)).join(', ')}]`,
-        )
+      if (EventSequenceNumber.Client.isGreaterThanOrEqual(batch[i - 1]!.seqNum, batch[i]!.seqNum) === true) {
+        return yield* NonMonotonicBatchError.make({
+          precedingSeqNum: batch[i - 1]!.seqNum,
+          violatingSeqNum: batch[i]!.seqNum,
+          violationIndex: i,
+          sessionId: batch[i]!.sessionId,
+        })
       }
     }
 
-    // Make sure smallest sequence number is > pushHead
-    if (EventSequenceNumber.isGreaterThanOrEqual(pushHead, batch[0]!.seqNum)) {
+    // Reject stale batches whose first event is at or behind the leader's push head.
+    if (EventSequenceNumber.Client.isGreaterThanOrEqual(pushHead, batch[0]!.seqNum) === true) {
       return yield* LeaderAheadError.make({
         minimumExpectedNum: pushHead,
         providedNum: batch[0]!.seqNum,
+        sessionId: batch[0]!.sessionId,
       })
     }
   })
+
+/**
+ * Handles a BackendIdMismatchError based on the configured behavior.
+ * This occurs when the sync backend has been reset and has a new identity.
+ */
+const handleBackendIdMismatch = Effect.fn('@livestore/common:LeaderSyncProcessor:handleBackendIdMismatch')(function* ({
+  error,
+  onBackendIdMismatch,
+  shutdownChannel,
+}: {
+  error: BackendIdMismatchError
+  onBackendIdMismatch: 'reset' | 'shutdown' | 'ignore'
+  shutdownChannel: ShutdownChannel
+}) {
+  const { dbEventlog, dbState } = yield* LeaderThreadCtx
+
+  if (onBackendIdMismatch === 'reset') {
+    yield* Effect.logWarning(
+      'Sync backend identity changed (backend was reset). Clearing local storage and shutting down.',
+      error,
+    )
+
+    // Clear local databases so the client can start fresh on next boot
+    yield* clearLocalDatabases({ dbEventlog, dbState })
+
+    // Send shutdown signal with special reason
+    yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'backend-id-mismatch' })).pipe(Effect.orDie)
+
+    return yield* Effect.die(error)
+  }
+
+  if (onBackendIdMismatch === 'shutdown') {
+    yield* Effect.logWarning(
+      'Sync backend identity changed (backend was reset). Shutting down without clearing local storage.',
+      error,
+    )
+
+    yield* shutdownChannel.send(error).pipe(Effect.orDie)
+
+    return yield* Effect.die(error)
+  }
+
+  // ignore mode
+  if (LS_DEV === true) {
+    yield* Effect.logDebug(
+      'Ignoring BackendIdMismatchError (sync backend was reset but client continues with stale data)',
+      error,
+    )
+  }
+})
+
+/**
+ * Clears local databases (eventlog and state) so the client can start fresh on next boot.
+ * This is used when the sync backend identity has changed (i.e. backend was reset).
+ */
+const clearLocalDatabases = ({ dbEventlog, dbState }: { dbEventlog: SqliteDb; dbState: SqliteDb }) =>
+  Effect.sync(() => {
+    // Clear eventlog tables
+    dbEventlog.execute(sql`DELETE FROM ${EVENTLOG_META_TABLE}`)
+    dbEventlog.execute(sql`DELETE FROM ${SYNC_STATUS_TABLE}`)
+
+    // Drop all state tables - they'll be recreated on next boot
+    const tables = dbState.select<{ name: string }>(
+      sql`SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`,
+    )
+    for (const { name } of tables) {
+      dbState.execute(`DROP TABLE IF EXISTS "${name}"`)
+    }
+  })