@livestore/common 0.4.0-dev.2 → 0.4.0-dev.20

package/src/leader-thread/LeaderSyncProcessor.ts

@@ -2,27 +2,34 @@ import { casesHandled, isNotUndefined, LS_DEV, shouldNeverHappen, TRACE_VERBOSE
 import type { HttpClient, Runtime, Scope, Tracer } from '@livestore/utils/effect'
 import {
   BucketQueue,
+  Cause,
   Deferred,
+  Duration,
   Effect,
   Exit,
   FiberHandle,
+  Layer,
   Option,
   OtelTracer,
-  pipe,
   Queue,
   ReadonlyArray,
+  Schedule,
   Stream,
   Subscribable,
   SubscriptionRef,
 } from '@livestore/utils/effect'
 import type * as otel from '@opentelemetry/api'
-
-import type { SqliteDb } from '../adapter-types.ts'
-import { SyncError, UnexpectedError } from '../adapter-types.ts'
+import { type IntentionalShutdownCause, type MaterializeError, type SqliteDb, UnknownError } from '../adapter-types.ts'
 import { makeMaterializerHash } from '../materializer-helper.ts'
 import type { LiveStoreSchema } from '../schema/mod.ts'
-import { EventSequenceNumber, getEventDef, LiveStoreEvent, SystemTables } from '../schema/mod.ts'
-import { LeaderAheadError } from '../sync/sync.ts'
+import { EventSequenceNumber, LiveStoreEvent, resolveEventDef, SystemTables } from '../schema/mod.ts'
+import {
+  type InvalidPullError,
+  type InvalidPushError,
+  type IsOfflineError,
+  LeaderAheadError,
+  type SyncBackend,
+} from '../sync/sync.ts'
 import * as SyncState from '../sync/syncstate.ts'
 import { sql } from '../util.ts'
 import * as Eventlog from './eventlog.ts'
@@ -31,7 +38,7 @@ import type { InitialBlockingSyncContext, LeaderSyncProcessor } from './types.ts
 import { LeaderThreadCtx } from './types.ts'
 
 type LocalPushQueueItem = [
-  event: LiveStoreEvent.EncodedWithMeta,
+  event: LiveStoreEvent.Client.EncodedWithMeta,
   deferred: Deferred.Deferred<void, LeaderAheadError> | undefined,
 ]
 
@@ -71,6 +78,7 @@ export const makeLeaderSyncProcessor = ({
   initialBlockingSyncContext,
   initialSyncState,
   onError,
+  livePull,
   params,
   testing,
 }: {
@@ -90,23 +98,26 @@ export const makeLeaderSyncProcessor = ({
      */
     backendPushBatchSize?: number
   }
+  /**
+   * Whether the sync backend should reactively pull new events from the sync backend
+   * When `false`, the sync processor will only do an initial pull
+   */
+  livePull: boolean
   testing: {
     delays?: {
       localPushProcessing?: Effect.Effect<void>
     }
   }
-}): Effect.Effect<LeaderSyncProcessor, UnexpectedError, Scope.Scope> =>
+}): Effect.Effect<LeaderSyncProcessor, UnknownError, Scope.Scope> =>
   Effect.gen(function* () {
-    const syncBackendPushQueue = yield* BucketQueue.make<LiveStoreEvent.EncodedWithMeta>()
+    const syncBackendPushQueue = yield* BucketQueue.make<LiveStoreEvent.Client.EncodedWithMeta>()
     const localPushBatchSize = params.localPushBatchSize ?? 1
     const backendPushBatchSize = params.backendPushBatchSize ?? 2
 
     const syncStateSref = yield* SubscriptionRef.make<SyncState.SyncState | undefined>(undefined)
 
-    const isClientEvent = (eventEncoded: LiveStoreEvent.EncodedWithMeta) => {
-      const { eventDef } = getEventDef(schema, eventEncoded.name)
-      return eventDef.options.clientOnly
-    }
+    const isClientEvent = (eventEncoded: LiveStoreEvent.Client.EncodedWithMeta) =>
+      schema.eventsDefsMap.get(eventEncoded.name)?.options.clientOnly ?? false
 
     const connectedClientSessionPullQueues = yield* makePullQueueSet
 
@@ -135,9 +146,9 @@ export const makeLeaderSyncProcessor = ({
      *
      * Thus the purpose of the pushHeadRef is the guard the integrity of the local push queue
      */
-    const pushHeadRef = { current: EventSequenceNumber.ROOT }
-    const advancePushHead = (eventNum: EventSequenceNumber.EventSequenceNumber) => {
-      pushHeadRef.current = EventSequenceNumber.max(pushHeadRef.current, eventNum)
+    const pushHeadRef = { current: EventSequenceNumber.Client.ROOT }
+    const advancePushHead = (eventNum: EventSequenceNumber.Client.Composite) => {
+      pushHeadRef.current = EventSequenceNumber.Client.max(pushHeadRef.current, eventNum)
     }
 
     // NOTE: New events are only pushed to sync backend after successful local push processing
@@ -180,14 +191,32 @@ export const makeLeaderSyncProcessor = ({
         const syncState = yield* syncStateSref
         if (syncState === undefined) return shouldNeverHappen('Not initialized')
 
-        const { eventDef } = getEventDef(schema, name)
+        const resolution = yield* resolveEventDef(schema, {
+          operation: '@livestore/common:LeaderSyncProcessor:pushPartial',
+          event: {
+            name,
+            args,
+            clientId,
+            sessionId,
+            seqNum: syncState.localHead,
+          },
+        }).pipe(UnknownError.mapToUnknownError)
 
-        const eventEncoded = new LiveStoreEvent.EncodedWithMeta({
+        if (resolution._tag === 'unknown') {
+          // Ignore partial pushes for unrecognised events – they are still
+          // persisted server-side once a schema update ships.
+          return
+        }
+
+        const eventEncoded = new LiveStoreEvent.Client.EncodedWithMeta({
           name,
           args,
           clientId,
           sessionId,
-          ...EventSequenceNumber.nextPair({ seqNum: syncState.localHead, isClient: eventDef.options.clientOnly }),
+          ...EventSequenceNumber.Client.nextPair({
+            seqNum: syncState.localHead,
+            isClient: resolution.eventDef.options.clientOnly,
+          }),
         })
 
         yield* push([eventEncoded])
@@ -213,10 +242,10 @@ export const makeLeaderSyncProcessor = ({
       // Rehydrate sync queue
       if (initialSyncState.pending.length > 0) {
         const globalPendingEvents = initialSyncState.pending
-          // Don't sync clientOnly events
+          // Don't sync client-local events
           .filter((eventEncoded) => {
-            const { eventDef } = getEventDef(schema, eventEncoded.name)
-            return eventDef.options.clientOnly === false
+            const eventDef = schema.eventsDefsMap.get(eventEncoded.name)
+            return eventDef === undefined ? true : eventDef.options.clientOnly === false
           })
 
         if (globalPendingEvents.length > 0) {
@@ -224,12 +253,31 @@ export const makeLeaderSyncProcessor = ({
         }
       }
 
-      const shutdownOnError = (cause: unknown) =>
+      const maybeShutdownOnError = (
+        cause: Cause.Cause<
+          | UnknownError
+          | IntentionalShutdownCause
+          | IsOfflineError
+          | InvalidPushError
+          | InvalidPullError
+          | MaterializeError
+        >,
+      ) =>
         Effect.gen(function* () {
-          if (onError === 'shutdown') {
-            yield* shutdownChannel.send(UnexpectedError.make({ cause }))
-            yield* Effect.die(cause)
+          if (onError === 'ignore') {
+            if (LS_DEV) {
+              yield* Effect.logDebug(
+                `Ignoring sync error (${cause._tag === 'Fail' ? cause.error._tag : cause._tag})`,
+                Cause.pretty(cause),
+              )
+            }
+            return
           }
+
+          const errorToSend = Cause.isFailType(cause) ? cause.error : UnknownError.make({ cause })
+          yield* shutdownChannel.send(errorToSend).pipe(Effect.orDie)
+
+          return yield* Effect.die(cause)
         })
 
       yield* backgroundApplyLocalPushes({
@@ -246,20 +294,19 @@ export const makeLeaderSyncProcessor = ({
         testing: {
           delay: testing?.delays?.localPushProcessing,
         },
-      }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped)
+      }).pipe(Effect.catchAllCause(maybeShutdownOnError), Effect.forkScoped)
 
-      const backendPushingFiberHandle = yield* FiberHandle.make()
+      const backendPushingFiberHandle = yield* FiberHandle.make<void, never>()
       const backendPushingEffect = backgroundBackendPushing({
         syncBackendPushQueue,
         otelSpan,
         devtoolsLatch: ctxRef.current?.devtoolsLatch,
         backendPushBatchSize,
-      }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError))
+      }).pipe(Effect.catchAllCause(maybeShutdownOnError))
 
       yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect)
 
       yield* backgroundBackendPulling({
-        initialBackendHead: initialSyncState.upstreamHead.global,
         isClientEvent,
         restartBackendPushing: (filteredRebasedPending) =>
           Effect.gen(function* () {
@@ -276,13 +323,24 @@ export const makeLeaderSyncProcessor = ({
         syncStateSref,
         localPushesLatch,
         pullLatch,
+        livePull,
         dbState,
         otelSpan,
         initialBlockingSyncContext,
         devtoolsLatch: ctxRef.current?.devtoolsLatch,
         connectedClientSessionPullQueues,
         advancePushHead,
-      }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped)
+      }).pipe(
+        Effect.retry({
+          // We want to retry pulling if we've lost connection to the sync backend
+          while: (cause) => cause._tag === 'IsOfflineError',
+        }),
+        Effect.catchAllCause(maybeShutdownOnError),
+        // Needed to avoid `Fiber terminated with an unhandled error` logs which seem to happen because of the `Effect.retry` above.
+        // This might be a bug in Effect. Only seems to happen in the browser.
+        Effect.provide(Layer.setUnhandledErrorLogLevel(Option.none())),
+        Effect.forkScoped,
+      )
 
       return { initialLeaderHead: initialSyncState.localHead }
     }).pipe(Effect.withSpanScoped('@livestore/common:LeaderSyncProcessor:boot'))
@@ -348,9 +406,9 @@ const backgroundApplyLocalPushes = ({
   localPushesLatch: Effect.Latch
   localPushesQueue: BucketQueue.BucketQueue<LocalPushQueueItem>
   syncStateSref: SubscriptionRef.SubscriptionRef<SyncState.SyncState | undefined>
-  syncBackendPushQueue: BucketQueue.BucketQueue<LiveStoreEvent.EncodedWithMeta>
+  syncBackendPushQueue: BucketQueue.BucketQueue<LiveStoreEvent.Client.EncodedWithMeta>
   schema: LiveStoreSchema
-  isClientEvent: (eventEncoded: LiveStoreEvent.EncodedWithMeta) => boolean
+  isClientEvent: (eventEncoded: LiveStoreEvent.Client.EncodedWithMeta) => boolean
   otelSpan: otel.Span | undefined
   connectedClientSessionPullQueues: PullQueueSet
   localPushBatchSize: number
@@ -379,33 +437,58 @@ const backgroundApplyLocalPushes = ({
 
       // Since the rebase generation might have changed since enqueuing, we need to filter out items with older generation
       // It's important that we filter after we got localPushesLatch, otherwise we might filter with the old generation
-      const [newEvents, deferreds] = pipe(
+      const [droppedItems, filteredItems] = ReadonlyArray.partition(
         batchItems,
-        ReadonlyArray.filter(([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration === currentRebaseGeneration),
-        ReadonlyArray.unzip,
+        ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= currentRebaseGeneration,
       )
 
-      if (newEvents.length === 0) {
-        // console.log('dropping old-gen batch', currentLocalPushGenerationRef.current)
-        // Allow the backend pulling to start
+      if (droppedItems.length > 0) {
+        otelSpan?.addEvent(`push:drop-old-generation`, {
+          droppedCount: droppedItems.length,
+          currentRebaseGeneration,
+        })
+
+        /**
+         * Dropped pushes may still have a deferred awaiting completion.
+         * Fail it so the caller learns the leader advanced and resubmits with the updated generation.
+         */
+        yield* Effect.forEach(
+          droppedItems.filter(
+            (item): item is [LiveStoreEvent.Client.EncodedWithMeta, Deferred.Deferred<void, LeaderAheadError>] =>
+              item[1] !== undefined,
+          ),
+          ([eventEncoded, deferred]) =>
+            Deferred.fail(
+              deferred,
+              LeaderAheadError.make({
+                minimumExpectedNum: syncState.localHead,
+                providedNum: eventEncoded.seqNum,
+              }),
+            ),
+        )
+      }
+
+      if (filteredItems.length === 0) {
         yield* pullLatch.open
         continue
       }
 
+      const [newEvents, deferreds] = ReadonlyArray.unzip(filteredItems)
+
       const mergeResult = SyncState.merge({
         syncState,
         payload: { _tag: 'local-push', newEvents },
         isClientEvent,
-        isEqualEvent: LiveStoreEvent.isEqualEncoded,
+        isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
       })
 
       switch (mergeResult._tag) {
-        case 'unexpected-error': {
-          otelSpan?.addEvent(`push:unexpected-error`, {
+        case 'unknown-error': {
+          otelSpan?.addEvent(`push:unknown-error`, {
             batchSize: newEvents.length,
             newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
           })
-          return yield* new SyncError({ cause: mergeResult.message })
+          return yield* new UnknownError({ cause: mergeResult.message })
         }
         case 'rebase': {
           return shouldNeverHappen('The leader thread should never have to rebase due to a local push')
@@ -474,10 +557,10 @@ const backgroundApplyLocalPushes = ({
         mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
       })
 
-      // Don't sync clientOnly events
+      // Don't sync client-local events
       const filteredBatch = mergeResult.newEvents.filter((eventEncoded) => {
-        const { eventDef } = getEventDef(schema, eventEncoded.name)
-        return eventDef.options.clientOnly === false
+        const eventDef = schema.eventsDefsMap.get(eventEncoded.name)
+        return eventDef === undefined ? true : eventDef.options.clientOnly === false
       })
 
       yield* BucketQueue.offerAll(syncBackendPushQueue, filteredBatch)
@@ -490,13 +573,13 @@ const backgroundApplyLocalPushes = ({
   })
 
 type MaterializeEventsBatch = (_: {
-  batchItems: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>
+  batchItems: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>
   /**
    * The deferreds are used by the caller to know when the mutation has been processed.
    * Indexes are aligned with `batchItems`
    */
   deferreds: ReadonlyArray<Deferred.Deferred<void, LeaderAheadError> | undefined> | undefined
-}) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
+}) => Effect.Effect<void, MaterializeError, LeaderThreadCtx>
 
 // TODO how to handle errors gracefully
 const materializeEventsBatch: MaterializeEventsBatch = ({ batchItems, deferreds }) =>
@@ -536,44 +619,46 @@ const materializeEventsBatch: MaterializeEventsBatch = ({ batchItems, deferreds
       attributes: { batchSize: batchItems.length },
     }),
     Effect.tapCauseLogPretty,
-    UnexpectedError.mapToUnexpectedError,
   )
 
 const backgroundBackendPulling = ({
-  initialBackendHead,
   isClientEvent,
   restartBackendPushing,
   otelSpan,
   dbState,
   syncStateSref,
   localPushesLatch,
+  livePull,
   pullLatch,
   devtoolsLatch,
   initialBlockingSyncContext,
   connectedClientSessionPullQueues,
   advancePushHead,
 }: {
-  initialBackendHead: EventSequenceNumber.GlobalEventSequenceNumber
-  isClientEvent: (eventEncoded: LiveStoreEvent.EncodedWithMeta) => boolean
+  isClientEvent: (eventEncoded: LiveStoreEvent.Client.EncodedWithMeta) => boolean
   restartBackendPushing: (
-    filteredRebasedPending: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>,
-  ) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx | HttpClient.HttpClient>
+    filteredRebasedPending: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>,
+  ) => Effect.Effect<void, UnknownError, LeaderThreadCtx | HttpClient.HttpClient>
   otelSpan: otel.Span | undefined
   syncStateSref: SubscriptionRef.SubscriptionRef<SyncState.SyncState | undefined>
   dbState: SqliteDb
   localPushesLatch: Effect.Latch
   pullLatch: Effect.Latch
+  livePull: boolean
   devtoolsLatch: Effect.Latch | undefined
   initialBlockingSyncContext: InitialBlockingSyncContext
   connectedClientSessionPullQueues: PullQueueSet
-  advancePushHead: (eventNum: EventSequenceNumber.EventSequenceNumber) => void
+  advancePushHead: (eventNum: EventSequenceNumber.Client.Composite) => void
 }) =>
   Effect.gen(function* () {
     const { syncBackend, dbState: db, dbEventlog, schema } = yield* LeaderThreadCtx
 
     if (syncBackend === undefined) return
 
-    const onNewPullChunk = (newEvents: LiveStoreEvent.EncodedWithMeta[], remaining: number) =>
+    const onNewPullChunk = (
+      newEvents: LiveStoreEvent.Client.EncodedWithMeta[],
+      pageInfo: SyncBackend.PullResPageInfo,
+    ) =>
       Effect.gen(function* () {
         if (newEvents.length === 0) return
 
@@ -594,18 +679,18 @@ const backgroundBackendPulling = ({
           syncState,
           payload: SyncState.PayloadUpstreamAdvance.make({ newEvents }),
           isClientEvent,
-          isEqualEvent: LiveStoreEvent.isEqualEncoded,
+          isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
           ignoreClientEvents: true,
         })
 
         if (mergeResult._tag === 'reject') {
           return shouldNeverHappen('The leader thread should never reject upstream advances')
-        } else if (mergeResult._tag === 'unexpected-error') {
-          otelSpan?.addEvent(`pull:unexpected-error`, {
+        } else if (mergeResult._tag === 'unknown-error') {
+          otelSpan?.addEvent(`pull:unknown-error`, {
             newEventsCount: newEvents.length,
             newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
           })
-          return yield* new SyncError({ cause: mergeResult.message })
+          return yield* new UnknownError({ cause: mergeResult.message })
         }
 
         const newBackendHead = newEvents.at(-1)!.seqNum
@@ -621,8 +706,8 @@ const backgroundBackendPulling = ({
           })
 
           const globalRebasedPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
-            const { eventDef } = getEventDef(schema, event.name)
-            return eventDef.options.clientOnly === false
+            const eventDef = schema.eventsDefsMap.get(event.name)
+            return eventDef === undefined ? true : eventDef.options.clientOnly === false
           })
           yield* restartBackendPushing(globalRebasedPendingEvents)
 
@@ -644,6 +729,13 @@ const backgroundBackendPulling = ({
             mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
           })
 
+          // Ensure push fiber is active after advance by restarting with current pending (non-client) events
+          const globalPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
+            const eventDef = schema.eventsDefsMap.get(event.name)
+            return eventDef === undefined ? true : eventDef.options.clientOnly === false
+          })
+          yield* restartBackendPushing(globalPendingEvents)
+
           yield* connectedClientSessionPullQueues.offer({
             payload: SyncState.payloadFromMergeResult(mergeResult),
             leaderHead: mergeResult.newSyncState.localHead,
@@ -654,10 +746,10 @@ const backgroundBackendPulling = ({
             // `newEvents` instead which we filter via `mergeResult.confirmedEvents`
             const confirmedNewEvents = newEvents.filter((event) =>
               mergeResult.confirmedEvents.some((confirmedEvent) =>
-                EventSequenceNumber.isEqual(event.seqNum, confirmedEvent.seqNum),
+                EventSequenceNumber.Client.isEqual(event.seqNum, confirmedEvent.seqNum),
               ),
             )
-            yield* Eventlog.updateSyncMetadata(confirmedNewEvents)
+            yield* Eventlog.updateSyncMetadata(confirmedNewEvents).pipe(UnknownError.mapToUnknownError)
           }
         }
 
@@ -671,18 +763,20 @@ const backgroundBackendPulling = ({
         yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState)
 
         // Allow local pushes to be processed again
-        if (remaining === 0) {
+        if (pageInfo._tag === 'NoMore') {
           yield* localPushesLatch.open
         }
       })
 
-    const cursorInfo = yield* Eventlog.getSyncBackendCursorInfo({ remoteHead: initialBackendHead })
+    const syncState = yield* syncStateSref
+    if (syncState === undefined) return shouldNeverHappen('Not initialized')
+    const cursorInfo = yield* Eventlog.getSyncBackendCursorInfo({ remoteHead: syncState.upstreamHead.global })
 
     const hashMaterializerResult = makeMaterializerHash({ schema, dbState })
 
-    yield* syncBackend.pull(cursorInfo).pipe(
+    yield* syncBackend.pull(cursorInfo, { live: livePull }).pipe(
       // TODO only take from queue while connected
-      Stream.tap(({ batch, remaining }) =>
+      Stream.tap(({ batch, pageInfo }) =>
         Effect.gen(function* () {
           // yield* Effect.spanEvent('batch', {
           //   attributes: {
@@ -690,31 +784,31 @@ const backgroundBackendPulling = ({
           //     batch: TRACE_VERBOSE ? batch : undefined,
           //   },
           // })
-
           // NOTE we only want to take process events when the sync backend is connected
           // (e.g. needed for simulating being offline)
           // TODO remove when there's a better way to handle this in stream above
           yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
-
           yield* onNewPullChunk(
             batch.map((_) =>
-              LiveStoreEvent.EncodedWithMeta.fromGlobal(_.eventEncoded, {
+              LiveStoreEvent.Client.EncodedWithMeta.fromGlobal(_.eventEncoded, {
                 syncMetadata: _.metadata,
                 // TODO we can't really know the materializer result here yet beyond the first event batch item as we need to materialize it one by one first
                 // This is a bug and needs to be fixed https://github.com/livestorejs/livestore/issues/503#issuecomment-3114533165
-                materializerHashLeader: hashMaterializerResult(LiveStoreEvent.encodedFromGlobal(_.eventEncoded)),
+                materializerHashLeader: hashMaterializerResult(LiveStoreEvent.Global.toClientEncoded(_.eventEncoded)),
                 materializerHashSession: Option.none(),
               }),
             ),
-            remaining,
+            pageInfo,
           )
-
-          yield* initialBlockingSyncContext.update({ processed: batch.length, remaining })
+          yield* initialBlockingSyncContext.update({ processed: batch.length, pageInfo })
         }),
       ),
       Stream.runDrain,
       Effect.interruptible,
     )
+
+    // Should only ever happen when livePull is false
+    yield* Effect.logDebug('backend-pulling finished', { livePull })
   }).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pulling'))
 
 const backgroundBackendPushing = ({
@@ -723,7 +817,7 @@ const backgroundBackendPushing = ({
   devtoolsLatch,
   backendPushBatchSize,
 }: {
-  syncBackendPushQueue: BucketQueue.BucketQueue<LiveStoreEvent.EncodedWithMeta>
+  syncBackendPushQueue: BucketQueue.BucketQueue<LiveStoreEvent.Client.EncodedWithMeta>
   otelSpan: otel.Span | undefined
   devtoolsLatch: Effect.Latch | undefined
   backendPushBatchSize: number
@@ -748,21 +842,57 @@ const backgroundBackendPushing = ({
         batch: TRACE_VERBOSE ? JSON.stringify(queueItems) : undefined,
       })
 
-      // TODO handle push errors (should only happen during concurrent pull+push)
-      const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either)
+      // Push with declarative retry/backoff using Effect schedules
+      // - Exponential backoff starting at 1s and doubling (1s, 2s, 4s, 8s, 16s, 30s ...)
+      // - Delay clamped at 30s (continues retrying at 30s)
+      // - Resets automatically after successful push
+      // TODO(metrics): expose counters/gauges for retry attempts and queue health via devtools/metrics
+
+      // Only retry for transient UnknownError cases
+      const isRetryable = (err: InvalidPushError | IsOfflineError) =>
+        err._tag === 'InvalidPushError' && err.cause._tag === 'LiveStore.UnknownError'
+
+      // Input: InvalidPushError | IsOfflineError, Output: Duration
+      const retrySchedule: Schedule.Schedule<Duration.DurationInput, InvalidPushError | IsOfflineError> =
+        Schedule.exponential(Duration.seconds(1)).pipe(
+          Schedule.andThenEither(Schedule.spaced(Duration.seconds(30))), // clamp at 30 second intervals
+          Schedule.compose(Schedule.elapsed),
+          Schedule.whileInput(isRetryable),
+        )
+
+      yield* Effect.gen(function* () {
+        const iteration = yield* Schedule.CurrentIterationMetadata
 
-      if (pushResult._tag === 'Left') {
-        if (LS_DEV) {
-          yield* Effect.logDebug('handled backend-push-error', { error: pushResult.left.toString() })
+        const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either)
+
+        const retries = iteration.recurrence
+        if (retries > 0 && pushResult._tag === 'Right') {
+          otelSpan?.addEvent('backend-push-retry-success', { retries, batchSize: queueItems.length })
         }
-        otelSpan?.addEvent('backend-push-error', { error: pushResult.left.toString() })
-        // wait for interrupt caused by background pulling which will then restart pushing
-        return yield* Effect.never
-      }
+
+        if (pushResult._tag === 'Left') {
+          otelSpan?.addEvent('backend-push-error', {
+            error: pushResult.left.toString(),
+            retries,
+            batchSize: queueItems.length,
+          })
+          const error = pushResult.left
+          if (
+            error._tag === 'IsOfflineError' ||
+            (error._tag === 'InvalidPushError' && error.cause._tag === 'ServerAheadError')
+          ) {
+            // It's a core part of the sync protocol that the sync backend will emit a new pull chunk alongside the ServerAheadError
+            yield* Effect.logDebug('handled backend-push-error (waiting for interupt caused by pull)', { error })
+            return yield* Effect.never
+          }
+
+          return yield* error
+        }
+      }).pipe(Effect.retry(retrySchedule))
     }
   }).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pushing'))
 
-const trimChangesetRows = (db: SqliteDb, newHead: EventSequenceNumber.EventSequenceNumber) => {
+const trimChangesetRows = (db: SqliteDb, newHead: EventSequenceNumber.Client.Composite) => {
   // Since we're using the session changeset rows to query for the current head,
   // we're keeping at least one row for the current head, and thus are using `<` instead of `<=`
   db.execute(sql`DELETE FROM ${SystemTables.SESSION_CHANGESET_META_TABLE} WHERE seqNumGlobal < ${newHead.global}`)
@@ -770,16 +900,16 @@ const trimChangesetRows = (db: SqliteDb, newHead: EventSequenceNumber.EventSeque
 
 interface PullQueueSet {
   makeQueue: (
-    cursor: EventSequenceNumber.EventSequenceNumber,
+    cursor: EventSequenceNumber.Client.Composite,
   ) => Effect.Effect<
     Queue.Queue<{ payload: typeof SyncState.PayloadUpstream.Type }>,
-    UnexpectedError,
+    UnknownError,
     Scope.Scope | LeaderThreadCtx
   >
   offer: (item: {
     payload: typeof SyncState.PayloadUpstream.Type
-    leaderHead: EventSequenceNumber.EventSequenceNumber
-  }) => Effect.Effect<void, UnexpectedError>
+    leaderHead: EventSequenceNumber.Client.Composite
+  }) => Effect.Effect<void, UnknownError>
 }
 
 const makePullQueueSet = Effect.gen(function* () {
@@ -809,17 +939,17 @@ const makePullQueueSet = Effect.gen(function* () {
 
       const payloadsSinceCursor = Array.from(cachedPayloads.entries())
         .flatMap(([seqNumStr, payloads]) =>
-          payloads.map((payload) => ({ payload, seqNum: EventSequenceNumber.fromString(seqNumStr) })),
+          payloads.map((payload) => ({ payload, seqNum: EventSequenceNumber.Client.fromString(seqNumStr) })),
         )
-        .filter(({ seqNum }) => EventSequenceNumber.isGreaterThan(seqNum, cursor))
-        .toSorted((a, b) => EventSequenceNumber.compare(a.seqNum, b.seqNum))
+        .filter(({ seqNum }) => EventSequenceNumber.Client.isGreaterThan(seqNum, cursor))
+        .toSorted((a, b) => EventSequenceNumber.Client.compare(a.seqNum, b.seqNum))
         .map(({ payload }) => {
           if (payload._tag === 'upstream-advance') {
             return {
               payload: {
                 _tag: 'upstream-advance' as const,
                 newEvents: ReadonlyArray.dropWhile(payload.newEvents, (eventEncoded) =>
-                  EventSequenceNumber.isGreaterThanOrEqual(cursor, eventEncoded.seqNum),
+                  EventSequenceNumber.Client.isGreaterThanOrEqual(cursor, eventEncoded.seqNum),
                 ),
               },
             }
@@ -865,7 +995,7 @@ const makePullQueueSet = Effect.gen(function* () {
 
   const offer: PullQueueSet['offer'] = (item) =>
     Effect.gen(function* () {
-      const seqNumStr = EventSequenceNumber.toString(item.leaderHead)
+      const seqNumStr = EventSequenceNumber.Client.toString(item.leaderHead)
       if (cachedPayloads.has(seqNumStr)) {
         cachedPayloads.get(seqNumStr)!.push(item.payload)
       } else {
@@ -890,26 +1020,35 @@ const makePullQueueSet = Effect.gen(function* () {
   }
 })
 
+/**
+ * Validate a client-provided batch before it is admitted to the leader queue.
+ * Ensures the numbers form a strictly increasing chain and that the first
+ * event sits ahead of the current push head.
+ */
 const validatePushBatch = (
-  batch: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>,
-  pushHead: EventSequenceNumber.EventSequenceNumber,
+  batch: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>,
+  pushHead: EventSequenceNumber.Client.Composite,
 ) =>
   Effect.gen(function* () {
     if (batch.length === 0) {
       return
     }
 
-    // Make sure batch is monotonically increasing
+    // Example: session A already enqueued e1…e6 while session B (same client, different
+    // session) still believes the head is e1 and submits [e2, e7, e8]. The numbers look
+    // monotonic from B’s perspective, but we must reject and force B to rebase locally
+    // so the leader never regresses.
     for (let i = 1; i < batch.length; i++) {
-      if (EventSequenceNumber.isGreaterThanOrEqual(batch[i - 1]!.seqNum, batch[i]!.seqNum)) {
-        shouldNeverHappen(
-          `Events must be ordered in monotonically ascending order by eventNum. Received: [${batch.map((e) => EventSequenceNumber.toString(e.seqNum)).join(', ')}]`,
-        )
+      if (EventSequenceNumber.Client.isGreaterThanOrEqual(batch[i - 1]!.seqNum, batch[i]!.seqNum)) {
+        return yield* LeaderAheadError.make({
+          minimumExpectedNum: batch[i - 1]!.seqNum,
+          providedNum: batch[i]!.seqNum,
+        })
       }
     }
 
     // Make sure smallest sequence number is > pushHead
-    if (EventSequenceNumber.isGreaterThanOrEqual(pushHead, batch[0]!.seqNum)) {
+    if (EventSequenceNumber.Client.isGreaterThanOrEqual(pushHead, batch[0]!.seqNum)) {
       return yield* LeaderAheadError.make({
         minimumExpectedNum: pushHead,
         providedNum: batch[0]!.seqNum,