@livestore/common 0.3.0-dev.8 → 0.3.0

package/src/leader-thread/LeaderSyncProcessor.ts

@@ -1,528 +1,763 @@
-import { shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
-import type { HttpClient, Scope } from '@livestore/utils/effect'
+import { casesHandled, isNotUndefined, LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
+import type { HttpClient, Runtime, Scope, Tracer } from '@livestore/utils/effect'
 import {
   BucketQueue,
   Deferred,
   Effect,
   Exit,
-  Fiber,
   FiberHandle,
-  Option,
   OtelTracer,
+  Queue,
   ReadonlyArray,
-  Ref,
-  Schema,
   Stream,
+  Subscribable,
   SubscriptionRef,
 } from '@livestore/utils/effect'
 import type * as otel from '@opentelemetry/api'
 
-import type { SynchronousDatabase } from '../adapter-types.js'
+import type { SqliteDb } from '../adapter-types.js'
 import { UnexpectedError } from '../adapter-types.js'
-import type { LiveStoreSchema, SessionChangesetMetaRow } from '../schema/mod.js'
-import {
-  EventId,
-  MUTATION_LOG_META_TABLE,
-  MutationEvent,
-  mutationLogMetaTable,
-  SESSION_CHANGESET_META_TABLE,
-} from '../schema/mod.js'
-import { updateRows } from '../sql-queries/index.js'
-import { InvalidPushError } from '../sync/sync.js'
+import type { LiveStoreSchema } from '../schema/mod.js'
+import { EventSequenceNumber, getEventDef, LiveStoreEvent, SystemTables } from '../schema/mod.js'
+import { LeaderAheadError } from '../sync/sync.js'
 import * as SyncState from '../sync/syncstate.js'
 import { sql } from '../util.js'
-import { makeApplyMutation } from './apply-mutation.js'
-import { execSql } from './connection.js'
-import { getBackendHeadFromDb, getLocalHeadFromDb, getMutationEventsSince, updateBackendHead } from './mutationlog.js'
-import type { InitialBlockingSyncContext, InitialSyncInfo, LeaderSyncProcessor } from './types.js'
+import * as Eventlog from './eventlog.js'
+import { rollback } from './materialize-event.js'
+import type { InitialBlockingSyncContext, LeaderSyncProcessor } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
-type ProcessorStateInit = {
-  _tag: 'init'
-}
-
-type ProcessorStateInSync = {
-  _tag: 'in-sync'
-  syncState: SyncState.SyncState
-}
-
-type ProcessorStateApplyingSyncStateAdvance = {
-  _tag: 'applying-syncstate-advance'
-  origin: 'pull' | 'push'
-  syncState: SyncState.SyncState
-  // TODO re-introduce this
-  // proccesHead: EventId
-  fiber: Fiber.RuntimeFiber<void, UnexpectedError>
-}
-
-type ProcessorState = ProcessorStateInit | ProcessorStateInSync | ProcessorStateApplyingSyncStateAdvance
+type LocalPushQueueItem = [
+  event: LiveStoreEvent.EncodedWithMeta,
+  deferred: Deferred.Deferred<void, LeaderAheadError> | undefined,
+  /** Used to determine whether the batch has become invalid due to a rejected local push batch */
+  generation: number,
+]
 
 /**
- * The general idea of the sync processor is to "follow the sync state"
- * and apply/rollback mutations as needed to the read model and mutation log.
- * The leader sync processor is also responsible for
- * - broadcasting mutations to client sessions via the pull queues.
- * - pushing mutations to the sync backend
+ * The LeaderSyncProcessor manages synchronization of events between
+ * the local state and the sync backend, ensuring efficient and orderly processing.
  *
- * In the leader sync processor, pulling always has precedence over pushing.
+ * In the LeaderSyncProcessor, pulling always has precedence over pushing.
  *
- * External events:
- * - Mutation pushed from client session
- * - Mutation pushed from devtools (via pushPartial)
- * - Mutation pulled from sync backend
+ * Responsibilities:
+ * - Queueing incoming local events in a localPushesQueue.
+ * - Broadcasting events to client sessions via pull queues.
+ * - Pushing events to the sync backend.
  *
- * The machine can be in the following states:
- * - in-sync: fully synced with remote, now idling
- * - applying-syncstate-advance (with pointer to current progress in case of rebase interrupt)
+ * Notes:
  *
- * Transitions:
- * - in-sync -> applying-syncstate-advance
- * - applying-syncstate-advance -> in-sync
- * - applying-syncstate-advance -> applying-syncstate-advance (need to interrupt previous operation)
+ * local push processing:
+ * - localPushesQueue:
+ *   - Maintains events in ascending order.
+ *   - Uses `Deferred` objects to resolve/reject events based on application success.
+ * - Processes events from the queue, applying events in batches.
+ * - Controlled by a `Latch` to manage execution flow.
+ * - The latch closes on pull receipt and re-opens post-pull completion.
+ * - Processes up to `maxBatchSize` events per cycle.
  *
- * Queuing vs interrupting behaviour:
- * - Operations caused by pull can never be interrupted
- * - Incoming pull can interrupt current push
- * - Incoming pull needs to wait to previous pull to finish
- * - Incoming push needs to wait to previous push to finish
+ * Currently we're advancing the db read model and eventlog in lockstep, but we could also decouple this in the future
+ *
+ * Tricky concurrency scenarios:
+ * - Queued local push batches becoming invalid due to a prior local push item being rejected.
+ *   Solution: Introduce a generation number for local push batches which is used to filter out old batches items in case of rejection.
  *
- * Backend pushing:
- * - continously push to backend
- * - only interrupted and restarted on rebase
  */
 export const makeLeaderSyncProcessor = ({
   schema,
-  dbMissing,
-  dbLog,
+  dbEventlogMissing,
+  dbEventlog,
+  dbState,
+  dbStateMissing,
   initialBlockingSyncContext,
+  onError,
+  params,
+  testing,
 }: {
   schema: LiveStoreSchema
-  /** Only used to know whether we can safely query dbLog during setup execution */
-  dbMissing: boolean
-  dbLog: SynchronousDatabase
+  /** Only used to know whether we can safely query dbEventlog during setup execution */
+  dbEventlogMissing: boolean
+  dbEventlog: SqliteDb
+  dbState: SqliteDb
+  /** Only used to know whether we can safely query dbState during setup execution */
+  dbStateMissing: boolean
   initialBlockingSyncContext: InitialBlockingSyncContext
+  onError: 'shutdown' | 'ignore'
+  params: {
+    /**
+     * @default 10
+     */
+    localPushBatchSize?: number
+    /**
+     * @default 50
+     */
+    backendPushBatchSize?: number
+  }
+  testing: {
+    delays?: {
+      localPushProcessing?: Effect.Effect<void>
+    }
+  }
 }): Effect.Effect<LeaderSyncProcessor, UnexpectedError, Scope.Scope> =>
   Effect.gen(function* () {
-    const syncBackendQueue = yield* BucketQueue.make<MutationEvent.EncodedWithMeta>()
+    const syncBackendPushQueue = yield* BucketQueue.make<LiveStoreEvent.EncodedWithMeta>()
+    const localPushBatchSize = params.localPushBatchSize ?? 10
+    const backendPushBatchSize = params.backendPushBatchSize ?? 50
 
-    const stateRef = yield* Ref.make<ProcessorState>({ _tag: 'init' })
+    const syncStateSref = yield* SubscriptionRef.make<SyncState.SyncState | undefined>(undefined)
 
-    const semaphore = yield* Effect.makeSemaphore(1)
-
-    const isLocalEvent = (mutationEventEncoded: MutationEvent.EncodedWithMeta) => {
-      const mutationDef = schema.mutations.get(mutationEventEncoded.mutation)!
-      return mutationDef.options.localOnly
+    const isClientEvent = (eventEncoded: LiveStoreEvent.EncodedWithMeta) => {
+      const { eventDef } = getEventDef(schema, eventEncoded.name)
+      return eventDef.options.clientOnly
     }
 
-    const spanRef = { current: undefined as otel.Span | undefined }
-    const applyMutationItemsRef = { current: undefined as ApplyMutationItems | undefined }
-
-    // TODO get rid of counters once Effect semaphore ordering is fixed
-    let counterRef = 0
-    let expectedCounter = 0
-
-    /*
-    TODO: refactor
-    - Pushes go directly into a Mailbox
-    - Have a worker fiber that takes from the mailbox (wouldn't need a semaphore)
-    */
+    const connectedClientSessionPullQueues = yield* makePullQueueSet
+
+    /**
+     * Tracks generations of queued local push events.
+     * If a local-push batch is rejected, all subsequent push queue items with the same generation are also rejected,
+     * even if they would be valid on their own.
+     */
+    // TODO get rid of this in favour of the `mergeGeneration` event sequence number field
+    const currentLocalPushGenerationRef = { current: 0 }
+
+    type MergeCounter = number
+    const mergeCounterRef = { current: dbStateMissing ? 0 : yield* getMergeCounterFromDb(dbState) }
+    const mergePayloads = new Map<MergeCounter, typeof SyncState.PayloadUpstream.Type>()
+
+    // This context depends on data from `boot`, we should find a better implementation to avoid this ref indirection.
+    const ctxRef = {
+      current: undefined as
+        | undefined
+        | {
+            otelSpan: otel.Span | undefined
+            span: Tracer.Span
+            devtoolsLatch: Effect.Latch | undefined
+            runtime: Runtime.Runtime<LeaderThreadCtx>
+          },
+    }
 
-    const waitForSyncState = (counter: number): Effect.Effect<ProcessorStateInSync> =>
-      Effect.gen(function* () {
-        // console.log('waitForSyncState: waiting for semaphore', counter)
-        yield* semaphore.take(1)
-        // NOTE this is a workaround to ensure the semaphore take-order is respected
-        // TODO this needs to be fixed upstream in Effect
-        if (counter !== expectedCounter) {
-          console.log(
-            `waitForSyncState: counter mismatch (expected: ${expectedCounter}, got: ${counter}), releasing semaphore`,
-          )
-          yield* semaphore.release(1)
-          yield* Effect.yieldNow()
-          // Retrying...
-          return yield* waitForSyncState(counter)
-        }
-        // console.log('waitForSyncState: took semaphore', counter)
-        const state = yield* Ref.get(stateRef)
-        if (state._tag !== 'in-sync') {
-          return shouldNeverHappen('Expected to be in-sync but got ' + state._tag)
-        }
-        expectedCounter = counter + 1
-        return state
-      }).pipe(Effect.withSpan(`@livestore/common:leader-thread:syncing:waitForSyncState(${counter})`))
+    const localPushesQueue = yield* BucketQueue.make<LocalPushQueueItem>()
+    const localPushesLatch = yield* Effect.makeLatch(true)
+    const pullLatch = yield* Effect.makeLatch(true)
+
+    /**
+     * Additionally to the `syncStateSref` we also need the `pushHeadRef` in order to prevent old/duplicate
+     * events from being pushed in a scenario like this:
+     * - client session A pushes e1
+     * - leader sync processor takes a bit and hasn't yet taken e1 from the localPushesQueue
+     * - client session B also pushes e1 (which should be rejected)
+     *
+     * Thus the purpoe of the pushHeadRef is the guard the integrity of the local push queue
+     */
+    const pushHeadRef = { current: EventSequenceNumber.ROOT }
+    const advancePushHead = (eventNum: EventSequenceNumber.EventSequenceNumber) => {
+      pushHeadRef.current = EventSequenceNumber.max(pushHeadRef.current, eventNum)
+    }
 
-    const push = (newEvents: ReadonlyArray<MutationEvent.EncodedWithMeta>) =>
+    // NOTE: New events are only pushed to sync backend after successful local push processing
+    const push: LeaderSyncProcessor['push'] = (newEvents, options) =>
       Effect.gen(function* () {
-        const counter = counterRef
-        counterRef++
-        // TODO validate batch
         if (newEvents.length === 0) return
 
-        const { connectedClientSessionPullQueues } = yield* LeaderThreadCtx
-
-        // TODO if there are multiple pending pushes, we should batch them together
-        const state = yield* waitForSyncState(counter)
-
-        const updateResult = SyncState.updateSyncState({
-          syncState: state.syncState,
-          payload: { _tag: 'local-push', newEvents },
-          isLocalEvent,
-          isEqualEvent: MutationEvent.isEqualEncoded,
-        })
+        yield* validatePushBatch(newEvents, pushHeadRef.current)
 
-        if (updateResult._tag === 'rebase') {
-          return shouldNeverHappen('The leader thread should never have to rebase due to a local push')
-        } else if (updateResult._tag === 'reject') {
-          return yield* Effect.fail(
-            InvalidPushError.make({
-              reason: {
-                _tag: 'LeaderAhead',
-                minimumExpectedId: updateResult.expectedMinimumId,
-                providedId: newEvents.at(0)!.id,
-              },
-            }),
-          )
-        }
-
-        const fiber = yield* applyMutationItemsRef.current!({ batchItems: updateResult.newEvents }).pipe(Effect.fork)
+        advancePushHead(newEvents.at(-1)!.seqNum)
 
-        yield* Ref.set(stateRef, {
-          _tag: 'applying-syncstate-advance',
-          origin: 'push',
-          syncState: updateResult.newSyncState,
-          fiber,
-        })
+        const waitForProcessing = options?.waitForProcessing ?? false
+        const generation = currentLocalPushGenerationRef.current
 
-        // console.log('setRef:applying-syncstate-advance after push', counter)
+        if (waitForProcessing) {
+          const deferreds = yield* Effect.forEach(newEvents, () => Deferred.make<void, LeaderAheadError>())
 
-        yield* connectedClientSessionPullQueues.offer({
-          payload: { _tag: 'upstream-advance', newEvents: updateResult.newEvents },
-          remaining: 0,
-        })
-
-        spanRef.current?.addEvent('local-push', {
-          batchSize: newEvents.length,
-          updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
-        })
-
-        // Don't sync localOnly mutations
-        const filteredBatch = updateResult.newEvents.filter((mutationEventEncoded) => {
-          const mutationDef = schema.mutations.get(mutationEventEncoded.mutation)!
-          return mutationDef.options.localOnly === false
-        })
+          const items = newEvents.map(
+            (eventEncoded, i) => [eventEncoded, deferreds[i], generation] as LocalPushQueueItem,
+          )
 
-        yield* BucketQueue.offerAll(syncBackendQueue, filteredBatch)
+          yield* BucketQueue.offerAll(localPushesQueue, items)
 
-        yield* fiber // Waiting for the mutation to be applied
+          yield* Effect.all(deferreds)
+        } else {
+          const items = newEvents.map((eventEncoded) => [eventEncoded, undefined, generation] as LocalPushQueueItem)
+          yield* BucketQueue.offerAll(localPushesQueue, items)
+        }
       }).pipe(
-        Effect.withSpan('@livestore/common:leader-thread:syncing:local-push', {
+        Effect.withSpan('@livestore/common:LeaderSyncProcessor:push', {
           attributes: {
             batchSize: newEvents.length,
             batch: TRACE_VERBOSE ? newEvents : undefined,
           },
-          links: spanRef.current
-            ? [{ _tag: 'SpanLink', span: OtelTracer.makeExternalSpan(spanRef.current.spanContext()), attributes: {} }]
-            : undefined,
+          links: ctxRef.current?.span ? [{ _tag: 'SpanLink', span: ctxRef.current.span, attributes: {} }] : undefined,
         }),
       )
 
-    const pushPartial: LeaderSyncProcessor['pushPartial'] = (mutationEventEncoded_) =>
+    const pushPartial: LeaderSyncProcessor['pushPartial'] = ({ event: { name, args }, clientId, sessionId }) =>
       Effect.gen(function* () {
-        const state = yield* Ref.get(stateRef)
-        if (state._tag === 'init') return shouldNeverHappen('Not initialized')
+        const syncState = yield* syncStateSref
+        if (syncState === undefined) return shouldNeverHappen('Not initialized')
 
-        const mutationDef =
-          schema.mutations.get(mutationEventEncoded_.mutation) ??
-          shouldNeverHappen(`Unknown mutation: ${mutationEventEncoded_.mutation}`)
+        const { eventDef } = getEventDef(schema, name)
 
-        const mutationEventEncoded = new MutationEvent.EncodedWithMeta({
-          ...mutationEventEncoded_,
-          ...EventId.nextPair(state.syncState.localHead, mutationDef.options.localOnly),
+        const eventEncoded = new LiveStoreEvent.EncodedWithMeta({
+          name,
+          args,
+          clientId,
+          sessionId,
+          ...EventSequenceNumber.nextPair(syncState.localHead, eventDef.options.clientOnly),
         })
 
-        yield* push([mutationEventEncoded])
-      }).pipe(Effect.catchTag('InvalidPushError', Effect.orDie))
+        yield* push([eventEncoded])
+      }).pipe(Effect.catchTag('LeaderAheadError', Effect.orDie))
 
     // Starts various background loops
-    const boot: LeaderSyncProcessor['boot'] = ({ dbReady }) =>
-      Effect.gen(function* () {
-        const span = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)))
-        spanRef.current = span
+    const boot: LeaderSyncProcessor['boot'] = Effect.gen(function* () {
+      const span = yield* Effect.currentSpan.pipe(Effect.orDie)
+      const otelSpan = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)))
+      const { devtools, shutdownChannel } = yield* LeaderThreadCtx
+      const runtime = yield* Effect.runtime<LeaderThreadCtx>()
+
+      ctxRef.current = {
+        otelSpan,
+        span,
+        devtoolsLatch: devtools.enabled ? devtools.syncBackendLatch : undefined,
+        runtime,
+      }
 
-        const initialBackendHead = dbMissing ? EventId.ROOT.global : getBackendHeadFromDb(dbLog)
-        const initialLocalHead = dbMissing ? EventId.ROOT : getLocalHeadFromDb(dbLog)
+      const initialLocalHead = dbEventlogMissing ? EventSequenceNumber.ROOT : Eventlog.getClientHeadFromDb(dbEventlog)
 
-        if (initialBackendHead > initialLocalHead.global) {
-          return shouldNeverHappen(
-            `During boot the backend head (${initialBackendHead}) should never be greater than the local head (${initialLocalHead.global})`,
-          )
-        }
+      const initialBackendHead = dbEventlogMissing
+        ? EventSequenceNumber.ROOT.global
+        : Eventlog.getBackendHeadFromDb(dbEventlog)
 
-        const pendingMutationEvents = yield* getMutationEventsSince({
-          global: initialBackendHead,
-          local: EventId.localDefault,
-        }).pipe(Effect.map(ReadonlyArray.map((_) => new MutationEvent.EncodedWithMeta(_))))
-
-        const initialSyncState = {
-          pending: pendingMutationEvents,
-          // On the leader we don't need a rollback tail beyond `pending` items
-          rollbackTail: [],
-          upstreamHead: { global: initialBackendHead, local: EventId.localDefault },
-          localHead: initialLocalHead,
-        } as SyncState.SyncState
-
-        /** State transitions need to happen atomically, so we use a Ref to track the state */
-        yield* Ref.set(stateRef, { _tag: 'in-sync', syncState: initialSyncState })
-
-        applyMutationItemsRef.current = yield* makeApplyMutationItems({ stateRef, semaphore })
-
-        // Rehydrate sync queue
-        if (pendingMutationEvents.length > 0) {
-          const filteredBatch = pendingMutationEvents
-            // Don't sync localOnly mutations
-            .filter((mutationEventEncoded) => {
-              const mutationDef = schema.mutations.get(mutationEventEncoded.mutation)!
-              return mutationDef.options.localOnly === false
-            })
+      if (initialBackendHead > initialLocalHead.global) {
+        return shouldNeverHappen(
+          `During boot the backend head (${initialBackendHead}) should never be greater than the local head (${initialLocalHead.global})`,
+        )
+      }
+
+      const pendingEvents = dbEventlogMissing
+        ? []
+        : yield* Eventlog.getEventsSince({ global: initialBackendHead, client: EventSequenceNumber.clientDefault })
+
+      const initialSyncState = new SyncState.SyncState({
+        pending: pendingEvents,
+        upstreamHead: { global: initialBackendHead, client: EventSequenceNumber.clientDefault },
+        localHead: initialLocalHead,
+      })
+
+      /** State transitions need to happen atomically, so we use a Ref to track the state */
+      yield* SubscriptionRef.set(syncStateSref, initialSyncState)
+
+      // Rehydrate sync queue
+      if (pendingEvents.length > 0) {
+        const globalPendingEvents = pendingEvents
+          // Don't sync clientOnly events
+          .filter((eventEncoded) => {
+            const { eventDef } = getEventDef(schema, eventEncoded.name)
+            return eventDef.options.clientOnly === false
+          })
 
-          yield* BucketQueue.offerAll(syncBackendQueue, filteredBatch)
+        if (globalPendingEvents.length > 0) {
+          yield* BucketQueue.offerAll(syncBackendPushQueue, globalPendingEvents)
         }
+      }
 
-        const backendPushingFiberHandle = yield* FiberHandle.make()
+      const shutdownOnError = (cause: unknown) =>
+        Effect.gen(function* () {
+          if (onError === 'shutdown') {
+            yield* shutdownChannel.send(UnexpectedError.make({ cause }))
+            yield* Effect.die(cause)
+          }
+        })
 
-        yield* FiberHandle.run(
-          backendPushingFiberHandle,
-          backgroundBackendPushing({ dbReady, syncBackendQueue, span }).pipe(Effect.tapCauseLogPretty),
-        )
+      yield* backgroundApplyLocalPushes({
+        localPushesLatch,
+        localPushesQueue,
+        pullLatch,
+        syncStateSref,
+        syncBackendPushQueue,
+        schema,
+        isClientEvent,
+        otelSpan,
+        currentLocalPushGenerationRef,
+        connectedClientSessionPullQueues,
+        mergeCounterRef,
+        mergePayloads,
+        localPushBatchSize,
+        testing: {
+          delay: testing?.delays?.localPushProcessing,
+        },
+      }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped)
+
+      const backendPushingFiberHandle = yield* FiberHandle.make()
+      const backendPushingEffect = backgroundBackendPushing({
+        syncBackendPushQueue,
+        otelSpan,
+        devtoolsLatch: ctxRef.current?.devtoolsLatch,
+        backendPushBatchSize,
+      }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError))
+
+      yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect)
+
+      yield* backgroundBackendPulling({
+        initialBackendHead,
+        isClientEvent,
+        restartBackendPushing: (filteredRebasedPending) =>
+          Effect.gen(function* () {
+            // Stop current pushing fiber
+            yield* FiberHandle.clear(backendPushingFiberHandle)
 
-        yield* backgroundBackendPulling({
-          dbReady,
-          initialBackendHead,
-          isLocalEvent,
-          restartBackendPushing: (filteredRebasedPending) =>
-            Effect.gen(function* () {
-              // Stop current pushing fiber
-              yield* FiberHandle.clear(backendPushingFiberHandle)
-
-              // Reset the sync queue
-              yield* BucketQueue.clear(syncBackendQueue)
-              yield* BucketQueue.offerAll(syncBackendQueue, filteredRebasedPending)
-
-              // Restart pushing fiber
-              yield* FiberHandle.run(
-                backendPushingFiberHandle,
-                backgroundBackendPushing({ dbReady, syncBackendQueue, span }).pipe(Effect.tapCauseLogPretty),
-              )
-            }),
-          applyMutationItemsRef,
-          stateRef,
-          semaphore,
-          span,
-          initialBlockingSyncContext,
-        }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
-      }).pipe(Effect.withSpanScoped('@livestore/common:leader-thread:syncing'))
+            // Reset the sync backend push queue
+            yield* BucketQueue.clear(syncBackendPushQueue)
+            yield* BucketQueue.offerAll(syncBackendPushQueue, filteredRebasedPending)
+
+            // Restart pushing fiber
+            yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect)
+          }),
+        syncStateSref,
+        localPushesLatch,
+        pullLatch,
+        otelSpan,
+        initialBlockingSyncContext,
+        devtoolsLatch: ctxRef.current?.devtoolsLatch,
+        connectedClientSessionPullQueues,
+        mergeCounterRef,
+        mergePayloads,
+        advancePushHead,
+      }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped)
+
+      return { initialLeaderHead: initialLocalHead }
+    }).pipe(Effect.withSpanScoped('@livestore/common:LeaderSyncProcessor:boot'))
+
+    const pull: LeaderSyncProcessor['pull'] = ({ cursor }) =>
+      Effect.gen(function* () {
+        const queue = yield* pullQueue({ cursor })
+        return Stream.fromQueue(queue)
+      }).pipe(Stream.unwrapScoped)
+
+    const pullQueue: LeaderSyncProcessor['pullQueue'] = ({ cursor }) => {
+      const runtime = ctxRef.current?.runtime ?? shouldNeverHappen('Not initialized')
+      return Effect.gen(function* () {
+        const queue = yield* connectedClientSessionPullQueues.makeQueue
+        const payloadsSinceCursor = Array.from(mergePayloads.entries())
+          .map(([mergeCounter, payload]) => ({ payload, mergeCounter }))
+          .filter(({ mergeCounter }) => mergeCounter > cursor.mergeCounter)
+          .toSorted((a, b) => a.mergeCounter - b.mergeCounter)
+          .map(({ payload, mergeCounter }) => {
+            if (payload._tag === 'upstream-advance') {
+              return {
+                payload: {
+                  _tag: 'upstream-advance' as const,
+                  newEvents: ReadonlyArray.dropWhile(payload.newEvents, (eventEncoded) =>
+                    EventSequenceNumber.isGreaterThanOrEqual(cursor.eventNum, eventEncoded.seqNum),
+                  ),
+                },
+                mergeCounter,
+              }
+            } else {
+              return { payload, mergeCounter }
+            }
+          })
+
+        yield* queue.offerAll(payloadsSinceCursor)
+
+        return queue
+      }).pipe(Effect.provide(runtime))
+    }
+
+    const syncState = Subscribable.make({
+      get: Effect.gen(function* () {
+        const syncState = yield* syncStateSref
+        if (syncState === undefined) return shouldNeverHappen('Not initialized')
+        return syncState
+      }),
+      changes: syncStateSref.changes.pipe(Stream.filter(isNotUndefined)),
+    })
 
     return {
+      pull,
+      pullQueue,
       push,
       pushPartial,
       boot,
-      syncState: Effect.gen(function* () {
-        const state = yield* Ref.get(stateRef)
-        if (state._tag === 'init') return shouldNeverHappen('Not initialized')
-        return state.syncState
-      }),
+      syncState,
+      getMergeCounter: () => mergeCounterRef.current,
     } satisfies LeaderSyncProcessor
   })
 
-type ApplyMutationItems = (_: {
-  batchItems: ReadonlyArray<MutationEvent.EncodedWithMeta>
-}) => Effect.Effect<void, UnexpectedError>
-
-// TODO how to handle errors gracefully
-const makeApplyMutationItems = ({
-  stateRef,
-  semaphore,
+const backgroundApplyLocalPushes = ({
+  localPushesLatch,
+  localPushesQueue,
+  pullLatch,
+  syncStateSref,
+  syncBackendPushQueue,
+  schema,
+  isClientEvent,
+  otelSpan,
+  currentLocalPushGenerationRef,
+  connectedClientSessionPullQueues,
+  mergeCounterRef,
+  mergePayloads,
+  localPushBatchSize,
+  testing,
 }: {
-  stateRef: Ref.Ref<ProcessorState>
-  semaphore: Effect.Semaphore
-}): Effect.Effect<ApplyMutationItems, UnexpectedError, LeaderThreadCtx | Scope.Scope> =>
+  pullLatch: Effect.Latch
+  localPushesLatch: Effect.Latch
+  localPushesQueue: BucketQueue.BucketQueue<LocalPushQueueItem>
+  syncStateSref: SubscriptionRef.SubscriptionRef<SyncState.SyncState | undefined>
+  syncBackendPushQueue: BucketQueue.BucketQueue<LiveStoreEvent.EncodedWithMeta>
+  schema: LiveStoreSchema
+  isClientEvent: (eventEncoded: LiveStoreEvent.EncodedWithMeta) => boolean
+  otelSpan: otel.Span | undefined
+  currentLocalPushGenerationRef: { current: number }
+  connectedClientSessionPullQueues: PullQueueSet
+  mergeCounterRef: { current: number }
+  mergePayloads: Map<number, typeof SyncState.PayloadUpstream.Type>
+  localPushBatchSize: number
+  testing: {
+    delay: Effect.Effect<void> | undefined
+  }
+}) =>
   Effect.gen(function* () {
-    const leaderThreadCtx = yield* LeaderThreadCtx
-    const { db, dbLog } = leaderThreadCtx
+    while (true) {
+      if (testing.delay !== undefined) {
+        yield* testing.delay.pipe(Effect.withSpan('localPushProcessingDelay'))
+      }
 
-    const applyMutation = yield* makeApplyMutation
+      const batchItems = yield* BucketQueue.takeBetween(localPushesQueue, 1, localPushBatchSize)
 
-    return ({ batchItems }) =>
-      Effect.gen(function* () {
-        const state = yield* Ref.get(stateRef)
-        if (state._tag !== 'applying-syncstate-advance') {
-          // console.log('applyMutationItems: counter', counter)
-          return shouldNeverHappen(`Expected to be applying-syncstate-advance but got ${state._tag}`)
-        }
+      // Wait for the backend pulling to finish
+      yield* localPushesLatch.await
 
-        db.execute('BEGIN TRANSACTION', undefined) // Start the transaction
-        dbLog.execute('BEGIN TRANSACTION', undefined) // Start the transaction
+      // Prevent backend pull processing until this local push is finished
+      yield* pullLatch.close
 
-        yield* Effect.addFinalizer((exit) =>
-          Effect.gen(function* () {
-            if (Exit.isSuccess(exit)) return
+      // Since the generation might have changed since enqueuing, we need to filter out items with older generation
+      // It's important that we filter after we got localPushesLatch, otherwise we might filter with the old generation
+      const filteredBatchItems = batchItems
+        .filter(([_1, _2, generation]) => generation === currentLocalPushGenerationRef.current)
+        .map(([eventEncoded, deferred]) => [eventEncoded, deferred] as const)
 
-            // Rollback in case of an error
-            db.execute('ROLLBACK', undefined)
-            dbLog.execute('ROLLBACK', undefined)
-          }),
-        )
+      if (filteredBatchItems.length === 0) {
+        // console.log('dropping old-gen batch', currentLocalPushGenerationRef.current)
+        // Allow the backend pulling to start
+        yield* pullLatch.open
+        continue
+      }
+
+      const [newEvents, deferreds] = ReadonlyArray.unzip(filteredBatchItems)
+
+      const syncState = yield* syncStateSref
+      if (syncState === undefined) return shouldNeverHappen('Not initialized')
+
+      const mergeResult = SyncState.merge({
+        syncState,
+        payload: { _tag: 'local-push', newEvents },
+        isClientEvent,
+        isEqualEvent: LiveStoreEvent.isEqualEncoded,
+      })
+
+      const mergeCounter = yield* incrementMergeCounter(mergeCounterRef)
+
+      switch (mergeResult._tag) {
+        case 'unexpected-error': {
+          otelSpan?.addEvent(`[${mergeCounter}]:push:unexpected-error`, {
+            batchSize: newEvents.length,
+            newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
+          })
+          return yield* Effect.fail(mergeResult.cause)
+        }
+        case 'rebase': {
+          return shouldNeverHappen('The leader thread should never have to rebase due to a local push')
+        }
+        case 'reject': {
+          otelSpan?.addEvent(`[${mergeCounter}]:push:reject`, {
+            batchSize: newEvents.length,
+            mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
+          })
 
-        for (let i = 0; i < batchItems.length; i++) {
-          const { meta, ...mutationEventEncoded } = batchItems[i]!
+          // TODO: how to test this?
+          currentLocalPushGenerationRef.current++
 
-          yield* applyMutation(mutationEventEncoded)
+          const nextGeneration = currentLocalPushGenerationRef.current
 
-          if (meta?.deferred) {
-            yield* Deferred.succeed(meta.deferred, void 0)
+          const providedNum = newEvents.at(0)!.seqNum
+          // All subsequent pushes with same generation should be rejected as well
+          // We're also handling the case where the localPushQueue already contains events
+          // from the next generation which we preserve in the queue
+          const remainingEventsMatchingGeneration = yield* BucketQueue.takeSplitWhere(
+            localPushesQueue,
+            (item) => item[2] >= nextGeneration,
+          )
+
+          // TODO we still need to better understand and handle this scenario
+          if (LS_DEV && (yield* BucketQueue.size(localPushesQueue)) > 0) {
+            console.log('localPushesQueue is not empty', yield* BucketQueue.size(localPushesQueue))
+            debugger
           }
 
-          // TODO re-introduce this
-          // if (i < batchItems.length - 1) {
-          //   yield* Ref.set(stateRef, { ...state, proccesHead: batchItems[i + 1]!.id })
-          // }
+          const allDeferredsToReject = [
+            ...deferreds,
+            ...remainingEventsMatchingGeneration.map(([_, deferred]) => deferred),
+          ].filter(isNotUndefined)
+
+          yield* Effect.forEach(allDeferredsToReject, (deferred) =>
+            Deferred.fail(
+              deferred,
+              LeaderAheadError.make({
+                minimumExpectedNum: mergeResult.expectedMinimumId,
+                providedNum,
+                // nextGeneration,
+              }),
+            ),
+          )
+
+          // Allow the backend pulling to start
+          yield* pullLatch.open
+
+          // In this case we're skipping state update and down/upstream processing
+          // We've cleared the local push queue and are now waiting for new local pushes / backend pulls
+          continue
+        }
+        case 'advance': {
+          break
+        }
+        default: {
+          casesHandled(mergeResult)
         }
+      }
 
-        db.execute('COMMIT', undefined) // Commit the transaction
-        dbLog.execute('COMMIT', undefined) // Commit the transaction
+      yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState)
 
-        yield* Ref.set(stateRef, { _tag: 'in-sync', syncState: state.syncState })
-        // console.log('setRef:sync after applyMutationItems', counter)
-        yield* semaphore.release(1)
-      }).pipe(
-        Effect.scoped,
-        Effect.withSpan('@livestore/common:leader-thread:syncing:applyMutationItems', {
-          attributes: { count: batchItems.length },
-        }),
-        Effect.tapCauseLogPretty,
-        UnexpectedError.mapToUnexpectedError,
-      )
+      yield* connectedClientSessionPullQueues.offer({
+        payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
+        mergeCounter,
+      })
+      mergePayloads.set(mergeCounter, SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }))
+
+      otelSpan?.addEvent(`[${mergeCounter}]:push:advance`, {
+        batchSize: newEvents.length,
+        mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
+      })
+
+      // Don't sync clientOnly events
+      const filteredBatch = mergeResult.newEvents.filter((eventEncoded) => {
+        const { eventDef } = getEventDef(schema, eventEncoded.name)
+        return eventDef.options.clientOnly === false
+      })
+
+      yield* BucketQueue.offerAll(syncBackendPushQueue, filteredBatch)
+
+      yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds })
+
+      // Allow the backend pulling to start
+      yield* pullLatch.open
+    }
   })
 
+type MaterializeEventsBatch = (_: {
+  batchItems: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>
+  /**
+   * The deferreds are used by the caller to know when the mutation has been processed.
+   * Indexes are aligned with `batchItems`
+   */
+  deferreds: ReadonlyArray<Deferred.Deferred<void, LeaderAheadError> | undefined> | undefined
+}) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
+
+// TODO how to handle errors gracefully
+const materializeEventsBatch: MaterializeEventsBatch = ({ batchItems, deferreds }) =>
+  Effect.gen(function* () {
+    const { dbState: db, dbEventlog, materializeEvent } = yield* LeaderThreadCtx
+
+    // NOTE We always start a transaction to ensure consistency between db and eventlog (even for single-item batches)
+    db.execute('BEGIN TRANSACTION', undefined) // Start the transaction
+    dbEventlog.execute('BEGIN TRANSACTION', undefined) // Start the transaction
+
+    yield* Effect.addFinalizer((exit) =>
+      Effect.gen(function* () {
+        if (Exit.isSuccess(exit)) return
+
+        // Rollback in case of an error
+        db.execute('ROLLBACK', undefined)
+        dbEventlog.execute('ROLLBACK', undefined)
+      }),
+    )
+
+    for (let i = 0; i < batchItems.length; i++) {
+      const { sessionChangeset } = yield* materializeEvent(batchItems[i]!)
+      batchItems[i]!.meta.sessionChangeset = sessionChangeset
+
+      if (deferreds?.[i] !== undefined) {
+        yield* Deferred.succeed(deferreds[i]!, void 0)
+      }
+    }
+
+    db.execute('COMMIT', undefined) // Commit the transaction
+    dbEventlog.execute('COMMIT', undefined) // Commit the transaction
+  }).pipe(
+    Effect.uninterruptible,
+    Effect.scoped,
+    Effect.withSpan('@livestore/common:LeaderSyncProcessor:materializeEventItems', {
+      attributes: { batchSize: batchItems.length },
+    }),
+    Effect.tapCauseLogPretty,
+    UnexpectedError.mapToUnexpectedError,
+  )
+
 const backgroundBackendPulling = ({
-  dbReady,
   initialBackendHead,
-  isLocalEvent,
+  isClientEvent,
   restartBackendPushing,
-  span,
-  stateRef,
-  applyMutationItemsRef,
-  semaphore,
+  otelSpan,
+  syncStateSref,
+  localPushesLatch,
+  pullLatch,
+  devtoolsLatch,
   initialBlockingSyncContext,
+  connectedClientSessionPullQueues,
+  mergeCounterRef,
+  mergePayloads,
+  advancePushHead,
 }: {
-  dbReady: Deferred.Deferred<void>
-  initialBackendHead: EventId.GlobalEventId
-  isLocalEvent: (mutationEventEncoded: MutationEvent.EncodedWithMeta) => boolean
+  initialBackendHead: EventSequenceNumber.GlobalEventSequenceNumber
+  isClientEvent: (eventEncoded: LiveStoreEvent.EncodedWithMeta) => boolean
   restartBackendPushing: (
-    filteredRebasedPending: ReadonlyArray<MutationEvent.EncodedWithMeta>,
+    filteredRebasedPending: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>,
   ) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx | HttpClient.HttpClient>
-  span: otel.Span | undefined
-  stateRef: Ref.Ref<ProcessorState>
-  applyMutationItemsRef: { current: ApplyMutationItems | undefined }
-  semaphore: Effect.Semaphore
+  otelSpan: otel.Span | undefined
+  syncStateSref: SubscriptionRef.SubscriptionRef<SyncState.SyncState | undefined>
+  localPushesLatch: Effect.Latch
+  pullLatch: Effect.Latch
+  devtoolsLatch: Effect.Latch | undefined
   initialBlockingSyncContext: InitialBlockingSyncContext
+  connectedClientSessionPullQueues: PullQueueSet
+  mergeCounterRef: { current: number }
+  mergePayloads: Map<number, typeof SyncState.PayloadUpstream.Type>
+  advancePushHead: (eventNum: EventSequenceNumber.EventSequenceNumber) => void
 }) =>
   Effect.gen(function* () {
-    const { syncBackend, db, dbLog, connectedClientSessionPullQueues, schema } = yield* LeaderThreadCtx
+    const { syncBackend, dbState: db, dbEventlog, schema } = yield* LeaderThreadCtx
 
     if (syncBackend === undefined) return
 
-    const cursorInfo = yield* getCursorInfo(initialBackendHead)
-
-    const onNewPullChunk = (newEvents: MutationEvent.EncodedWithMeta[], remaining: number) =>
+    const onNewPullChunk = (newEvents: LiveStoreEvent.EncodedWithMeta[], remaining: number) =>
       Effect.gen(function* () {
         if (newEvents.length === 0) return
 
-        const state = yield* Ref.get(stateRef)
-        if (state._tag === 'init') return shouldNeverHappen('Not initialized')
+        if (devtoolsLatch !== undefined) {
+          yield* devtoolsLatch.await
+        }
 
-        // const counter = state.counter + 1
+        // Prevent more local pushes from being processed until this pull is finished
+        yield* localPushesLatch.close
 
-        if (state._tag === 'applying-syncstate-advance') {
-          if (state.origin === 'push') {
-            yield* Fiber.interrupt(state.fiber)
-            // In theory we should force-take the semaphore here, but as it's still taken,
-            // it's already in the right state we want it to be in
-          } else {
-            // Wait for previous advance to finish
-            yield* semaphore.take(1)
-          }
-        }
+        // Wait for pending local pushes to finish
+        yield* pullLatch.await
 
-        const trimRollbackUntil = newEvents.at(-1)!.id
+        const syncState = yield* syncStateSref
+        if (syncState === undefined) return shouldNeverHappen('Not initialized')
 
-        const updateResult = SyncState.updateSyncState({
-          syncState: state.syncState,
-          payload: { _tag: 'upstream-advance', newEvents, trimRollbackUntil },
-          isLocalEvent,
-          isEqualEvent: MutationEvent.isEqualEncoded,
-          ignoreLocalEvents: true,
+        const mergeResult = SyncState.merge({
+          syncState,
+          payload: SyncState.PayloadUpstreamAdvance.make({ newEvents }),
+          isClientEvent,
+          isEqualEvent: LiveStoreEvent.isEqualEncoded,
+          ignoreClientEvents: true,
         })
 
-        if (updateResult._tag === 'reject') {
+        const mergeCounter = yield* incrementMergeCounter(mergeCounterRef)
+
+        if (mergeResult._tag === 'reject') {
           return shouldNeverHappen('The leader thread should never reject upstream advances')
+        } else if (mergeResult._tag === 'unexpected-error') {
+          otelSpan?.addEvent(`[${mergeCounter}]:pull:unexpected-error`, {
+            newEventsCount: newEvents.length,
+            newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
+          })
+          return yield* Effect.fail(mergeResult.cause)
         }
 
-        const newBackendHead = newEvents.at(-1)!.id
+        const newBackendHead = newEvents.at(-1)!.seqNum
 
-        updateBackendHead(dbLog, newBackendHead)
+        Eventlog.updateBackendHead(dbEventlog, newBackendHead)
 
-        if (updateResult._tag === 'rebase') {
-          span?.addEvent('backend-pull:rebase', {
+        if (mergeResult._tag === 'rebase') {
+          otelSpan?.addEvent(`[${mergeCounter}]:pull:rebase`, {
             newEventsCount: newEvents.length,
             newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
-            rollbackCount: updateResult.eventsToRollback.length,
-            updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
+            rollbackCount: mergeResult.rollbackEvents.length,
+            mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
           })
 
-          const filteredRebasedPending = updateResult.newSyncState.pending.filter((mutationEvent) => {
-            const mutationDef = schema.mutations.get(mutationEvent.mutation)!
-            return mutationDef.options.localOnly === false
+          const globalRebasedPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
+            const { eventDef } = getEventDef(schema, event.name)
+            return eventDef.options.clientOnly === false
           })
-          yield* restartBackendPushing(filteredRebasedPending)
+          yield* restartBackendPushing(globalRebasedPendingEvents)
 
-          if (updateResult.eventsToRollback.length > 0) {
-            yield* rollback({ db, dbLog, eventIdsToRollback: updateResult.eventsToRollback.map((_) => _.id) })
+          if (mergeResult.rollbackEvents.length > 0) {
+            yield* rollback({
+              dbState: db,
+              dbEventlog,
+              eventNumsToRollback: mergeResult.rollbackEvents.map((_) => _.seqNum),
+            })
           }
 
           yield* connectedClientSessionPullQueues.offer({
-            payload: {
-              _tag: 'upstream-rebase',
-              newEvents: updateResult.newEvents,
-              rollbackUntil: updateResult.eventsToRollback.at(0)!.id,
-              trimRollbackUntil,
-            },
-            remaining,
+            payload: SyncState.PayloadUpstreamRebase.make({
+              newEvents: mergeResult.newEvents,
+              rollbackEvents: mergeResult.rollbackEvents,
+            }),
+            mergeCounter,
           })
+          mergePayloads.set(
+            mergeCounter,
+            SyncState.PayloadUpstreamRebase.make({
+              newEvents: mergeResult.newEvents,
+              rollbackEvents: mergeResult.rollbackEvents,
+            }),
+          )
         } else {
-          span?.addEvent('backend-pull:advance', {
+          otelSpan?.addEvent(`[${mergeCounter}]:pull:advance`, {
             newEventsCount: newEvents.length,
-            updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
+            mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
           })
 
           yield* connectedClientSessionPullQueues.offer({
-            payload: { _tag: 'upstream-advance', newEvents: updateResult.newEvents, trimRollbackUntil },
-            remaining,
+            payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
+            mergeCounter,
           })
+          mergePayloads.set(mergeCounter, SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }))
+
+          if (mergeResult.confirmedEvents.length > 0) {
+            // `mergeResult.confirmedEvents` don't contain the correct sync metadata, so we need to use
+            // `newEvents` instead which we filter via `mergeResult.confirmedEvents`
+            const confirmedNewEvents = newEvents.filter((event) =>
+              mergeResult.confirmedEvents.some((confirmedEvent) =>
+                EventSequenceNumber.isEqual(event.seqNum, confirmedEvent.seqNum),
+              ),
+            )
+            yield* Eventlog.updateSyncMetadata(confirmedNewEvents)
+          }
         }
 
+        // Removes the changeset rows which are no longer needed as we'll never have to rollback beyond this point
         trimChangesetRows(db, newBackendHead)
 
-        const fiber = yield* applyMutationItemsRef.current!({ batchItems: updateResult.newEvents }).pipe(Effect.fork)
+        advancePushHead(mergeResult.newSyncState.localHead)
 
-        yield* Ref.set(stateRef, {
-          _tag: 'applying-syncstate-advance',
-          origin: 'pull',
-          syncState: updateResult.newSyncState,
-          fiber,
-        })
+        yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds: undefined })
+
+        yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState)
+
+        // Allow local pushes to be processed again
+        if (remaining === 0) {
+          yield* localPushesLatch.open
+        }
       })
 
+    const cursorInfo = yield* Eventlog.getSyncBackendCursorInfo(initialBackendHead)
+
     yield* syncBackend.pull(cursorInfo).pipe(
       // TODO only take from queue while connected
       Stream.tap(({ batch, remaining }) =>
@@ -534,16 +769,13 @@ const backgroundBackendPulling = ({
           //   },
           // })
 
-          // Wait for the db to be initially created
-          yield* dbReady
-
-          // NOTE we only want to take process mutations when the sync backend is connected
+          // NOTE we only want to take process events when the sync backend is connected
           // (e.g. needed for simulating being offline)
           // TODO remove when there's a better way to handle this in stream above
           yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
 
           yield* onNewPullChunk(
-            batch.map((_) => MutationEvent.EncodedWithMeta.fromGlobal(_.mutationEventEncoded)),
+            batch.map((_) => LiveStoreEvent.EncodedWithMeta.fromGlobal(_.eventEncoded, _.metadata)),
             remaining,
           )
 
@@ -553,94 +785,35 @@ const backgroundBackendPulling = ({
       Stream.runDrain,
       Effect.interruptible,
     )
-  }).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:backend-pulling'))
-
-const rollback = ({
-  db,
-  dbLog,
-  eventIdsToRollback,
-}: {
-  db: SynchronousDatabase
-  dbLog: SynchronousDatabase
-  eventIdsToRollback: EventId.EventId[]
-}) =>
-  Effect.gen(function* () {
-    const rollbackEvents = db
-      .select<SessionChangesetMetaRow>(
-        sql`SELECT * FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idLocal) IN (${eventIdsToRollback.map((id) => `(${id.global}, ${id.local})`).join(', ')})`,
-      )
-      .map((_) => ({ id: { global: _.idGlobal, local: _.idLocal }, changeset: _.changeset, debug: _.debug }))
-      .toSorted((a, b) => EventId.compare(a.id, b.id))
-
-    // Apply changesets in reverse order
-    for (let i = rollbackEvents.length - 1; i >= 0; i--) {
-      const { changeset } = rollbackEvents[i]!
-      if (changeset !== null) {
-        db.makeChangeset(changeset).invert().apply()
-      }
-    }
-
-    // Delete the changeset rows
-    db.execute(
-      sql`DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idLocal) IN (${eventIdsToRollback.map((id) => `(${id.global}, ${id.local})`).join(', ')})`,
-    )
-
-    // Delete the mutation log rows
-    dbLog.execute(
-      sql`DELETE FROM ${MUTATION_LOG_META_TABLE} WHERE (idGlobal, idLocal) IN (${eventIdsToRollback.map((id) => `(${id.global}, ${id.local})`).join(', ')})`,
-    )
-  }).pipe(
-    Effect.withSpan('@livestore/common:leader-thread:syncing:rollback', {
-      attributes: { count: eventIdsToRollback.length },
-    }),
-  )
-
-const getCursorInfo = (remoteHead: EventId.GlobalEventId) =>
-  Effect.gen(function* () {
-    const { dbLog } = yield* LeaderThreadCtx
-
-    if (remoteHead === EventId.ROOT.global) return Option.none()
-
-    const MutationlogQuerySchema = Schema.Struct({
-      syncMetadataJson: Schema.parseJson(Schema.Option(Schema.JsonValue)),
-    }).pipe(Schema.pluck('syncMetadataJson'), Schema.Array, Schema.head)
-
-    const syncMetadataOption = yield* Effect.sync(() =>
-      dbLog.select<{ syncMetadataJson: string }>(
-        sql`SELECT syncMetadataJson FROM ${MUTATION_LOG_META_TABLE} WHERE idGlobal = ${remoteHead} ORDER BY idLocal ASC LIMIT 1`,
-      ),
-    ).pipe(Effect.andThen(Schema.decode(MutationlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie)
-
-    return Option.some({
-      cursor: { global: remoteHead, local: EventId.localDefault },
-      metadata: syncMetadataOption,
-    }) satisfies InitialSyncInfo
-  }).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:getCursorInfo', { attributes: { remoteHead } }))
+  }).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pulling'))
 
 const backgroundBackendPushing = ({
-  dbReady,
-  syncBackendQueue,
-  span,
+  syncBackendPushQueue,
+  otelSpan,
+  devtoolsLatch,
+  backendPushBatchSize,
 }: {
-  dbReady: Deferred.Deferred<void>
-  syncBackendQueue: BucketQueue.BucketQueue<MutationEvent.EncodedWithMeta>
-  span: otel.Span | undefined
+  syncBackendPushQueue: BucketQueue.BucketQueue<LiveStoreEvent.EncodedWithMeta>
+  otelSpan: otel.Span | undefined
+  devtoolsLatch: Effect.Latch | undefined
+  backendPushBatchSize: number
 }) =>
   Effect.gen(function* () {
-    const { syncBackend, dbLog } = yield* LeaderThreadCtx
+    const { syncBackend } = yield* LeaderThreadCtx
     if (syncBackend === undefined) return
 
-    yield* dbReady
-
     while (true) {
       yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
 
-      // TODO make batch size configurable
-      const queueItems = yield* BucketQueue.takeBetween(syncBackendQueue, 1, 50)
+      const queueItems = yield* BucketQueue.takeBetween(syncBackendPushQueue, 1, backendPushBatchSize)
 
       yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
 
-      span?.addEvent('backend-push', {
+      if (devtoolsLatch !== undefined) {
+        yield* devtoolsLatch.await
+      }
+
+      otelSpan?.addEvent('backend-push', {
         batchSize: queueItems.length,
         batch: TRACE_VERBOSE ? JSON.stringify(queueItems) : undefined,
       })
@@ -649,31 +822,119 @@ const backgroundBackendPushing = ({
       const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either)
 
       if (pushResult._tag === 'Left') {
-        span?.addEvent('backend-push-error', { error: pushResult.left.toString() })
+        if (LS_DEV) {
+          yield* Effect.logDebug('handled backend-push-error', { error: pushResult.left.toString() })
+        }
+        otelSpan?.addEvent('backend-push-error', { error: pushResult.left.toString() })
         // wait for interrupt caused by background pulling which will then restart pushing
         return yield* Effect.never
       }
-
-      const { metadata } = pushResult.right
-
-      // TODO try to do this in a single query
-      for (let i = 0; i < queueItems.length; i++) {
-        const mutationEventEncoded = queueItems[i]!
-        yield* execSql(
-          dbLog,
-          ...updateRows({
-            tableName: MUTATION_LOG_META_TABLE,
-            columns: mutationLogMetaTable.sqliteDef.columns,
-            where: { idGlobal: mutationEventEncoded.id.global, idLocal: mutationEventEncoded.id.local },
-            updateValues: { syncMetadataJson: metadata[i]! },
-          }),
-        )
-      }
     }
-  }).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:leader-thread:syncing:backend-pushing'))
+  }).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pushing'))
 
-const trimChangesetRows = (db: SynchronousDatabase, newHead: EventId.EventId) => {
+const trimChangesetRows = (db: SqliteDb, newHead: EventSequenceNumber.EventSequenceNumber) => {
   // Since we're using the session changeset rows to query for the current head,
   // we're keeping at least one row for the current head, and thus are using `<` instead of `<=`
-  db.execute(sql`DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE idGlobal < ${newHead.global}`)
+  db.execute(sql`DELETE FROM ${SystemTables.SESSION_CHANGESET_META_TABLE} WHERE seqNumGlobal < ${newHead.global}`)
 }
+
+interface PullQueueSet {
+  makeQueue: Effect.Effect<
+    Queue.Queue<{ payload: typeof SyncState.PayloadUpstream.Type; mergeCounter: number }>,
+    UnexpectedError,
+    Scope.Scope | LeaderThreadCtx
+  >
+  offer: (item: {
+    payload: typeof SyncState.PayloadUpstream.Type
+    mergeCounter: number
+  }) => Effect.Effect<void, UnexpectedError>
+}
+
+const makePullQueueSet = Effect.gen(function* () {
+  const set = new Set<Queue.Queue<{ payload: typeof SyncState.PayloadUpstream.Type; mergeCounter: number }>>()
+
+  yield* Effect.addFinalizer(() =>
+    Effect.gen(function* () {
+      for (const queue of set) {
+        yield* Queue.shutdown(queue)
+      }
+
+      set.clear()
+    }),
+  )
+
+  const makeQueue: PullQueueSet['makeQueue'] = Effect.gen(function* () {
+    const queue = yield* Queue.unbounded<{
+      payload: typeof SyncState.PayloadUpstream.Type
+      mergeCounter: number
+    }>().pipe(Effect.acquireRelease(Queue.shutdown))
+
+    yield* Effect.addFinalizer(() => Effect.sync(() => set.delete(queue)))
+
+    set.add(queue)
+
+    return queue
+  })
+
+  const offer: PullQueueSet['offer'] = (item) =>
+    Effect.gen(function* () {
+      // Short-circuit if the payload is an empty upstream advance
+      if (item.payload._tag === 'upstream-advance' && item.payload.newEvents.length === 0) {
+        return
+      }
+
+      for (const queue of set) {
+        yield* Queue.offer(queue, item)
+      }
+    })
+
+  return {
+    makeQueue,
+    offer,
+  }
+})
+
+const incrementMergeCounter = (mergeCounterRef: { current: number }) =>
+  Effect.gen(function* () {
+    const { dbState } = yield* LeaderThreadCtx
+    mergeCounterRef.current++
+    dbState.execute(
+      sql`INSERT OR REPLACE INTO ${SystemTables.LEADER_MERGE_COUNTER_TABLE} (id, mergeCounter) VALUES (0, ${mergeCounterRef.current})`,
+    )
+    return mergeCounterRef.current
+  })
+
+const getMergeCounterFromDb = (dbState: SqliteDb) =>
+  Effect.gen(function* () {
+    const result = dbState.select<{ mergeCounter: number }>(
+      sql`SELECT mergeCounter FROM ${SystemTables.LEADER_MERGE_COUNTER_TABLE} WHERE id = 0`,
+    )
+    return result[0]?.mergeCounter ?? 0
+  })
+
+const validatePushBatch = (
+  batch: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>,
+  pushHead: EventSequenceNumber.EventSequenceNumber,
+) =>
+  Effect.gen(function* () {
+    if (batch.length === 0) {
+      return
+    }
+
+    // Make sure batch is monotonically increasing
+    for (let i = 1; i < batch.length; i++) {
+      if (EventSequenceNumber.isGreaterThanOrEqual(batch[i - 1]!.seqNum, batch[i]!.seqNum)) {
+        shouldNeverHappen(
+          `Events must be ordered in monotonically ascending order by eventNum. Received: [${batch.map((e) => EventSequenceNumber.toString(e.seqNum)).join(', ')}]`,
+        )
+      }
+    }
+
+    // Make sure smallest sequence number is > pushHead
+    if (EventSequenceNumber.isGreaterThanOrEqual(pushHead, batch[0]!.seqNum)) {
+      return yield* LeaderAheadError.make({
+        minimumExpectedNum: pushHead,
+        providedNum: batch[0]!.seqNum,
+      })
+    }
+  })