@livestore/common 0.0.0-snapshot-2b8a9de3ec1a701aca891ebc2c98eb328274ae9e → 0.0.0-snapshot-2ef046b02334f52613d31dbe06af53487685edc0

package/src/leader-thread/LeaderSyncProcessor.ts

@@ -1,15 +1,14 @@
 import { casesHandled, isNotUndefined, LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
-import type { HttpClient, Scope, Tracer } from '@livestore/utils/effect'
+import type { HttpClient, Runtime, Scope, Tracer } from '@livestore/utils/effect'
 import {
   BucketQueue,
   Deferred,
   Effect,
   Exit,
   FiberHandle,
-  Option,
   OtelTracer,
+  Queue,
   ReadonlyArray,
-  Schema,
   Stream,
   Subscribable,
   SubscriptionRef,
@@ -18,30 +17,29 @@ import type * as otel from '@opentelemetry/api'
 
 import type { SqliteDb } from '../adapter-types.js'
 import { UnexpectedError } from '../adapter-types.js'
-import type { LiveStoreSchema, SessionChangesetMetaRow } from '../schema/mod.js'
+import type { LiveStoreSchema } from '../schema/mod.js'
 import {
   EventId,
   getMutationDef,
-  MUTATION_LOG_META_TABLE,
+  LEADER_MERGE_COUNTER_TABLE,
   MutationEvent,
-  mutationLogMetaTable,
   SESSION_CHANGESET_META_TABLE,
 } from '../schema/mod.js'
-import { updateRows } from '../sql-queries/index.js'
 import { LeaderAheadError } from '../sync/sync.js'
 import * as SyncState from '../sync/syncstate.js'
 import { sql } from '../util.js'
-import { makeApplyMutation } from './apply-mutation.js'
-import { execSql } from './connection.js'
-import { getBackendHeadFromDb, getClientHeadFromDb, getMutationEventsSince, updateBackendHead } from './mutationlog.js'
-import type { InitialBlockingSyncContext, InitialSyncInfo, LeaderSyncProcessor } from './types.js'
+import { rollback } from './apply-mutation.js'
+import * as Mutationlog from './mutationlog.js'
+import type { InitialBlockingSyncContext, LeaderSyncProcessor } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
 export const BACKEND_PUSH_BATCH_SIZE = 50
+export const LOCAL_PUSH_BATCH_SIZE = 10
 
 type LocalPushQueueItem = [
   mutationEvent: MutationEvent.EncodedWithMeta,
   deferred: Deferred.Deferred<void, LeaderAheadError> | undefined,
+  /** Used to determine whether the batch has become invalid due to a rejected local push batch */
   generation: number,
 ]
 
@@ -52,40 +50,49 @@ type LocalPushQueueItem = [
  * In the LeaderSyncProcessor, pulling always has precedence over pushing.
  *
  * Responsibilities:
- * - Queueing incoming local mutations in a localPushMailbox.
+ * - Queueing incoming local mutations in a localPushesQueue.
  * - Broadcasting mutations to client sessions via pull queues.
  * - Pushing mutations to the sync backend.
  *
  * Notes:
  *
  * local push processing:
- * - localPushMailbox:
+ * - localPushesQueue:
  *   - Maintains events in ascending order.
  *   - Uses `Deferred` objects to resolve/reject events based on application success.
- * - Processes events from the mailbox, applying mutations in batches.
+ * - Processes events from the queue, applying mutations in batches.
  * - Controlled by a `Latch` to manage execution flow.
  * - The latch closes on pull receipt and re-opens post-pull completion.
  * - Processes up to `maxBatchSize` events per cycle.
  *
+ * Currently we're advancing the db read model and mutation log in lockstep, but we could also decouple this in the future
+ *
+ * Tricky concurrency scenarios:
+ * - Queued local push batches becoming invalid due to a prior local push item being rejected.
+ *   Solution: Introduce a generation number for local push batches which is used to filter out old batches items in case of rejection.
+ *
  */
 export const makeLeaderSyncProcessor = ({
   schema,
-  dbMissing,
+  dbMutationLogMissing,
   dbMutationLog,
-  clientId,
+  dbReadModel,
+  dbReadModelMissing,
   initialBlockingSyncContext,
   onError,
 }: {
   schema: LiveStoreSchema
   /** Only used to know whether we can safely query dbMutationLog during setup execution */
-  dbMissing: boolean
+  dbMutationLogMissing: boolean
   dbMutationLog: SqliteDb
-  clientId: string
+  dbReadModel: SqliteDb
+  /** Only used to know whether we can safely query dbReadModel during setup execution */
+  dbReadModelMissing: boolean
   initialBlockingSyncContext: InitialBlockingSyncContext
   onError: 'shutdown' | 'ignore'
 }): Effect.Effect<LeaderSyncProcessor, UnexpectedError, Scope.Scope> =>
   Effect.gen(function* () {
-    const syncBackendQueue = yield* BucketQueue.make<MutationEvent.EncodedWithMeta>()
+    const syncBackendPushQueue = yield* BucketQueue.make<MutationEvent.EncodedWithMeta>()
 
     const syncStateSref = yield* SubscriptionRef.make<SyncState.SyncState | undefined>(undefined)
 
@@ -94,13 +101,18 @@ export const makeLeaderSyncProcessor = ({
       return mutationDef.options.clientOnly
     }
 
+    const connectedClientSessionPullQueues = yield* makePullQueueSet
+
     /**
      * Tracks generations of queued local push events.
-     * If a batch is rejected, all subsequent push queue items with the same generation are also rejected,
+     * If a local-push batch is rejected, all subsequent push queue items with the same generation are also rejected,
      * even if they would be valid on their own.
      */
     const currentLocalPushGenerationRef = { current: 0 }
 
+    const mergeCounterRef = { current: dbReadModelMissing ? 0 : yield* getMergeCounterFromDb(dbReadModel) }
+    const mergePayloads = new Map<number, typeof SyncState.PayloadUpstream.Type>()
+
     // This context depends on data from `boot`, we should find a better implementation to avoid this ref indirection.
     const ctxRef = {
       current: undefined as
@@ -109,6 +121,7 @@ export const makeLeaderSyncProcessor = ({
             otelSpan: otel.Span | undefined
             span: Tracer.Span
             devtoolsLatch: Effect.Latch | undefined
+            runtime: Runtime.Runtime<LeaderThreadCtx>
           },
     }
 
@@ -116,24 +129,12 @@ export const makeLeaderSyncProcessor = ({
     const localPushesLatch = yield* Effect.makeLatch(true)
     const pullLatch = yield* Effect.makeLatch(true)
 
+    // NOTE: New events are only pushed to sync backend after successful local push processing
     const push: LeaderSyncProcessor['push'] = (newEvents, options) =>
       Effect.gen(function* () {
         // TODO validate batch
         if (newEvents.length === 0) return
 
-        // if (options.generation < currentLocalPushGenerationRef.current) {
-        //   debugger
-        //   // We can safely drop this batch as it's from a previous push generation
-        //   return
-        // }
-
-        if (clientId === 'client-b') {
-          // console.log(
-          //   'push from client session',
-          //   newEvents.map((item) => item.toJSON()),
-          // )
-        }
-
         const waitForProcessing = options?.waitForProcessing ?? false
         const generation = currentLocalPushGenerationRef.current
 
@@ -154,7 +155,7 @@ export const makeLeaderSyncProcessor = ({
           yield* BucketQueue.offerAll(localPushesQueue, items)
         }
       }).pipe(
-        Effect.withSpan('@livestore/common:leader-thread:syncing:local-push', {
+        Effect.withSpan('@livestore/common:LeaderSyncProcessor:local-push', {
           attributes: {
             batchSize: newEvents.length,
             batch: TRACE_VERBOSE ? newEvents : undefined,
@@ -164,7 +165,7 @@ export const makeLeaderSyncProcessor = ({
       )
 
     const pushPartial: LeaderSyncProcessor['pushPartial'] = ({
-      mutationEvent: partialMutationEvent,
+      mutationEvent: { mutation, args },
       clientId,
       sessionId,
     }) =>
@@ -172,10 +173,11 @@ export const makeLeaderSyncProcessor = ({
         const syncState = yield* syncStateSref
         if (syncState === undefined) return shouldNeverHappen('Not initialized')
 
-        const mutationDef = getMutationDef(schema, partialMutationEvent.mutation)
+        const mutationDef = getMutationDef(schema, mutation)
 
         const mutationEventEncoded = new MutationEvent.EncodedWithMeta({
-          ...partialMutationEvent,
+          mutation,
+          args,
           clientId,
           sessionId,
           ...EventId.nextPair(syncState.localHead, mutationDef.options.clientOnly),
@@ -185,134 +187,166 @@ export const makeLeaderSyncProcessor = ({
       }).pipe(Effect.catchTag('LeaderAheadError', Effect.orDie))
 
     // Starts various background loops
-    const boot: LeaderSyncProcessor['boot'] = ({ dbReady }) =>
-      Effect.gen(function* () {
-        const span = yield* Effect.currentSpan.pipe(Effect.orDie)
-        const otelSpan = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)))
-        const { devtools, shutdownChannel } = yield* LeaderThreadCtx
+    const boot: LeaderSyncProcessor['boot'] = Effect.gen(function* () {
+      const span = yield* Effect.currentSpan.pipe(Effect.orDie)
+      const otelSpan = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)))
+      const { devtools, shutdownChannel } = yield* LeaderThreadCtx
+      const runtime = yield* Effect.runtime<LeaderThreadCtx>()
+
+      ctxRef.current = {
+        otelSpan,
+        span,
+        devtoolsLatch: devtools.enabled ? devtools.syncBackendLatch : undefined,
+        runtime,
+      }
 
-        ctxRef.current = {
-          otelSpan,
-          span,
-          devtoolsLatch: devtools.enabled ? devtools.syncBackendLatch : undefined,
-        }
+      const initialBackendHead = dbMutationLogMissing
+        ? EventId.ROOT.global
+        : Mutationlog.getBackendHeadFromDb(dbMutationLog)
+      const initialLocalHead = dbMutationLogMissing ? EventId.ROOT : Mutationlog.getClientHeadFromDb(dbMutationLog)
 
-        const initialBackendHead = dbMissing ? EventId.ROOT.global : getBackendHeadFromDb(dbMutationLog)
-        const initialLocalHead = dbMissing ? EventId.ROOT : getClientHeadFromDb(dbMutationLog)
+      if (initialBackendHead > initialLocalHead.global) {
+        return shouldNeverHappen(
+          `During boot the backend head (${initialBackendHead}) should never be greater than the local head (${initialLocalHead.global})`,
+        )
+      }
 
-        if (initialBackendHead > initialLocalHead.global) {
-          return shouldNeverHappen(
-            `During boot the backend head (${initialBackendHead}) should never be greater than the local head (${initialLocalHead.global})`,
-          )
-        }
+      const pendingMutationEvents = dbMutationLogMissing
+        ? []
+        : yield* Mutationlog.getMutationEventsSince({ global: initialBackendHead, client: EventId.clientDefault })
 
-        const pendingMutationEvents = yield* getMutationEventsSince({
-          global: initialBackendHead,
-          client: EventId.clientDefault,
-        }).pipe(Effect.map(ReadonlyArray.map((_) => new MutationEvent.EncodedWithMeta(_))))
-
-        const initialSyncState = new SyncState.SyncState({
-          pending: pendingMutationEvents,
-          // On the leader we don't need a rollback tail beyond `pending` items
-          rollbackTail: [],
-          upstreamHead: { global: initialBackendHead, client: EventId.clientDefault },
-          localHead: initialLocalHead,
-        })
+      const initialSyncState = new SyncState.SyncState({
+        pending: pendingMutationEvents,
+        upstreamHead: { global: initialBackendHead, client: EventId.clientDefault },
+        localHead: initialLocalHead,
+      })
 
-        /** State transitions need to happen atomically, so we use a Ref to track the state */
-        yield* SubscriptionRef.set(syncStateSref, initialSyncState)
+      /** State transitions need to happen atomically, so we use a Ref to track the state */
+      yield* SubscriptionRef.set(syncStateSref, initialSyncState)
 
-        // Rehydrate sync queue
-        if (pendingMutationEvents.length > 0) {
-          const filteredBatch = pendingMutationEvents
-            // Don't sync clientOnly mutations
-            .filter((mutationEventEncoded) => {
-              const mutationDef = getMutationDef(schema, mutationEventEncoded.mutation)
-              return mutationDef.options.clientOnly === false
-            })
+      // Rehydrate sync queue
+      if (pendingMutationEvents.length > 0) {
+        const globalPendingMutationEvents = pendingMutationEvents
+          // Don't sync clientOnly mutations
+          .filter((mutationEventEncoded) => {
+            const mutationDef = getMutationDef(schema, mutationEventEncoded.mutation)
+            return mutationDef.options.clientOnly === false
+          })
 
-          yield* BucketQueue.offerAll(syncBackendQueue, filteredBatch)
+        if (globalPendingMutationEvents.length > 0) {
+          yield* BucketQueue.offerAll(syncBackendPushQueue, globalPendingMutationEvents)
         }
+      }
 
-        const shutdownOnError = (cause: unknown) =>
-          Effect.gen(function* () {
-            if (onError === 'shutdown') {
-              yield* shutdownChannel.send(UnexpectedError.make({ cause }))
-              yield* Effect.die(cause)
-            }
-          })
-
-        yield* backgroundApplyLocalPushes({
-          localPushesLatch,
-          localPushesQueue,
-          pullLatch,
-          syncStateSref,
-          syncBackendQueue,
-          schema,
-          isClientEvent,
-          otelSpan,
-          currentLocalPushGenerationRef,
-        }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped)
-
-        const backendPushingFiberHandle = yield* FiberHandle.make()
-
-        yield* FiberHandle.run(
-          backendPushingFiberHandle,
-          backgroundBackendPushing({
-            dbReady,
-            syncBackendQueue,
-            otelSpan,
-            devtoolsLatch: ctxRef.current?.devtoolsLatch,
-          }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError)),
-        )
+      const shutdownOnError = (cause: unknown) =>
+        Effect.gen(function* () {
+          if (onError === 'shutdown') {
+            yield* shutdownChannel.send(UnexpectedError.make({ cause }))
+            yield* Effect.die(cause)
+          }
+        })
 
-        yield* backgroundBackendPulling({
-          dbReady,
-          initialBackendHead,
-          isClientEvent,
-          restartBackendPushing: (filteredRebasedPending) =>
-            Effect.gen(function* () {
-              // Stop current pushing fiber
-              yield* FiberHandle.clear(backendPushingFiberHandle)
-
-              // Reset the sync queue
-              yield* BucketQueue.clear(syncBackendQueue)
-              yield* BucketQueue.offerAll(syncBackendQueue, filteredRebasedPending)
-
-              // Restart pushing fiber
-              yield* FiberHandle.run(
-                backendPushingFiberHandle,
-                backgroundBackendPushing({
-                  dbReady,
-                  syncBackendQueue,
-                  otelSpan,
-                  devtoolsLatch: ctxRef.current?.devtoolsLatch,
-                }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError)),
-              )
-            }),
-          syncStateSref,
-          localPushesLatch,
-          pullLatch,
+      yield* backgroundApplyLocalPushes({
+        localPushesLatch,
+        localPushesQueue,
+        pullLatch,
+        syncStateSref,
+        syncBackendPushQueue,
+        schema,
+        isClientEvent,
+        otelSpan,
+        currentLocalPushGenerationRef,
+        connectedClientSessionPullQueues,
+        mergeCounterRef,
+        mergePayloads,
+      }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped)
+
+      const backendPushingFiberHandle = yield* FiberHandle.make()
+
+      yield* FiberHandle.run(
+        backendPushingFiberHandle,
+        backgroundBackendPushing({
+          syncBackendPushQueue,
           otelSpan,
-          initialBlockingSyncContext,
           devtoolsLatch: ctxRef.current?.devtoolsLatch,
-        }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped)
+        }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError)),
+      )
+
+      yield* backgroundBackendPulling({
+        initialBackendHead,
+        isClientEvent,
+        restartBackendPushing: (filteredRebasedPending) =>
+          Effect.gen(function* () {
+            // Stop current pushing fiber
+            yield* FiberHandle.clear(backendPushingFiberHandle)
+
+            // Reset the sync backend push queue
+            yield* BucketQueue.clear(syncBackendPushQueue)
+            yield* BucketQueue.offerAll(syncBackendPushQueue, filteredRebasedPending)
+
+            // Restart pushing fiber
+            yield* FiberHandle.run(
+              backendPushingFiberHandle,
+              backgroundBackendPushing({
+                syncBackendPushQueue,
+                otelSpan,
+                devtoolsLatch: ctxRef.current?.devtoolsLatch,
+              }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError)),
+            )
+          }),
+        syncStateSref,
+        localPushesLatch,
+        pullLatch,
+        otelSpan,
+        initialBlockingSyncContext,
+        devtoolsLatch: ctxRef.current?.devtoolsLatch,
+        connectedClientSessionPullQueues,
+        mergeCounterRef,
+        mergePayloads,
+      }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped)
+
+      return { initialLeaderHead: initialLocalHead }
+    }).pipe(Effect.withSpanScoped('@livestore/common:LeaderSyncProcessor:boot'))
+
+    const pull: LeaderSyncProcessor['pull'] = ({ cursor }) => {
+      return Effect.gen(function* () {
+        const queue = yield* pullQueue({ cursor })
+        return Stream.fromQueue(queue)
+      }).pipe(Stream.unwrapScoped)
+    }
+
+    const pullQueue: LeaderSyncProcessor['pullQueue'] = ({ cursor }) => {
+      const runtime = ctxRef.current?.runtime ?? shouldNeverHappen('Not initialized')
+      return Effect.gen(function* () {
+        const queue = yield* connectedClientSessionPullQueues.makeQueue(cursor)
+        const payloadsSinceCursor = Array.from(mergePayloads.entries())
+          .map(([mergeCounter, payload]) => ({ payload, mergeCounter }))
+          .filter(({ mergeCounter }) => mergeCounter > cursor)
+          .toSorted((a, b) => a.mergeCounter - b.mergeCounter)
 
-        return { initialLeaderHead: initialLocalHead }
-      }).pipe(Effect.withSpanScoped('@livestore/common:leader-thread:syncing'))
+        yield* queue.offerAll(payloadsSinceCursor)
+
+        return queue
+      }).pipe(Effect.provide(runtime))
+    }
+
+    const syncState = Subscribable.make({
+      get: Effect.gen(function* () {
+        const syncState = yield* syncStateSref
+        if (syncState === undefined) return shouldNeverHappen('Not initialized')
+        return syncState
+      }),
+      changes: syncStateSref.changes.pipe(Stream.filter(isNotUndefined)),
+    })
 
     return {
+      pull,
+      pullQueue,
       push,
       pushPartial,
       boot,
-      syncState: Subscribable.make({
-        get: Effect.gen(function* () {
-          const syncState = yield* syncStateSref
-          if (syncState === undefined) return shouldNeverHappen('Not initialized')
-          return syncState
-        }),
-        changes: syncStateSref.changes.pipe(Stream.filter(isNotUndefined)),
-      }),
+      syncState,
+      getMergeCounter: () => mergeCounterRef.current,
     } satisfies LeaderSyncProcessor
   })
 
@@ -321,30 +355,32 @@ const backgroundApplyLocalPushes = ({
   localPushesQueue,
   pullLatch,
   syncStateSref,
-  syncBackendQueue,
+  syncBackendPushQueue,
   schema,
   isClientEvent,
   otelSpan,
   currentLocalPushGenerationRef,
+  connectedClientSessionPullQueues,
+  mergeCounterRef,
+  mergePayloads,
 }: {
   pullLatch: Effect.Latch
   localPushesLatch: Effect.Latch
   localPushesQueue: BucketQueue.BucketQueue<LocalPushQueueItem>
   syncStateSref: SubscriptionRef.SubscriptionRef<SyncState.SyncState | undefined>
-  syncBackendQueue: BucketQueue.BucketQueue<MutationEvent.EncodedWithMeta>
+  syncBackendPushQueue: BucketQueue.BucketQueue<MutationEvent.EncodedWithMeta>
   schema: LiveStoreSchema
   isClientEvent: (mutationEventEncoded: MutationEvent.EncodedWithMeta) => boolean
   otelSpan: otel.Span | undefined
   currentLocalPushGenerationRef: { current: number }
+  connectedClientSessionPullQueues: PullQueueSet
+  mergeCounterRef: { current: number }
+  mergePayloads: Map<number, typeof SyncState.PayloadUpstream.Type>
 }) =>
   Effect.gen(function* () {
-    const { connectedClientSessionPullQueues, clientId } = yield* LeaderThreadCtx
-
-    const applyMutationItems = yield* makeApplyMutationItems
-
     while (true) {
       // TODO make batch size configurable
-      const batchItems = yield* BucketQueue.takeBetween(localPushesQueue, 1, 10)
+      const batchItems = yield* BucketQueue.takeBetween(localPushesQueue, 1, LOCAL_PUSH_BATCH_SIZE)
 
       // Wait for the backend pulling to finish
       yield* localPushesLatch.await
@@ -377,9 +413,11 @@ const backgroundApplyLocalPushes = ({
         isEqualEvent: MutationEvent.isEqualEncoded,
       })
 
+      const mergeCounter = yield* incrementMergeCounter(mergeCounterRef)
+
       switch (mergeResult._tag) {
         case 'unexpected-error': {
-          otelSpan?.addEvent('local-push:unexpected-error', {
+          otelSpan?.addEvent(`merge[${mergeCounter}]:local-push:unexpected-error`, {
             batchSize: newEvents.length,
             newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
           })
@@ -389,15 +427,12 @@ const backgroundApplyLocalPushes = ({
           return shouldNeverHappen('The leader thread should never have to rebase due to a local push')
         }
         case 'reject': {
-          otelSpan?.addEvent('local-push:reject', {
+          otelSpan?.addEvent(`merge[${mergeCounter}]:local-push:reject`, {
             batchSize: newEvents.length,
             mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
           })
 
-          /*
-
-          TODO: how to test this?
-          */
+          // TODO: how to test this?
           currentLocalPushGenerationRef.current++
 
           const nextGeneration = currentLocalPushGenerationRef.current
@@ -411,7 +446,8 @@ const backgroundApplyLocalPushes = ({
             (item) => item[2] >= nextGeneration,
           )
 
-          if ((yield* BucketQueue.size(localPushesQueue)) > 0) {
+          // TODO we still need to better understand and handle this scenario
+          if (LS_DEV && (yield* BucketQueue.size(localPushesQueue)) > 0) {
             console.log('localPushesQueue is not empty', yield* BucketQueue.size(localPushesQueue))
             debugger
           }
@@ -449,16 +485,13 @@ const backgroundApplyLocalPushes = ({
 
       yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState)
 
-      if (clientId === 'client-b') {
-        // yield* Effect.log('offer upstream-advance due to local-push')
-        // debugger
-      }
       yield* connectedClientSessionPullQueues.offer({
-        payload: { _tag: 'upstream-advance', newEvents: mergeResult.newEvents },
-        remaining: 0,
+        payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
+        mergeCounter,
       })
+      mergePayloads.set(mergeCounter, SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }))
 
-      otelSpan?.addEvent('local-push', {
+      otelSpan?.addEvent(`merge[${mergeCounter}]:local-push:advance`, {
         batchSize: newEvents.length,
         mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
       })
@@ -469,67 +502,65 @@ const backgroundApplyLocalPushes = ({
         return mutationDef.options.clientOnly === false
       })
 
-      yield* BucketQueue.offerAll(syncBackendQueue, filteredBatch)
+      yield* BucketQueue.offerAll(syncBackendPushQueue, filteredBatch)
 
-      yield* applyMutationItems({ batchItems: newEvents, deferreds })
+      yield* applyMutationsBatch({ batchItems: newEvents, deferreds })
 
       // Allow the backend pulling to start
       yield* pullLatch.open
     }
   })
 
-type ApplyMutationItems = (_: {
+type ApplyMutationsBatch = (_: {
   batchItems: ReadonlyArray<MutationEvent.EncodedWithMeta>
-  /** Indexes are aligned with `batchItems` */
+  /**
+   * The deferreds are used by the caller to know when the mutation has been processed.
+   * Indexes are aligned with `batchItems`
+   */
   deferreds: ReadonlyArray<Deferred.Deferred<void, LeaderAheadError> | undefined> | undefined
-}) => Effect.Effect<void, UnexpectedError>
+}) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
 
 // TODO how to handle errors gracefully
-const makeApplyMutationItems: Effect.Effect<ApplyMutationItems, UnexpectedError, LeaderThreadCtx | Scope.Scope> =
+const applyMutationsBatch: ApplyMutationsBatch = ({ batchItems, deferreds }) =>
   Effect.gen(function* () {
-    const leaderThreadCtx = yield* LeaderThreadCtx
-    const { dbReadModel: db, dbMutationLog } = leaderThreadCtx
+    const { dbReadModel: db, dbMutationLog, applyMutation } = yield* LeaderThreadCtx
 
-    const applyMutation = yield* makeApplyMutation
+    // NOTE We always start a transaction to ensure consistency between db and mutation log (even for single-item batches)
+    db.execute('BEGIN TRANSACTION', undefined) // Start the transaction
+    dbMutationLog.execute('BEGIN TRANSACTION', undefined) // Start the transaction
 
-    return ({ batchItems, deferreds }) =>
+    yield* Effect.addFinalizer((exit) =>
       Effect.gen(function* () {
-        db.execute('BEGIN TRANSACTION', undefined) // Start the transaction
-        dbMutationLog.execute('BEGIN TRANSACTION', undefined) // Start the transaction
-
-        yield* Effect.addFinalizer((exit) =>
-          Effect.gen(function* () {
-            if (Exit.isSuccess(exit)) return
+        if (Exit.isSuccess(exit)) return
 
-            // Rollback in case of an error
-            db.execute('ROLLBACK', undefined)
-            dbMutationLog.execute('ROLLBACK', undefined)
-          }),
-        )
+        // Rollback in case of an error
+        db.execute('ROLLBACK', undefined)
+        dbMutationLog.execute('ROLLBACK', undefined)
+      }),
+    )
 
-        for (let i = 0; i < batchItems.length; i++) {
-          yield* applyMutation(batchItems[i]!)
+    for (let i = 0; i < batchItems.length; i++) {
+      const { sessionChangeset } = yield* applyMutation(batchItems[i]!)
+      batchItems[i]!.meta.sessionChangeset = sessionChangeset
 
-          if (deferreds?.[i] !== undefined) {
-            yield* Deferred.succeed(deferreds[i]!, void 0)
-          }
-        }
+      if (deferreds?.[i] !== undefined) {
+        yield* Deferred.succeed(deferreds[i]!, void 0)
+      }
+    }
 
-        db.execute('COMMIT', undefined) // Commit the transaction
-        dbMutationLog.execute('COMMIT', undefined) // Commit the transaction
-      }).pipe(
-        Effect.uninterruptible,
-        Effect.scoped,
-        Effect.withSpan('@livestore/common:leader-thread:syncing:applyMutationItems', {
-          attributes: { count: batchItems.length },
-        }),
-        Effect.tapCauseLogPretty,
-        UnexpectedError.mapToUnexpectedError,
-      )
-  })
+    db.execute('COMMIT', undefined) // Commit the transaction
+    dbMutationLog.execute('COMMIT', undefined) // Commit the transaction
+  }).pipe(
+    Effect.uninterruptible,
+    Effect.scoped,
+    Effect.withSpan('@livestore/common:LeaderSyncProcessor:applyMutationItems', {
+      attributes: { batchSize: batchItems.length },
+    }),
+    Effect.tapCauseLogPretty,
+    UnexpectedError.mapToUnexpectedError,
+  )
 
 const backgroundBackendPulling = ({
-  dbReady,
   initialBackendHead,
   isClientEvent,
   restartBackendPushing,
@@ -539,8 +570,10 @@ const backgroundBackendPulling = ({
   pullLatch,
   devtoolsLatch,
   initialBlockingSyncContext,
+  connectedClientSessionPullQueues,
+  mergeCounterRef,
+  mergePayloads,
 }: {
-  dbReady: Deferred.Deferred<void>
   initialBackendHead: EventId.GlobalEventId
   isClientEvent: (mutationEventEncoded: MutationEvent.EncodedWithMeta) => boolean
   restartBackendPushing: (
@@ -552,23 +585,15 @@ const backgroundBackendPulling = ({
   pullLatch: Effect.Latch
   devtoolsLatch: Effect.Latch | undefined
   initialBlockingSyncContext: InitialBlockingSyncContext
+  connectedClientSessionPullQueues: PullQueueSet
+  mergeCounterRef: { current: number }
+  mergePayloads: Map<number, typeof SyncState.PayloadUpstream.Type>
 }) =>
   Effect.gen(function* () {
-    const {
-      syncBackend,
-      dbReadModel: db,
-      dbMutationLog,
-      connectedClientSessionPullQueues,
-      schema,
-      clientId,
-    } = yield* LeaderThreadCtx
+    const { syncBackend, dbReadModel: db, dbMutationLog, schema } = yield* LeaderThreadCtx
 
     if (syncBackend === undefined) return
 
-    const cursorInfo = yield* getCursorInfo(initialBackendHead)
-
-    const applyMutationItems = yield* makeApplyMutationItems
-
     const onNewPullChunk = (newEvents: MutationEvent.EncodedWithMeta[], remaining: number) =>
       Effect.gen(function* () {
         if (newEvents.length === 0) return
@@ -586,20 +611,20 @@ const backgroundBackendPulling = ({
         const syncState = yield* syncStateSref
         if (syncState === undefined) return shouldNeverHappen('Not initialized')
 
-        const trimRollbackUntil = newEvents.at(-1)!.id
-
         const mergeResult = SyncState.merge({
           syncState,
-          payload: { _tag: 'upstream-advance', newEvents, trimRollbackUntil },
+          payload: SyncState.PayloadUpstreamAdvance.make({ newEvents }),
           isClientEvent,
           isEqualEvent: MutationEvent.isEqualEncoded,
           ignoreClientEvents: true,
         })
 
+        const mergeCounter = yield* incrementMergeCounter(mergeCounterRef)
+
         if (mergeResult._tag === 'reject') {
           return shouldNeverHappen('The leader thread should never reject upstream advances')
         } else if (mergeResult._tag === 'unexpected-error') {
-          otelSpan?.addEvent('backend-pull:unexpected-error', {
+          otelSpan?.addEvent(`merge[${mergeCounter}]:backend-pull:unexpected-error`, {
             newEventsCount: newEvents.length,
             newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
           })
@@ -608,62 +633,79 @@ const backgroundBackendPulling = ({
 
         const newBackendHead = newEvents.at(-1)!.id
 
-        updateBackendHead(dbMutationLog, newBackendHead)
+        Mutationlog.updateBackendHead(dbMutationLog, newBackendHead)
 
         if (mergeResult._tag === 'rebase') {
-          otelSpan?.addEvent('backend-pull:rebase', {
+          otelSpan?.addEvent(`merge[${mergeCounter}]:backend-pull:rebase`, {
             newEventsCount: newEvents.length,
             newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
-            rollbackCount: mergeResult.eventsToRollback.length,
+            rollbackCount: mergeResult.rollbackEvents.length,
             mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
           })
 
-          const filteredRebasedPending = mergeResult.newSyncState.pending.filter((mutationEvent) => {
+          const globalRebasedPendingEvents = mergeResult.newSyncState.pending.filter((mutationEvent) => {
             const mutationDef = getMutationDef(schema, mutationEvent.mutation)
             return mutationDef.options.clientOnly === false
           })
-          yield* restartBackendPushing(filteredRebasedPending)
+          yield* restartBackendPushing(globalRebasedPendingEvents)
 
-          if (mergeResult.eventsToRollback.length > 0) {
-            yield* rollback({ db, dbMutationLog, eventIdsToRollback: mergeResult.eventsToRollback.map((_) => _.id) })
+          if (mergeResult.rollbackEvents.length > 0) {
+            yield* rollback({ db, dbMutationLog, eventIdsToRollback: mergeResult.rollbackEvents.map((_) => _.id) })
           }
 
           yield* connectedClientSessionPullQueues.offer({
-            payload: {
-              _tag: 'upstream-rebase',
+            payload: SyncState.PayloadUpstreamRebase.make({
               newEvents: mergeResult.newEvents,
-              rollbackUntil: mergeResult.eventsToRollback.at(0)!.id,
-              trimRollbackUntil,
-            },
-            remaining,
+              rollbackEvents: mergeResult.rollbackEvents,
+            }),
+            mergeCounter,
           })
+          mergePayloads.set(
+            mergeCounter,
+            SyncState.PayloadUpstreamRebase.make({
+              newEvents: mergeResult.newEvents,
+              rollbackEvents: mergeResult.rollbackEvents,
+            }),
+          )
         } else {
-          otelSpan?.addEvent('backend-pull:advance', {
+          otelSpan?.addEvent(`merge[${mergeCounter}]:backend-pull:advance`, {
             newEventsCount: newEvents.length,
             mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
           })
 
-          if (clientId === 'client-b') {
-            // yield* Effect.log('offer upstream-advance due to pull')
-          }
           yield* connectedClientSessionPullQueues.offer({
-            payload: { _tag: 'upstream-advance', newEvents: mergeResult.newEvents, trimRollbackUntil },
-            remaining,
+            payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
+            mergeCounter,
           })
+          mergePayloads.set(mergeCounter, SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }))
+
+          if (mergeResult.confirmedEvents.length > 0) {
+            // `mergeResult.confirmedEvents` don't contain the correct sync metadata, so we need to use
+            // `newEvents` instead which we filter via `mergeResult.confirmedEvents`
+            const confirmedNewEvents = newEvents.filter((mutationEvent) =>
+              mergeResult.confirmedEvents.some((confirmedEvent) =>
+                EventId.isEqual(mutationEvent.id, confirmedEvent.id),
+              ),
+            )
+            yield* Mutationlog.updateSyncMetadata(confirmedNewEvents)
+          }
         }
 
+        // Removes the changeset rows which are no longer needed as we'll never have to rollback beyond this point
         trimChangesetRows(db, newBackendHead)
 
-        yield* applyMutationItems({ batchItems: mergeResult.newEvents, deferreds: undefined })
+        yield* applyMutationsBatch({ batchItems: mergeResult.newEvents, deferreds: undefined })
 
         yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState)
 
+        // Allow local pushes to be processed again
         if (remaining === 0) {
-          // Allow local pushes to be processed again
           yield* localPushesLatch.open
         }
       })
 
+    const cursorInfo = yield* Mutationlog.getSyncBackendCursorInfo(initialBackendHead)
+
     yield* syncBackend.pull(cursorInfo).pipe(
       // TODO only take from queue while connected
       Stream.tap(({ batch, remaining }) =>
@@ -675,16 +717,13 @@ const backgroundBackendPulling = ({
           //   },
           // })
 
-          // Wait for the db to be initially created
-          yield* dbReady
-
           // NOTE we only want to take process mutations when the sync backend is connected
           // (e.g. needed for simulating being offline)
           // TODO remove when there's a better way to handle this in stream above
           yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
 
           yield* onNewPullChunk(
-            batch.map((_) => MutationEvent.EncodedWithMeta.fromGlobal(_.mutationEventEncoded)),
+            batch.map((_) => MutationEvent.EncodedWithMeta.fromGlobal(_.mutationEventEncoded, _.metadata)),
             remaining,
           )
 
@@ -694,102 +733,26 @@ const backgroundBackendPulling = ({
       Stream.runDrain,
       Effect.interruptible,
     )
-  }).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:backend-pulling'))
-
-const rollback = ({
-  db,
-  dbMutationLog,
-  eventIdsToRollback,
-}: {
-  db: SqliteDb
-  dbMutationLog: SqliteDb
-  eventIdsToRollback: EventId.EventId[]
-}) =>
-  Effect.gen(function* () {
-    const rollbackEvents = db
-      .select<SessionChangesetMetaRow>(
-        sql`SELECT * FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdsToRollback.map((id) => `(${id.global}, ${id.client})`).join(', ')})`,
-      )
-      .map((_) => ({ id: { global: _.idGlobal, client: _.idClient }, changeset: _.changeset, debug: _.debug }))
-      .sort((a, b) => EventId.compare(a.id, b.id))
-    // TODO bring back `.toSorted` once Expo supports it
-    // .toSorted((a, b) => EventId.compare(a.id, b.id))
-
-    // Apply changesets in reverse order
-    for (let i = rollbackEvents.length - 1; i >= 0; i--) {
-      const { changeset } = rollbackEvents[i]!
-      if (changeset !== null) {
-        db.makeChangeset(changeset).invert().apply()
-      }
-    }
-
-    const eventIdPairChunks = ReadonlyArray.chunksOf(100)(
-      eventIdsToRollback.map((id) => `(${id.global}, ${id.client})`),
-    )
-
-    // Delete the changeset rows
-    for (const eventIdPairChunk of eventIdPairChunks) {
-      db.execute(
-        sql`DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdPairChunk.join(', ')})`,
-      )
-    }
-
-    // Delete the mutation log rows
-    for (const eventIdPairChunk of eventIdPairChunks) {
-      dbMutationLog.execute(
-        sql`DELETE FROM ${MUTATION_LOG_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdPairChunk.join(', ')})`,
-      )
-    }
-  }).pipe(
-    Effect.withSpan('@livestore/common:leader-thread:syncing:rollback', {
-      attributes: { count: eventIdsToRollback.length },
-    }),
-  )
-
-const getCursorInfo = (remoteHead: EventId.GlobalEventId) =>
-  Effect.gen(function* () {
-    const { dbMutationLog } = yield* LeaderThreadCtx
-
-    if (remoteHead === EventId.ROOT.global) return Option.none()
-
-    const MutationlogQuerySchema = Schema.Struct({
-      syncMetadataJson: Schema.parseJson(Schema.Option(Schema.JsonValue)),
-    }).pipe(Schema.pluck('syncMetadataJson'), Schema.Array, Schema.head)
-
-    const syncMetadataOption = yield* Effect.sync(() =>
-      dbMutationLog.select<{ syncMetadataJson: string }>(
-        sql`SELECT syncMetadataJson FROM ${MUTATION_LOG_META_TABLE} WHERE idGlobal = ${remoteHead} ORDER BY idClient ASC LIMIT 1`,
-      ),
-    ).pipe(Effect.andThen(Schema.decode(MutationlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie)
-
-    return Option.some({
-      cursor: { global: remoteHead, client: EventId.clientDefault },
-      metadata: syncMetadataOption,
-    }) satisfies InitialSyncInfo
-  }).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:getCursorInfo', { attributes: { remoteHead } }))
+  }).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pulling'))
 
 const backgroundBackendPushing = ({
-  dbReady,
-  syncBackendQueue,
+  syncBackendPushQueue,
   otelSpan,
   devtoolsLatch,
 }: {
-  dbReady: Deferred.Deferred<void>
-  syncBackendQueue: BucketQueue.BucketQueue<MutationEvent.EncodedWithMeta>
+  syncBackendPushQueue: BucketQueue.BucketQueue<MutationEvent.EncodedWithMeta>
   otelSpan: otel.Span | undefined
   devtoolsLatch: Effect.Latch | undefined
 }) =>
   Effect.gen(function* () {
-    const { syncBackend, dbMutationLog } = yield* LeaderThreadCtx
+    const { syncBackend } = yield* LeaderThreadCtx
     if (syncBackend === undefined) return
 
-    yield* dbReady
-
     while (true) {
       yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
 
       // TODO make batch size configurable
-      const queueItems = yield* BucketQueue.takeBetween(syncBackendQueue, 1, BACKEND_PUSH_BATCH_SIZE)
+      const queueItems = yield* BucketQueue.takeBetween(syncBackendPushQueue, 1, BACKEND_PUSH_BATCH_SIZE)
 
       yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
 
@@ -813,27 +776,88 @@ const backgroundBackendPushing = ({
         // wait for interrupt caused by background pulling which will then restart pushing
         return yield* Effect.never
       }
-
-      const { metadata } = pushResult.right
-
-      // TODO try to do this in a single query
-      for (let i = 0; i < queueItems.length; i++) {
-        const mutationEventEncoded = queueItems[i]!
-        yield* execSql(
-          dbMutationLog,
-          ...updateRows({
-            tableName: MUTATION_LOG_META_TABLE,
-            columns: mutationLogMetaTable.sqliteDef.columns,
-            where: { idGlobal: mutationEventEncoded.id.global, idClient: mutationEventEncoded.id.client },
-            updateValues: { syncMetadataJson: metadata[i]! },
-          }),
-        )
-      }
     }
-  }).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:leader-thread:syncing:backend-pushing'))
+  }).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pushing'))
 
 const trimChangesetRows = (db: SqliteDb, newHead: EventId.EventId) => {
   // Since we're using the session changeset rows to query for the current head,
   // we're keeping at least one row for the current head, and thus are using `<` instead of `<=`
   db.execute(sql`DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE idGlobal < ${newHead.global}`)
 }
+
+interface PullQueueSet {
+  makeQueue: (
+    cursor: number,
+  ) => Effect.Effect<
+    Queue.Queue<{ payload: typeof SyncState.PayloadUpstream.Type; mergeCounter: number }>,
+    UnexpectedError,
+    Scope.Scope | LeaderThreadCtx
+  >
+  offer: (item: {
+    payload: typeof SyncState.PayloadUpstream.Type
+    mergeCounter: number
+  }) => Effect.Effect<void, UnexpectedError>
+}
+
+const makePullQueueSet = Effect.gen(function* () {
+  const set = new Set<Queue.Queue<{ payload: typeof SyncState.PayloadUpstream.Type; mergeCounter: number }>>()
+
+  yield* Effect.addFinalizer(() =>
+    Effect.gen(function* () {
+      for (const queue of set) {
+        yield* Queue.shutdown(queue)
+      }
+
+      set.clear()
+    }),
+  )
+
+  const makeQueue: PullQueueSet['makeQueue'] = () =>
+    Effect.gen(function* () {
+      const queue = yield* Queue.unbounded<{
+        payload: typeof SyncState.PayloadUpstream.Type
+        mergeCounter: number
+      }>().pipe(Effect.acquireRelease(Queue.shutdown))
+
+      yield* Effect.addFinalizer(() => Effect.sync(() => set.delete(queue)))
+
+      set.add(queue)
+
+      return queue
+    })
+
+  const offer: PullQueueSet['offer'] = (item) =>
+    Effect.gen(function* () {
+      // Short-circuit if the payload is an empty upstream advance
+      if (item.payload._tag === 'upstream-advance' && item.payload.newEvents.length === 0) {
+        return
+      }
+
+      for (const queue of set) {
+        yield* Queue.offer(queue, item)
+      }
+    })
+
+  return {
+    makeQueue,
+    offer,
+  }
+})
+
+const incrementMergeCounter = (mergeCounterRef: { current: number }) =>
+  Effect.gen(function* () {
+    const { dbReadModel } = yield* LeaderThreadCtx
+    mergeCounterRef.current++
+    dbReadModel.execute(
+      sql`INSERT OR REPLACE INTO ${LEADER_MERGE_COUNTER_TABLE} (id, mergeCounter) VALUES (0, ${mergeCounterRef.current})`,
+    )
+    return mergeCounterRef.current
+  })
+
+const getMergeCounterFromDb = (dbReadModel: SqliteDb) =>
+  Effect.gen(function* () {
+    const result = dbReadModel.select<{ mergeCounter: number }>(
+      sql`SELECT mergeCounter FROM ${LEADER_MERGE_COUNTER_TABLE} WHERE id = 0`,
+    )
+    return result[0]?.mergeCounter ?? 0
+  })