@livestore/common 0.3.0-dev.11 → 0.3.0-dev.2

package/src/leader-thread/{LeaderSyncProcessor.ts → leader-sync-processor.ts}

@@ -1,22 +1,22 @@
-import { isNotUndefined, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
-import type { HttpClient, Scope, Tracer } from '@livestore/utils/effect'
+import { shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
+import type { HttpClient, Scope } from '@livestore/utils/effect'
 import {
   BucketQueue,
   Deferred,
   Effect,
   Exit,
+  Fiber,
   FiberHandle,
   Option,
   OtelTracer,
-  ReadonlyArray,
+  Ref,
   Schema,
   Stream,
-  Subscribable,
   SubscriptionRef,
 } from '@livestore/utils/effect'
 import type * as otel from '@opentelemetry/api'
 
-import type { SqliteDb } from '../adapter-types.js'
+import type { SynchronousDatabase } from '../adapter-types.js'
 import { UnexpectedError } from '../adapter-types.js'
 import type { LiveStoreSchema, SessionChangesetMetaRow } from '../schema/mod.js'
 import {
@@ -33,114 +33,202 @@ import { sql } from '../util.js'
 import { makeApplyMutation } from './apply-mutation.js'
 import { execSql } from './connection.js'
 import { getBackendHeadFromDb, getLocalHeadFromDb, getMutationEventsSince, updateBackendHead } from './mutationlog.js'
-import type { InitialBlockingSyncContext, InitialSyncInfo, LeaderSyncProcessor } from './types.js'
+import type { InitialBlockingSyncContext, InitialSyncInfo, SyncProcessor } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
-type PushQueueItem = [
-  mutationEvent: MutationEvent.EncodedWithMeta,
-  deferred: Deferred.Deferred<void, InvalidPushError> | undefined,
-]
+type ProcessorStateInit = {
+  _tag: 'init'
+}
+
+type ProcessorStateInSync = {
+  _tag: 'in-sync'
+  syncState: SyncState.SyncState
+}
+
+type ProcessorStateApplyingSyncStateAdvance = {
+  _tag: 'applying-syncstate-advance'
+  origin: 'pull' | 'push'
+  syncState: SyncState.SyncState
+  // TODO re-introduce this
+  // proccesHead: EventId
+  fiber: Fiber.RuntimeFiber<void, UnexpectedError>
+}
+
+type ProcessorState = ProcessorStateInit | ProcessorStateInSync | ProcessorStateApplyingSyncStateAdvance
 
 /**
- * The LeaderSyncProcessor manages synchronization of mutations between
- * the local state and the sync backend, ensuring efficient and orderly processing.
+ * The general idea of the sync processor is to "follow the sync state"
+ * and apply/rollback mutations as needed to the read model and mutation log.
+ * The leader sync processor is also responsible for
+ * - broadcasting mutations to client sessions via the pull queues.
+ * - pushing mutations to the sync backend
  *
- * In the LeaderSyncProcessor, pulling always has precedence over pushing.
+ * In the leader sync processor, pulling always has precedence over pushing.
  *
- * Responsibilities:
- * - Queueing incoming local mutations in a localPushMailbox.
- * - Broadcasting mutations to client sessions via pull queues.
- * - Pushing mutations to the sync backend.
+ * External events:
+ * - Mutation pushed from client session
+ * - Mutation pushed from devtools (via pushPartial)
+ * - Mutation pulled from sync backend
  *
- * Notes:
+ * The machine can be in the following states:
+ * - in-sync: fully synced with remote, now idling
+ * - applying-syncstate-advance (with pointer to current progress in case of rebase interrupt)
  *
- * local push processing:
- * - localPushMailbox:
- *   - Maintains events in ascending order.
- *   - Uses `Deferred` objects to resolve/reject events based on application success.
- * - Processes events from the mailbox, applying mutations in batches.
- * - Controlled by a `Latch` to manage execution flow.
- * - The latch closes on pull receipt and re-opens post-pull completion.
- * - Processes up to `maxBatchSize` events per cycle.
+ * Transitions:
+ * - in-sync -> applying-syncstate-advance
+ * - applying-syncstate-advance -> in-sync
+ * - applying-syncstate-advance -> applying-syncstate-advance (need to interrupt previous operation)
  *
+ * Queuing vs interrupting behaviour:
+ * - Operations caused by pull can never be interrupted
+ * - Incoming pull can interrupt current push
+ * - Incoming pull needs to wait to previous pull to finish
+ * - Incoming push needs to wait to previous push to finish
+ *
+ * Backend pushing:
+ * - continously push to backend
+ * - only interrupted and restarted on rebase
  */
 export const makeLeaderSyncProcessor = ({
   schema,
   dbMissing,
-  dbMutationLog,
+  dbLog,
   initialBlockingSyncContext,
 }: {
   schema: LiveStoreSchema
-  /** Only used to know whether we can safely query dbMutationLog during setup execution */
+  /** Only used to know whether we can safely query dbLog during setup execution */
   dbMissing: boolean
-  dbMutationLog: SqliteDb
+  dbLog: SynchronousDatabase
   initialBlockingSyncContext: InitialBlockingSyncContext
-}): Effect.Effect<LeaderSyncProcessor, UnexpectedError, Scope.Scope> =>
+}): Effect.Effect<SyncProcessor, UnexpectedError, Scope.Scope> =>
   Effect.gen(function* () {
-    const syncBackendQueue = yield* BucketQueue.make<MutationEvent.EncodedWithMeta>()
+    const syncBackendQueue = yield* BucketQueue.make<MutationEvent.AnyEncoded>()
+
+    const stateRef = yield* Ref.make<ProcessorState>({ _tag: 'init' })
 
-    const syncStateSref = yield* SubscriptionRef.make<SyncState.SyncState | undefined>(undefined)
+    const semaphore = yield* Effect.makeSemaphore(1)
 
     const isLocalEvent = (mutationEventEncoded: MutationEvent.EncodedWithMeta) => {
       const mutationDef = schema.mutations.get(mutationEventEncoded.mutation)!
       return mutationDef.options.localOnly
     }
 
-    // This context depends on data from `boot`, we should find a better implementation to avoid this ref indirection.
-    const ctxRef = {
-      current: undefined as
-        | undefined
-        | {
-            otelSpan: otel.Span | undefined
-            span: Tracer.Span
-            devtoolsPullLatch: Effect.Latch | undefined
-            devtoolsPushLatch: Effect.Latch | undefined
-          },
-    }
+    const spanRef = { current: undefined as otel.Span | undefined }
+    const applyMutationItemsRef = { current: undefined as ApplyMutationItems | undefined }
+
+    // TODO get rid of counters once Effect semaphore ordering is fixed
+    let counterRef = 0
+    let expectedCounter = 0
+
+    /*
+    TODO: refactor
+    - Pushes go directly into a Mailbox
+    - Have a worker fiber that takes from the mailbox (wouldn't need a semaphore)
+    */
 
-    const localPushesQueue = yield* BucketQueue.make<PushQueueItem>()
-    const localPushesLatch = yield* Effect.makeLatch(true)
-    const pullLatch = yield* Effect.makeLatch(true)
+    const waitForSyncState = (counter: number): Effect.Effect<ProcessorStateInSync> =>
+      Effect.gen(function* () {
+        // console.log('waitForSyncState: waiting for semaphore', counter)
+        yield* semaphore.take(1)
+        // NOTE this is a workaround to ensure the semaphore take-order is respected
+        // TODO this needs to be fixed upstream in Effect
+        if (counter !== expectedCounter) {
+          console.log(
+            `waitForSyncState: counter mismatch (expected: ${expectedCounter}, got: ${counter}), releasing semaphore`,
+          )
+          yield* semaphore.release(1)
+          yield* Effect.yieldNow()
+          // Retrying...
+          return yield* waitForSyncState(counter)
+        }
+        // console.log('waitForSyncState: took semaphore', counter)
+        const state = yield* Ref.get(stateRef)
+        if (state._tag !== 'in-sync') {
+          return shouldNeverHappen('Expected to be in-sync but got ' + state._tag)
+        }
+        expectedCounter = counter + 1
+        return state
+      }).pipe(Effect.withSpan(`@livestore/common:leader-thread:syncing:waitForSyncState(${counter})`))
 
-    const push: LeaderSyncProcessor['push'] = (newEvents, options) =>
+    const push = (newEvents: ReadonlyArray<MutationEvent.EncodedWithMeta>) =>
       Effect.gen(function* () {
+        const counter = counterRef
+        counterRef++
         // TODO validate batch
         if (newEvents.length === 0) return
 
-        if (ctxRef.current?.devtoolsPushLatch !== undefined) {
-          yield* ctxRef.current.devtoolsPushLatch.await
-        }
+        const { connectedClientSessionPullQueues } = yield* LeaderThreadCtx
 
-        const waitForProcessing = options?.waitForProcessing ?? false
+        // TODO if there are multiple pending pushes, we should batch them together
+        const state = yield* waitForSyncState(counter)
 
-        if (waitForProcessing) {
-          const deferreds = yield* Effect.forEach(newEvents, () => Deferred.make<void, InvalidPushError>())
+        const updateResult = SyncState.updateSyncState({
+          syncState: state.syncState,
+          payload: { _tag: 'local-push', newEvents },
+          isLocalEvent,
+          isEqualEvent: MutationEvent.isEqualEncoded,
+        })
 
-          const items = newEvents.map(
-            (mutationEventEncoded, i) => [mutationEventEncoded, deferreds[i]] as PushQueueItem,
+        if (updateResult._tag === 'rebase') {
+          return shouldNeverHappen('The leader thread should never have to rebase due to a local push')
+        } else if (updateResult._tag === 'reject') {
+          return yield* Effect.fail(
+            InvalidPushError.make({
+              reason: {
+                _tag: 'LeaderAhead',
+                minimumExpectedId: updateResult.expectedMinimumId,
+                providedId: newEvents.at(0)!.id,
+              },
+            }),
           )
+        }
 
-          yield* BucketQueue.offerAll(localPushesQueue, items)
+        const fiber = yield* applyMutationItemsRef.current!({ batchItems: updateResult.newEvents }).pipe(Effect.fork)
 
-          yield* Effect.all(deferreds)
-        } else {
-          const items = newEvents.map((mutationEventEncoded) => [mutationEventEncoded, undefined] as PushQueueItem)
-          yield* BucketQueue.offerAll(localPushesQueue, items)
-        }
+        yield* Ref.set(stateRef, {
+          _tag: 'applying-syncstate-advance',
+          origin: 'push',
+          syncState: updateResult.newSyncState,
+          fiber,
+        })
+
+        // console.log('setRef:applying-syncstate-advance after push', counter)
+
+        yield* connectedClientSessionPullQueues.offer({
+          payload: { _tag: 'upstream-advance', newEvents: updateResult.newEvents },
+          remaining: 0,
+        })
+
+        spanRef.current?.addEvent('local-push', {
+          batchSize: newEvents.length,
+          updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
+        })
+
+        // Don't sync localOnly mutations
+        const filteredBatch = updateResult.newEvents.filter((mutationEventEncoded) => {
+          const mutationDef = schema.mutations.get(mutationEventEncoded.mutation)!
+          return mutationDef.options.localOnly === false
+        })
+
+        yield* BucketQueue.offerAll(syncBackendQueue, filteredBatch)
+
+        yield* fiber // Waiting for the mutation to be applied
       }).pipe(
         Effect.withSpan('@livestore/common:leader-thread:syncing:local-push', {
           attributes: {
             batchSize: newEvents.length,
             batch: TRACE_VERBOSE ? newEvents : undefined,
           },
-          links: ctxRef.current?.span ? [{ _tag: 'SpanLink', span: ctxRef.current.span, attributes: {} }] : undefined,
+          links: spanRef.current
+            ? [{ _tag: 'SpanLink', span: OtelTracer.makeExternalSpan(spanRef.current.spanContext()), attributes: {} }]
+            : undefined,
         }),
       )
 
-    const pushPartial: LeaderSyncProcessor['pushPartial'] = (mutationEventEncoded_) =>
+    const pushPartial: SyncProcessor['pushPartial'] = (mutationEventEncoded_) =>
       Effect.gen(function* () {
-        const syncState = yield* syncStateSref
-        if (syncState === undefined) return shouldNeverHappen('Not initialized')
+        const state = yield* Ref.get(stateRef)
+        if (state._tag === 'init') return shouldNeverHappen('Not initialized')
 
         const mutationDef =
           schema.mutations.get(mutationEventEncoded_.mutation) ??
@@ -148,28 +236,20 @@ export const makeLeaderSyncProcessor = ({
 
         const mutationEventEncoded = new MutationEvent.EncodedWithMeta({
           ...mutationEventEncoded_,
-          ...EventId.nextPair(syncState.localHead, mutationDef.options.localOnly),
+          ...EventId.nextPair(state.syncState.localHead, mutationDef.options.localOnly),
         })
 
         yield* push([mutationEventEncoded])
       }).pipe(Effect.catchTag('InvalidPushError', Effect.orDie))
 
     // Starts various background loops
-    const boot: LeaderSyncProcessor['boot'] = ({ dbReady }) =>
+    const boot: SyncProcessor['boot'] = ({ dbReady }) =>
       Effect.gen(function* () {
-        const span = yield* Effect.currentSpan.pipe(Effect.orDie)
-        const otelSpan = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)))
-        const { devtools } = yield* LeaderThreadCtx
+        const span = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)))
+        spanRef.current = span
 
-        ctxRef.current = {
-          otelSpan,
-          span,
-          devtoolsPullLatch: devtools.enabled ? devtools.syncBackendPullLatch : undefined,
-          devtoolsPushLatch: devtools.enabled ? devtools.syncBackendPushLatch : undefined,
-        }
-
-        const initialBackendHead = dbMissing ? EventId.ROOT.global : getBackendHeadFromDb(dbMutationLog)
-        const initialLocalHead = dbMissing ? EventId.ROOT : getLocalHeadFromDb(dbMutationLog)
+        const initialBackendHead = dbMissing ? EventId.ROOT.global : getBackendHeadFromDb(dbLog)
+        const initialLocalHead = dbMissing ? EventId.ROOT : getLocalHeadFromDb(dbLog)
 
         if (initialBackendHead > initialLocalHead.global) {
           return shouldNeverHappen(
@@ -177,21 +257,20 @@ export const makeLeaderSyncProcessor = ({
           )
         }
 
-        const pendingMutationEvents = yield* getMutationEventsSince({
-          global: initialBackendHead,
-          local: EventId.localDefault,
-        }).pipe(Effect.map(ReadonlyArray.map((_) => new MutationEvent.EncodedWithMeta(_))))
+        const pendingMutationEvents = yield* getMutationEventsSince({ global: initialBackendHead, local: 0 })
 
-        const initialSyncState = new SyncState.SyncState({
-          pending: pendingMutationEvents,
+        const initialSyncState = {
+          pending: pendingMutationEvents.map((_) => new MutationEvent.EncodedWithMeta(_)),
           // On the leader we don't need a rollback tail beyond `pending` items
           rollbackTail: [],
-          upstreamHead: { global: initialBackendHead, local: EventId.localDefault },
+          upstreamHead: { global: initialBackendHead, local: 0 },
           localHead: initialLocalHead,
-        })
+        } as SyncState.SyncState
 
         /** State transitions need to happen atomically, so we use a Ref to track the state */
-        yield* SubscriptionRef.set(syncStateSref, initialSyncState)
+        yield* Ref.set(stateRef, { _tag: 'in-sync', syncState: initialSyncState })
+
+        applyMutationItemsRef.current = yield* makeApplyMutationItems({ stateRef, semaphore })
 
         // Rehydrate sync queue
         if (pendingMutationEvents.length > 0) {
@@ -205,22 +284,11 @@ export const makeLeaderSyncProcessor = ({
           yield* BucketQueue.offerAll(syncBackendQueue, filteredBatch)
         }
 
-        yield* backgroundApplyLocalPushes({
-          localPushesLatch,
-          localPushesQueue,
-          pullLatch,
-          syncStateSref,
-          syncBackendQueue,
-          schema,
-          isLocalEvent,
-          otelSpan,
-        }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
-
         const backendPushingFiberHandle = yield* FiberHandle.make()
 
         yield* FiberHandle.run(
           backendPushingFiberHandle,
-          backgroundBackendPushing({ dbReady, syncBackendQueue, otelSpan }).pipe(Effect.tapCauseLogPretty),
+          backgroundBackendPushing({ dbReady, syncBackendQueue, span }).pipe(Effect.tapCauseLogPretty),
         )
 
         yield* backgroundBackendPulling({
@@ -239,15 +307,14 @@ export const makeLeaderSyncProcessor = ({
               // Restart pushing fiber
               yield* FiberHandle.run(
                 backendPushingFiberHandle,
-                backgroundBackendPushing({ dbReady, syncBackendQueue, otelSpan }).pipe(Effect.tapCauseLogPretty),
+                backgroundBackendPushing({ dbReady, syncBackendQueue, span }).pipe(Effect.tapCauseLogPretty),
               )
             }),
-          syncStateSref,
-          localPushesLatch,
-          pullLatch,
-          otelSpan,
+          applyMutationItemsRef,
+          stateRef,
+          semaphore,
+          span,
           initialBlockingSyncContext,
-          devtoolsPullLatch: ctxRef.current?.devtoolsPullLatch,
         }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
       }).pipe(Effect.withSpanScoped('@livestore/common:leader-thread:syncing'))
 
@@ -255,141 +322,42 @@ export const makeLeaderSyncProcessor = ({
       push,
       pushPartial,
       boot,
-      syncState: Subscribable.make({
-        get: Effect.gen(function* () {
-          const syncState = yield* syncStateSref
-          if (syncState === undefined) return shouldNeverHappen('Not initialized')
-          return syncState
-        }),
-        changes: syncStateSref.changes.pipe(Stream.filter(isNotUndefined)),
+      syncState: Effect.gen(function* () {
+        const state = yield* Ref.get(stateRef)
+        if (state._tag === 'init') return shouldNeverHappen('Not initialized')
+        return state.syncState
       }),
-    } satisfies LeaderSyncProcessor
-  })
-
-const backgroundApplyLocalPushes = ({
-  localPushesLatch,
-  localPushesQueue,
-  pullLatch,
-  syncStateSref,
-  syncBackendQueue,
-  schema,
-  isLocalEvent,
-  otelSpan,
-}: {
-  pullLatch: Effect.Latch
-  localPushesLatch: Effect.Latch
-  localPushesQueue: BucketQueue.BucketQueue<PushQueueItem>
-  syncStateSref: SubscriptionRef.SubscriptionRef<SyncState.SyncState | undefined>
-  syncBackendQueue: BucketQueue.BucketQueue<MutationEvent.EncodedWithMeta>
-  schema: LiveStoreSchema
-  isLocalEvent: (mutationEventEncoded: MutationEvent.EncodedWithMeta) => boolean
-  otelSpan: otel.Span | undefined
-}) =>
-  Effect.gen(function* () {
-    const { connectedClientSessionPullQueues } = yield* LeaderThreadCtx
-
-    const applyMutationItems = yield* makeApplyMutationItems
-
-    while (true) {
-      // TODO make batch size configurable
-      const batchItems = yield* BucketQueue.takeBetween(localPushesQueue, 1, 10)
-      const [newEvents, deferreds] = ReadonlyArray.unzip(batchItems)
-
-      // Wait for the backend pulling to finish
-      yield* localPushesLatch.await
-
-      // Prevent the backend pulling from starting until this local push is finished
-      yield* pullLatch.close
-
-      const syncState = yield* syncStateSref
-      if (syncState === undefined) return shouldNeverHappen('Not initialized')
-
-      const updateResult = SyncState.updateSyncState({
-        syncState,
-        payload: { _tag: 'local-push', newEvents },
-        isLocalEvent,
-        isEqualEvent: MutationEvent.isEqualEncoded,
-      })
-
-      if (updateResult._tag === 'rebase') {
-        return shouldNeverHappen('The leader thread should never have to rebase due to a local push')
-      } else if (updateResult._tag === 'reject') {
-        otelSpan?.addEvent('local-push:reject', {
-          batchSize: newEvents.length,
-          updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
-        })
-
-        const providedId = newEvents.at(0)!.id
-        const remainingEvents = yield* BucketQueue.takeAll(localPushesQueue)
-        const allDeferreds = [...deferreds, ...remainingEvents.map(([_, deferred]) => deferred)].filter(isNotUndefined)
-        yield* Effect.forEach(allDeferreds, (deferred) =>
-          Deferred.fail(
-            deferred,
-            InvalidPushError.make({
-              // TODO improve error handling so it differentiates between a push being rejected
-              // because of itself or because of another push
-              reason: {
-                _tag: 'LeaderAhead',
-                minimumExpectedId: updateResult.expectedMinimumId,
-                providedId,
-              },
-            }),
-          ),
-        )
-
-        // Allow the backend pulling to start
-        yield* pullLatch.open
-
-        // In this case we're skipping state update and down/upstream processing
-        // We've cleared the local push queue and are now waiting for new local pushes / backend pulls
-        continue
-      }
-
-      yield* SubscriptionRef.set(syncStateSref, updateResult.newSyncState)
-
-      yield* connectedClientSessionPullQueues.offer({
-        payload: { _tag: 'upstream-advance', newEvents: updateResult.newEvents },
-        remaining: 0,
-      })
-
-      otelSpan?.addEvent('local-push', {
-        batchSize: newEvents.length,
-        updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
-      })
-
-      // Don't sync localOnly mutations
-      const filteredBatch = updateResult.newEvents.filter((mutationEventEncoded) => {
-        const mutationDef = schema.mutations.get(mutationEventEncoded.mutation)!
-        return mutationDef.options.localOnly === false
-      })
-
-      yield* BucketQueue.offerAll(syncBackendQueue, filteredBatch)
-
-      yield* applyMutationItems({ batchItems: newEvents, deferreds })
-
-      // Allow the backend pulling to start
-      yield* pullLatch.open
-    }
+    } satisfies SyncProcessor
   })
 
 type ApplyMutationItems = (_: {
   batchItems: ReadonlyArray<MutationEvent.EncodedWithMeta>
-  /** Indexes are aligned with `batchItems` */
-  deferreds: ReadonlyArray<Deferred.Deferred<void, InvalidPushError> | undefined> | undefined
 }) => Effect.Effect<void, UnexpectedError>
 
 // TODO how to handle errors gracefully
-const makeApplyMutationItems: Effect.Effect<ApplyMutationItems, UnexpectedError, LeaderThreadCtx | Scope.Scope> =
+const makeApplyMutationItems = ({
+  stateRef,
+  semaphore,
+}: {
+  stateRef: Ref.Ref<ProcessorState>
+  semaphore: Effect.Semaphore
+}): Effect.Effect<ApplyMutationItems, UnexpectedError, LeaderThreadCtx | Scope.Scope> =>
   Effect.gen(function* () {
     const leaderThreadCtx = yield* LeaderThreadCtx
-    const { dbReadModel: db, dbMutationLog } = leaderThreadCtx
+    const { db, dbLog } = leaderThreadCtx
 
     const applyMutation = yield* makeApplyMutation
 
-    return ({ batchItems, deferreds }) =>
+    return ({ batchItems }) =>
       Effect.gen(function* () {
+        const state = yield* Ref.get(stateRef)
+        if (state._tag !== 'applying-syncstate-advance') {
+          // console.log('applyMutationItems: counter', counter)
+          return shouldNeverHappen(`Expected to be applying-syncstate-advance but got ${state._tag}`)
+        }
+
         db.execute('BEGIN TRANSACTION', undefined) // Start the transaction
-        dbMutationLog.execute('BEGIN TRANSACTION', undefined) // Start the transaction
+        dbLog.execute('BEGIN TRANSACTION', undefined) // Start the transaction
 
         yield* Effect.addFinalizer((exit) =>
           Effect.gen(function* () {
@@ -397,26 +365,34 @@ const makeApplyMutationItems: Effect.Effect<ApplyMutationItems, UnexpectedError,
 
             // Rollback in case of an error
             db.execute('ROLLBACK', undefined)
-            dbMutationLog.execute('ROLLBACK', undefined)
+            dbLog.execute('ROLLBACK', undefined)
           }),
         )
 
         for (let i = 0; i < batchItems.length; i++) {
-          yield* applyMutation(batchItems[i]!)
+          const { meta, ...mutationEventEncoded } = batchItems[i]!
+
+          yield* applyMutation(mutationEventEncoded)
 
-          if (deferreds?.[i] !== undefined) {
-            yield* Deferred.succeed(deferreds[i]!, void 0)
+          if (meta?.deferred) {
+            yield* Deferred.succeed(meta.deferred, void 0)
           }
+
+          // TODO re-introduce this
+          // if (i < batchItems.length - 1) {
+          //   yield* Ref.set(stateRef, { ...state, proccesHead: batchItems[i + 1]!.id })
+          // }
         }
 
         db.execute('COMMIT', undefined) // Commit the transaction
-        dbMutationLog.execute('COMMIT', undefined) // Commit the transaction
+        dbLog.execute('COMMIT', undefined) // Commit the transaction
+
+        yield* Ref.set(stateRef, { _tag: 'in-sync', syncState: state.syncState })
+        // console.log('setRef:sync after applyMutationItems', counter)
+        yield* semaphore.release(1)
       }).pipe(
-        Effect.uninterruptible,
         Effect.scoped,
-        Effect.withSpan('@livestore/common:leader-thread:syncing:applyMutationItems', {
-          attributes: { count: batchItems.length },
-        }),
+        Effect.withSpan('@livestore/common:leader-thread:syncing:applyMutationItems'),
         Effect.tapCauseLogPretty,
         UnexpectedError.mapToUnexpectedError,
       )
@@ -427,62 +403,55 @@ const backgroundBackendPulling = ({
   initialBackendHead,
   isLocalEvent,
   restartBackendPushing,
-  otelSpan,
-  syncStateSref,
-  localPushesLatch,
-  pullLatch,
-  devtoolsPullLatch,
+  span,
+  stateRef,
+  applyMutationItemsRef,
+  semaphore,
   initialBlockingSyncContext,
 }: {
   dbReady: Deferred.Deferred<void>
-  initialBackendHead: EventId.GlobalEventId
+  initialBackendHead: number
   isLocalEvent: (mutationEventEncoded: MutationEvent.EncodedWithMeta) => boolean
   restartBackendPushing: (
     filteredRebasedPending: ReadonlyArray<MutationEvent.EncodedWithMeta>,
   ) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx | HttpClient.HttpClient>
-  otelSpan: otel.Span | undefined
-  syncStateSref: SubscriptionRef.SubscriptionRef<SyncState.SyncState | undefined>
-  localPushesLatch: Effect.Latch
-  pullLatch: Effect.Latch
-  devtoolsPullLatch: Effect.Latch | undefined
+  span: otel.Span | undefined
+  stateRef: Ref.Ref<ProcessorState>
+  applyMutationItemsRef: { current: ApplyMutationItems | undefined }
+  semaphore: Effect.Semaphore
   initialBlockingSyncContext: InitialBlockingSyncContext
 }) =>
   Effect.gen(function* () {
-    const {
-      syncBackend,
-      dbReadModel: db,
-      dbMutationLog,
-      connectedClientSessionPullQueues,
-      schema,
-    } = yield* LeaderThreadCtx
+    const { syncBackend, db, dbLog, connectedClientSessionPullQueues, schema } = yield* LeaderThreadCtx
 
     if (syncBackend === undefined) return
 
     const cursorInfo = yield* getCursorInfo(initialBackendHead)
 
-    const applyMutationItems = yield* makeApplyMutationItems
-
     const onNewPullChunk = (newEvents: MutationEvent.EncodedWithMeta[], remaining: number) =>
       Effect.gen(function* () {
         if (newEvents.length === 0) return
 
-        if (devtoolsPullLatch !== undefined) {
-          yield* devtoolsPullLatch.await
-        }
+        const state = yield* Ref.get(stateRef)
+        if (state._tag === 'init') return shouldNeverHappen('Not initialized')
 
-        // Prevent more local pushes from being processed until this pull is finished
-        yield* localPushesLatch.close
+        // const counter = state.counter + 1
 
-        // Wait for pending local pushes to finish
-        yield* pullLatch.await
-
-        const syncState = yield* syncStateSref
-        if (syncState === undefined) return shouldNeverHappen('Not initialized')
+        if (state._tag === 'applying-syncstate-advance') {
+          if (state.origin === 'push') {
+            yield* Fiber.interrupt(state.fiber)
+            // In theory we should force-take the semaphore here, but as it's still taken,
+            // it's already in the right state we want it to be in
+          } else {
+            // Wait for previous advance to finish
+            yield* semaphore.take(1)
+          }
+        }
 
         const trimRollbackUntil = newEvents.at(-1)!.id
 
         const updateResult = SyncState.updateSyncState({
-          syncState,
+          syncState: state.syncState,
           payload: { _tag: 'upstream-advance', newEvents, trimRollbackUntil },
           isLocalEvent,
           isEqualEvent: MutationEvent.isEqualEncoded,
@@ -495,10 +464,10 @@ const backgroundBackendPulling = ({
 
         const newBackendHead = newEvents.at(-1)!.id
 
-        updateBackendHead(dbMutationLog, newBackendHead)
+        updateBackendHead(dbLog, newBackendHead)
 
         if (updateResult._tag === 'rebase') {
-          otelSpan?.addEvent('backend-pull:rebase', {
+          span?.addEvent('backend-pull:rebase', {
             newEventsCount: newEvents.length,
             newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
             rollbackCount: updateResult.eventsToRollback.length,
@@ -512,7 +481,7 @@ const backgroundBackendPulling = ({
           yield* restartBackendPushing(filteredRebasedPending)
 
           if (updateResult.eventsToRollback.length > 0) {
-            yield* rollback({ db, dbMutationLog, eventIdsToRollback: updateResult.eventsToRollback.map((_) => _.id) })
+            yield* rollback({ db, dbLog, eventIdsToRollback: updateResult.eventsToRollback.map((_) => _.id) })
           }
 
           yield* connectedClientSessionPullQueues.offer({
@@ -525,7 +494,7 @@ const backgroundBackendPulling = ({
             remaining,
           })
         } else {
-          otelSpan?.addEvent('backend-pull:advance', {
+          span?.addEvent('backend-pull:advance', {
             newEventsCount: newEvents.length,
             updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
           })
@@ -536,16 +505,17 @@ const backgroundBackendPulling = ({
           })
         }
 
-        trimChangesetRows(db, newBackendHead)
+        const fiber = yield* applyMutationItemsRef.current!({
+          batchItems: updateResult.newEvents,
+        }).pipe(Effect.fork)
 
-        yield* applyMutationItems({ batchItems: updateResult.newEvents, deferreds: undefined })
-
-        yield* SubscriptionRef.set(syncStateSref, updateResult.newSyncState)
-
-        if (remaining === 0) {
-          // Allow local pushes to be processed again
-          yield* localPushesLatch.open
-        }
+        yield* Ref.set(stateRef, {
+          _tag: 'applying-syncstate-advance',
+          origin: 'pull',
+          syncState: updateResult.newSyncState,
+          fiber,
+        })
+        // console.log('setRef:applying-syncstate-advance after backgroundBackendPulling', -1)
       })
 
     yield* syncBackend.pull(cursorInfo).pipe(
@@ -568,7 +538,7 @@ const backgroundBackendPulling = ({
           yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
 
           yield* onNewPullChunk(
-            batch.map((_) => MutationEvent.EncodedWithMeta.fromGlobal(_.mutationEventEncoded)),
+            batch.map((_) => new MutationEvent.EncodedWithMeta(_.mutationEventEncoded)),
             remaining,
           )
 
@@ -582,11 +552,11 @@ const backgroundBackendPulling = ({
 
 const rollback = ({
   db,
-  dbMutationLog,
+  dbLog,
   eventIdsToRollback,
 }: {
-  db: SqliteDb
-  dbMutationLog: SqliteDb
+  db: SynchronousDatabase
+  dbLog: SynchronousDatabase
   eventIdsToRollback: EventId.EventId[]
 }) =>
   Effect.gen(function* () {
@@ -600,9 +570,7 @@ const rollback = ({
     // Apply changesets in reverse order
     for (let i = rollbackEvents.length - 1; i >= 0; i--) {
       const { changeset } = rollbackEvents[i]!
-      if (changeset !== null) {
-        db.makeChangeset(changeset).invert().apply()
-      }
+      db.makeChangeset(changeset).invert().apply()
     }
 
     // Delete the changeset rows
@@ -611,7 +579,7 @@ const rollback = ({
     )
 
     // Delete the mutation log rows
-    dbMutationLog.execute(
+    dbLog.execute(
       sql`DELETE FROM ${MUTATION_LOG_META_TABLE} WHERE (idGlobal, idLocal) IN (${eventIdsToRollback.map((id) => `(${id.global}, ${id.local})`).join(', ')})`,
     )
   }).pipe(
@@ -620,9 +588,9 @@ const rollback = ({
     }),
   )
 
-const getCursorInfo = (remoteHead: EventId.GlobalEventId) =>
+const getCursorInfo = (remoteHead: number) =>
   Effect.gen(function* () {
-    const { dbMutationLog } = yield* LeaderThreadCtx
+    const { dbLog } = yield* LeaderThreadCtx
 
     if (remoteHead === EventId.ROOT.global) return Option.none()
 
@@ -631,13 +599,13 @@ const getCursorInfo = (remoteHead: EventId.GlobalEventId) =>
     }).pipe(Schema.pluck('syncMetadataJson'), Schema.Array, Schema.head)
 
     const syncMetadataOption = yield* Effect.sync(() =>
-      dbMutationLog.select<{ syncMetadataJson: string }>(
+      dbLog.select<{ syncMetadataJson: string }>(
         sql`SELECT syncMetadataJson FROM ${MUTATION_LOG_META_TABLE} WHERE idGlobal = ${remoteHead} ORDER BY idLocal ASC LIMIT 1`,
       ),
     ).pipe(Effect.andThen(Schema.decode(MutationlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie)
 
     return Option.some({
-      cursor: { global: remoteHead, local: EventId.localDefault },
+      cursor: { global: remoteHead, local: 0 },
       metadata: syncMetadataOption,
     }) satisfies InitialSyncInfo
   }).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:getCursorInfo', { attributes: { remoteHead } }))
@@ -645,14 +613,14 @@ const getCursorInfo = (remoteHead: EventId.GlobalEventId) =>
 const backgroundBackendPushing = ({
   dbReady,
   syncBackendQueue,
-  otelSpan,
+  span,
 }: {
   dbReady: Deferred.Deferred<void>
-  syncBackendQueue: BucketQueue.BucketQueue<MutationEvent.EncodedWithMeta>
-  otelSpan: otel.Span | undefined
+  syncBackendQueue: BucketQueue.BucketQueue<MutationEvent.AnyEncoded>
+  span: otel.Span | undefined
 }) =>
   Effect.gen(function* () {
-    const { syncBackend, dbMutationLog } = yield* LeaderThreadCtx
+    const { syncBackend, dbLog } = yield* LeaderThreadCtx
     if (syncBackend === undefined) return
 
     yield* dbReady
@@ -665,17 +633,17 @@ const backgroundBackendPushing = ({
 
       yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
 
-      otelSpan?.addEvent('backend-push', {
+      span?.addEvent('backend-push', {
         batchSize: queueItems.length,
         batch: TRACE_VERBOSE ? JSON.stringify(queueItems) : undefined,
       })
 
       // TODO handle push errors (should only happen during concurrent pull+push)
-      const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either)
+      const pushResult = yield* syncBackend.push(queueItems).pipe(Effect.either)
 
       if (pushResult._tag === 'Left') {
-        otelSpan?.addEvent('backend-push-error', { error: pushResult.left.toString() })
-        // wait for interrupt caused by background pulling which will then restart pushing
+        span?.addEvent('backend-push-error', { error: pushResult.left.toString() })
+        // wait for interrupt and restarting of pushing
         return yield* Effect.never
       }
 
@@ -685,7 +653,7 @@ const backgroundBackendPushing = ({
       for (let i = 0; i < queueItems.length; i++) {
         const mutationEventEncoded = queueItems[i]!
         yield* execSql(
-          dbMutationLog,
+          dbLog,
           ...updateRows({
             tableName: MUTATION_LOG_META_TABLE,
             columns: mutationLogMetaTable.sqliteDef.columns,
@@ -696,9 +664,3 @@ const backgroundBackendPushing = ({
       }
     }
   }).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:leader-thread:syncing:backend-pushing'))
-
-const trimChangesetRows = (db: SqliteDb, newHead: EventId.EventId) => {
-  // Since we're using the session changeset rows to query for the current head,
-  // we're keeping at least one row for the current head, and thus are using `<` instead of `<=`
-  db.execute(sql`DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE idGlobal < ${newHead.global}`)
-}