@livestore/common 0.3.0-dev.26 → 0.3.0-dev.28

package/dist/leader-thread/LeaderSyncProcessor.js

@@ -1,16 +1,13 @@
 import { casesHandled, isNotUndefined, LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils';
-import { BucketQueue, Deferred, Effect, Exit, FiberHandle, Option, OtelTracer, ReadonlyArray, Schema, Stream, Subscribable, SubscriptionRef, } from '@livestore/utils/effect';
+import { BucketQueue, Deferred, Effect, Exit, FiberHandle, OtelTracer, Queue, ReadonlyArray, Stream, Subscribable, SubscriptionRef, } from '@livestore/utils/effect';
 import { UnexpectedError } from '../adapter-types.js';
-import { EventId, getMutationDef, MUTATION_LOG_META_TABLE, MutationEvent, mutationLogMetaTable, SESSION_CHANGESET_META_TABLE, } from '../schema/mod.js';
-import { updateRows } from '../sql-queries/index.js';
+import { EventId, getMutationDef, LEADER_MERGE_COUNTER_TABLE, MutationEvent, SESSION_CHANGESET_META_TABLE, } from '../schema/mod.js';
 import { LeaderAheadError } from '../sync/sync.js';
 import * as SyncState from '../sync/syncstate.js';
 import { sql } from '../util.js';
-import { makeApplyMutation } from './apply-mutation.js';
-import { execSql } from './connection.js';
-import { getBackendHeadFromDb, getClientHeadFromDb, getMutationEventsSince, updateBackendHead } from './mutationlog.js';
+import { rollback } from './apply-mutation.js';
+import * as Mutationlog from './mutationlog.js';
 import { LeaderThreadCtx } from './types.js';
-export const BACKEND_PUSH_BATCH_SIZE = 50;
 /**
  * The LeaderSyncProcessor manages synchronization of mutations between
  * the local state and the sync backend, ensuring efficient and orderly processing.
@@ -18,35 +15,47 @@ export const BACKEND_PUSH_BATCH_SIZE = 50;
  * In the LeaderSyncProcessor, pulling always has precedence over pushing.
  *
  * Responsibilities:
- * - Queueing incoming local mutations in a localPushMailbox.
+ * - Queueing incoming local mutations in a localPushesQueue.
  * - Broadcasting mutations to client sessions via pull queues.
  * - Pushing mutations to the sync backend.
  *
  * Notes:
  *
  * local push processing:
- * - localPushMailbox:
+ * - localPushesQueue:
  *   - Maintains events in ascending order.
  *   - Uses `Deferred` objects to resolve/reject events based on application success.
- * - Processes events from the mailbox, applying mutations in batches.
+ * - Processes events from the queue, applying mutations in batches.
  * - Controlled by a `Latch` to manage execution flow.
  * - The latch closes on pull receipt and re-opens post-pull completion.
  * - Processes up to `maxBatchSize` events per cycle.
  *
+ * Currently we're advancing the db read model and mutation log in lockstep, but we could also decouple this in the future
+ *
+ * Tricky concurrency scenarios:
+ * - Queued local push batches becoming invalid due to a prior local push item being rejected.
+ *   Solution: Introduce a generation number for local push batches which is used to filter out old batches items in case of rejection.
+ *
  */
-export const makeLeaderSyncProcessor = ({ schema, dbMissing, dbMutationLog, clientId, initialBlockingSyncContext, onError, }) => Effect.gen(function* () {
-    const syncBackendQueue = yield* BucketQueue.make();
+export const makeLeaderSyncProcessor = ({ schema, dbMutationLogMissing, dbMutationLog, dbReadModel, dbReadModelMissing, initialBlockingSyncContext, onError, params, testing, }) => Effect.gen(function* () {
+    const syncBackendPushQueue = yield* BucketQueue.make();
+    const localPushBatchSize = params.localPushBatchSize ?? 10;
+    const backendPushBatchSize = params.backendPushBatchSize ?? 50;
     const syncStateSref = yield* SubscriptionRef.make(undefined);
     const isClientEvent = (mutationEventEncoded) => {
         const mutationDef = getMutationDef(schema, mutationEventEncoded.mutation);
         return mutationDef.options.clientOnly;
     };
+    const connectedClientSessionPullQueues = yield* makePullQueueSet;
     /**
      * Tracks generations of queued local push events.
-     * If a batch is rejected, all subsequent push queue items with the same generation are also rejected,
+     * If a local-push batch is rejected, all subsequent push queue items with the same generation are also rejected,
      * even if they would be valid on their own.
      */
+    // TODO get rid of this in favour of the `mergeGeneration` event id field
     const currentLocalPushGenerationRef = { current: 0 };
+    const mergeCounterRef = { current: dbReadModelMissing ? 0 : yield* getMergeCounterFromDb(dbReadModel) };
+    const mergePayloads = new Map();
     // This context depends on data from `boot`, we should find a better implementation to avoid this ref indirection.
     const ctxRef = {
         current: undefined,
@@ -54,21 +63,25 @@ export const makeLeaderSyncProcessor = ({ schema, dbMissing, dbMutationLog, clie
     const localPushesQueue = yield* BucketQueue.make();
     const localPushesLatch = yield* Effect.makeLatch(true);
     const pullLatch = yield* Effect.makeLatch(true);
+    /**
+     * Additionally to the `syncStateSref` we also need the `pushHeadRef` in order to prevent old/duplicate
+     * events from being pushed in a scenario like this:
+     * - client session A pushes e1
+     * - leader sync processor takes a bit and hasn't yet taken e1 from the localPushesQueue
+     * - client session B also pushes e1 (which should be rejected)
+     *
+     * Thus the purpoe of the pushHeadRef is the guard the integrity of the local push queue
+     */
+    const pushHeadRef = { current: EventId.ROOT };
+    const advancePushHead = (eventId) => {
+        pushHeadRef.current = EventId.max(pushHeadRef.current, eventId);
+    };
+    // NOTE: New events are only pushed to sync backend after successful local push processing
     const push = (newEvents, options) => Effect.gen(function* () {
-        // TODO validate batch
         if (newEvents.length === 0)
             return;
-        // if (options.generation < currentLocalPushGenerationRef.current) {
-        //   debugger
-        //   // We can safely drop this batch as it's from a previous push generation
-        //   return
-        // }
-        if (clientId === 'client-b') {
-            // console.log(
-            //   'push from client session',
-            //   newEvents.map((item) => item.toJSON()),
-            // )
-        }
+        yield* validatePushBatch(newEvents, pushHeadRef.current);
+        advancePushHead(newEvents.at(-1).id);
         const waitForProcessing = options?.waitForProcessing ?? false;
         const generation = currentLocalPushGenerationRef.current;
         if (waitForProcessing) {
@@ -81,20 +94,21 @@ export const makeLeaderSyncProcessor = ({ schema, dbMissing, dbMutationLog, clie
             const items = newEvents.map((mutationEventEncoded) => [mutationEventEncoded, undefined, generation]);
             yield* BucketQueue.offerAll(localPushesQueue, items);
         }
-    }).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:local-push', {
+    }).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:push', {
         attributes: {
             batchSize: newEvents.length,
             batch: TRACE_VERBOSE ? newEvents : undefined,
         },
         links: ctxRef.current?.span ? [{ _tag: 'SpanLink', span: ctxRef.current.span, attributes: {} }] : undefined,
     }));
-    const pushPartial = ({ mutationEvent: partialMutationEvent, clientId, sessionId, }) => Effect.gen(function* () {
+    const pushPartial = ({ mutationEvent: { mutation, args }, clientId, sessionId, }) => Effect.gen(function* () {
         const syncState = yield* syncStateSref;
         if (syncState === undefined)
             return shouldNeverHappen('Not initialized');
-        const mutationDef = getMutationDef(schema, partialMutationEvent.mutation);
+        const mutationDef = getMutationDef(schema, mutation);
         const mutationEventEncoded = new MutationEvent.EncodedWithMeta({
-            ...partialMutationEvent,
+            mutation,
+            args,
             clientId,
             sessionId,
             ...EventId.nextPair(syncState.localHead, mutationDef.options.clientOnly),
@@ -102,28 +116,29 @@ export const makeLeaderSyncProcessor = ({ schema, dbMissing, dbMutationLog, clie
         yield* push([mutationEventEncoded]);
     }).pipe(Effect.catchTag('LeaderAheadError', Effect.orDie));
     // Starts various background loops
-    const boot = ({ dbReady }) => Effect.gen(function* () {
+    const boot = Effect.gen(function* () {
         const span = yield* Effect.currentSpan.pipe(Effect.orDie);
         const otelSpan = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)));
         const { devtools, shutdownChannel } = yield* LeaderThreadCtx;
+        const runtime = yield* Effect.runtime();
         ctxRef.current = {
             otelSpan,
             span,
             devtoolsLatch: devtools.enabled ? devtools.syncBackendLatch : undefined,
+            runtime,
         };
-        const initialBackendHead = dbMissing ? EventId.ROOT.global : getBackendHeadFromDb(dbMutationLog);
-        const initialLocalHead = dbMissing ? EventId.ROOT : getClientHeadFromDb(dbMutationLog);
+        const initialLocalHead = dbMutationLogMissing ? EventId.ROOT : Mutationlog.getClientHeadFromDb(dbMutationLog);
+        const initialBackendHead = dbMutationLogMissing
+            ? EventId.ROOT.global
+            : Mutationlog.getBackendHeadFromDb(dbMutationLog);
         if (initialBackendHead > initialLocalHead.global) {
             return shouldNeverHappen(`During boot the backend head (${initialBackendHead}) should never be greater than the local head (${initialLocalHead.global})`);
         }
-        const pendingMutationEvents = yield* getMutationEventsSince({
-            global: initialBackendHead,
-            client: EventId.clientDefault,
-        }).pipe(Effect.map(ReadonlyArray.map((_) => new MutationEvent.EncodedWithMeta(_))));
+        const pendingMutationEvents = dbMutationLogMissing
+            ? []
+            : yield* Mutationlog.getMutationEventsSince({ global: initialBackendHead, client: EventId.clientDefault });
         const initialSyncState = new SyncState.SyncState({
             pending: pendingMutationEvents,
-            // On the leader we don't need a rollback tail beyond `pending` items
-            rollbackTail: [],
             upstreamHead: { global: initialBackendHead, client: EventId.clientDefault },
             localHead: initialLocalHead,
         });
@@ -131,13 +146,15 @@ export const makeLeaderSyncProcessor = ({ schema, dbMissing, dbMutationLog, clie
         yield* SubscriptionRef.set(syncStateSref, initialSyncState);
         // Rehydrate sync queue
         if (pendingMutationEvents.length > 0) {
-            const filteredBatch = pendingMutationEvents
+            const globalPendingMutationEvents = pendingMutationEvents
                 // Don't sync clientOnly mutations
                 .filter((mutationEventEncoded) => {
                 const mutationDef = getMutationDef(schema, mutationEventEncoded.mutation);
                 return mutationDef.options.clientOnly === false;
             });
-            yield* BucketQueue.offerAll(syncBackendQueue, filteredBatch);
+            if (globalPendingMutationEvents.length > 0) {
+                yield* BucketQueue.offerAll(syncBackendPushQueue, globalPendingMutationEvents);
+            }
         }
         const shutdownOnError = (cause) => Effect.gen(function* () {
             if (onError === 'shutdown') {
@@ -150,36 +167,38 @@ export const makeLeaderSyncProcessor = ({ schema, dbMissing, dbMutationLog, clie
             localPushesQueue,
             pullLatch,
             syncStateSref,
-            syncBackendQueue,
+            syncBackendPushQueue,
             schema,
             isClientEvent,
             otelSpan,
             currentLocalPushGenerationRef,
+            connectedClientSessionPullQueues,
+            mergeCounterRef,
+            mergePayloads,
+            localPushBatchSize,
+            testing: {
+                delay: testing?.delays?.localPushProcessing,
+            },
         }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped);
         const backendPushingFiberHandle = yield* FiberHandle.make();
-        yield* FiberHandle.run(backendPushingFiberHandle, backgroundBackendPushing({
-            dbReady,
-            syncBackendQueue,
+        const backendPushingEffect = backgroundBackendPushing({
+            syncBackendPushQueue,
             otelSpan,
             devtoolsLatch: ctxRef.current?.devtoolsLatch,
-        }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError)));
+            backendPushBatchSize,
+        }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError));
+        yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect);
         yield* backgroundBackendPulling({
-            dbReady,
             initialBackendHead,
             isClientEvent,
             restartBackendPushing: (filteredRebasedPending) => Effect.gen(function* () {
                 // Stop current pushing fiber
                 yield* FiberHandle.clear(backendPushingFiberHandle);
-                // Reset the sync queue
-                yield* BucketQueue.clear(syncBackendQueue);
-                yield* BucketQueue.offerAll(syncBackendQueue, filteredRebasedPending);
+                // Reset the sync backend push queue
+                yield* BucketQueue.clear(syncBackendPushQueue);
+                yield* BucketQueue.offerAll(syncBackendPushQueue, filteredRebasedPending);
                 // Restart pushing fiber
-                yield* FiberHandle.run(backendPushingFiberHandle, backgroundBackendPushing({
-                    dbReady,
-                    syncBackendQueue,
-                    otelSpan,
-                    devtoolsLatch: ctxRef.current?.devtoolsLatch,
-                }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError)));
+                yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect);
             }),
             syncStateSref,
             localPushesLatch,
@@ -187,30 +206,68 @@ export const makeLeaderSyncProcessor = ({ schema, dbMissing, dbMutationLog, clie
             otelSpan,
             initialBlockingSyncContext,
             devtoolsLatch: ctxRef.current?.devtoolsLatch,
+            connectedClientSessionPullQueues,
+            mergeCounterRef,
+            mergePayloads,
+            advancePushHead,
         }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped);
         return { initialLeaderHead: initialLocalHead };
-    }).pipe(Effect.withSpanScoped('@livestore/common:leader-thread:syncing'));
+    }).pipe(Effect.withSpanScoped('@livestore/common:LeaderSyncProcessor:boot'));
+    const pull = ({ cursor }) => Effect.gen(function* () {
+        const queue = yield* pullQueue({ cursor });
+        return Stream.fromQueue(queue);
+    }).pipe(Stream.unwrapScoped);
+    const pullQueue = ({ cursor }) => {
+        const runtime = ctxRef.current?.runtime ?? shouldNeverHappen('Not initialized');
+        return Effect.gen(function* () {
+            const queue = yield* connectedClientSessionPullQueues.makeQueue;
+            const payloadsSinceCursor = Array.from(mergePayloads.entries())
+                .map(([mergeCounter, payload]) => ({ payload, mergeCounter }))
+                .filter(({ mergeCounter }) => mergeCounter > cursor.mergeCounter)
+                .toSorted((a, b) => a.mergeCounter - b.mergeCounter)
+                .map(({ payload, mergeCounter }) => {
+                if (payload._tag === 'upstream-advance') {
+                    return {
+                        payload: {
+                            _tag: 'upstream-advance',
+                            newEvents: ReadonlyArray.dropWhile(payload.newEvents, (mutationEventEncoded) => EventId.isGreaterThanOrEqual(cursor.eventId, mutationEventEncoded.id)),
+                        },
+                        mergeCounter,
+                    };
+                }
+                else {
+                    return { payload, mergeCounter };
+                }
+            });
+            yield* queue.offerAll(payloadsSinceCursor);
+            return queue;
+        }).pipe(Effect.provide(runtime));
+    };
+    const syncState = Subscribable.make({
+        get: Effect.gen(function* () {
+            const syncState = yield* syncStateSref;
+            if (syncState === undefined)
+                return shouldNeverHappen('Not initialized');
+            return syncState;
+        }),
+        changes: syncStateSref.changes.pipe(Stream.filter(isNotUndefined)),
+    });
     return {
+        pull,
+        pullQueue,
         push,
         pushPartial,
         boot,
-        syncState: Subscribable.make({
-            get: Effect.gen(function* () {
-                const syncState = yield* syncStateSref;
-                if (syncState === undefined)
-                    return shouldNeverHappen('Not initialized');
-                return syncState;
-            }),
-            changes: syncStateSref.changes.pipe(Stream.filter(isNotUndefined)),
-        }),
+        syncState,
+        getMergeCounter: () => mergeCounterRef.current,
     };
 });
-const backgroundApplyLocalPushes = ({ localPushesLatch, localPushesQueue, pullLatch, syncStateSref, syncBackendQueue, schema, isClientEvent, otelSpan, currentLocalPushGenerationRef, }) => Effect.gen(function* () {
-    const { connectedClientSessionPullQueues, clientId } = yield* LeaderThreadCtx;
-    const applyMutationItems = yield* makeApplyMutationItems;
+const backgroundApplyLocalPushes = ({ localPushesLatch, localPushesQueue, pullLatch, syncStateSref, syncBackendPushQueue, schema, isClientEvent, otelSpan, currentLocalPushGenerationRef, connectedClientSessionPullQueues, mergeCounterRef, mergePayloads, localPushBatchSize, testing, }) => Effect.gen(function* () {
     while (true) {
-        // TODO make batch size configurable
-        const batchItems = yield* BucketQueue.takeBetween(localPushesQueue, 1, 10);
+        if (testing.delay !== undefined) {
+            yield* testing.delay.pipe(Effect.withSpan('localPushProcessingDelay'));
+        }
+        const batchItems = yield* BucketQueue.takeBetween(localPushesQueue, 1, localPushBatchSize);
         // Wait for the backend pulling to finish
         yield* localPushesLatch.await;
         // Prevent backend pull processing until this local push is finished
@@ -236,9 +293,10 @@ const backgroundApplyLocalPushes = ({ localPushesLatch, localPushesQueue, pullLa
             isClientEvent,
             isEqualEvent: MutationEvent.isEqualEncoded,
         });
+        const mergeCounter = yield* incrementMergeCounter(mergeCounterRef);
         switch (mergeResult._tag) {
             case 'unexpected-error': {
-                otelSpan?.addEvent('local-push:unexpected-error', {
+                otelSpan?.addEvent(`[${mergeCounter}]:push:unexpected-error`, {
                     batchSize: newEvents.length,
                     newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
                 });
@@ -248,14 +306,11 @@ const backgroundApplyLocalPushes = ({ localPushesLatch, localPushesQueue, pullLa
                 return shouldNeverHappen('The leader thread should never have to rebase due to a local push');
             }
             case 'reject': {
-                otelSpan?.addEvent('local-push:reject', {
+                otelSpan?.addEvent(`[${mergeCounter}]:push:reject`, {
                     batchSize: newEvents.length,
                     mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
                 });
-                /*
-      
-                TODO: how to test this?
-                */
+                // TODO: how to test this?
                 currentLocalPushGenerationRef.current++;
                 const nextGeneration = currentLocalPushGenerationRef.current;
                 const providedId = newEvents.at(0).id;
@@ -263,7 +318,8 @@ const backgroundApplyLocalPushes = ({ localPushesLatch, localPushesQueue, pullLa
                 // We're also handling the case where the localPushQueue already contains events
                 // from the next generation which we preserve in the queue
                 const remainingEventsMatchingGeneration = yield* BucketQueue.takeSplitWhere(localPushesQueue, (item) => item[2] >= nextGeneration);
-                if ((yield* BucketQueue.size(localPushesQueue)) > 0) {
+                // TODO we still need to better understand and handle this scenario
+                if (LS_DEV && (yield* BucketQueue.size(localPushesQueue)) > 0) {
                     console.log('localPushesQueue is not empty', yield* BucketQueue.size(localPushesQueue));
                     debugger;
                 }
@@ -290,15 +346,12 @@ const backgroundApplyLocalPushes = ({ localPushesLatch, localPushesQueue, pullLa
             }
         }
         yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
-        if (clientId === 'client-b') {
-            // yield* Effect.log('offer upstream-advance due to local-push')
-            // debugger
-        }
         yield* connectedClientSessionPullQueues.offer({
-            payload: { _tag: 'upstream-advance', newEvents: mergeResult.newEvents },
-            remaining: 0,
+            payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
+            mergeCounter,
         });
-        otelSpan?.addEvent('local-push', {
+        mergePayloads.set(mergeCounter, SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }));
+        otelSpan?.addEvent(`[${mergeCounter}]:push:advance`, {
             batchSize: newEvents.length,
             mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
         });
@@ -307,45 +360,41 @@ const backgroundApplyLocalPushes = ({ localPushesLatch, localPushesQueue, pullLa
             const mutationDef = getMutationDef(schema, mutationEventEncoded.mutation);
             return mutationDef.options.clientOnly === false;
         });
-        yield* BucketQueue.offerAll(syncBackendQueue, filteredBatch);
-        yield* applyMutationItems({ batchItems: newEvents, deferreds });
+        yield* BucketQueue.offerAll(syncBackendPushQueue, filteredBatch);
+        yield* applyMutationsBatch({ batchItems: mergeResult.newEvents, deferreds });
         // Allow the backend pulling to start
         yield* pullLatch.open;
     }
 });
 // TODO how to handle errors gracefully
-const makeApplyMutationItems = Effect.gen(function* () {
-    const leaderThreadCtx = yield* LeaderThreadCtx;
-    const { dbReadModel: db, dbMutationLog } = leaderThreadCtx;
-    const applyMutation = yield* makeApplyMutation;
-    return ({ batchItems, deferreds }) => Effect.gen(function* () {
-        db.execute('BEGIN TRANSACTION', undefined); // Start the transaction
-        dbMutationLog.execute('BEGIN TRANSACTION', undefined); // Start the transaction
-        yield* Effect.addFinalizer((exit) => Effect.gen(function* () {
-            if (Exit.isSuccess(exit))
-                return;
-            // Rollback in case of an error
-            db.execute('ROLLBACK', undefined);
-            dbMutationLog.execute('ROLLBACK', undefined);
-        }));
-        for (let i = 0; i < batchItems.length; i++) {
-            yield* applyMutation(batchItems[i]);
-            if (deferreds?.[i] !== undefined) {
-                yield* Deferred.succeed(deferreds[i], void 0);
-            }
+const applyMutationsBatch = ({ batchItems, deferreds }) => Effect.gen(function* () {
+    const { dbReadModel: db, dbMutationLog, applyMutation } = yield* LeaderThreadCtx;
+    // NOTE We always start a transaction to ensure consistency between db and mutation log (even for single-item batches)
+    db.execute('BEGIN TRANSACTION', undefined); // Start the transaction
+    dbMutationLog.execute('BEGIN TRANSACTION', undefined); // Start the transaction
+    yield* Effect.addFinalizer((exit) => Effect.gen(function* () {
+        if (Exit.isSuccess(exit))
+            return;
+        // Rollback in case of an error
+        db.execute('ROLLBACK', undefined);
+        dbMutationLog.execute('ROLLBACK', undefined);
+    }));
+    for (let i = 0; i < batchItems.length; i++) {
+        const { sessionChangeset } = yield* applyMutation(batchItems[i]);
+        batchItems[i].meta.sessionChangeset = sessionChangeset;
+        if (deferreds?.[i] !== undefined) {
+            yield* Deferred.succeed(deferreds[i], void 0);
         }
-        db.execute('COMMIT', undefined); // Commit the transaction
-        dbMutationLog.execute('COMMIT', undefined); // Commit the transaction
-    }).pipe(Effect.uninterruptible, Effect.scoped, Effect.withSpan('@livestore/common:leader-thread:syncing:applyMutationItems', {
-        attributes: { count: batchItems.length },
-    }), Effect.tapCauseLogPretty, UnexpectedError.mapToUnexpectedError);
-});
-const backgroundBackendPulling = ({ dbReady, initialBackendHead, isClientEvent, restartBackendPushing, otelSpan, syncStateSref, localPushesLatch, pullLatch, devtoolsLatch, initialBlockingSyncContext, }) => Effect.gen(function* () {
-    const { syncBackend, dbReadModel: db, dbMutationLog, connectedClientSessionPullQueues, schema, clientId, } = yield* LeaderThreadCtx;
+    }
+    db.execute('COMMIT', undefined); // Commit the transaction
+    dbMutationLog.execute('COMMIT', undefined); // Commit the transaction
+}).pipe(Effect.uninterruptible, Effect.scoped, Effect.withSpan('@livestore/common:LeaderSyncProcessor:applyMutationItems', {
+    attributes: { batchSize: batchItems.length },
+}), Effect.tapCauseLogPretty, UnexpectedError.mapToUnexpectedError);
+const backgroundBackendPulling = ({ initialBackendHead, isClientEvent, restartBackendPushing, otelSpan, syncStateSref, localPushesLatch, pullLatch, devtoolsLatch, initialBlockingSyncContext, connectedClientSessionPullQueues, mergeCounterRef, mergePayloads, advancePushHead, }) => Effect.gen(function* () {
+    const { syncBackend, dbReadModel: db, dbMutationLog, schema } = yield* LeaderThreadCtx;
     if (syncBackend === undefined)
         return;
-    const cursorInfo = yield* getCursorInfo(initialBackendHead);
-    const applyMutationItems = yield* makeApplyMutationItems;
     const onNewPullChunk = (newEvents, remaining) => Effect.gen(function* () {
         if (newEvents.length === 0)
             return;
@@ -359,72 +408,81 @@ const backgroundBackendPulling = ({ dbReady, initialBackendHead, isClientEvent,
         const syncState = yield* syncStateSref;
         if (syncState === undefined)
             return shouldNeverHappen('Not initialized');
-        const trimRollbackUntil = newEvents.at(-1).id;
         const mergeResult = SyncState.merge({
             syncState,
-            payload: { _tag: 'upstream-advance', newEvents, trimRollbackUntil },
+            payload: SyncState.PayloadUpstreamAdvance.make({ newEvents }),
             isClientEvent,
             isEqualEvent: MutationEvent.isEqualEncoded,
             ignoreClientEvents: true,
         });
+        const mergeCounter = yield* incrementMergeCounter(mergeCounterRef);
         if (mergeResult._tag === 'reject') {
             return shouldNeverHappen('The leader thread should never reject upstream advances');
         }
         else if (mergeResult._tag === 'unexpected-error') {
-            otelSpan?.addEvent('backend-pull:unexpected-error', {
+            otelSpan?.addEvent(`[${mergeCounter}]:pull:unexpected-error`, {
                 newEventsCount: newEvents.length,
                 newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
             });
             return yield* Effect.fail(mergeResult.cause);
         }
         const newBackendHead = newEvents.at(-1).id;
-        updateBackendHead(dbMutationLog, newBackendHead);
+        Mutationlog.updateBackendHead(dbMutationLog, newBackendHead);
         if (mergeResult._tag === 'rebase') {
-            otelSpan?.addEvent('backend-pull:rebase', {
+            otelSpan?.addEvent(`[${mergeCounter}]:pull:rebase`, {
                 newEventsCount: newEvents.length,
                 newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
-                rollbackCount: mergeResult.eventsToRollback.length,
+                rollbackCount: mergeResult.rollbackEvents.length,
                 mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
             });
-            const filteredRebasedPending = mergeResult.newSyncState.pending.filter((mutationEvent) => {
+            const globalRebasedPendingEvents = mergeResult.newSyncState.pending.filter((mutationEvent) => {
                 const mutationDef = getMutationDef(schema, mutationEvent.mutation);
                 return mutationDef.options.clientOnly === false;
             });
-            yield* restartBackendPushing(filteredRebasedPending);
-            if (mergeResult.eventsToRollback.length > 0) {
-                yield* rollback({ db, dbMutationLog, eventIdsToRollback: mergeResult.eventsToRollback.map((_) => _.id) });
+            yield* restartBackendPushing(globalRebasedPendingEvents);
+            if (mergeResult.rollbackEvents.length > 0) {
+                yield* rollback({ db, dbMutationLog, eventIdsToRollback: mergeResult.rollbackEvents.map((_) => _.id) });
             }
             yield* connectedClientSessionPullQueues.offer({
-                payload: {
-                    _tag: 'upstream-rebase',
+                payload: SyncState.PayloadUpstreamRebase.make({
                     newEvents: mergeResult.newEvents,
-                    rollbackUntil: mergeResult.eventsToRollback.at(0).id,
-                    trimRollbackUntil,
-                },
-                remaining,
+                    rollbackEvents: mergeResult.rollbackEvents,
+                }),
+                mergeCounter,
             });
+            mergePayloads.set(mergeCounter, SyncState.PayloadUpstreamRebase.make({
+                newEvents: mergeResult.newEvents,
+                rollbackEvents: mergeResult.rollbackEvents,
+            }));
         }
         else {
-            otelSpan?.addEvent('backend-pull:advance', {
+            otelSpan?.addEvent(`[${mergeCounter}]:pull:advance`, {
                 newEventsCount: newEvents.length,
                 mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
             });
-            if (clientId === 'client-b') {
-                // yield* Effect.log('offer upstream-advance due to pull')
-            }
             yield* connectedClientSessionPullQueues.offer({
-                payload: { _tag: 'upstream-advance', newEvents: mergeResult.newEvents, trimRollbackUntil },
-                remaining,
+                payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
+                mergeCounter,
             });
+            mergePayloads.set(mergeCounter, SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }));
+            if (mergeResult.confirmedEvents.length > 0) {
+                // `mergeResult.confirmedEvents` don't contain the correct sync metadata, so we need to use
+                // `newEvents` instead which we filter via `mergeResult.confirmedEvents`
+                const confirmedNewEvents = newEvents.filter((mutationEvent) => mergeResult.confirmedEvents.some((confirmedEvent) => EventId.isEqual(mutationEvent.id, confirmedEvent.id)));
+                yield* Mutationlog.updateSyncMetadata(confirmedNewEvents);
+            }
         }
+        // Removes the changeset rows which are no longer needed as we'll never have to rollback beyond this point
         trimChangesetRows(db, newBackendHead);
-        yield* applyMutationItems({ batchItems: mergeResult.newEvents, deferreds: undefined });
+        advancePushHead(mergeResult.newSyncState.localHead);
+        yield* applyMutationsBatch({ batchItems: mergeResult.newEvents, deferreds: undefined });
         yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
+        // Allow local pushes to be processed again
         if (remaining === 0) {
-            // Allow local pushes to be processed again
             yield* localPushesLatch.open;
         }
     });
+    const cursorInfo = yield* Mutationlog.getSyncBackendCursorInfo(initialBackendHead);
     yield* syncBackend.pull(cursorInfo).pipe(
     // TODO only take from queue while connected
     Stream.tap(({ batch, remaining }) => Effect.gen(function* () {
@@ -434,64 +492,21 @@ const backgroundBackendPulling = ({ dbReady, initialBackendHead, isClientEvent,
         //     batch: TRACE_VERBOSE ? batch : undefined,
         //   },
         // })
-        // Wait for the db to be initially created
-        yield* dbReady;
         // NOTE we only want to take process mutations when the sync backend is connected
         // (e.g. needed for simulating being offline)
         // TODO remove when there's a better way to handle this in stream above
         yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
-        yield* onNewPullChunk(batch.map((_) => MutationEvent.EncodedWithMeta.fromGlobal(_.mutationEventEncoded)), remaining);
+        yield* onNewPullChunk(batch.map((_) => MutationEvent.EncodedWithMeta.fromGlobal(_.mutationEventEncoded, _.metadata)), remaining);
         yield* initialBlockingSyncContext.update({ processed: batch.length, remaining });
     })), Stream.runDrain, Effect.interruptible);
-}).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:backend-pulling'));
-const rollback = ({ db, dbMutationLog, eventIdsToRollback, }) => Effect.gen(function* () {
-    const rollbackEvents = db
-        .select(sql `SELECT * FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdsToRollback.map((id) => `(${id.global}, ${id.client})`).join(', ')})`)
-        .map((_) => ({ id: { global: _.idGlobal, client: _.idClient }, changeset: _.changeset, debug: _.debug }))
-        .sort((a, b) => EventId.compare(a.id, b.id));
-    // TODO bring back `.toSorted` once Expo supports it
-    // .toSorted((a, b) => EventId.compare(a.id, b.id))
-    // Apply changesets in reverse order
-    for (let i = rollbackEvents.length - 1; i >= 0; i--) {
-        const { changeset } = rollbackEvents[i];
-        if (changeset !== null) {
-            db.makeChangeset(changeset).invert().apply();
-        }
-    }
-    const eventIdPairChunks = ReadonlyArray.chunksOf(100)(eventIdsToRollback.map((id) => `(${id.global}, ${id.client})`));
-    // Delete the changeset rows
-    for (const eventIdPairChunk of eventIdPairChunks) {
-        db.execute(sql `DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdPairChunk.join(', ')})`);
-    }
-    // Delete the mutation log rows
-    for (const eventIdPairChunk of eventIdPairChunks) {
-        dbMutationLog.execute(sql `DELETE FROM ${MUTATION_LOG_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdPairChunk.join(', ')})`);
-    }
-}).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:rollback', {
-    attributes: { count: eventIdsToRollback.length },
-}));
-const getCursorInfo = (remoteHead) => Effect.gen(function* () {
-    const { dbMutationLog } = yield* LeaderThreadCtx;
-    if (remoteHead === EventId.ROOT.global)
-        return Option.none();
-    const MutationlogQuerySchema = Schema.Struct({
-        syncMetadataJson: Schema.parseJson(Schema.Option(Schema.JsonValue)),
-    }).pipe(Schema.pluck('syncMetadataJson'), Schema.Array, Schema.head);
-    const syncMetadataOption = yield* Effect.sync(() => dbMutationLog.select(sql `SELECT syncMetadataJson FROM ${MUTATION_LOG_META_TABLE} WHERE idGlobal = ${remoteHead} ORDER BY idClient ASC LIMIT 1`)).pipe(Effect.andThen(Schema.decode(MutationlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie);
-    return Option.some({
-        cursor: { global: remoteHead, client: EventId.clientDefault },
-        metadata: syncMetadataOption,
-    });
-}).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:getCursorInfo', { attributes: { remoteHead } }));
-const backgroundBackendPushing = ({ dbReady, syncBackendQueue, otelSpan, devtoolsLatch, }) => Effect.gen(function* () {
-    const { syncBackend, dbMutationLog } = yield* LeaderThreadCtx;
+}).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pulling'));
+const backgroundBackendPushing = ({ syncBackendPushQueue, otelSpan, devtoolsLatch, backendPushBatchSize, }) => Effect.gen(function* () {
+    const { syncBackend } = yield* LeaderThreadCtx;
     if (syncBackend === undefined)
         return;
-    yield* dbReady;
     while (true) {
         yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
-        // TODO make batch size configurable
-        const queueItems = yield* BucketQueue.takeBetween(syncBackendQueue, 1, BACKEND_PUSH_BATCH_SIZE);
+        const queueItems = yield* BucketQueue.takeBetween(syncBackendPushQueue, 1, backendPushBatchSize);
         yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
         if (devtoolsLatch !== undefined) {
             yield* devtoolsLatch.await;
@@ -510,22 +525,67 @@ const backgroundBackendPushing = ({ dbReady, syncBackendQueue, otelSpan, devtool
             // wait for interrupt caused by background pulling which will then restart pushing
             return yield* Effect.never;
         }
-        const { metadata } = pushResult.right;
-        // TODO try to do this in a single query
-        for (let i = 0; i < queueItems.length; i++) {
-            const mutationEventEncoded = queueItems[i];
-            yield* execSql(dbMutationLog, ...updateRows({
-                tableName: MUTATION_LOG_META_TABLE,
-                columns: mutationLogMetaTable.sqliteDef.columns,
-                where: { idGlobal: mutationEventEncoded.id.global, idClient: mutationEventEncoded.id.client },
-                updateValues: { syncMetadataJson: metadata[i] },
-            }));
-        }
     }
-}).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:leader-thread:syncing:backend-pushing'));
+}).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pushing'));
 const trimChangesetRows = (db, newHead) => {
     // Since we're using the session changeset rows to query for the current head,
     // we're keeping at least one row for the current head, and thus are using `<` instead of `<=`
     db.execute(sql `DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE idGlobal < ${newHead.global}`);
 };
+const makePullQueueSet = Effect.gen(function* () {
+    const set = new Set();
+    yield* Effect.addFinalizer(() => Effect.gen(function* () {
+        for (const queue of set) {
+            yield* Queue.shutdown(queue);
+        }
+        set.clear();
+    }));
+    const makeQueue = Effect.gen(function* () {
+        const queue = yield* Queue.unbounded().pipe(Effect.acquireRelease(Queue.shutdown));
+        yield* Effect.addFinalizer(() => Effect.sync(() => set.delete(queue)));
+        set.add(queue);
+        return queue;
+    });
+    const offer = (item) => Effect.gen(function* () {
+        // Short-circuit if the payload is an empty upstream advance
+        if (item.payload._tag === 'upstream-advance' && item.payload.newEvents.length === 0) {
+            return;
+        }
+        for (const queue of set) {
+            yield* Queue.offer(queue, item);
+        }
+    });
+    return {
+        makeQueue,
+        offer,
+    };
+});
+const incrementMergeCounter = (mergeCounterRef) => Effect.gen(function* () {
+    const { dbReadModel } = yield* LeaderThreadCtx;
+    mergeCounterRef.current++;
+    dbReadModel.execute(sql `INSERT OR REPLACE INTO ${LEADER_MERGE_COUNTER_TABLE} (id, mergeCounter) VALUES (0, ${mergeCounterRef.current})`);
+    return mergeCounterRef.current;
+});
+const getMergeCounterFromDb = (dbReadModel) => Effect.gen(function* () {
+    const result = dbReadModel.select(sql `SELECT mergeCounter FROM ${LEADER_MERGE_COUNTER_TABLE} WHERE id = 0`);
+    return result[0]?.mergeCounter ?? 0;
+});
+const validatePushBatch = (batch, pushHead) => Effect.gen(function* () {
+    if (batch.length === 0) {
+        return;
+    }
+    // Make sure batch is monotonically increasing
+    for (let i = 1; i < batch.length; i++) {
+        if (EventId.isGreaterThanOrEqual(batch[i - 1].id, batch[i].id)) {
+            shouldNeverHappen(`Events must be ordered in monotonically ascending order by eventId. Received: [${batch.map((e) => EventId.toString(e.id)).join(', ')}]`);
+        }
+    }
+    // Make sure smallest event id is > pushHead
+    if (EventId.isGreaterThanOrEqual(pushHead, batch[0].id)) {
+        return yield* LeaderAheadError.make({
+            minimumExpectedId: pushHead,
+            providedId: batch[0].id,
+        });
+    }
+});
 //# sourceMappingURL=LeaderSyncProcessor.js.map