@livestore/common 0.3.0-dev.9 → 0.3.0

package/dist/leader-thread/LeaderSyncProcessor.js

@@ -1,270 +1,406 @@
-import { isNotUndefined, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils';
-import { BucketQueue, Deferred, Effect, Exit, FiberHandle, Option, OtelTracer, ReadonlyArray, Schema, Stream, Subscribable, SubscriptionRef, } from '@livestore/utils/effect';
+import { casesHandled, isNotUndefined, LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils';
+import { BucketQueue, Deferred, Effect, Exit, FiberHandle, OtelTracer, Queue, ReadonlyArray, Stream, Subscribable, SubscriptionRef, } from '@livestore/utils/effect';
 import { UnexpectedError } from '../adapter-types.js';
-import { EventId, MUTATION_LOG_META_TABLE, MutationEvent, mutationLogMetaTable, SESSION_CHANGESET_META_TABLE, } from '../schema/mod.js';
-import { updateRows } from '../sql-queries/index.js';
-import { InvalidPushError } from '../sync/sync.js';
+import { EventSequenceNumber, getEventDef, LiveStoreEvent, SystemTables } from '../schema/mod.js';
+import { LeaderAheadError } from '../sync/sync.js';
 import * as SyncState from '../sync/syncstate.js';
 import { sql } from '../util.js';
-import { makeApplyMutation } from './apply-mutation.js';
-import { execSql } from './connection.js';
-import { getBackendHeadFromDb, getLocalHeadFromDb, getMutationEventsSince, updateBackendHead } from './mutationlog.js';
+import * as Eventlog from './eventlog.js';
+import { rollback } from './materialize-event.js';
 import { LeaderThreadCtx } from './types.js';
 /**
- * The LeaderSyncProcessor manages synchronization of mutations between
+ * The LeaderSyncProcessor manages synchronization of events between
  * the local state and the sync backend, ensuring efficient and orderly processing.
  *
  * In the LeaderSyncProcessor, pulling always has precedence over pushing.
  *
  * Responsibilities:
- * - Queueing incoming local mutations in a localPushMailbox.
- * - Broadcasting mutations to client sessions via pull queues.
- * - Pushing mutations to the sync backend.
+ * - Queueing incoming local events in a localPushesQueue.
+ * - Broadcasting events to client sessions via pull queues.
+ * - Pushing events to the sync backend.
  *
  * Notes:
  *
  * local push processing:
- * - localPushMailbox:
+ * - localPushesQueue:
  *   - Maintains events in ascending order.
  *   - Uses `Deferred` objects to resolve/reject events based on application success.
- * - Processes events from the mailbox, applying mutations in batches.
+ * - Processes events from the queue, applying events in batches.
  * - Controlled by a `Latch` to manage execution flow.
  * - The latch closes on pull receipt and re-opens post-pull completion.
  * - Processes up to `maxBatchSize` events per cycle.
  *
+ * Currently we're advancing the db read model and eventlog in lockstep, but we could also decouple this in the future
+ *
+ * Tricky concurrency scenarios:
+ * - Queued local push batches becoming invalid due to a prior local push item being rejected.
+ *   Solution: Introduce a generation number for local push batches which is used to filter out old batches items in case of rejection.
+ *
  */
-export const makeLeaderSyncProcessor = ({ schema, dbMissing, dbLog, initialBlockingSyncContext, }) => Effect.gen(function* () {
-    const syncBackendQueue = yield* BucketQueue.make();
+export const makeLeaderSyncProcessor = ({ schema, dbEventlogMissing, dbEventlog, dbState, dbStateMissing, initialBlockingSyncContext, onError, params, testing, }) => Effect.gen(function* () {
+    const syncBackendPushQueue = yield* BucketQueue.make();
+    const localPushBatchSize = params.localPushBatchSize ?? 10;
+    const backendPushBatchSize = params.backendPushBatchSize ?? 50;
     const syncStateSref = yield* SubscriptionRef.make(undefined);
-    const isLocalEvent = (mutationEventEncoded) => {
-        const mutationDef = schema.mutations.get(mutationEventEncoded.mutation);
-        return mutationDef.options.localOnly;
+    const isClientEvent = (eventEncoded) => {
+        const { eventDef } = getEventDef(schema, eventEncoded.name);
+        return eventDef.options.clientOnly;
+    };
+    const connectedClientSessionPullQueues = yield* makePullQueueSet;
+    /**
+     * Tracks generations of queued local push events.
+     * If a local-push batch is rejected, all subsequent push queue items with the same generation are also rejected,
+     * even if they would be valid on their own.
+     */
+    // TODO get rid of this in favour of the `mergeGeneration` event sequence number field
+    const currentLocalPushGenerationRef = { current: 0 };
+    const mergeCounterRef = { current: dbStateMissing ? 0 : yield* getMergeCounterFromDb(dbState) };
+    const mergePayloads = new Map();
+    // This context depends on data from `boot`, we should find a better implementation to avoid this ref indirection.
+    const ctxRef = {
+        current: undefined,
     };
-    const spanRef = { current: undefined };
     const localPushesQueue = yield* BucketQueue.make();
     const localPushesLatch = yield* Effect.makeLatch(true);
     const pullLatch = yield* Effect.makeLatch(true);
+    /**
+     * Additionally to the `syncStateSref` we also need the `pushHeadRef` in order to prevent old/duplicate
+     * events from being pushed in a scenario like this:
+     * - client session A pushes e1
+     * - leader sync processor takes a bit and hasn't yet taken e1 from the localPushesQueue
+     * - client session B also pushes e1 (which should be rejected)
+     *
+     * Thus the purpoe of the pushHeadRef is the guard the integrity of the local push queue
+     */
+    const pushHeadRef = { current: EventSequenceNumber.ROOT };
+    const advancePushHead = (eventNum) => {
+        pushHeadRef.current = EventSequenceNumber.max(pushHeadRef.current, eventNum);
+    };
+    // NOTE: New events are only pushed to sync backend after successful local push processing
     const push = (newEvents, options) => Effect.gen(function* () {
-        // TODO validate batch
         if (newEvents.length === 0)
             return;
+        yield* validatePushBatch(newEvents, pushHeadRef.current);
+        advancePushHead(newEvents.at(-1).seqNum);
         const waitForProcessing = options?.waitForProcessing ?? false;
-        const deferreds = waitForProcessing
-            ? yield* Effect.forEach(newEvents, () => Deferred.make())
-            : newEvents.map((_) => undefined);
-        // TODO validate batch ordering
-        const mappedEvents = newEvents.map((mutationEventEncoded, i) => new MutationEvent.EncodedWithMeta({
-            ...mutationEventEncoded,
-            meta: { deferred: deferreds[i] },
-        }));
-        yield* BucketQueue.offerAll(localPushesQueue, mappedEvents);
+        const generation = currentLocalPushGenerationRef.current;
         if (waitForProcessing) {
+            const deferreds = yield* Effect.forEach(newEvents, () => Deferred.make());
+            const items = newEvents.map((eventEncoded, i) => [eventEncoded, deferreds[i], generation]);
+            yield* BucketQueue.offerAll(localPushesQueue, items);
             yield* Effect.all(deferreds);
         }
-    }).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:local-push', {
+        else {
+            const items = newEvents.map((eventEncoded) => [eventEncoded, undefined, generation]);
+            yield* BucketQueue.offerAll(localPushesQueue, items);
+        }
+    }).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:push', {
         attributes: {
             batchSize: newEvents.length,
             batch: TRACE_VERBOSE ? newEvents : undefined,
         },
-        links: spanRef.current
-            ? [{ _tag: 'SpanLink', span: OtelTracer.makeExternalSpan(spanRef.current.spanContext()), attributes: {} }]
-            : undefined,
+        links: ctxRef.current?.span ? [{ _tag: 'SpanLink', span: ctxRef.current.span, attributes: {} }] : undefined,
     }));
-    const pushPartial = (mutationEventEncoded_) => Effect.gen(function* () {
+    const pushPartial = ({ event: { name, args }, clientId, sessionId }) => Effect.gen(function* () {
         const syncState = yield* syncStateSref;
         if (syncState === undefined)
             return shouldNeverHappen('Not initialized');
-        const mutationDef = schema.mutations.get(mutationEventEncoded_.mutation) ??
-            shouldNeverHappen(`Unknown mutation: ${mutationEventEncoded_.mutation}`);
-        const mutationEventEncoded = new MutationEvent.EncodedWithMeta({
-            ...mutationEventEncoded_,
-            ...EventId.nextPair(syncState.localHead, mutationDef.options.localOnly),
+        const { eventDef } = getEventDef(schema, name);
+        const eventEncoded = new LiveStoreEvent.EncodedWithMeta({
+            name,
+            args,
+            clientId,
+            sessionId,
+            ...EventSequenceNumber.nextPair(syncState.localHead, eventDef.options.clientOnly),
         });
-        yield* push([mutationEventEncoded]);
-    }).pipe(Effect.catchTag('InvalidPushError', Effect.orDie));
+        yield* push([eventEncoded]);
+    }).pipe(Effect.catchTag('LeaderAheadError', Effect.orDie));
     // Starts various background loops
-    const boot = ({ dbReady }) => Effect.gen(function* () {
-        const span = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)));
-        spanRef.current = span;
-        const initialBackendHead = dbMissing ? EventId.ROOT.global : getBackendHeadFromDb(dbLog);
-        const initialLocalHead = dbMissing ? EventId.ROOT : getLocalHeadFromDb(dbLog);
+    const boot = Effect.gen(function* () {
+        const span = yield* Effect.currentSpan.pipe(Effect.orDie);
+        const otelSpan = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)));
+        const { devtools, shutdownChannel } = yield* LeaderThreadCtx;
+        const runtime = yield* Effect.runtime();
+        ctxRef.current = {
+            otelSpan,
+            span,
+            devtoolsLatch: devtools.enabled ? devtools.syncBackendLatch : undefined,
+            runtime,
+        };
+        const initialLocalHead = dbEventlogMissing ? EventSequenceNumber.ROOT : Eventlog.getClientHeadFromDb(dbEventlog);
+        const initialBackendHead = dbEventlogMissing
+            ? EventSequenceNumber.ROOT.global
+            : Eventlog.getBackendHeadFromDb(dbEventlog);
         if (initialBackendHead > initialLocalHead.global) {
             return shouldNeverHappen(`During boot the backend head (${initialBackendHead}) should never be greater than the local head (${initialLocalHead.global})`);
         }
-        const pendingMutationEvents = yield* getMutationEventsSince({
-            global: initialBackendHead,
-            local: EventId.localDefault,
-        }).pipe(Effect.map(ReadonlyArray.map((_) => new MutationEvent.EncodedWithMeta(_))));
+        const pendingEvents = dbEventlogMissing
+            ? []
+            : yield* Eventlog.getEventsSince({ global: initialBackendHead, client: EventSequenceNumber.clientDefault });
         const initialSyncState = new SyncState.SyncState({
-            pending: pendingMutationEvents,
-            // On the leader we don't need a rollback tail beyond `pending` items
-            rollbackTail: [],
-            upstreamHead: { global: initialBackendHead, local: EventId.localDefault },
+            pending: pendingEvents,
+            upstreamHead: { global: initialBackendHead, client: EventSequenceNumber.clientDefault },
             localHead: initialLocalHead,
         });
         /** State transitions need to happen atomically, so we use a Ref to track the state */
         yield* SubscriptionRef.set(syncStateSref, initialSyncState);
         // Rehydrate sync queue
-        if (pendingMutationEvents.length > 0) {
-            const filteredBatch = pendingMutationEvents
-                // Don't sync localOnly mutations
-                .filter((mutationEventEncoded) => {
-                const mutationDef = schema.mutations.get(mutationEventEncoded.mutation);
-                return mutationDef.options.localOnly === false;
+        if (pendingEvents.length > 0) {
+            const globalPendingEvents = pendingEvents
+                // Don't sync clientOnly events
+                .filter((eventEncoded) => {
+                const { eventDef } = getEventDef(schema, eventEncoded.name);
+                return eventDef.options.clientOnly === false;
             });
-            yield* BucketQueue.offerAll(syncBackendQueue, filteredBatch);
+            if (globalPendingEvents.length > 0) {
+                yield* BucketQueue.offerAll(syncBackendPushQueue, globalPendingEvents);
+            }
         }
+        const shutdownOnError = (cause) => Effect.gen(function* () {
+            if (onError === 'shutdown') {
+                yield* shutdownChannel.send(UnexpectedError.make({ cause }));
+                yield* Effect.die(cause);
+            }
+        });
         yield* backgroundApplyLocalPushes({
             localPushesLatch,
             localPushesQueue,
             pullLatch,
             syncStateSref,
-            syncBackendQueue,
+            syncBackendPushQueue,
             schema,
-            isLocalEvent,
-            span,
-        }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped);
+            isClientEvent,
+            otelSpan,
+            currentLocalPushGenerationRef,
+            connectedClientSessionPullQueues,
+            mergeCounterRef,
+            mergePayloads,
+            localPushBatchSize,
+            testing: {
+                delay: testing?.delays?.localPushProcessing,
+            },
+        }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped);
         const backendPushingFiberHandle = yield* FiberHandle.make();
-        yield* FiberHandle.run(backendPushingFiberHandle, backgroundBackendPushing({ dbReady, syncBackendQueue, span }).pipe(Effect.tapCauseLogPretty));
+        const backendPushingEffect = backgroundBackendPushing({
+            syncBackendPushQueue,
+            otelSpan,
+            devtoolsLatch: ctxRef.current?.devtoolsLatch,
+            backendPushBatchSize,
+        }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError));
+        yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect);
         yield* backgroundBackendPulling({
-            dbReady,
             initialBackendHead,
-            isLocalEvent,
+            isClientEvent,
             restartBackendPushing: (filteredRebasedPending) => Effect.gen(function* () {
                 // Stop current pushing fiber
                 yield* FiberHandle.clear(backendPushingFiberHandle);
-                // Reset the sync queue
-                yield* BucketQueue.clear(syncBackendQueue);
-                yield* BucketQueue.offerAll(syncBackendQueue, filteredRebasedPending);
+                // Reset the sync backend push queue
+                yield* BucketQueue.clear(syncBackendPushQueue);
+                yield* BucketQueue.offerAll(syncBackendPushQueue, filteredRebasedPending);
                 // Restart pushing fiber
-                yield* FiberHandle.run(backendPushingFiberHandle, backgroundBackendPushing({ dbReady, syncBackendQueue, span }).pipe(Effect.tapCauseLogPretty));
+                yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect);
             }),
             syncStateSref,
             localPushesLatch,
             pullLatch,
-            span,
+            otelSpan,
             initialBlockingSyncContext,
-        }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped);
-    }).pipe(Effect.withSpanScoped('@livestore/common:leader-thread:syncing'));
+            devtoolsLatch: ctxRef.current?.devtoolsLatch,
+            connectedClientSessionPullQueues,
+            mergeCounterRef,
+            mergePayloads,
+            advancePushHead,
+        }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped);
+        return { initialLeaderHead: initialLocalHead };
+    }).pipe(Effect.withSpanScoped('@livestore/common:LeaderSyncProcessor:boot'));
+    const pull = ({ cursor }) => Effect.gen(function* () {
+        const queue = yield* pullQueue({ cursor });
+        return Stream.fromQueue(queue);
+    }).pipe(Stream.unwrapScoped);
+    const pullQueue = ({ cursor }) => {
+        const runtime = ctxRef.current?.runtime ?? shouldNeverHappen('Not initialized');
+        return Effect.gen(function* () {
+            const queue = yield* connectedClientSessionPullQueues.makeQueue;
+            const payloadsSinceCursor = Array.from(mergePayloads.entries())
+                .map(([mergeCounter, payload]) => ({ payload, mergeCounter }))
+                .filter(({ mergeCounter }) => mergeCounter > cursor.mergeCounter)
+                .toSorted((a, b) => a.mergeCounter - b.mergeCounter)
+                .map(({ payload, mergeCounter }) => {
+                if (payload._tag === 'upstream-advance') {
+                    return {
+                        payload: {
+                            _tag: 'upstream-advance',
+                            newEvents: ReadonlyArray.dropWhile(payload.newEvents, (eventEncoded) => EventSequenceNumber.isGreaterThanOrEqual(cursor.eventNum, eventEncoded.seqNum)),
+                        },
+                        mergeCounter,
+                    };
+                }
+                else {
+                    return { payload, mergeCounter };
+                }
+            });
+            yield* queue.offerAll(payloadsSinceCursor);
+            return queue;
+        }).pipe(Effect.provide(runtime));
+    };
+    const syncState = Subscribable.make({
+        get: Effect.gen(function* () {
+            const syncState = yield* syncStateSref;
+            if (syncState === undefined)
+                return shouldNeverHappen('Not initialized');
+            return syncState;
+        }),
+        changes: syncStateSref.changes.pipe(Stream.filter(isNotUndefined)),
+    });
     return {
+        pull,
+        pullQueue,
         push,
         pushPartial,
         boot,
-        syncState: Subscribable.make({
-            get: Effect.gen(function* () {
-                const syncState = yield* syncStateSref;
-                if (syncState === undefined)
-                    return shouldNeverHappen('Not initialized');
-                return syncState;
-            }),
-            changes: syncStateSref.changes.pipe(Stream.filter(isNotUndefined)),
-        }),
+        syncState,
+        getMergeCounter: () => mergeCounterRef.current,
     };
 });
-const backgroundApplyLocalPushes = ({ localPushesLatch, localPushesQueue, pullLatch, syncStateSref, syncBackendQueue, schema, isLocalEvent, span, }) => Effect.gen(function* () {
-    const { connectedClientSessionPullQueues } = yield* LeaderThreadCtx;
-    const applyMutationItems = yield* makeApplyMutationItems;
+const backgroundApplyLocalPushes = ({ localPushesLatch, localPushesQueue, pullLatch, syncStateSref, syncBackendPushQueue, schema, isClientEvent, otelSpan, currentLocalPushGenerationRef, connectedClientSessionPullQueues, mergeCounterRef, mergePayloads, localPushBatchSize, testing, }) => Effect.gen(function* () {
     while (true) {
-        // TODO make this configurable
-        const newEvents = yield* BucketQueue.takeBetween(localPushesQueue, 1, 10);
+        if (testing.delay !== undefined) {
+            yield* testing.delay.pipe(Effect.withSpan('localPushProcessingDelay'));
+        }
+        const batchItems = yield* BucketQueue.takeBetween(localPushesQueue, 1, localPushBatchSize);
         // Wait for the backend pulling to finish
         yield* localPushesLatch.await;
-        // Prevent the backend pulling from starting until this local push is finished
+        // Prevent backend pull processing until this local push is finished
         yield* pullLatch.close;
+        // Since the generation might have changed since enqueuing, we need to filter out items with older generation
+        // It's important that we filter after we got localPushesLatch, otherwise we might filter with the old generation
+        const filteredBatchItems = batchItems
+            .filter(([_1, _2, generation]) => generation === currentLocalPushGenerationRef.current)
+            .map(([eventEncoded, deferred]) => [eventEncoded, deferred]);
+        if (filteredBatchItems.length === 0) {
+            // console.log('dropping old-gen batch', currentLocalPushGenerationRef.current)
+            // Allow the backend pulling to start
+            yield* pullLatch.open;
+            continue;
+        }
+        const [newEvents, deferreds] = ReadonlyArray.unzip(filteredBatchItems);
         const syncState = yield* syncStateSref;
         if (syncState === undefined)
             return shouldNeverHappen('Not initialized');
-        const updateResult = SyncState.updateSyncState({
+        const mergeResult = SyncState.merge({
             syncState,
             payload: { _tag: 'local-push', newEvents },
-            isLocalEvent,
-            isEqualEvent: MutationEvent.isEqualEncoded,
+            isClientEvent,
+            isEqualEvent: LiveStoreEvent.isEqualEncoded,
         });
-        if (updateResult._tag === 'rebase') {
-            return shouldNeverHappen('The leader thread should never have to rebase due to a local push');
-        }
-        else if (updateResult._tag === 'reject') {
-            span?.addEvent('local-push:reject', {
-                batchSize: newEvents.length,
-                updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
-            });
-            const providedId = newEvents.at(0).id;
-            const remainingEvents = yield* BucketQueue.takeAll(localPushesQueue);
-            const allEvents = [...newEvents, ...remainingEvents];
-            yield* Effect.forEach(allEvents, (mutationEventEncoded) => mutationEventEncoded.meta.deferred
-                ? Deferred.fail(mutationEventEncoded.meta.deferred, InvalidPushError.make({
-                    // TODO improve error handling so it differentiates between a push being rejected
-                    // because of itself or because of another push
-                    reason: {
-                        _tag: 'LeaderAhead',
-                        minimumExpectedId: updateResult.expectedMinimumId,
-                        providedId,
-                    },
-                }))
-                : Effect.void);
-            // Allow the backend pulling to start
-            yield* pullLatch.open;
-            // In this case we're skipping state update and down/upstream processing
-            // We've cleared the local push queue and are now waiting for new local pushes / backend pulls
-            continue;
+        const mergeCounter = yield* incrementMergeCounter(mergeCounterRef);
+        switch (mergeResult._tag) {
+            case 'unexpected-error': {
+                otelSpan?.addEvent(`[${mergeCounter}]:push:unexpected-error`, {
+                    batchSize: newEvents.length,
+                    newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
+                });
+                return yield* Effect.fail(mergeResult.cause);
+            }
+            case 'rebase': {
+                return shouldNeverHappen('The leader thread should never have to rebase due to a local push');
+            }
+            case 'reject': {
+                otelSpan?.addEvent(`[${mergeCounter}]:push:reject`, {
+                    batchSize: newEvents.length,
+                    mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
+                });
+                // TODO: how to test this?
+                currentLocalPushGenerationRef.current++;
+                const nextGeneration = currentLocalPushGenerationRef.current;
+                const providedNum = newEvents.at(0).seqNum;
+                // All subsequent pushes with same generation should be rejected as well
+                // We're also handling the case where the localPushQueue already contains events
+                // from the next generation which we preserve in the queue
+                const remainingEventsMatchingGeneration = yield* BucketQueue.takeSplitWhere(localPushesQueue, (item) => item[2] >= nextGeneration);
+                // TODO we still need to better understand and handle this scenario
+                if (LS_DEV && (yield* BucketQueue.size(localPushesQueue)) > 0) {
+                    console.log('localPushesQueue is not empty', yield* BucketQueue.size(localPushesQueue));
+                    debugger;
+                }
+                const allDeferredsToReject = [
+                    ...deferreds,
+                    ...remainingEventsMatchingGeneration.map(([_, deferred]) => deferred),
+                ].filter(isNotUndefined);
+                yield* Effect.forEach(allDeferredsToReject, (deferred) => Deferred.fail(deferred, LeaderAheadError.make({
+                    minimumExpectedNum: mergeResult.expectedMinimumId,
+                    providedNum,
+                    // nextGeneration,
+                })));
+                // Allow the backend pulling to start
+                yield* pullLatch.open;
+                // In this case we're skipping state update and down/upstream processing
+                // We've cleared the local push queue and are now waiting for new local pushes / backend pulls
+                continue;
+            }
+            case 'advance': {
+                break;
+            }
+            default: {
+                casesHandled(mergeResult);
+            }
         }
-        yield* SubscriptionRef.set(syncStateSref, updateResult.newSyncState);
+        yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
         yield* connectedClientSessionPullQueues.offer({
-            payload: { _tag: 'upstream-advance', newEvents: updateResult.newEvents },
-            remaining: 0,
+            payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
+            mergeCounter,
         });
-        span?.addEvent('local-push', {
+        mergePayloads.set(mergeCounter, SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }));
+        otelSpan?.addEvent(`[${mergeCounter}]:push:advance`, {
             batchSize: newEvents.length,
-            updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
+            mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
         });
-        // Don't sync localOnly mutations
-        const filteredBatch = updateResult.newEvents.filter((mutationEventEncoded) => {
-            const mutationDef = schema.mutations.get(mutationEventEncoded.mutation);
-            return mutationDef.options.localOnly === false;
+        // Don't sync clientOnly events
+        const filteredBatch = mergeResult.newEvents.filter((eventEncoded) => {
+            const { eventDef } = getEventDef(schema, eventEncoded.name);
+            return eventDef.options.clientOnly === false;
         });
-        yield* BucketQueue.offerAll(syncBackendQueue, filteredBatch);
-        yield* applyMutationItems({ batchItems: newEvents });
+        yield* BucketQueue.offerAll(syncBackendPushQueue, filteredBatch);
+        yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds });
         // Allow the backend pulling to start
         yield* pullLatch.open;
     }
 });
 // TODO how to handle errors gracefully
-const makeApplyMutationItems = Effect.gen(function* () {
-    const leaderThreadCtx = yield* LeaderThreadCtx;
-    const { db, dbLog } = leaderThreadCtx;
-    const applyMutation = yield* makeApplyMutation;
-    return ({ batchItems }) => Effect.gen(function* () {
-        db.execute('BEGIN TRANSACTION', undefined); // Start the transaction
-        dbLog.execute('BEGIN TRANSACTION', undefined); // Start the transaction
-        yield* Effect.addFinalizer((exit) => Effect.gen(function* () {
-            if (Exit.isSuccess(exit))
-                return;
-            // Rollback in case of an error
-            db.execute('ROLLBACK', undefined);
-            dbLog.execute('ROLLBACK', undefined);
-        }));
-        for (let i = 0; i < batchItems.length; i++) {
-            const { meta, ...mutationEventEncoded } = batchItems[i];
-            yield* applyMutation(mutationEventEncoded);
-            if (meta?.deferred) {
-                yield* Deferred.succeed(meta.deferred, void 0);
-            }
+const materializeEventsBatch = ({ batchItems, deferreds }) => Effect.gen(function* () {
+    const { dbState: db, dbEventlog, materializeEvent } = yield* LeaderThreadCtx;
+    // NOTE We always start a transaction to ensure consistency between db and eventlog (even for single-item batches)
+    db.execute('BEGIN TRANSACTION', undefined); // Start the transaction
+    dbEventlog.execute('BEGIN TRANSACTION', undefined); // Start the transaction
+    yield* Effect.addFinalizer((exit) => Effect.gen(function* () {
+        if (Exit.isSuccess(exit))
+            return;
+        // Rollback in case of an error
+        db.execute('ROLLBACK', undefined);
+        dbEventlog.execute('ROLLBACK', undefined);
+    }));
+    for (let i = 0; i < batchItems.length; i++) {
+        const { sessionChangeset } = yield* materializeEvent(batchItems[i]);
+        batchItems[i].meta.sessionChangeset = sessionChangeset;
+        if (deferreds?.[i] !== undefined) {
+            yield* Deferred.succeed(deferreds[i], void 0);
         }
-        db.execute('COMMIT', undefined); // Commit the transaction
-        dbLog.execute('COMMIT', undefined); // Commit the transaction
-    }).pipe(Effect.uninterruptible, Effect.scoped, Effect.withSpan('@livestore/common:leader-thread:syncing:applyMutationItems', {
-        attributes: { count: batchItems.length },
-    }), Effect.tapCauseLogPretty, UnexpectedError.mapToUnexpectedError);
-});
-const backgroundBackendPulling = ({ dbReady, initialBackendHead, isLocalEvent, restartBackendPushing, span, syncStateSref, localPushesLatch, pullLatch, initialBlockingSyncContext, }) => Effect.gen(function* () {
-    const { syncBackend, db, dbLog, connectedClientSessionPullQueues, schema } = yield* LeaderThreadCtx;
+    }
+    db.execute('COMMIT', undefined); // Commit the transaction
+    dbEventlog.execute('COMMIT', undefined); // Commit the transaction
+}).pipe(Effect.uninterruptible, Effect.scoped, Effect.withSpan('@livestore/common:LeaderSyncProcessor:materializeEventItems', {
+    attributes: { batchSize: batchItems.length },
+}), Effect.tapCauseLogPretty, UnexpectedError.mapToUnexpectedError);
+const backgroundBackendPulling = ({ initialBackendHead, isClientEvent, restartBackendPushing, otelSpan, syncStateSref, localPushesLatch, pullLatch, devtoolsLatch, initialBlockingSyncContext, connectedClientSessionPullQueues, mergeCounterRef, mergePayloads, advancePushHead, }) => Effect.gen(function* () {
+    const { syncBackend, dbState: db, dbEventlog, schema } = yield* LeaderThreadCtx;
     if (syncBackend === undefined)
         return;
-    const cursorInfo = yield* getCursorInfo(initialBackendHead);
-    const applyMutationItems = yield* makeApplyMutationItems;
     const onNewPullChunk = (newEvents, remaining) => Effect.gen(function* () {
         if (newEvents.length === 0)
             return;
+        if (devtoolsLatch !== undefined) {
+            yield* devtoolsLatch.await;
+        }
         // Prevent more local pushes from being processed until this pull is finished
         yield* localPushesLatch.close;
         // Wait for pending local pushes to finish
@@ -272,62 +408,85 @@ const backgroundBackendPulling = ({ dbReady, initialBackendHead, isLocalEvent, r
         const syncState = yield* syncStateSref;
         if (syncState === undefined)
             return shouldNeverHappen('Not initialized');
-        const trimRollbackUntil = newEvents.at(-1).id;
-        const updateResult = SyncState.updateSyncState({
+        const mergeResult = SyncState.merge({
             syncState,
-            payload: { _tag: 'upstream-advance', newEvents, trimRollbackUntil },
-            isLocalEvent,
-            isEqualEvent: MutationEvent.isEqualEncoded,
-            ignoreLocalEvents: true,
+            payload: SyncState.PayloadUpstreamAdvance.make({ newEvents }),
+            isClientEvent,
+            isEqualEvent: LiveStoreEvent.isEqualEncoded,
+            ignoreClientEvents: true,
         });
-        if (updateResult._tag === 'reject') {
+        const mergeCounter = yield* incrementMergeCounter(mergeCounterRef);
+        if (mergeResult._tag === 'reject') {
             return shouldNeverHappen('The leader thread should never reject upstream advances');
         }
-        const newBackendHead = newEvents.at(-1).id;
-        updateBackendHead(dbLog, newBackendHead);
-        if (updateResult._tag === 'rebase') {
-            span?.addEvent('backend-pull:rebase', {
+        else if (mergeResult._tag === 'unexpected-error') {
+            otelSpan?.addEvent(`[${mergeCounter}]:pull:unexpected-error`, {
                 newEventsCount: newEvents.length,
                 newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
-                rollbackCount: updateResult.eventsToRollback.length,
-                updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
             });
-            const filteredRebasedPending = updateResult.newSyncState.pending.filter((mutationEvent) => {
-                const mutationDef = schema.mutations.get(mutationEvent.mutation);
-                return mutationDef.options.localOnly === false;
+            return yield* Effect.fail(mergeResult.cause);
+        }
+        const newBackendHead = newEvents.at(-1).seqNum;
+        Eventlog.updateBackendHead(dbEventlog, newBackendHead);
+        if (mergeResult._tag === 'rebase') {
+            otelSpan?.addEvent(`[${mergeCounter}]:pull:rebase`, {
+                newEventsCount: newEvents.length,
+                newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
+                rollbackCount: mergeResult.rollbackEvents.length,
+                mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
+            });
+            const globalRebasedPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
+                const { eventDef } = getEventDef(schema, event.name);
+                return eventDef.options.clientOnly === false;
             });
-            yield* restartBackendPushing(filteredRebasedPending);
-            if (updateResult.eventsToRollback.length > 0) {
-                yield* rollback({ db, dbLog, eventIdsToRollback: updateResult.eventsToRollback.map((_) => _.id) });
+            yield* restartBackendPushing(globalRebasedPendingEvents);
+            if (mergeResult.rollbackEvents.length > 0) {
+                yield* rollback({
+                    dbState: db,
+                    dbEventlog,
+                    eventNumsToRollback: mergeResult.rollbackEvents.map((_) => _.seqNum),
+                });
             }
             yield* connectedClientSessionPullQueues.offer({
-                payload: {
-                    _tag: 'upstream-rebase',
-                    newEvents: updateResult.newEvents,
-                    rollbackUntil: updateResult.eventsToRollback.at(0).id,
-                    trimRollbackUntil,
-                },
-                remaining,
+                payload: SyncState.PayloadUpstreamRebase.make({
+                    newEvents: mergeResult.newEvents,
+                    rollbackEvents: mergeResult.rollbackEvents,
+                }),
+                mergeCounter,
             });
+            mergePayloads.set(mergeCounter, SyncState.PayloadUpstreamRebase.make({
+                newEvents: mergeResult.newEvents,
+                rollbackEvents: mergeResult.rollbackEvents,
+            }));
         }
         else {
-            span?.addEvent('backend-pull:advance', {
+            otelSpan?.addEvent(`[${mergeCounter}]:pull:advance`, {
                 newEventsCount: newEvents.length,
-                updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
+                mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
             });
             yield* connectedClientSessionPullQueues.offer({
-                payload: { _tag: 'upstream-advance', newEvents: updateResult.newEvents, trimRollbackUntil },
-                remaining,
+                payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
+                mergeCounter,
             });
+            mergePayloads.set(mergeCounter, SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }));
+            if (mergeResult.confirmedEvents.length > 0) {
+                // `mergeResult.confirmedEvents` don't contain the correct sync metadata, so we need to use
+                // `newEvents` instead which we filter via `mergeResult.confirmedEvents`
+                const confirmedNewEvents = newEvents.filter((event) => mergeResult.confirmedEvents.some((confirmedEvent) => EventSequenceNumber.isEqual(event.seqNum, confirmedEvent.seqNum)));
+                yield* Eventlog.updateSyncMetadata(confirmedNewEvents);
+            }
         }
+        // Removes the changeset rows which are no longer needed as we'll never have to rollback beyond this point
         trimChangesetRows(db, newBackendHead);
-        yield* applyMutationItems({ batchItems: updateResult.newEvents });
-        yield* SubscriptionRef.set(syncStateSref, updateResult.newSyncState);
+        advancePushHead(mergeResult.newSyncState.localHead);
+        yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds: undefined });
+        yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
+        // Allow local pushes to be processed again
         if (remaining === 0) {
-            // Allow local pushes to be processed again
             yield* localPushesLatch.open;
         }
     });
+    const cursorInfo = yield* Eventlog.getSyncBackendCursorInfo(initialBackendHead);
     yield* syncBackend.pull(cursorInfo).pipe(
     // TODO only take from queue while connected
     Stream.tap(({ batch, remaining }) => Effect.gen(function* () {
@@ -337,85 +496,100 @@ const backgroundBackendPulling = ({ dbReady, initialBackendHead, isLocalEvent, r
         //     batch: TRACE_VERBOSE ? batch : undefined,
         //   },
         // })
-        // Wait for the db to be initially created
-        yield* dbReady;
-        // NOTE we only want to take process mutations when the sync backend is connected
+        // NOTE we only want to take process events when the sync backend is connected
         // (e.g. needed for simulating being offline)
         // TODO remove when there's a better way to handle this in stream above
         yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
-        yield* onNewPullChunk(batch.map((_) => MutationEvent.EncodedWithMeta.fromGlobal(_.mutationEventEncoded)), remaining);
+        yield* onNewPullChunk(batch.map((_) => LiveStoreEvent.EncodedWithMeta.fromGlobal(_.eventEncoded, _.metadata)), remaining);
         yield* initialBlockingSyncContext.update({ processed: batch.length, remaining });
     })), Stream.runDrain, Effect.interruptible);
-}).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:backend-pulling'));
-const rollback = ({ db, dbLog, eventIdsToRollback, }) => Effect.gen(function* () {
-    const rollbackEvents = db
-        .select(sql `SELECT * FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idLocal) IN (${eventIdsToRollback.map((id) => `(${id.global}, ${id.local})`).join(', ')})`)
-        .map((_) => ({ id: { global: _.idGlobal, local: _.idLocal }, changeset: _.changeset, debug: _.debug }))
-        .toSorted((a, b) => EventId.compare(a.id, b.id));
-    // Apply changesets in reverse order
-    for (let i = rollbackEvents.length - 1; i >= 0; i--) {
-        const { changeset } = rollbackEvents[i];
-        if (changeset !== null) {
-            db.makeChangeset(changeset).invert().apply();
-        }
-    }
-    // Delete the changeset rows
-    db.execute(sql `DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idLocal) IN (${eventIdsToRollback.map((id) => `(${id.global}, ${id.local})`).join(', ')})`);
-    // Delete the mutation log rows
-    dbLog.execute(sql `DELETE FROM ${MUTATION_LOG_META_TABLE} WHERE (idGlobal, idLocal) IN (${eventIdsToRollback.map((id) => `(${id.global}, ${id.local})`).join(', ')})`);
-}).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:rollback', {
-    attributes: { count: eventIdsToRollback.length },
-}));
-const getCursorInfo = (remoteHead) => Effect.gen(function* () {
-    const { dbLog } = yield* LeaderThreadCtx;
-    if (remoteHead === EventId.ROOT.global)
-        return Option.none();
-    const MutationlogQuerySchema = Schema.Struct({
-        syncMetadataJson: Schema.parseJson(Schema.Option(Schema.JsonValue)),
-    }).pipe(Schema.pluck('syncMetadataJson'), Schema.Array, Schema.head);
-    const syncMetadataOption = yield* Effect.sync(() => dbLog.select(sql `SELECT syncMetadataJson FROM ${MUTATION_LOG_META_TABLE} WHERE idGlobal = ${remoteHead} ORDER BY idLocal ASC LIMIT 1`)).pipe(Effect.andThen(Schema.decode(MutationlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie);
-    return Option.some({
-        cursor: { global: remoteHead, local: EventId.localDefault },
-        metadata: syncMetadataOption,
-    });
-}).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:getCursorInfo', { attributes: { remoteHead } }));
-const backgroundBackendPushing = ({ dbReady, syncBackendQueue, span, }) => Effect.gen(function* () {
-    const { syncBackend, dbLog } = yield* LeaderThreadCtx;
+}).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pulling'));
+const backgroundBackendPushing = ({ syncBackendPushQueue, otelSpan, devtoolsLatch, backendPushBatchSize, }) => Effect.gen(function* () {
+    const { syncBackend } = yield* LeaderThreadCtx;
     if (syncBackend === undefined)
         return;
-    yield* dbReady;
     while (true) {
         yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
-        // TODO make batch size configurable
-        const queueItems = yield* BucketQueue.takeBetween(syncBackendQueue, 1, 50);
+        const queueItems = yield* BucketQueue.takeBetween(syncBackendPushQueue, 1, backendPushBatchSize);
         yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
-        span?.addEvent('backend-push', {
+        if (devtoolsLatch !== undefined) {
+            yield* devtoolsLatch.await;
+        }
+        otelSpan?.addEvent('backend-push', {
             batchSize: queueItems.length,
             batch: TRACE_VERBOSE ? JSON.stringify(queueItems) : undefined,
         });
         // TODO handle push errors (should only happen during concurrent pull+push)
         const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either);
         if (pushResult._tag === 'Left') {
-            span?.addEvent('backend-push-error', { error: pushResult.left.toString() });
+            if (LS_DEV) {
+                yield* Effect.logDebug('handled backend-push-error', { error: pushResult.left.toString() });
+            }
+            otelSpan?.addEvent('backend-push-error', { error: pushResult.left.toString() });
             // wait for interrupt caused by background pulling which will then restart pushing
             return yield* Effect.never;
         }
-        const { metadata } = pushResult.right;
-        // TODO try to do this in a single query
-        for (let i = 0; i < queueItems.length; i++) {
-            const mutationEventEncoded = queueItems[i];
-            yield* execSql(dbLog, ...updateRows({
-                tableName: MUTATION_LOG_META_TABLE,
-                columns: mutationLogMetaTable.sqliteDef.columns,
-                where: { idGlobal: mutationEventEncoded.id.global, idLocal: mutationEventEncoded.id.local },
-                updateValues: { syncMetadataJson: metadata[i] },
-            }));
-        }
     }
-}).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:leader-thread:syncing:backend-pushing'));
+}).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pushing'));
 const trimChangesetRows = (db, newHead) => {
     // Since we're using the session changeset rows to query for the current head,
     // we're keeping at least one row for the current head, and thus are using `<` instead of `<=`
-    db.execute(sql `DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE idGlobal < ${newHead.global}`);
+    db.execute(sql `DELETE FROM ${SystemTables.SESSION_CHANGESET_META_TABLE} WHERE seqNumGlobal < ${newHead.global}`);
 };
+const makePullQueueSet = Effect.gen(function* () {
+    const set = new Set();
+    yield* Effect.addFinalizer(() => Effect.gen(function* () {
+        for (const queue of set) {
+            yield* Queue.shutdown(queue);
+        }
+        set.clear();
+    }));
+    const makeQueue = Effect.gen(function* () {
+        const queue = yield* Queue.unbounded().pipe(Effect.acquireRelease(Queue.shutdown));
+        yield* Effect.addFinalizer(() => Effect.sync(() => set.delete(queue)));
+        set.add(queue);
+        return queue;
+    });
+    const offer = (item) => Effect.gen(function* () {
+        // Short-circuit if the payload is an empty upstream advance
+        if (item.payload._tag === 'upstream-advance' && item.payload.newEvents.length === 0) {
+            return;
+        }
+        for (const queue of set) {
+            yield* Queue.offer(queue, item);
+        }
+    });
+    return {
+        makeQueue,
+        offer,
+    };
+});
+const incrementMergeCounter = (mergeCounterRef) => Effect.gen(function* () {
+    const { dbState } = yield* LeaderThreadCtx;
+    mergeCounterRef.current++;
+    dbState.execute(sql `INSERT OR REPLACE INTO ${SystemTables.LEADER_MERGE_COUNTER_TABLE} (id, mergeCounter) VALUES (0, ${mergeCounterRef.current})`);
+    return mergeCounterRef.current;
+});
+const getMergeCounterFromDb = (dbState) => Effect.gen(function* () {
+    const result = dbState.select(sql `SELECT mergeCounter FROM ${SystemTables.LEADER_MERGE_COUNTER_TABLE} WHERE id = 0`);
+    return result[0]?.mergeCounter ?? 0;
+});
+const validatePushBatch = (batch, pushHead) => Effect.gen(function* () {
+    if (batch.length === 0) {
+        return;
+    }
+    // Make sure batch is monotonically increasing
+    for (let i = 1; i < batch.length; i++) {
+        if (EventSequenceNumber.isGreaterThanOrEqual(batch[i - 1].seqNum, batch[i].seqNum)) {
+            shouldNeverHappen(`Events must be ordered in monotonically ascending order by eventNum. Received: [${batch.map((e) => EventSequenceNumber.toString(e.seqNum)).join(', ')}]`);
+        }
+    }
+    // Make sure smallest sequence number is > pushHead
+    if (EventSequenceNumber.isGreaterThanOrEqual(pushHead, batch[0].seqNum)) {
+        return yield* LeaderAheadError.make({
+            minimumExpectedNum: pushHead,
+            providedNum: batch[0].seqNum,
+        });
+    }
+});
 //# sourceMappingURL=LeaderSyncProcessor.js.map