@livestore/common 0.4.0-dev.9 → 0.5.0-dev.0

package/dist/leader-thread/LeaderSyncProcessor.js

@@ -1,14 +1,18 @@
-import { casesHandled, isNotUndefined, LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils';
-import { BucketQueue, Cause, Deferred, Duration, Effect, Exit, FiberHandle, Layer, Option, OtelTracer, pipe, Queue, ReadonlyArray, Schedule, Stream, Subscribable, SubscriptionRef, } from '@livestore/utils/effect';
-import { UnexpectedError, } from "../adapter-types.js";
+import { casesHandled, LS_DEV, TRACE_VERBOSE } from '@livestore/utils';
+import { Cause, Context, Deferred, Duration, Effect, Exit, FiberHandle, Layer, Option, Predicate, Queue, ReadonlyArray, References, Result, Schedule, Schema, Semaphore, Stream, Subscribable, SubscriptionRef, TxQueue, } from '@livestore/utils/effect';
+import { MaterializeError, UnknownError } from "../adapter-types.js";
+import { IntentionalShutdownCause } from "../errors.js";
 import { makeMaterializerHash } from "../materializer-helper.js";
-import { EventSequenceNumber, getEventDef, LiveStoreEvent, SystemTables } from "../schema/mod.js";
-import { LeaderAheadError, } from "../sync/sync.js";
+import { EventSequenceNumber, LiveStoreEvent, resolveEventDef, SystemTables } from "../schema/mod.js";
+import { EVENTLOG_META_TABLE, SYNC_STATUS_TABLE } from "../schema/state/sqlite/system-tables/eventlog-tables.js";
+import * as StateHead from "../StateHead.js";
 import * as SyncState from "../sync/syncstate.js";
 import { sql } from "../util.js";
 import * as Eventlog from "./eventlog.js";
 import { rollback } from "./materialize-event.js";
+import { isRejectedPushError, LeaderAheadError, NonContiguousBatchError, NonMonotonicBatchError, StaleRebaseGenerationError, } from "./RejectedPushError.js";
 import { LeaderThreadCtx } from "./types.js";
+export const TypeId = '~@livestore/common/LeaderSyncProcessor';
 /**
  * The LeaderSyncProcessor manages synchronization of events between
  * the local state and the sync backend, ensuring efficient and orderly processing.
@@ -27,11 +31,11 @@ import { LeaderThreadCtx } from "./types.js";
  *   - Maintains events in ascending order.
  *   - Uses `Deferred` objects to resolve/reject events based on application success.
  * - Processes events from the queue, applying events in batches.
- * - Controlled by a `Latch` to manage execution flow.
- * - The latch closes on pull receipt and re-opens post-pull completion.
+ * - Controlled by a mutex (`Semaphore(1)`) to ensure mutual exclusion between local push and backend pull processing.
+ * - The backend pull side acquires the mutex before processing and releases it on post-pull completion.
  * - Processes up to `maxBatchSize` events per cycle.
  *
- * Currently we're advancing the state db and eventlog in lockstep, but we could also decouple this in the future
+ * Currently, we're advancing the state db and eventlog in lockstep, but we could also decouple this in the future
  *
  * Tricky concurrency scenarios:
  * - Queued local push batches becoming invalid due to a prior local push item being rejected.
@@ -39,309 +43,505 @@ import { LeaderThreadCtx } from "./types.js";
  *
  * See ClientSessionSyncProcessor for how the leader and session sync processors are similar/different.
  */
-export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncContext, initialSyncState, onError, livePull, params, testing, }) => Effect.gen(function* () {
-    const syncBackendPushQueue = yield* BucketQueue.make();
-    const localPushBatchSize = params.localPushBatchSize ?? 1;
-    const backendPushBatchSize = params.backendPushBatchSize ?? 2;
+export class LeaderSyncProcessor extends Context.Service()('@livestore/common/LeaderSyncProcessor') {
+}
+export const make = Effect.fnUntraced(function* ({ schema, dbState, initialBlockingSyncContext, initialSyncState, onError, onBackendIdMismatch, livePull, params, testing, }) {
+    const stateHead = yield* StateHead.StateHead;
+    const syncBackendPushQueue = yield* TxQueue.unbounded();
+    const localPushBatchSize = params.localPushBatchSize ?? 10;
+    const backendPushBatchSize = params.backendPushBatchSize ?? 50;
     const syncStateSref = yield* SubscriptionRef.make(undefined);
-    const isClientEvent = (eventEncoded) => {
-        const { eventDef } = getEventDef(schema, eventEncoded.name);
-        return eventDef.options.clientOnly;
-    };
+    const isClientOnlyEvent = (eventEncoded) => schema.eventsDefsMap.get(eventEncoded.name)?.options.clientOnly ?? false;
     const connectedClientSessionPullQueues = yield* makePullQueueSet;
     // This context depends on data from `boot`, we should find a better implementation to avoid this ref indirection.
     const ctxRef = {
         current: undefined,
     };
-    const localPushesQueue = yield* BucketQueue.make();
-    const localPushesLatch = yield* Effect.makeLatch(true);
-    const pullLatch = yield* Effect.makeLatch(true);
+    const localPushesQueue = yield* TxQueue.unbounded();
+    // Reservations cover admitted pushes from validation until they are applied or rejected. The Set's
+    // insertion order mirrors admission order, so its last relevant item is the optimistic sequence head.
+    const reservedLocalPushItems = new Set();
+    // Ensures mutual exclusion between local push and backend pull processing.
+    const localPushBackendPullMutex = yield* Semaphore.make(1);
+    // Serializes validation, queue admission, and prefix-fence reconciliation.
+    const pushAdmissionSemaphore = yield* Semaphore.make(1);
     /**
-     * Additionally to the `syncStateSref` we also need the `pushHeadRef` in order to prevent old/duplicate
-     * events from being pushed in a scenario like this:
-     * - client session A pushes e1
-     * - leader sync processor takes a bit and hasn't yet taken e1 from the localPushesQueue
-     * - client session B also pushes e1 (which should be rejected)
+     * Admission fence for local pushes. Unlike `syncState.localHead`, this advances as soon as an event
+     * is queued, so another session cannot claim the same sequence number while that event is waiting
+     * to be applied. With authoritative head e0 and reserved pushes e1/e2, this points to e2.
      *
-     * Thus the purpose of the pushHeadRef is the guard the integrity of the local push queue
+     * All reads and writes are protected by `pushAdmissionSemaphore` so validation and reservation are
+     * one atomic operation from the perspective of concurrent sessions and backend pulls.
      */
-    const pushHeadRef = { current: EventSequenceNumber.ROOT };
-    const advancePushHead = (eventNum) => {
-        pushHeadRef.current = EventSequenceNumber.max(pushHeadRef.current, eventNum);
-    };
-    // NOTE: New events are only pushed to sync backend after successful local push processing
-    const push = (newEvents, options) => Effect.gen(function* () {
-        if (newEvents.length === 0)
-            return;
-        // console.debug('push', newEvents)
-        yield* validatePushBatch(newEvents, pushHeadRef.current);
-        advancePushHead(newEvents.at(-1).seqNum);
-        const waitForProcessing = options?.waitForProcessing ?? false;
-        if (waitForProcessing) {
-            const deferreds = yield* Effect.forEach(newEvents, () => Deferred.make());
-            const items = newEvents.map((eventEncoded, i) => [eventEncoded, deferreds[i]]);
-            yield* BucketQueue.offerAll(localPushesQueue, items);
-            yield* Effect.all(deferreds);
-        }
-        else {
-            const items = newEvents.map((eventEncoded) => [eventEncoded, undefined]);
-            yield* BucketQueue.offerAll(localPushesQueue, items);
+    const pushHeadRef = { current: initialSyncState.localHead };
+    /**
+     * A backend pull may advance or rebase authoritative history while local pushes remain reserved.
+     * Keep the fence at the newest reservation that is still valid for that history; if none remains,
+     * fall back to the authoritative head. Stale reservations are released later by the queue worker.
+     */
+    const reconcilePushHead = (authoritativeHead) => pushAdmissionSemaphore.withPermits(1)(Effect.gen(function* () {
+        const latestCurrentGenerationItem = [...reservedLocalPushItems].findLast(([event]) => event.seqNum.rebaseGeneration >= authoritativeHead.rebaseGeneration);
+        pushHeadRef.current = latestCurrentGenerationItem?.[0].seqNum ?? authoritativeHead;
+    }).pipe(Effect.uninterruptible));
+    /**
+     * Stop completed, rejected, or stale queue items from extending the admission fence, then rebuild
+     * the fence from any pushes that are still reserved.
+     */
+    const releasePushReservations = (items, authoritativeHead) => pushAdmissionSemaphore.withPermits(1)(Effect.gen(function* () {
+        for (const item of items)
+            reservedLocalPushItems.delete(item);
+        const latestCurrentGenerationItem = [...reservedLocalPushItems].findLast(([event]) => event.seqNum.rebaseGeneration >= authoritativeHead.rebaseGeneration);
+        pushHeadRef.current = latestCurrentGenerationItem?.[0].seqNum ?? authoritativeHead;
+    }).pipe(Effect.uninterruptible));
+    const backgroundApplyLocalPushes = Effect.gen(function* () {
+        while (true) {
+            if (testing.delays?.localPushProcessing !== undefined) {
+                yield* testing.delays.localPushProcessing.pipe(Effect.withSpan('localPushProcessingDelay'));
+            }
+            const batchItems = yield* TxQueue.takeBetween(localPushesQueue, 1, localPushBatchSize);
+            // Applies a batch of local pushes, guarded by the localPushBackendPullMutex to ensure mutual exclusion with backend pulling
+            yield* Effect.gen(function* () {
+                const syncState = yield* Effect.fromNullishOr(yield* SubscriptionRef.get(syncStateSref)).pipe(Effect.orDieDebugger);
+                const currentRebaseGeneration = syncState.localHead.rebaseGeneration;
+                // Since the rebase generation might have changed since enqueuing, we need to filter out items with older generation
+                // It's important that we filter after acquiring the localPushBackendPullMutex, otherwise we might filter with the old generation
+                const [droppedItems, filteredItems] = ReadonlyArray.partition(batchItems, (batchItem) => batchItem[0].seqNum.rebaseGeneration >= currentRebaseGeneration
+                    ? Result.succeed(batchItem)
+                    : Result.fail(batchItem));
+                if (droppedItems.length > 0) {
+                    yield* Effect.spanEvent(`push:drop-old-generation`, {
+                        droppedCount: droppedItems.length,
+                        currentRebaseGeneration,
+                    });
+                    yield* Effect.forEach(droppedItems, ([eventEncoded, deferred]) => Deferred.fail(deferred, StaleRebaseGenerationError.make({
+                        currentRebaseGeneration,
+                        providedRebaseGeneration: eventEncoded.seqNum.rebaseGeneration,
+                        sessionId: eventEncoded.sessionId,
+                    })));
+                    yield* releasePushReservations(droppedItems, syncState.localHead);
+                }
+                if (filteredItems.length === 0) {
+                    return;
+                }
+                const [newEvents, deferreds] = ReadonlyArray.unzip(filteredItems);
+                yield* Effect.annotateCurrentSpan({
+                    batchSize: newEvents.length,
+                    ...(TRACE_VERBOSE === true ? { newEvents: jsonStringify(newEvents) } : {}),
+                });
+                const mergeResult = yield* SyncState.merge({
+                    syncState,
+                    payload: { _tag: 'local-push', newEvents },
+                    isClientOnlyEvent,
+                    isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
+                });
+                switch (mergeResult._tag) {
+                    case 'rebase': {
+                        return yield* Effect.dieDebugger('The leader thread should never have to rebase due to a local push');
+                    }
+                    case 'reject': {
+                        yield* Effect.spanEvent(`push:reject`, {
+                            batchSize: newEvents.length,
+                            ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
+                        });
+                        // TODO: how to test this?
+                        const nextRebaseGeneration = currentRebaseGeneration + 1;
+                        const providedNum = newEvents.at(0).seqNum;
+                        // All subsequent pushes with same generation should be rejected as well
+                        // We're also handling the case where the localPushQueue already contains events
+                        // from the next generation which we preserve in the queue
+                        const remainingEventsMatchingGeneration = yield* takePrefixUntil(localPushesQueue, ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= nextRebaseGeneration);
+                        // TODO we still need to better understand and handle this scenario
+                        const remainingLocalPushes = yield* snapshotTxQueue(localPushesQueue);
+                        if (LS_DEV === true && remainingLocalPushes.length > 0) {
+                            console.log('localPushesQueue is not empty', remainingLocalPushes.length);
+                            // oxlint-disable-next-line eslint(no-debugger) -- intentional breakpoint for unexpected queue state
+                            debugger;
+                        }
+                        const allDeferredsToReject = [
+                            ...deferreds,
+                            ...remainingEventsMatchingGeneration.map(([_, deferred]) => deferred),
+                        ];
+                        // The rejected batch and its drained same-generation suffix will never advance leader state,
+                        // so release their sequence-number reservations before clients retry from the authoritative head.
+                        yield* releasePushReservations([...filteredItems, ...remainingEventsMatchingGeneration], syncState.localHead);
+                        yield* Effect.forEach(allDeferredsToReject, (deferred) => Deferred.fail(deferred, LeaderAheadError.make({
+                            minimumExpectedNum: mergeResult.expectedMinimumId,
+                            providedNum,
+                            sessionId: newEvents.at(0).sessionId,
+                        })));
+                        // In this case we're skipping state update and down/upstream processing
+                        // We've cleared the local push queue and are now waiting for new local pushes / backend pulls
+                        return;
+                    }
+                    case 'advance': {
+                        break;
+                    }
+                    default: {
+                        casesHandled(mergeResult);
+                    }
+                }
+                // For a local-push advance, `newEvents` and the appended pending suffix describe the same logical
+                // events but serve different roles and may be distinct instances. Materialize the retained pending
+                // instances so their rollback metadata remains available if a later backend event causes a rebase.
+                const acceptedPendingEvents = mergeResult.newSyncState.pending.slice(syncState.pending.length);
+                if (acceptedPendingEvents.length !== mergeResult.newEvents.length) {
+                    return yield* Effect.dieDebugger('Local push events must be retained in pending state');
+                }
+                yield* materializeEventsBatch({ batchItems: acceptedPendingEvents });
+                yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
+                yield* connectedClientSessionPullQueues.offer({
+                    payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: acceptedPendingEvents }),
+                    leaderHead: mergeResult.newSyncState.localHead,
+                });
+                yield* Effect.spanEvent(`push:advance`, {
+                    batchSize: newEvents.length,
+                    ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
+                });
+                // Don't sync client-only events
+                const globalOrUnknownEvents = acceptedPendingEvents.filter((e) => !isClientOnlyEvent(e));
+                yield* TxQueue.offerAll(syncBackendPushQueue, globalOrUnknownEvents);
+                yield* releasePushReservations(filteredItems, mergeResult.newSyncState.localHead);
+                // A push is acknowledged only after the complete batch is materialized, published in
+                // leader sync state, exposed to sessions, and queued for backend propagation.
+                yield* Effect.forEach(deferreds, (deferred) => Deferred.succeed(deferred, void 0));
+            }).pipe(localPushBackendPullMutex.withPermits(1));
         }
-    }).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:push', {
-        attributes: {
-            batchSize: newEvents.length,
-            batch: TRACE_VERBOSE ? newEvents : undefined,
-        },
-        links: ctxRef.current?.span ? [{ _tag: 'SpanLink', span: ctxRef.current.span, attributes: {} }] : undefined,
-    }));
-    const pushPartial = ({ event: { name, args }, clientId, sessionId }) => Effect.gen(function* () {
-        const syncState = yield* syncStateSref;
-        if (syncState === undefined)
-            return shouldNeverHappen('Not initialized');
-        const { eventDef } = getEventDef(schema, name);
-        const eventEncoded = new LiveStoreEvent.EncodedWithMeta({
-            name,
-            args,
-            clientId,
-            sessionId,
-            ...EventSequenceNumber.nextPair({ seqNum: syncState.localHead, isClient: eventDef.options.clientOnly }),
+    });
+    const backgroundBackendPulling = Effect.fn('@livestore/common:LeaderSyncProcessor:backend-pulling')(function* ({ restartBackendPushing, }) {
+        const { syncBackend, dbState: db, dbEventlog, schema } = yield* LeaderThreadCtx;
+        if (syncBackend === undefined)
+            return;
+        let pullMutexHeld = false;
+        const releasePullMutexIfHeld = Effect.gen(function* () {
+            if (pullMutexHeld === false)
+                return;
+            pullMutexHeld = false;
+            yield* localPushBackendPullMutex.release(1);
         });
-        yield* push([eventEncoded]);
-    }).pipe(Effect.catchTag('LeaderAheadError', Effect.orDie));
-    // Starts various background loops
-    const boot = Effect.gen(function* () {
-        const span = yield* Effect.currentSpan.pipe(Effect.orDie);
-        const otelSpan = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)));
-        const { devtools, shutdownChannel } = yield* LeaderThreadCtx;
-        const runtime = yield* Effect.runtime();
-        ctxRef.current = {
-            otelSpan,
-            span,
-            devtoolsLatch: devtools.enabled ? devtools.syncBackendLatch : undefined,
-            runtime,
-        };
-        /** State transitions need to happen atomically, so we use a Ref to track the state */
-        yield* SubscriptionRef.set(syncStateSref, initialSyncState);
-        // Rehydrate sync queue
-        if (initialSyncState.pending.length > 0) {
-            const globalPendingEvents = initialSyncState.pending
-                // Don't sync clientOnly events
-                .filter((eventEncoded) => {
-                const { eventDef } = getEventDef(schema, eventEncoded.name);
-                return eventDef.options.clientOnly === false;
-            });
-            if (globalPendingEvents.length > 0) {
-                yield* BucketQueue.offerAll(syncBackendPushQueue, globalPendingEvents);
+        const isPullPaginationComplete = (pageInfo) => pageInfo._tag === 'NoMore';
+        const onNewPullChunk = (newEvents, pageInfo) => Effect.gen(function* () {
+            if (ctxRef.current?.devtoolsLatch !== undefined) {
+                yield* ctxRef.current.devtoolsLatch.await;
             }
-        }
-        const maybeShutdownOnError = (cause) => Effect.gen(function* () {
-            if (onError === 'ignore') {
-                if (LS_DEV) {
-                    yield* Effect.logDebug(`Ignoring sync error (${cause._tag === 'Fail' ? cause.error._tag : cause._tag})`, Cause.pretty(cause));
+            if (newEvents.length === 0) {
+                if (isPullPaginationComplete(pageInfo) === true) {
+                    yield* releasePullMutexIfHeld;
                 }
                 return;
             }
-            const errorToSend = Cause.isFailType(cause) ? cause.error : UnexpectedError.make({ cause });
-            yield* shutdownChannel.send(errorToSend).pipe(Effect.orDie);
-            return yield* Effect.die(cause);
-        });
-        yield* backgroundApplyLocalPushes({
-            localPushesLatch,
-            localPushesQueue,
-            pullLatch,
-            syncStateSref,
-            syncBackendPushQueue,
-            schema,
-            isClientEvent,
-            otelSpan,
-            connectedClientSessionPullQueues,
-            localPushBatchSize,
-            testing: {
-                delay: testing?.delays?.localPushProcessing,
-            },
-        }).pipe(Effect.catchAllCause(maybeShutdownOnError), Effect.forkScoped);
-        const backendPushingFiberHandle = yield* FiberHandle.make();
-        const backendPushingEffect = backgroundBackendPushing({
-            syncBackendPushQueue,
-            otelSpan,
-            devtoolsLatch: ctxRef.current?.devtoolsLatch,
-            backendPushBatchSize,
-        }).pipe(Effect.catchAllCause(maybeShutdownOnError));
-        yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect);
-        yield* backgroundBackendPulling({
-            isClientEvent,
-            restartBackendPushing: (filteredRebasedPending) => Effect.gen(function* () {
-                // Stop current pushing fiber
-                yield* FiberHandle.clear(backendPushingFiberHandle);
-                // Reset the sync backend push queue
-                yield* BucketQueue.clear(syncBackendPushQueue);
-                yield* BucketQueue.offerAll(syncBackendPushQueue, filteredRebasedPending);
-                // Restart pushing fiber
-                yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect);
-            }),
-            syncStateSref,
-            localPushesLatch,
-            pullLatch,
-            livePull,
-            dbState,
-            otelSpan,
-            initialBlockingSyncContext,
-            devtoolsLatch: ctxRef.current?.devtoolsLatch,
-            connectedClientSessionPullQueues,
-            advancePushHead,
-        }).pipe(Effect.retry({
-            // We want to retry pulling if we've lost connection to the sync backend
-            while: (cause) => cause._tag === 'IsOfflineError',
-        }), Effect.catchAllCause(maybeShutdownOnError), 
-        // Needed to avoid `Fiber terminated with an unhandled error` logs which seem to happen because of the `Effect.retry` above.
-        // This might be a bug in Effect. Only seems to happen in the browser.
-        Effect.provide(Layer.setUnhandledErrorLogLevel(Option.none())), Effect.forkScoped);
-        return { initialLeaderHead: initialSyncState.localHead };
-    }).pipe(Effect.withSpanScoped('@livestore/common:LeaderSyncProcessor:boot'));
-    const pull = ({ cursor }) => Effect.gen(function* () {
-        const queue = yield* pullQueue({ cursor });
-        return Stream.fromQueue(queue);
-    }).pipe(Stream.unwrapScoped);
-    /*
-    Notes for a potential new `LeaderSyncProcessor.pull` implementation:
-
-    - Doesn't take cursor but is "atomically called" in the leader during the snapshot phase
-      - TODO: how is this done "atomically" in the web adapter where the snapshot is read optimistically?
-    - Would require a new kind of "boot-phase" API which is stream based:
-      - initial message: state snapshot + seq num head
-      - subsequent messages: sync state payloads
-
-    - alternative: instead of session pulling sync state payloads from leader, we could send
-      - events in the "advance" case
-      - full new state db snapshot in the "rebase" case
-        - downside: importing the snapshot is expensive
-    */
-    const pullQueue = ({ cursor }) => {
-        const runtime = ctxRef.current?.runtime ?? shouldNeverHappen('Not initialized');
-        return connectedClientSessionPullQueues.makeQueue(cursor).pipe(Effect.provide(runtime));
-    };
-    const syncState = Subscribable.make({
-        get: Effect.gen(function* () {
-            const syncState = yield* syncStateSref;
-            if (syncState === undefined)
-                return shouldNeverHappen('Not initialized');
-            return syncState;
-        }),
-        changes: syncStateSref.changes.pipe(Stream.filter(isNotUndefined)),
-    });
-    return {
-        pull,
-        pullQueue,
-        push,
-        pushPartial,
-        boot,
-        syncState,
-    };
-});
-const backgroundApplyLocalPushes = ({ localPushesLatch, localPushesQueue, pullLatch, syncStateSref, syncBackendPushQueue, schema, isClientEvent, otelSpan, connectedClientSessionPullQueues, localPushBatchSize, testing, }) => Effect.gen(function* () {
-    while (true) {
-        if (testing.delay !== undefined) {
-            yield* testing.delay.pipe(Effect.withSpan('localPushProcessingDelay'));
-        }
-        const batchItems = yield* BucketQueue.takeBetween(localPushesQueue, 1, localPushBatchSize);
-        // Wait for the backend pulling to finish
-        yield* localPushesLatch.await;
-        // Prevent backend pull processing until this local push is finished
-        yield* pullLatch.close;
-        const syncState = yield* syncStateSref;
-        if (syncState === undefined)
-            return shouldNeverHappen('Not initialized');
-        const currentRebaseGeneration = syncState.localHead.rebaseGeneration;
-        // Since the rebase generation might have changed since enqueuing, we need to filter out items with older generation
-        // It's important that we filter after we got localPushesLatch, otherwise we might filter with the old generation
-        const [newEvents, deferreds] = pipe(batchItems, ReadonlyArray.filter(([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration === currentRebaseGeneration), ReadonlyArray.unzip);
-        if (newEvents.length === 0) {
-            // console.log('dropping old-gen batch', currentLocalPushGenerationRef.current)
-            // Allow the backend pulling to start
-            yield* pullLatch.open;
-            continue;
-        }
-        const mergeResult = SyncState.merge({
-            syncState,
-            payload: { _tag: 'local-push', newEvents },
-            isClientEvent,
-            isEqualEvent: LiveStoreEvent.isEqualEncoded,
-        });
-        switch (mergeResult._tag) {
-            case 'unexpected-error': {
-                otelSpan?.addEvent(`push:unexpected-error`, {
-                    batchSize: newEvents.length,
-                    newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
-                });
-                return yield* new UnexpectedError({ cause: mergeResult.message });
-            }
-            case 'rebase': {
-                return shouldNeverHappen('The leader thread should never have to rebase due to a local push');
+            // Prevent more local pushes from being processed until this pull pagination sequence is finished.
+            if (pullMutexHeld === false) {
+                yield* localPushBackendPullMutex.take(1);
+                pullMutexHeld = true;
             }
-            case 'reject': {
-                otelSpan?.addEvent(`push:reject`, {
-                    batchSize: newEvents.length,
-                    mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
+            const chunkExit = yield* Effect.gen(function* () {
+                const syncState = yield* Effect.fromNullishOr(yield* SubscriptionRef.get(syncStateSref)).pipe(Effect.orDieDebugger);
+                yield* Effect.annotateCurrentSpan({
+                    'merge.newEventsCount': newEvents.length,
+                    ...(TRACE_VERBOSE === true ? { 'merge.newEvents': jsonStringify(newEvents) } : {}),
+                });
+                const mergeResult = yield* SyncState.merge({
+                    syncState,
+                    payload: SyncState.PayloadUpstreamAdvance.make({ newEvents }),
+                    isClientOnlyEvent,
+                    isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
+                    ignoreClientOnlyEvents: true,
                 });
-                // TODO: how to test this?
-                const nextRebaseGeneration = currentRebaseGeneration + 1;
-                const providedNum = newEvents.at(0).seqNum;
-                // All subsequent pushes with same generation should be rejected as well
-                // We're also handling the case where the localPushQueue already contains events
-                // from the next generation which we preserve in the queue
-                const remainingEventsMatchingGeneration = yield* BucketQueue.takeSplitWhere(localPushesQueue, ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= nextRebaseGeneration);
-                // TODO we still need to better understand and handle this scenario
-                if (LS_DEV && (yield* BucketQueue.size(localPushesQueue)) > 0) {
-                    console.log('localPushesQueue is not empty', yield* BucketQueue.size(localPushesQueue));
-                    // biome-ignore lint/suspicious/noDebugger: debugging
-                    debugger;
+                if (mergeResult._tag === 'reject') {
+                    return yield* Effect.dieDebugger('The leader thread should never reject upstream advances');
+                }
+                const newBackendHead = newEvents.at(-1).seqNum;
+                Eventlog.updateBackendHead(dbEventlog, newBackendHead);
+                if (mergeResult._tag === 'rebase') {
+                    yield* Effect.spanEvent(`pull:rebase[${mergeResult.newSyncState.localHead.rebaseGeneration}]`, {
+                        newEventsCount: newEvents.length,
+                        ...(TRACE_VERBOSE === true ? { newEvents: jsonStringify(newEvents) } : {}),
+                        rollbackCount: mergeResult.rollbackEvents.length,
+                        ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
+                    });
+                    const globalOrUnknownRebasedPendingEvents = mergeResult.newSyncState.pending.filter((e) => !isClientOnlyEvent(e));
+                    yield* restartBackendPushing(globalOrUnknownRebasedPendingEvents);
+                    if (mergeResult.rollbackEvents.length > 0) {
+                        yield* rollback({
+                            dbState: db,
+                            dbEventlog,
+                            eventNumsToRollback: mergeResult.rollbackEvents.map((_) => _.seqNum),
+                        });
+                        yield* stateHead
+                            .set(mergeResult.rollbackEvents[0].parentSeqNum)
+                            .pipe(Effect.mapError((cause) => MaterializeError.make({ cause })));
+                    }
+                    yield* connectedClientSessionPullQueues.offer({
+                        payload: SyncState.payloadFromMergeResult(mergeResult),
+                        leaderHead: mergeResult.newSyncState.localHead,
+                    });
+                }
+                else {
+                    yield* Effect.spanEvent(`pull:advance`, {
+                        newEventsCount: newEvents.length,
+                        ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
+                    });
+                    // Ensure push fiber is active after advance by restarting with current pending (non-client-only) events
+                    const globalOrUnknownPendingEvents = mergeResult.newSyncState.pending.filter((e) => !isClientOnlyEvent(e));
+                    yield* restartBackendPushing(globalOrUnknownPendingEvents);
+                    yield* connectedClientSessionPullQueues.offer({
+                        payload: SyncState.payloadFromMergeResult(mergeResult),
+                        leaderHead: mergeResult.newSyncState.localHead,
+                    });
+                    if (mergeResult.confirmedEvents.length > 0) {
+                        // `mergeResult.confirmedEvents` don't contain the correct sync metadata, so we need to use
+                        // `newEvents` instead which we filter via `mergeResult.confirmedEvents`
+                        const confirmedNewEvents = newEvents.filter((event) => mergeResult.confirmedEvents.some((confirmedEvent) => EventSequenceNumber.Client.isEqual(event.seqNum, confirmedEvent.seqNum)));
+                        yield* Eventlog.updateSyncMetadata(confirmedNewEvents).pipe(Effect.orDieDebugger);
+                    }
                 }
-                const allDeferredsToReject = [
-                    ...deferreds,
-                    ...remainingEventsMatchingGeneration.map(([_, deferred]) => deferred),
-                ].filter(isNotUndefined);
-                yield* Effect.forEach(allDeferredsToReject, (deferred) => Deferred.fail(deferred, LeaderAheadError.make({ minimumExpectedNum: mergeResult.expectedMinimumId, providedNum })));
-                // Allow the backend pulling to start
-                yield* pullLatch.open;
-                // In this case we're skipping state update and down/upstream processing
-                // We've cleared the local push queue and are now waiting for new local pushes / backend pulls
-                continue;
+                // Removes the changeset rows which are no longer needed as we'll never have to rollback beyond this point
+                trimChangesetRows(db, newBackendHead);
+                // The backend merge may advance or rebase the authoritative head. Realign the admission
+                // fence now so newly arriving pushes are validated against that history, not the pre-pull head.
+                yield* reconcilePushHead(mergeResult.newSyncState.localHead);
+                // Apply the merged events to storage before publishing the new sync state below, so readers
+                // cannot observe a leader head whose events have not yet been materialized.
+                yield* materializeEventsBatch({ batchItems: mergeResult.newEvents });
+                yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
+            }).pipe(Effect.exit);
+            if (Exit.isFailure(chunkExit) === true) {
+                yield* releasePullMutexIfHeld;
+                return yield* Effect.failCause(chunkExit.cause);
             }
-            case 'advance': {
-                break;
+            if (isPullPaginationComplete(pageInfo) === true) {
+                yield* releasePullMutexIfHeld;
             }
-            default: {
-                casesHandled(mergeResult);
+        });
+        const syncState = yield* Effect.fromNullishOr(yield* SubscriptionRef.get(syncStateSref)).pipe(Effect.orDieDebugger);
+        const cursorInfo = yield* Eventlog.getSyncBackendCursorInfo({ remoteHead: syncState.upstreamHead.global });
+        const hashMaterializerResult = makeMaterializerHash({ schema, dbState });
+        yield* syncBackend.pull(cursorInfo, { live: livePull }).pipe(
+        // TODO only take from queue while connected
+        Stream.tap(({ batch, pageInfo }) => Effect.gen(function* () {
+            // NOTE we only want to take process events when the sync backend is connected
+            // (e.g. needed for simulating being offline)
+            // TODO remove when there's a better way to handle this in stream above
+            yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
+            yield* onNewPullChunk(batch.map((_) => LiveStoreEvent.Client.EncodedWithMeta.fromGlobal(_.eventEncoded, {
+                syncMetadata: _.metadata,
+                // TODO we can't really know the materializer result here yet beyond the first event batch item as we need to materialize it one by one first
+                // This is a bug and needs to be fixed https://github.com/livestorejs/livestore/issues/503#issuecomment-3114533165
+                materializerHashLeader: hashMaterializerResult(LiveStoreEvent.Global.toClientEncoded(_.eventEncoded)),
+                materializerHashSession: Option.none(),
+            })), pageInfo);
+            yield* initialBlockingSyncContext.update({ processed: batch.length, pageInfo });
+        })), Stream.runDrain, Effect.interruptible, Effect.ensuring(releasePullMutexIfHeld));
+        // Should only ever happen when livePull is false
+        yield* Effect.logDebug('backend-pulling finished', { livePull });
+    });
+    const backgroundBackendPushing = Effect.gen(function* () {
+        const { syncBackend } = yield* LeaderThreadCtx;
+        if (syncBackend === undefined)
+            return;
+        while (true) {
+            yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
+            const queueItems = yield* TxQueue.takeBetween(syncBackendPushQueue, 1, backendPushBatchSize);
+            yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
+            if (ctxRef.current?.devtoolsLatch !== undefined) {
+                yield* ctxRef.current.devtoolsLatch.await;
             }
+            yield* Effect.spanEvent('backend-push', {
+                batchSize: queueItems.length,
+                ...(TRACE_VERBOSE === true ? { batch: jsonStringify(queueItems) } : {}),
+            });
+            // Push with declarative retry/backoff using Effect schedules
+            // - Exponential backoff starting at 1s and doubling (1s, 2s, 4s, 8s, 16s, 30s ...)
+            // - Delay clamped at 30s (continues retrying at 30s)
+            // - Resets automatically after successful push
+            // TODO(metrics): expose counters/gauges for retry attempts and queue health via devtools/metrics
+            yield* Effect.gen(function* () {
+                const iteration = yield* Schedule.CurrentMetadata;
+                const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.result);
+                const retries = iteration.attempt;
+                if (retries > 0 && Result.isSuccess(pushResult) === true) {
+                    yield* Effect.spanEvent('backend-push-retry-success', { retries, batchSize: queueItems.length });
+                }
+                if (Result.isFailure(pushResult) === true) {
+                    yield* Effect.spanEvent('backend-push-error', {
+                        error: pushResult.failure.toString(),
+                        retries,
+                        batchSize: queueItems.length,
+                    });
+                    const error = pushResult.failure;
+                    if (error._tag === 'ServerAheadError') {
+                        // It's a core part of the sync protocol that the sync backend will emit a new pull chunk alongside the ServerAheadError
+                        yield* Effect.logDebug('handled backend-push-error (waiting for interupt caused by pull)', { error });
+                        return yield* Effect.never;
+                    }
+                    return yield* error;
+                }
+            }).pipe(
+            // Retry transient errors
+            Effect.retry({
+                schedule: Schedule.exponential(Duration.seconds(1)).pipe(Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(30))))),
+                while: (error) => error._tag === 'IsOfflineError' || error._tag === 'UnknownError',
+            }), 
+            // This is needed to narrow the Error type. Our retry policy runs indefinitely, but Effect.retry does not narrow the Error type.
+            Effect.catchIf((error) => error._tag === 'IsOfflineError' || error._tag === 'UnknownError', Effect.die));
         }
-        yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
-        yield* connectedClientSessionPullQueues.offer({
-            payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
-            leaderHead: mergeResult.newSyncState.localHead,
-        });
-        otelSpan?.addEvent(`push:advance`, {
+    }).pipe(Effect.interruptible);
+    // NOTE: New events are only pushed to sync backend after successful local push processing
+    const push = (newEvents) => Effect.gen(function* () {
+        if (newEvents.length === 0)
+            return;
+        const deferreds = yield* Effect.forEach(newEvents, () => Deferred.make());
+        const items = newEvents.map((eventEncoded, i) => [eventEncoded, deferreds[i]]);
+        // Validation, reservation, enqueueing, and fence advancement form one admission transaction.
+        // Serializing them prevents concurrent sessions from both validating against the same head and
+        // prevents backend reconciliation from changing the fence midway through admission.
+        yield* pushAdmissionSemaphore.withPermits(1)(
+        // Cancellation must not leave reservations, queue contents, and the admission fence disagreeing.
+        Effect.gen(function* () {
+            yield* validatePushBatch(newEvents, pushHeadRef.current, isClientOnlyEvent);
+            for (const item of items)
+                reservedLocalPushItems.add(item);
+            yield* TxQueue.offerAll(localPushesQueue, items);
+            pushHeadRef.current = newEvents.at(-1).seqNum;
+            if (testing.hooks?.localPushAdmitted !== undefined) {
+                yield* testing.hooks.localPushAdmitted(newEvents);
+            }
+        }).pipe(Effect.uninterruptible));
+        yield* Effect.all(deferreds.map(Deferred.await));
+    }).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:push', {
+        attributes: {
             batchSize: newEvents.length,
-            mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
-        });
-        // Don't sync clientOnly events
-        const filteredBatch = mergeResult.newEvents.filter((eventEncoded) => {
-            const { eventDef } = getEventDef(schema, eventEncoded.name);
-            return eventDef.options.clientOnly === false;
-        });
-        yield* BucketQueue.offerAll(syncBackendPushQueue, filteredBatch);
-        yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds });
-        // Allow the backend pulling to start
-        yield* pullLatch.open;
-    }
+            batch: TRACE_VERBOSE === true ? newEvents : undefined,
+        },
+        links: ctxRef.current?.span !== undefined ? [{ span: ctxRef.current.span, attributes: {} }] : undefined,
+    }));
+    return LeaderSyncProcessor.of({
+        [TypeId]: TypeId,
+        // Starts various background loops
+        boot: Effect.gen(function* () {
+            const span = yield* Effect.currentSpan.pipe(Effect.orDie);
+            const { devtools, shutdownChannel } = yield* LeaderThreadCtx;
+            const services = yield* Effect.context();
+            ctxRef.current = {
+                span,
+                devtoolsLatch: devtools.enabled === true ? devtools.syncBackendLatch : undefined,
+                services,
+            };
+            /** State transitions need to happen atomically, so we use a Ref to track the state */
+            yield* SubscriptionRef.set(syncStateSref, initialSyncState);
+            // Rehydrate sync queue
+            if (initialSyncState.pending.length > 0) {
+                const globalOrUnknownPendingEvents = initialSyncState.pending
+                    // Don't sync client-only events
+                    .filter((eventEncoded) => !isClientOnlyEvent(eventEncoded));
+                if (globalOrUnknownPendingEvents.length > 0) {
+                    yield* TxQueue.offerAll(syncBackendPushQueue, globalOrUnknownPendingEvents);
+                }
+            }
+            const handleBackendIdMismatchError = (error) => handleBackendIdMismatch({ error, onBackendIdMismatch, shutdownChannel });
+            const maybeShutdownOnError = (cause) => Effect.gen(function* () {
+                if (onError === 'ignore') {
+                    if (LS_DEV === true) {
+                        yield* Effect.logDebug(`Ignoring sync error (${Option.getOrUndefined(Cause.findErrorOption(cause))?._tag ?? cause.toString()})`, Cause.pretty(cause));
+                    }
+                    return;
+                }
+                const error = Option.getOrUndefined(Cause.findErrorOption(cause));
+                const errorToSend = error === undefined ? UnknownError.make({ cause }) : error;
+                yield* shutdownChannel.send(errorToSend).pipe(Effect.orDie);
+                return yield* Effect.failCause(cause).pipe(Effect.orDie);
+            });
+            yield* backgroundApplyLocalPushes.pipe(Effect.catchCause(maybeShutdownOnError), Effect.forkScoped);
+            const backendPushingFiberHandle = yield* FiberHandle.make();
+            const backendPushingEffect = backgroundBackendPushing.pipe(Effect.catchTag('BackendIdMismatchError', handleBackendIdMismatchError), Effect.catchCause(maybeShutdownOnError));
+            yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect);
+            yield* backgroundBackendPulling({
+                restartBackendPushing: (filteredRebasedPending) => Effect.gen(function* () {
+                    // Stop current pushing fiber
+                    yield* FiberHandle.clear(backendPushingFiberHandle);
+                    // Reset the sync backend push queue
+                    yield* TxQueue.clear(syncBackendPushQueue);
+                    yield* TxQueue.offerAll(syncBackendPushQueue, filteredRebasedPending);
+                    // Restart pushing fiber
+                    yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect);
+                }),
+            }).pipe(Effect.retry({
+                // Retry pulling when we've lost connection to the sync backend
+                // We're using `until` with a refinement instead of `while` to narrow `IsOfflineError` out of the error type.
+                // See https://github.com/Effect-TS/effect/issues/6122
+                until: (error) => error._tag !== 'IsOfflineError',
+            }), Effect.catchTag('BackendIdMismatchError', handleBackendIdMismatchError), Effect.catchCause(maybeShutdownOnError), 
+            // Needed to avoid `Fiber terminated with an unhandled error` logs which seem to happen because of the `Effect.retry` above.
+            // This might be a bug in Effect. Only seems to happen in the browser.
+            Effect.provideService(References.UnhandledLogLevel, undefined), Effect.forkScoped);
+            return { initialLeaderHead: initialSyncState.localHead };
+        }).pipe(Effect.withSpanScoped('@livestore/common:LeaderSyncProcessor:boot')),
+        push,
+        pushPartial: ({ event: { name, args }, clientId, sessionId }) => Effect.gen(function* () {
+            const syncState = yield* Effect.fromNullishOr(yield* SubscriptionRef.get(syncStateSref)).pipe(Effect.orDieDebugger);
+            const resolution = yield* resolveEventDef(schema, {
+                operation: '@livestore/common:LeaderSyncProcessor:pushPartial',
+                event: {
+                    name,
+                    args,
+                    clientId,
+                    sessionId,
+                    seqNum: syncState.localHead,
+                },
+            });
+            if (resolution._tag === 'unknown') {
+                // Ignore partial pushes for unrecognised events – they are still
+                // persisted server-side once a schema update ships.
+                return;
+            }
+            const eventEncoded = new LiveStoreEvent.Client.EncodedWithMeta({
+                name,
+                args,
+                clientId,
+                sessionId,
+                ...EventSequenceNumber.Client.nextPair({
+                    seqNum: syncState.localHead,
+                    isClientOnly: resolution.eventDef.options.clientOnly,
+                }),
+            });
+            yield* push([eventEncoded]);
+        }).pipe(
+        // pushPartial constructs the event sequence number internally, so these errors should never happen.
+        Effect.catchIf(isRejectedPushError, Effect.die)),
+        pull: ({ cursor }) => Effect.gen(function* () {
+            const queue = yield* Effect.fromNullishOr(ctxRef.current?.services).pipe(Effect.orDieDebugger, Effect.flatMap((services) => connectedClientSessionPullQueues.makeQueue(cursor).pipe(Effect.provide(services))));
+            return Stream.fromQueue(queue);
+        }).pipe(Stream.unwrap),
+        /*
+          Notes for a potential new `LeaderSyncProcessor.pull` implementation:
+    
+          - Doesn't take cursor but is "atomically called" in the leader during the snapshot phase
+            - TODO: how is this done "atomically" in the web adapter where the snapshot is read optimistically?
+          - Would require a new kind of "boot-phase" API which is stream based:
+            - initial message: state snapshot + seq num head
+            - subsequent messages: sync state payloads
+    
+          - alternative: instead of session pulling sync state payloads from leader, we could send
+            - events in the "advance" case
+            - full new state db snapshot in the "rebase" case
+              - downside: importing the snapshot is expensive
+          */
+        pullQueue: ({ cursor }) => Effect.fromNullishOr(ctxRef.current?.services).pipe(Effect.orDieDebugger, Effect.flatMap((services) => connectedClientSessionPullQueues.makeQueue(cursor).pipe(Effect.provide(services)))),
+        syncState: Subscribable.make({
+            get: SubscriptionRef.get(syncStateSref).pipe(Effect.flatMap(Effect.fromNullishOr), Effect.orDieDebugger),
+            changes: SubscriptionRef.changes(syncStateSref).pipe(Stream.filter(Predicate.isNotUndefined)),
+        }),
+    });
 });
+export const layer = (options) => Layer.effect(LeaderSyncProcessor, make(options));
 // TODO how to handle errors gracefully
-const materializeEventsBatch = ({ batchItems, deferreds }) => Effect.gen(function* () {
+const materializeEventsBatch = ({ batchItems }) => Effect.gen(function* () {
     const { dbState: db, dbEventlog, materializeEvent } = yield* LeaderThreadCtx;
     // NOTE We always start a transaction to ensure consistency between db and eventlog (even for single-item batches)
     db.execute('BEGIN TRANSACTION', undefined); // Start the transaction
     dbEventlog.execute('BEGIN TRANSACTION', undefined); // Start the transaction
     yield* Effect.addFinalizer((exit) => Effect.gen(function* () {
-        if (Exit.isSuccess(exit))
+        if (Exit.isSuccess(exit) === true)
             return;
         // Rollback in case of an error
         db.execute('ROLLBACK', undefined);
@@ -351,187 +551,12 @@ const materializeEventsBatch = ({ batchItems, deferreds }) => Effect.gen(functio
         const { sessionChangeset, hash } = yield* materializeEvent(batchItems[i]);
         batchItems[i].meta.sessionChangeset = sessionChangeset;
         batchItems[i].meta.materializerHashLeader = hash;
-        if (deferreds?.[i] !== undefined) {
-            yield* Deferred.succeed(deferreds[i], void 0);
-        }
     }
     db.execute('COMMIT', undefined); // Commit the transaction
     dbEventlog.execute('COMMIT', undefined); // Commit the transaction
 }).pipe(Effect.uninterruptible, Effect.scoped, Effect.withSpan('@livestore/common:LeaderSyncProcessor:materializeEventItems', {
     attributes: { batchSize: batchItems.length },
 }), Effect.tapCauseLogPretty);
-const backgroundBackendPulling = ({ isClientEvent, restartBackendPushing, otelSpan, dbState, syncStateSref, localPushesLatch, livePull, pullLatch, devtoolsLatch, initialBlockingSyncContext, connectedClientSessionPullQueues, advancePushHead, }) => Effect.gen(function* () {
-    const { syncBackend, dbState: db, dbEventlog, schema } = yield* LeaderThreadCtx;
-    if (syncBackend === undefined)
-        return;
-    const onNewPullChunk = (newEvents, pageInfo) => Effect.gen(function* () {
-        if (newEvents.length === 0)
-            return;
-        if (devtoolsLatch !== undefined) {
-            yield* devtoolsLatch.await;
-        }
-        // Prevent more local pushes from being processed until this pull is finished
-        yield* localPushesLatch.close;
-        // Wait for pending local pushes to finish
-        yield* pullLatch.await;
-        const syncState = yield* syncStateSref;
-        if (syncState === undefined)
-            return shouldNeverHappen('Not initialized');
-        const mergeResult = SyncState.merge({
-            syncState,
-            payload: SyncState.PayloadUpstreamAdvance.make({ newEvents }),
-            isClientEvent,
-            isEqualEvent: LiveStoreEvent.isEqualEncoded,
-            ignoreClientEvents: true,
-        });
-        if (mergeResult._tag === 'reject') {
-            return shouldNeverHappen('The leader thread should never reject upstream advances');
-        }
-        else if (mergeResult._tag === 'unexpected-error') {
-            otelSpan?.addEvent(`pull:unexpected-error`, {
-                newEventsCount: newEvents.length,
-                newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
-            });
-            return yield* new UnexpectedError({ cause: mergeResult.message });
-        }
-        const newBackendHead = newEvents.at(-1).seqNum;
-        Eventlog.updateBackendHead(dbEventlog, newBackendHead);
-        if (mergeResult._tag === 'rebase') {
-            otelSpan?.addEvent(`pull:rebase[${mergeResult.newSyncState.localHead.rebaseGeneration}]`, {
-                newEventsCount: newEvents.length,
-                newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
-                rollbackCount: mergeResult.rollbackEvents.length,
-                mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
-            });
-            const globalRebasedPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
-                const { eventDef } = getEventDef(schema, event.name);
-                return eventDef.options.clientOnly === false;
-            });
-            yield* restartBackendPushing(globalRebasedPendingEvents);
-            if (mergeResult.rollbackEvents.length > 0) {
-                yield* rollback({
-                    dbState: db,
-                    dbEventlog,
-                    eventNumsToRollback: mergeResult.rollbackEvents.map((_) => _.seqNum),
-                });
-            }
-            yield* connectedClientSessionPullQueues.offer({
-                payload: SyncState.payloadFromMergeResult(mergeResult),
-                leaderHead: mergeResult.newSyncState.localHead,
-            });
-        }
-        else {
-            otelSpan?.addEvent(`pull:advance`, {
-                newEventsCount: newEvents.length,
-                mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
-            });
-            // Ensure push fiber is active after advance by restarting with current pending (non-client) events
-            const globalPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
-                const { eventDef } = getEventDef(schema, event.name);
-                return eventDef.options.clientOnly === false;
-            });
-            yield* restartBackendPushing(globalPendingEvents);
-            yield* connectedClientSessionPullQueues.offer({
-                payload: SyncState.payloadFromMergeResult(mergeResult),
-                leaderHead: mergeResult.newSyncState.localHead,
-            });
-            if (mergeResult.confirmedEvents.length > 0) {
-                // `mergeResult.confirmedEvents` don't contain the correct sync metadata, so we need to use
-                // `newEvents` instead which we filter via `mergeResult.confirmedEvents`
-                const confirmedNewEvents = newEvents.filter((event) => mergeResult.confirmedEvents.some((confirmedEvent) => EventSequenceNumber.isEqual(event.seqNum, confirmedEvent.seqNum)));
-                yield* Eventlog.updateSyncMetadata(confirmedNewEvents).pipe(UnexpectedError.mapToUnexpectedError);
-            }
-        }
-        // Removes the changeset rows which are no longer needed as we'll never have to rollback beyond this point
-        trimChangesetRows(db, newBackendHead);
-        advancePushHead(mergeResult.newSyncState.localHead);
-        yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds: undefined });
-        yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
-        // Allow local pushes to be processed again
-        if (pageInfo._tag === 'NoMore') {
-            yield* localPushesLatch.open;
-        }
-    });
-    const syncState = yield* syncStateSref;
-    if (syncState === undefined)
-        return shouldNeverHappen('Not initialized');
-    const cursorInfo = yield* Eventlog.getSyncBackendCursorInfo({ remoteHead: syncState.upstreamHead.global });
-    const hashMaterializerResult = makeMaterializerHash({ schema, dbState });
-    yield* syncBackend.pull(cursorInfo, { live: livePull }).pipe(
-    // TODO only take from queue while connected
-    Stream.tap(({ batch, pageInfo }) => Effect.gen(function* () {
-        // yield* Effect.spanEvent('batch', {
-        //   attributes: {
-        //     batchSize: batch.length,
-        //     batch: TRACE_VERBOSE ? batch : undefined,
-        //   },
-        // })
-        // NOTE we only want to take process events when the sync backend is connected
-        // (e.g. needed for simulating being offline)
-        // TODO remove when there's a better way to handle this in stream above
-        yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
-        yield* onNewPullChunk(batch.map((_) => LiveStoreEvent.EncodedWithMeta.fromGlobal(_.eventEncoded, {
-            syncMetadata: _.metadata,
-            // TODO we can't really know the materializer result here yet beyond the first event batch item as we need to materialize it one by one first
-            // This is a bug and needs to be fixed https://github.com/livestorejs/livestore/issues/503#issuecomment-3114533165
-            materializerHashLeader: hashMaterializerResult(LiveStoreEvent.encodedFromGlobal(_.eventEncoded)),
-            materializerHashSession: Option.none(),
-        })), pageInfo);
-        yield* initialBlockingSyncContext.update({ processed: batch.length, pageInfo });
-    })), Stream.runDrain, Effect.interruptible);
-    // Should only ever happen when livePull is false
-    yield* Effect.logDebug('backend-pulling finished', { livePull });
-}).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pulling'));
-const backgroundBackendPushing = ({ syncBackendPushQueue, otelSpan, devtoolsLatch, backendPushBatchSize, }) => Effect.gen(function* () {
-    const { syncBackend } = yield* LeaderThreadCtx;
-    if (syncBackend === undefined)
-        return;
-    while (true) {
-        yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
-        const queueItems = yield* BucketQueue.takeBetween(syncBackendPushQueue, 1, backendPushBatchSize);
-        yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true);
-        if (devtoolsLatch !== undefined) {
-            yield* devtoolsLatch.await;
-        }
-        otelSpan?.addEvent('backend-push', {
-            batchSize: queueItems.length,
-            batch: TRACE_VERBOSE ? JSON.stringify(queueItems) : undefined,
-        });
-        // Push with declarative retry/backoff using Effect schedules
-        // - Exponential backoff starting at 1s and doubling (1s, 2s, 4s, 8s, 16s, 30s ...)
-        // - Delay clamped at 30s (continues retrying at 30s)
-        // - Resets automatically after successful push
-        // TODO(metrics): expose counters/gauges for retry attempts and queue health via devtools/metrics
-        // Only retry for transient UnexpectedError cases
-        const isRetryable = (err) => err._tag === 'InvalidPushError' && err.cause._tag === 'LiveStore.UnexpectedError';
-        // Input: InvalidPushError | IsOfflineError, Output: Duration
-        const retrySchedule = Schedule.exponential(Duration.seconds(1)).pipe(Schedule.andThenEither(Schedule.spaced(Duration.seconds(30))), // clamp at 30 second intervals
-        Schedule.compose(Schedule.elapsed), Schedule.whileInput(isRetryable));
-        yield* Effect.gen(function* () {
-            const iteration = yield* Schedule.CurrentIterationMetadata;
-            const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either);
-            const retries = iteration.recurrence;
-            if (retries > 0 && pushResult._tag === 'Right') {
-                otelSpan?.addEvent('backend-push-retry-success', { retries, batchSize: queueItems.length });
-            }
-            if (pushResult._tag === 'Left') {
-                otelSpan?.addEvent('backend-push-error', {
-                    error: pushResult.left.toString(),
-                    retries,
-                    batchSize: queueItems.length,
-                });
-                const error = pushResult.left;
-                if (error._tag === 'IsOfflineError' ||
-                    (error._tag === 'InvalidPushError' && error.cause._tag === 'ServerAheadError')) {
-                    // It's a core part of the sync protocol that the sync backend will emit a new pull chunk alongside the ServerAheadError
-                    yield* Effect.logDebug('handled backend-push-error (waiting for interupt caused by pull)', { error });
-                    return yield* Effect.never;
-                }
-                return yield* error;
-            }
-        }).pipe(Effect.retry(retrySchedule));
-    }
-}).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pushing'));
 const trimChangesetRows = (db, newHead) => {
     // Since we're using the session changeset rows to query for the current head,
     // we're keeping at least one row for the current head, and thus are using `<` instead of `<=`
@@ -548,18 +573,18 @@ const makePullQueueSet = Effect.gen(function* () {
         set.clear();
     }));
     const makeQueue = (cursor) => Effect.gen(function* () {
-        const queue = yield* Queue.unbounded().pipe(Effect.acquireRelease(Queue.shutdown));
+        const queue = yield* Effect.acquireRelease(Queue.unbounded(), Queue.shutdown);
         yield* Effect.addFinalizer(() => Effect.sync(() => set.delete(queue)));
         const payloadsSinceCursor = Array.from(cachedPayloads.entries())
-            .flatMap(([seqNumStr, payloads]) => payloads.map((payload) => ({ payload, seqNum: EventSequenceNumber.fromString(seqNumStr) })))
-            .filter(({ seqNum }) => EventSequenceNumber.isGreaterThan(seqNum, cursor))
-            .toSorted((a, b) => EventSequenceNumber.compare(a.seqNum, b.seqNum))
+            .flatMap(([seqNumStr, payloads]) => payloads.map((payload) => ({ payload, seqNum: EventSequenceNumber.Client.fromString(seqNumStr) })))
+            .filter(({ seqNum }) => EventSequenceNumber.Client.isGreaterThan(seqNum, cursor))
+            .toSorted((a, b) => EventSequenceNumber.Client.compare(a.seqNum, b.seqNum))
             .map(({ payload }) => {
             if (payload._tag === 'upstream-advance') {
                 return {
                     payload: {
                         _tag: 'upstream-advance',
-                        newEvents: ReadonlyArray.dropWhile(payload.newEvents, (eventEncoded) => EventSequenceNumber.isGreaterThanOrEqual(cursor, eventEncoded.seqNum)),
+                        newEvents: ReadonlyArray.dropWhile(payload.newEvents, (eventEncoded) => EventSequenceNumber.Client.isGreaterThanOrEqual(cursor, eventEncoded.seqNum)),
                     },
                 };
             }
@@ -594,13 +619,13 @@ const makePullQueueSet = Effect.gen(function* () {
         //     ...(payload._tag === 'upstream-rebase' ? payload.rollbackEvents.map((_) => _.toJSON()) : []),
         //   ]),
         // )
-        yield* queue.offerAll(payloadsSinceCursor);
+        yield* Queue.offerAll(queue, payloadsSinceCursor);
         set.add(queue);
         return queue;
     });
     const offer = (item) => Effect.gen(function* () {
-        const seqNumStr = EventSequenceNumber.toString(item.leaderHead);
-        if (cachedPayloads.has(seqNumStr)) {
+        const seqNumStr = EventSequenceNumber.Client.toString(item.leaderHead);
+        if (cachedPayloads.has(seqNumStr) === true) {
             cachedPayloads.get(seqNumStr).push(item.payload);
         }
         else {
@@ -620,22 +645,123 @@ const makePullQueueSet = Effect.gen(function* () {
         offer,
     };
 });
-const validatePushBatch = (batch, pushHead) => Effect.gen(function* () {
+/**
+ * Validate a client-provided batch before it is admitted to the leader queue.
+ * Ensures the numbers form a strictly increasing chain and that the first
+ * event sits ahead of the current push head.
+ */
+const validatePushBatch = (batch, pushHead, isClientOnlyEvent) => Effect.gen(function* () {
     if (batch.length === 0) {
         return;
     }
-    // Make sure batch is monotonically increasing
+    // Defensive check: callers should already provide a strictly increasing sequence
+    // of event numbers.
     for (let i = 1; i < batch.length; i++) {
-        if (EventSequenceNumber.isGreaterThanOrEqual(batch[i - 1].seqNum, batch[i].seqNum)) {
-            shouldNeverHappen(`Events must be ordered in monotonically ascending order by eventNum. Received: [${batch.map((e) => EventSequenceNumber.toString(e.seqNum)).join(', ')}]`);
+        if (EventSequenceNumber.Client.isGreaterThanOrEqual(batch[i - 1].seqNum, batch[i].seqNum) === true) {
+            return yield* NonMonotonicBatchError.make({
+                precedingSeqNum: batch[i - 1].seqNum,
+                violatingSeqNum: batch[i].seqNum,
+                violationIndex: i,
+                sessionId: batch[i].sessionId,
+            });
         }
     }
-    // Make sure smallest sequence number is > pushHead
-    if (EventSequenceNumber.isGreaterThanOrEqual(pushHead, batch[0].seqNum)) {
+    // Reject stale batches whose first event is at or behind the leader's push head.
+    if (EventSequenceNumber.Client.isGreaterThanOrEqual(pushHead, batch[0].seqNum) === true) {
         return yield* LeaderAheadError.make({
             minimumExpectedNum: pushHead,
             providedNum: batch[0].seqNum,
+            sessionId: batch[0].sessionId,
+        });
+    }
+    // A rebase replaces the optimistic history that the client built on. Events from an older
+    // generation may now have the wrong parent and must be recreated from the leader's current head.
+    if (batch[0].seqNum.rebaseGeneration < pushHead.rebaseGeneration) {
+        return yield* StaleRebaseGenerationError.make({
+            currentRebaseGeneration: pushHead.rebaseGeneration,
+            providedRebaseGeneration: batch[0].seqNum.rebaseGeneration,
+            sessionId: batch[0].sessionId,
         });
     }
+    // Validate the batch as one unbroken chain starting at the admission fence. Checking only that
+    // numbers increase would still allow gaps or events that point at an unrelated parent.
+    let precedingSeqNum = pushHead;
+    for (let i = 0; i < batch.length; i++) {
+        const event = batch[i];
+        // Global events advance the global position; client-only events advance its client-local suffix.
+        const expectedPair = EventSequenceNumber.Client.nextPair({
+            seqNum: precedingSeqNum,
+            isClientOnly: isClientOnlyEvent(event),
+            rebaseGeneration: event.seqNum.rebaseGeneration,
+        });
+        if (EventSequenceNumber.Client.isEqual(event.seqNum, expectedPair.seqNum) === false ||
+            isSameSequencePosition(event.parentSeqNum, expectedPair.parentSeqNum) === false) {
+            return yield* NonContiguousBatchError.make({
+                expectedSeqNum: expectedPair.seqNum,
+                providedSeqNum: event.seqNum,
+                expectedParentSeqNum: expectedPair.parentSeqNum,
+                providedParentSeqNum: event.parentSeqNum,
+                violationIndex: i,
+                sessionId: event.sessionId,
+            });
+        }
+        precedingSeqNum = event.seqNum;
+    }
+});
+/**
+ * Parent linkage identifies a position in the event chain. Rebase generation describes the version
+ * of optimistic history, so it is validated on the event itself rather than as part of parent identity.
+ */
+const isSameSequencePosition = (left, right) => left.global === right.global && left.client === right.client;
+/**
+ * Handles a BackendIdMismatchError based on the configured behavior.
+ * This occurs when the sync backend has been reset and has a new identity.
+ */
+const handleBackendIdMismatch = Effect.fn('@livestore/common:LeaderSyncProcessor:handleBackendIdMismatch')(function* ({ error, onBackendIdMismatch, shutdownChannel, }) {
+    const { dbEventlog, dbState } = yield* LeaderThreadCtx;
+    if (onBackendIdMismatch === 'reset') {
+        yield* Effect.logWarning('Sync backend identity changed (backend was reset). Clearing local storage and shutting down.', error);
+        // Clear local databases so the client can start fresh on next boot
+        yield* clearLocalDatabases({ dbEventlog, dbState });
+        // Send shutdown signal with special reason
+        yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'backend-id-mismatch' })).pipe(Effect.orDie);
+        return yield* Effect.die(error);
+    }
+    if (onBackendIdMismatch === 'shutdown') {
+        yield* Effect.logWarning('Sync backend identity changed (backend was reset). Shutting down without clearing local storage.', error);
+        yield* shutdownChannel.send(error).pipe(Effect.orDie);
+        return yield* Effect.die(error);
+    }
+    // ignore mode
+    if (LS_DEV === true) {
+        yield* Effect.logDebug('Ignoring BackendIdMismatchError (sync backend was reset but client continues with stale data)', error);
+    }
+});
+/**
+ * Clears local databases (eventlog and state) so the client can start fresh on next boot.
+ * This is used when the sync backend identity has changed (i.e. backend was reset).
+ */
+const clearLocalDatabases = ({ dbEventlog, dbState }) => Effect.sync(() => {
+    // Clear eventlog tables
+    dbEventlog.execute(sql `DELETE FROM ${EVENTLOG_META_TABLE}`);
+    dbEventlog.execute(sql `DELETE FROM ${SYNC_STATUS_TABLE}`);
+    // Drop all state tables - they'll be recreated on next boot
+    const tables = dbState.select(sql `SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`);
+    for (const { name } of tables) {
+        dbState.execute(`DROP TABLE IF EXISTS "${name}"`);
+    }
 });
+const snapshotTxQueue = (queue) => Effect.tx(Effect.gen(function* () {
+    const items = yield* TxQueue.clear(queue);
+    yield* TxQueue.offerAll(queue, items);
+    return items;
+}));
+const takePrefixUntil = (queue, predicate) => Effect.tx(Effect.gen(function* () {
+    const items = yield* TxQueue.clear(queue);
+    const [prefix, rest] = ReadonlyArray.splitWhere(items, predicate);
+    yield* TxQueue.offerAll(queue, rest);
+    return prefix;
+}));
+/** Serialize value to JSON string for trace attributes */
+const jsonStringify = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
 //# sourceMappingURL=LeaderSyncProcessor.js.map