@livestore/common 0.4.0-dev.21 → 0.4.0-dev.23

package/dist/leader-thread/LeaderSyncProcessor.js

@@ -1,14 +1,18 @@
-import { casesHandled, isNotUndefined, LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils';
-import { BucketQueue, Cause, Deferred, Duration, Effect, Exit, FiberHandle, Layer, Option, OtelTracer, Queue, ReadonlyArray, Schedule, Stream, Subscribable, SubscriptionRef, } from '@livestore/utils/effect';
+import { casesHandled, isNotUndefined, LS_DEV, TRACE_VERBOSE } from '@livestore/utils';
+import { BucketQueue, Cause, Deferred, Duration, Effect, Exit, FiberHandle, Layer, Option, Queue, ReadonlyArray, Schedule, Schema, Stream, Subscribable, SubscriptionRef, } from '@livestore/utils/effect';
 import { UnknownError } from "../adapter-types.js";
+import { IntentionalShutdownCause } from "../errors.js";
 import { makeMaterializerHash } from "../materializer-helper.js";
 import { EventSequenceNumber, LiveStoreEvent, resolveEventDef, SystemTables } from "../schema/mod.js";
-import { LeaderAheadError, } from "../sync/sync.js";
+import { EVENTLOG_META_TABLE, SYNC_STATUS_TABLE } from "../schema/state/sqlite/system-tables/eventlog-tables.js";
+import { isRejectedPushError, LeaderAheadError, NonMonotonicBatchError, StaleRebaseGenerationError } from "./RejectedPushError.js";
 import * as SyncState from "../sync/syncstate.js";
 import { sql } from "../util.js";
 import * as Eventlog from "./eventlog.js";
 import { rollback } from "./materialize-event.js";
 import { LeaderThreadCtx } from "./types.js";
+/** Serialize value to JSON string for trace attributes */
+const jsonStringify = Schema.encodeSync(Schema.parseJson());
 /**
  * The LeaderSyncProcessor manages synchronization of events between
  * the local state and the sync backend, ensuring efficient and orderly processing.
@@ -27,11 +31,11 @@ import { LeaderThreadCtx } from "./types.js";
  *   - Maintains events in ascending order.
  *   - Uses `Deferred` objects to resolve/reject events based on application success.
  * - Processes events from the queue, applying events in batches.
- * - Controlled by a `Latch` to manage execution flow.
- * - The latch closes on pull receipt and re-opens post-pull completion.
+ * - Controlled by a mutex (`Semaphore(1)`) to ensure mutual exclusion between local push and backend pull processing.
+ * - The backend pull side acquires the mutex before processing and releases it on post-pull completion.
  * - Processes up to `maxBatchSize` events per cycle.
  *
- * Currently we're advancing the state db and eventlog in lockstep, but we could also decouple this in the future
+ * Currently, we're advancing the state db and eventlog in lockstep, but we could also decouple this in the future
  *
  * Tricky concurrency scenarios:
  * - Queued local push batches becoming invalid due to a prior local push item being rejected.
@@ -39,7 +43,7 @@ import { LeaderThreadCtx } from "./types.js";
  *
  * See ClientSessionSyncProcessor for how the leader and session sync processors are similar/different.
  */
-export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncContext, initialSyncState, onError, livePull, params, testing, }) => Effect.gen(function* () {
+export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncContext, initialSyncState, onError, onBackendIdMismatch, livePull, params, testing, }) => Effect.gen(function* () {
     const syncBackendPushQueue = yield* BucketQueue.make();
     const localPushBatchSize = params.localPushBatchSize ?? 10;
     const backendPushBatchSize = params.backendPushBatchSize ?? 50;
@@ -51,8 +55,8 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
         current: undefined,
     };
     const localPushesQueue = yield* BucketQueue.make();
-    const localPushesLatch = yield* Effect.makeLatch(true);
-    const pullLatch = yield* Effect.makeLatch(true);
+    // Ensures mutual exclusion between local push and backend pull processing.
+    const localPushBackendPullMutex = yield* Effect.makeSemaphore(1);
     /**
      * Additionally to the `syncStateSref` we also need the `pushHeadRef` in order to prevent old/duplicate
      * events from being pushed in a scenario like this:
@@ -74,7 +78,7 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
         yield* validatePushBatch(newEvents, pushHeadRef.current);
         advancePushHead(newEvents.at(-1).seqNum);
         const waitForProcessing = options?.waitForProcessing ?? false;
-        if (waitForProcessing) {
+        if (waitForProcessing === true) {
             const deferreds = yield* Effect.forEach(newEvents, () => Deferred.make());
             const items = newEvents.map((eventEncoded, i) => [eventEncoded, deferreds[i]]);
             yield* BucketQueue.offerAll(localPushesQueue, items);
@@ -87,14 +91,14 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
     }).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:push', {
         attributes: {
             batchSize: newEvents.length,
-            batch: TRACE_VERBOSE ? newEvents : undefined,
+            batch: TRACE_VERBOSE === true ? newEvents : undefined,
         },
-        links: ctxRef.current?.span ? [{ _tag: 'SpanLink', span: ctxRef.current.span, attributes: {} }] : undefined,
+        links: ctxRef.current?.span !== undefined
+            ? [{ _tag: 'SpanLink', span: ctxRef.current.span, attributes: {} }]
+            : undefined,
     }));
     const pushPartial = ({ event: { name, args }, clientId, sessionId }) => Effect.gen(function* () {
-        const syncState = yield* syncStateSref;
-        if (syncState === undefined)
-            return shouldNeverHappen('Not initialized');
+        const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger);
         const resolution = yield* resolveEventDef(schema, {
             operation: '@livestore/common:LeaderSyncProcessor:pushPartial',
             event: {
@@ -104,7 +108,7 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
                 sessionId,
                 seqNum: syncState.localHead,
             },
-        }).pipe(UnknownError.mapToUnknownError);
+        });
         if (resolution._tag === 'unknown') {
             // Ignore partial pushes for unrecognised events – they are still
             // persisted server-side once a schema update ships.
@@ -121,17 +125,17 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
             }),
         });
         yield* push([eventEncoded]);
-    }).pipe(Effect.catchTag('LeaderAheadError', Effect.orDie));
+    }).pipe(
+    // pushPartial constructs the event sequence number internally, so these errors should never happen.
+    Effect.catchIf(isRejectedPushError, Effect.die));
     // Starts various background loops
     const boot = Effect.gen(function* () {
         const span = yield* Effect.currentSpan.pipe(Effect.orDie);
-        const otelSpan = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)));
         const { devtools, shutdownChannel } = yield* LeaderThreadCtx;
         const runtime = yield* Effect.runtime();
         ctxRef.current = {
-            otelSpan,
             span,
-            devtoolsLatch: devtools.enabled ? devtools.syncBackendLatch : undefined,
+            devtoolsLatch: devtools.enabled === true ? devtools.syncBackendLatch : undefined,
             runtime,
         };
         /** State transitions need to happen atomically, so we use a Ref to track the state */
@@ -148,26 +152,25 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
                 yield* BucketQueue.offerAll(syncBackendPushQueue, globalPendingEvents);
             }
         }
+        const handleBackendIdMismatchError = (error) => handleBackendIdMismatch({ error, onBackendIdMismatch, shutdownChannel });
         const maybeShutdownOnError = (cause) => Effect.gen(function* () {
             if (onError === 'ignore') {
-                if (LS_DEV) {
+                if (LS_DEV === true) {
                     yield* Effect.logDebug(`Ignoring sync error (${cause._tag === 'Fail' ? cause.error._tag : cause._tag})`, Cause.pretty(cause));
                 }
                 return;
             }
-            const errorToSend = Cause.isFailType(cause) ? cause.error : UnknownError.make({ cause });
+            const errorToSend = Cause.isFailType(cause) === true ? cause.error : UnknownError.make({ cause });
             yield* shutdownChannel.send(errorToSend).pipe(Effect.orDie);
-            return yield* Effect.die(cause);
+            return yield* Effect.failCause(cause).pipe(Effect.orDie);
         });
         yield* backgroundApplyLocalPushes({
-            localPushesLatch,
+            localPushBackendPullMutex,
             localPushesQueue,
-            pullLatch,
             syncStateSref,
             syncBackendPushQueue,
             schema,
             isClientEvent,
-            otelSpan,
             connectedClientSessionPullQueues,
             localPushBatchSize,
             testing: {
@@ -177,10 +180,9 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
         const backendPushingFiberHandle = yield* FiberHandle.make();
         const backendPushingEffect = backgroundBackendPushing({
             syncBackendPushQueue,
-            otelSpan,
             devtoolsLatch: ctxRef.current?.devtoolsLatch,
             backendPushBatchSize,
-        }).pipe(Effect.catchAllCause(maybeShutdownOnError));
+        }).pipe(Effect.catchTag('BackendIdMismatchError', handleBackendIdMismatchError), Effect.catchAllCause(maybeShutdownOnError));
         yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect);
         yield* backgroundBackendPulling({
             isClientEvent,
@@ -194,19 +196,19 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
                 yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect);
             }),
             syncStateSref,
-            localPushesLatch,
-            pullLatch,
+            localPushBackendPullMutex,
             livePull,
             dbState,
-            otelSpan,
             initialBlockingSyncContext,
             devtoolsLatch: ctxRef.current?.devtoolsLatch,
             connectedClientSessionPullQueues,
             advancePushHead,
         }).pipe(Effect.retry({
-            // We want to retry pulling if we've lost connection to the sync backend
-            while: (cause) => cause._tag === 'IsOfflineError',
-        }), Effect.catchAllCause(maybeShutdownOnError), 
+            // Retry pulling when we've lost connection to the sync backend
+            // We're using `until` with a refinement instead of `while` to narrow `IsOfflineError` out of the error type.
+            // See https://github.com/Effect-TS/effect/issues/6122
+            until: (error) => error._tag !== 'IsOfflineError',
+        }), Effect.catchTag('BackendIdMismatchError', handleBackendIdMismatchError), Effect.catchAllCause(maybeShutdownOnError), 
         // Needed to avoid `Fiber terminated with an unhandled error` logs which seem to happen because of the `Effect.retry` above.
         // This might be a bug in Effect. Only seems to happen in the browser.
         Effect.provide(Layer.setUnhandledErrorLogLevel(Option.none())), Effect.forkScoped);
@@ -230,17 +232,9 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
       - full new state db snapshot in the "rebase" case
         - downside: importing the snapshot is expensive
     */
-    const pullQueue = ({ cursor }) => {
-        const runtime = ctxRef.current?.runtime ?? shouldNeverHappen('Not initialized');
-        return connectedClientSessionPullQueues.makeQueue(cursor).pipe(Effect.provide(runtime));
-    };
+    const pullQueue = ({ cursor }) => Effect.fromNullable(ctxRef.current?.runtime).pipe(Effect.orDieDebugger, Effect.flatMap((runtime) => connectedClientSessionPullQueues.makeQueue(cursor).pipe(Effect.provide(runtime))));
     const syncState = Subscribable.make({
-        get: Effect.gen(function* () {
-            const syncState = yield* syncStateSref;
-            if (syncState === undefined)
-                return shouldNeverHappen('Not initialized');
-            return syncState;
-        }),
+        get: syncStateSref.pipe(Effect.flatMap(Effect.fromNullable), Effect.orDieDebugger),
         changes: syncStateSref.changes.pipe(Stream.filter(isNotUndefined)),
     });
     return {
@@ -252,113 +246,103 @@ export const makeLeaderSyncProcessor = ({ schema, dbState, initialBlockingSyncCo
         syncState,
     };
 });
-const backgroundApplyLocalPushes = ({ localPushesLatch, localPushesQueue, pullLatch, syncStateSref, syncBackendPushQueue, schema, isClientEvent, otelSpan, connectedClientSessionPullQueues, localPushBatchSize, testing, }) => Effect.gen(function* () {
+const backgroundApplyLocalPushes = ({ localPushBackendPullMutex, localPushesQueue, syncStateSref, syncBackendPushQueue, schema, isClientEvent, connectedClientSessionPullQueues, localPushBatchSize, testing, }) => Effect.gen(function* () {
     while (true) {
         if (testing.delay !== undefined) {
             yield* testing.delay.pipe(Effect.withSpan('localPushProcessingDelay'));
         }
         const batchItems = yield* BucketQueue.takeBetween(localPushesQueue, 1, localPushBatchSize);
-        // Wait for the backend pulling to finish
-        yield* localPushesLatch.await;
-        // Prevent backend pull processing until this local push is finished
-        yield* pullLatch.close;
-        const syncState = yield* syncStateSref;
-        if (syncState === undefined)
-            return shouldNeverHappen('Not initialized');
-        const currentRebaseGeneration = syncState.localHead.rebaseGeneration;
-        // Since the rebase generation might have changed since enqueuing, we need to filter out items with older generation
-        // It's important that we filter after we got localPushesLatch, otherwise we might filter with the old generation
-        const [droppedItems, filteredItems] = ReadonlyArray.partition(batchItems, ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= currentRebaseGeneration);
-        if (droppedItems.length > 0) {
-            otelSpan?.addEvent(`push:drop-old-generation`, {
-                droppedCount: droppedItems.length,
-                currentRebaseGeneration,
-            });
-            /**
-             * Dropped pushes may still have a deferred awaiting completion.
-             * Fail it so the caller learns the leader advanced and resubmits with the updated generation.
-             */
-            yield* Effect.forEach(droppedItems.filter((item) => item[1] !== undefined), ([eventEncoded, deferred]) => Deferred.fail(deferred, LeaderAheadError.make({
-                minimumExpectedNum: syncState.localHead,
-                providedNum: eventEncoded.seqNum,
-            })));
-        }
-        if (filteredItems.length === 0) {
-            yield* pullLatch.open;
-            continue;
-        }
-        const [newEvents, deferreds] = ReadonlyArray.unzip(filteredItems);
-        const mergeResult = SyncState.merge({
-            syncState,
-            payload: { _tag: 'local-push', newEvents },
-            isClientEvent,
-            isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
-        });
-        switch (mergeResult._tag) {
-            case 'unknown-error': {
-                otelSpan?.addEvent(`push:unknown-error`, {
-                    batchSize: newEvents.length,
-                    newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
+        // Applies a batch of local pushes, guarded by the localPushBackendPullMutex to ensure mutual exclusion with backend pulling
+        yield* Effect.gen(function* () {
+            const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger);
+            const currentRebaseGeneration = syncState.localHead.rebaseGeneration;
+            // Since the rebase generation might have changed since enqueuing, we need to filter out items with older generation
+            // It's important that we filter after acquiring the localPushBackendPullMutex, otherwise we might filter with the old generation
+            const [droppedItems, filteredItems] = ReadonlyArray.partition(batchItems, ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= currentRebaseGeneration);
+            if (droppedItems.length > 0) {
+                yield* Effect.spanEvent(`push:drop-old-generation`, {
+                    droppedCount: droppedItems.length,
+                    currentRebaseGeneration,
                 });
-                return yield* new UnknownError({ cause: mergeResult.message });
+                /**
+                 * Dropped pushes may still have a deferred awaiting completion.
+                 * Fail it so the caller learns the leader advanced and resubmits with the updated generation.
+                 */
+                yield* Effect.forEach(droppedItems.filter((item) => item[1] !== undefined), ([eventEncoded, deferred]) => Deferred.fail(deferred, StaleRebaseGenerationError.make({
+                    currentRebaseGeneration,
+                    providedRebaseGeneration: eventEncoded.seqNum.rebaseGeneration,
+                    sessionId: eventEncoded.sessionId,
+                })));
             }
-            case 'rebase': {
-                return shouldNeverHappen('The leader thread should never have to rebase due to a local push');
+            if (filteredItems.length === 0) {
+                return;
             }
-            case 'reject': {
-                otelSpan?.addEvent(`push:reject`, {
-                    batchSize: newEvents.length,
-                    mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
-                });
-                // TODO: how to test this?
-                const nextRebaseGeneration = currentRebaseGeneration + 1;
-                const providedNum = newEvents.at(0).seqNum;
-                // All subsequent pushes with same generation should be rejected as well
-                // We're also handling the case where the localPushQueue already contains events
-                // from the next generation which we preserve in the queue
-                const remainingEventsMatchingGeneration = yield* BucketQueue.takeSplitWhere(localPushesQueue, ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= nextRebaseGeneration);
-                // TODO we still need to better understand and handle this scenario
-                if (LS_DEV && (yield* BucketQueue.size(localPushesQueue)) > 0) {
-                    console.log('localPushesQueue is not empty', yield* BucketQueue.size(localPushesQueue));
-                    // biome-ignore lint/suspicious/noDebugger: debugging
-                    debugger;
+            const [newEvents, deferreds] = ReadonlyArray.unzip(filteredItems);
+            yield* Effect.annotateCurrentSpan({
+                'batchSize': newEvents.length,
+                ...(TRACE_VERBOSE === true ? { 'newEvents': jsonStringify(newEvents) } : {}),
+            });
+            const mergeResult = yield* SyncState.merge({
+                syncState,
+                payload: { _tag: 'local-push', newEvents },
+                isClientEvent,
+                isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
+            });
+            switch (mergeResult._tag) {
+                case 'rebase': {
+                    return yield* Effect.dieDebugger('The leader thread should never have to rebase due to a local push');
+                }
+                case 'reject': {
+                    yield* Effect.spanEvent(`push:reject`, {
+                        batchSize: newEvents.length,
+                        ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
+                    });
+                    // TODO: how to test this?
+                    const nextRebaseGeneration = currentRebaseGeneration + 1;
+                    const providedNum = newEvents.at(0).seqNum;
+                    // All subsequent pushes with same generation should be rejected as well
+                    // We're also handling the case where the localPushQueue already contains events
+                    // from the next generation which we preserve in the queue
+                    const remainingEventsMatchingGeneration = yield* BucketQueue.takeSplitWhere(localPushesQueue, ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= nextRebaseGeneration);
+                    // TODO we still need to better understand and handle this scenario
+                    if (LS_DEV === true && (yield* BucketQueue.size(localPushesQueue)) > 0) {
+                        console.log('localPushesQueue is not empty', yield* BucketQueue.size(localPushesQueue));
+                        // oxlint-disable-next-line eslint(no-debugger) -- intentional breakpoint for unexpected queue state
+                        debugger;
+                    }
+                    const allDeferredsToReject = [
+                        ...deferreds,
+                        ...remainingEventsMatchingGeneration.map(([_, deferred]) => deferred),
+                    ].filter(isNotUndefined);
+                    yield* Effect.forEach(allDeferredsToReject, (deferred) => Deferred.fail(deferred, LeaderAheadError.make({ minimumExpectedNum: mergeResult.expectedMinimumId, providedNum, sessionId: newEvents.at(0).sessionId })));
+                    // In this case we're skipping state update and down/upstream processing
+                    // We've cleared the local push queue and are now waiting for new local pushes / backend pulls
+                    return;
+                }
+                case 'advance': {
+                    break;
+                }
+                default: {
+                    casesHandled(mergeResult);
                 }
-                const allDeferredsToReject = [
-                    ...deferreds,
-                    ...remainingEventsMatchingGeneration.map(([_, deferred]) => deferred),
-                ].filter(isNotUndefined);
-                yield* Effect.forEach(allDeferredsToReject, (deferred) => Deferred.fail(deferred, LeaderAheadError.make({ minimumExpectedNum: mergeResult.expectedMinimumId, providedNum })));
-                // Allow the backend pulling to start
-                yield* pullLatch.open;
-                // In this case we're skipping state update and down/upstream processing
-                // We've cleared the local push queue and are now waiting for new local pushes / backend pulls
-                continue;
-            }
-            case 'advance': {
-                break;
-            }
-            default: {
-                casesHandled(mergeResult);
             }
-        }
-        yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
-        yield* connectedClientSessionPullQueues.offer({
-            payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
-            leaderHead: mergeResult.newSyncState.localHead,
-        });
-        otelSpan?.addEvent(`push:advance`, {
-            batchSize: newEvents.length,
-            mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
-        });
-        // Don't sync client-local events
-        const filteredBatch = mergeResult.newEvents.filter((eventEncoded) => {
-            const eventDef = schema.eventsDefsMap.get(eventEncoded.name);
-            return eventDef === undefined ? true : eventDef.options.clientOnly === false;
-        });
-        yield* BucketQueue.offerAll(syncBackendPushQueue, filteredBatch);
-        yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds });
-        // Allow the backend pulling to start
-        yield* pullLatch.open;
+            yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
+            yield* connectedClientSessionPullQueues.offer({
+                payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: mergeResult.newEvents }),
+                leaderHead: mergeResult.newSyncState.localHead,
+            });
+            yield* Effect.spanEvent(`push:advance`, {
+                batchSize: newEvents.length,
+                ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
+            });
+            // Don't sync client-local events
+            const filteredBatch = mergeResult.newEvents.filter((eventEncoded) => {
+                const eventDef = schema.eventsDefsMap.get(eventEncoded.name);
+                return eventDef === undefined ? true : eventDef.options.clientOnly === false;
+            });
+            yield* BucketQueue.offerAll(syncBackendPushQueue, filteredBatch);
+            yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds });
+        }).pipe(localPushBackendPullMutex.withPermits(1));
     }
 });
 // TODO how to handle errors gracefully
@@ -368,7 +352,7 @@ const materializeEventsBatch = ({ batchItems, deferreds }) => Effect.gen(functio
     db.execute('BEGIN TRANSACTION', undefined); // Start the transaction
     dbEventlog.execute('BEGIN TRANSACTION', undefined); // Start the transaction
     yield* Effect.addFinalizer((exit) => Effect.gen(function* () {
-        if (Exit.isSuccess(exit))
+        if (Exit.isSuccess(exit) === true)
             return;
         // Rollback in case of an error
         db.execute('ROLLBACK', undefined);
@@ -387,112 +371,117 @@ const materializeEventsBatch = ({ batchItems, deferreds }) => Effect.gen(functio
 }).pipe(Effect.uninterruptible, Effect.scoped, Effect.withSpan('@livestore/common:LeaderSyncProcessor:materializeEventItems', {
     attributes: { batchSize: batchItems.length },
 }), Effect.tapCauseLogPretty);
-const backgroundBackendPulling = ({ isClientEvent, restartBackendPushing, otelSpan, dbState, syncStateSref, localPushesLatch, livePull, pullLatch, devtoolsLatch, initialBlockingSyncContext, connectedClientSessionPullQueues, advancePushHead, }) => Effect.gen(function* () {
+const backgroundBackendPulling = Effect.fn('@livestore/common:LeaderSyncProcessor:backend-pulling')(function* ({ isClientEvent, restartBackendPushing, dbState, syncStateSref, localPushBackendPullMutex, livePull, devtoolsLatch, initialBlockingSyncContext, connectedClientSessionPullQueues, advancePushHead, }) {
     const { syncBackend, dbState: db, dbEventlog, schema } = yield* LeaderThreadCtx;
     if (syncBackend === undefined)
         return;
-    const onNewPullChunk = (newEvents, pageInfo) => Effect.gen(function* () {
-        if (newEvents.length === 0)
+    let pullMutexHeld = false;
+    const releasePullMutexIfHeld = Effect.gen(function* () {
+        if (pullMutexHeld === false)
             return;
+        pullMutexHeld = false;
+        yield* localPushBackendPullMutex.release(1);
+    });
+    const isPullPaginationComplete = (pageInfo) => pageInfo._tag === 'NoMore';
+    const onNewPullChunk = (newEvents, pageInfo) => Effect.gen(function* () {
         if (devtoolsLatch !== undefined) {
             yield* devtoolsLatch.await;
         }
-        // Prevent more local pushes from being processed until this pull is finished
-        yield* localPushesLatch.close;
-        // Wait for pending local pushes to finish
-        yield* pullLatch.await;
-        const syncState = yield* syncStateSref;
-        if (syncState === undefined)
-            return shouldNeverHappen('Not initialized');
-        const mergeResult = SyncState.merge({
-            syncState,
-            payload: SyncState.PayloadUpstreamAdvance.make({ newEvents }),
-            isClientEvent,
-            isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
-            ignoreClientEvents: true,
-        });
-        if (mergeResult._tag === 'reject') {
-            return shouldNeverHappen('The leader thread should never reject upstream advances');
+        if (newEvents.length === 0) {
+            if (isPullPaginationComplete(pageInfo) === true) {
+                yield* releasePullMutexIfHeld;
+            }
+            return;
         }
-        else if (mergeResult._tag === 'unknown-error') {
-            otelSpan?.addEvent(`pull:unknown-error`, {
-                newEventsCount: newEvents.length,
-                newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
-            });
-            return yield* new UnknownError({ cause: mergeResult.message });
+        // Prevent more local pushes from being processed until this pull pagination sequence is finished.
+        if (pullMutexHeld === false) {
+            yield* localPushBackendPullMutex.take(1);
+            pullMutexHeld = true;
         }
-        const newBackendHead = newEvents.at(-1).seqNum;
-        Eventlog.updateBackendHead(dbEventlog, newBackendHead);
-        if (mergeResult._tag === 'rebase') {
-            otelSpan?.addEvent(`pull:rebase[${mergeResult.newSyncState.localHead.rebaseGeneration}]`, {
-                newEventsCount: newEvents.length,
-                newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
-                rollbackCount: mergeResult.rollbackEvents.length,
-                mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
+        const chunkExit = yield* Effect.gen(function* () {
+            const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger);
+            yield* Effect.annotateCurrentSpan({
+                'merge.newEventsCount': newEvents.length,
+                ...(TRACE_VERBOSE === true ? { 'merge.newEvents': jsonStringify(newEvents) } : {}),
             });
-            const globalRebasedPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
-                const eventDef = schema.eventsDefsMap.get(event.name);
-                return eventDef === undefined ? true : eventDef.options.clientOnly === false;
+            const mergeResult = yield* SyncState.merge({
+                syncState,
+                payload: SyncState.PayloadUpstreamAdvance.make({ newEvents }),
+                isClientEvent,
+                isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
+                ignoreClientEvents: true,
             });
-            yield* restartBackendPushing(globalRebasedPendingEvents);
-            if (mergeResult.rollbackEvents.length > 0) {
-                yield* rollback({
-                    dbState: db,
-                    dbEventlog,
-                    eventNumsToRollback: mergeResult.rollbackEvents.map((_) => _.seqNum),
+            if (mergeResult._tag === 'reject') {
+                return yield* Effect.dieDebugger('The leader thread should never reject upstream advances');
+            }
+            const newBackendHead = newEvents.at(-1).seqNum;
+            Eventlog.updateBackendHead(dbEventlog, newBackendHead);
+            if (mergeResult._tag === 'rebase') {
+                yield* Effect.spanEvent(`pull:rebase[${mergeResult.newSyncState.localHead.rebaseGeneration}]`, {
+                    newEventsCount: newEvents.length,
+                    ...(TRACE_VERBOSE === true ? { newEvents: jsonStringify(newEvents) } : {}),
+                    rollbackCount: mergeResult.rollbackEvents.length,
+                    ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
+                });
+                const globalRebasedPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
+                    const eventDef = schema.eventsDefsMap.get(event.name);
+                    return eventDef === undefined ? true : eventDef.options.clientOnly === false;
+                });
+                yield* restartBackendPushing(globalRebasedPendingEvents);
+                if (mergeResult.rollbackEvents.length > 0) {
+                    yield* rollback({
+                        dbState: db,
+                        dbEventlog,
+                        eventNumsToRollback: mergeResult.rollbackEvents.map((_) => _.seqNum),
+                    });
+                }
+                yield* connectedClientSessionPullQueues.offer({
+                    payload: SyncState.payloadFromMergeResult(mergeResult),
+                    leaderHead: mergeResult.newSyncState.localHead,
                 });
             }
-            yield* connectedClientSessionPullQueues.offer({
-                payload: SyncState.payloadFromMergeResult(mergeResult),
-                leaderHead: mergeResult.newSyncState.localHead,
-            });
-        }
-        else {
-            otelSpan?.addEvent(`pull:advance`, {
-                newEventsCount: newEvents.length,
-                mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
-            });
-            // Ensure push fiber is active after advance by restarting with current pending (non-client) events
-            const globalPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
-                const eventDef = schema.eventsDefsMap.get(event.name);
-                return eventDef === undefined ? true : eventDef.options.clientOnly === false;
-            });
-            yield* restartBackendPushing(globalPendingEvents);
-            yield* connectedClientSessionPullQueues.offer({
-                payload: SyncState.payloadFromMergeResult(mergeResult),
-                leaderHead: mergeResult.newSyncState.localHead,
-            });
-            if (mergeResult.confirmedEvents.length > 0) {
-                // `mergeResult.confirmedEvents` don't contain the correct sync metadata, so we need to use
-                // `newEvents` instead which we filter via `mergeResult.confirmedEvents`
-                const confirmedNewEvents = newEvents.filter((event) => mergeResult.confirmedEvents.some((confirmedEvent) => EventSequenceNumber.Client.isEqual(event.seqNum, confirmedEvent.seqNum)));
-                yield* Eventlog.updateSyncMetadata(confirmedNewEvents).pipe(UnknownError.mapToUnknownError);
+            else {
+                yield* Effect.spanEvent(`pull:advance`, {
+                    newEventsCount: newEvents.length,
+                    ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
+                });
+                // Ensure push fiber is active after advance by restarting with current pending (non-client) events
+                const globalPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
+                    const eventDef = schema.eventsDefsMap.get(event.name);
+                    return eventDef === undefined ? true : eventDef.options.clientOnly === false;
+                });
+                yield* restartBackendPushing(globalPendingEvents);
+                yield* connectedClientSessionPullQueues.offer({
+                    payload: SyncState.payloadFromMergeResult(mergeResult),
+                    leaderHead: mergeResult.newSyncState.localHead,
+                });
+                if (mergeResult.confirmedEvents.length > 0) {
+                    // `mergeResult.confirmedEvents` don't contain the correct sync metadata, so we need to use
+                    // `newEvents` instead which we filter via `mergeResult.confirmedEvents`
+                    const confirmedNewEvents = newEvents.filter((event) => mergeResult.confirmedEvents.some((confirmedEvent) => EventSequenceNumber.Client.isEqual(event.seqNum, confirmedEvent.seqNum)));
+                    yield* Eventlog.updateSyncMetadata(confirmedNewEvents).pipe(Effect.orDieDebugger);
+                }
             }
+            // Removes the changeset rows which are no longer needed as we'll never have to rollback beyond this point
+            trimChangesetRows(db, newBackendHead);
+            advancePushHead(mergeResult.newSyncState.localHead);
+            yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds: undefined });
+            yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
+        }).pipe(Effect.exit);
+        if (Exit.isFailure(chunkExit) === true) {
+            yield* releasePullMutexIfHeld;
+            return yield* Effect.failCause(chunkExit.cause);
         }
-        // Removes the changeset rows which are no longer needed as we'll never have to rollback beyond this point
-        trimChangesetRows(db, newBackendHead);
-        advancePushHead(mergeResult.newSyncState.localHead);
-        yield* materializeEventsBatch({ batchItems: mergeResult.newEvents, deferreds: undefined });
-        yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState);
-        // Allow local pushes to be processed again
-        if (pageInfo._tag === 'NoMore') {
-            yield* localPushesLatch.open;
+        if (isPullPaginationComplete(pageInfo) === true) {
+            yield* releasePullMutexIfHeld;
         }
     });
-    const syncState = yield* syncStateSref;
-    if (syncState === undefined)
-        return shouldNeverHappen('Not initialized');
+    const syncState = yield* Effect.fromNullable(yield* syncStateSref).pipe(Effect.orDieDebugger);
     const cursorInfo = yield* Eventlog.getSyncBackendCursorInfo({ remoteHead: syncState.upstreamHead.global });
     const hashMaterializerResult = makeMaterializerHash({ schema, dbState });
     yield* syncBackend.pull(cursorInfo, { live: livePull }).pipe(
     // TODO only take from queue while connected
     Stream.tap(({ batch, pageInfo }) => Effect.gen(function* () {
-        // yield* Effect.spanEvent('batch', {
-        //   attributes: {
-        //     batchSize: batch.length,
-        //     batch: TRACE_VERBOSE ? batch : undefined,
-        //   },
-        // })
         // NOTE we only want to take process events when the sync backend is connected
         // (e.g. needed for simulating being offline)
         // TODO remove when there's a better way to handle this in stream above
@@ -505,11 +494,11 @@ const backgroundBackendPulling = ({ isClientEvent, restartBackendPushing, otelSp
             materializerHashSession: Option.none(),
         })), pageInfo);
         yield* initialBlockingSyncContext.update({ processed: batch.length, pageInfo });
-    })), Stream.runDrain, Effect.interruptible);
+    })), Stream.runDrain, Effect.interruptible, Effect.ensuring(releasePullMutexIfHeld));
     // Should only ever happen when livePull is false
     yield* Effect.logDebug('backend-pulling finished', { livePull });
-}).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pulling'));
-const backgroundBackendPushing = ({ syncBackendPushQueue, otelSpan, devtoolsLatch, backendPushBatchSize, }) => Effect.gen(function* () {
+});
+const backgroundBackendPushing = Effect.fn('@livestore/common:LeaderSyncProcessor:backend-pushing')(function* ({ syncBackendPushQueue, devtoolsLatch, backendPushBatchSize, }) {
     const { syncBackend } = yield* LeaderThreadCtx;
     if (syncBackend === undefined)
         return;
@@ -520,45 +509,47 @@ const backgroundBackendPushing = ({ syncBackendPushQueue, otelSpan, devtoolsLatc
         if (devtoolsLatch !== undefined) {
             yield* devtoolsLatch.await;
         }
-        otelSpan?.addEvent('backend-push', {
+        yield* Effect.spanEvent('backend-push', {
             batchSize: queueItems.length,
-            batch: TRACE_VERBOSE ? JSON.stringify(queueItems) : undefined,
+            ...(TRACE_VERBOSE === true ? { batch: jsonStringify(queueItems) } : {}),
         });
         // Push with declarative retry/backoff using Effect schedules
         // - Exponential backoff starting at 1s and doubling (1s, 2s, 4s, 8s, 16s, 30s ...)
         // - Delay clamped at 30s (continues retrying at 30s)
         // - Resets automatically after successful push
         // TODO(metrics): expose counters/gauges for retry attempts and queue health via devtools/metrics
-        // Only retry for transient UnknownError cases
-        const isRetryable = (err) => err._tag === 'InvalidPushError' && err.cause._tag === 'LiveStore.UnknownError';
-        // Input: InvalidPushError | IsOfflineError, Output: Duration
-        const retrySchedule = Schedule.exponential(Duration.seconds(1)).pipe(Schedule.andThenEither(Schedule.spaced(Duration.seconds(30))), // clamp at 30 second intervals
-        Schedule.compose(Schedule.elapsed), Schedule.whileInput(isRetryable));
         yield* Effect.gen(function* () {
             const iteration = yield* Schedule.CurrentIterationMetadata;
             const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either);
             const retries = iteration.recurrence;
             if (retries > 0 && pushResult._tag === 'Right') {
-                otelSpan?.addEvent('backend-push-retry-success', { retries, batchSize: queueItems.length });
+                yield* Effect.spanEvent('backend-push-retry-success', { retries, batchSize: queueItems.length });
             }
             if (pushResult._tag === 'Left') {
-                otelSpan?.addEvent('backend-push-error', {
+                yield* Effect.spanEvent('backend-push-error', {
                     error: pushResult.left.toString(),
                     retries,
                     batchSize: queueItems.length,
                 });
                 const error = pushResult.left;
-                if (error._tag === 'IsOfflineError' ||
-                    (error._tag === 'InvalidPushError' && error.cause._tag === 'ServerAheadError')) {
+                if (error._tag === 'ServerAheadError') {
                     // It's a core part of the sync protocol that the sync backend will emit a new pull chunk alongside the ServerAheadError
                     yield* Effect.logDebug('handled backend-push-error (waiting for interupt caused by pull)', { error });
                     return yield* Effect.never;
                 }
                 return yield* error;
             }
-        }).pipe(Effect.retry(retrySchedule));
+        }).pipe(
+        // Retry transient errors
+        Effect.retry({
+            schedule: Schedule.exponential(Duration.seconds(1)).pipe(Schedule.modifyDelay((_, delay) => Duration.min(delay, Duration.seconds(30))) // Cap delay at 30s intervals.
+            ),
+            while: (error) => error._tag === 'IsOfflineError' || error._tag === 'UnknownError',
+        }), 
+        // This is needed to narrow the Error type. Our retry policy runs indefinitely, but Effect.retry does not narrow the Error type.
+        Effect.catchIf((error) => error._tag === 'IsOfflineError' || error._tag === 'UnknownError', Effect.die));
     }
-}).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pushing'));
+}, Effect.interruptible);
 const trimChangesetRows = (db, newHead) => {
     // Since we're using the session changeset rows to query for the current head,
     // we're keeping at least one row for the current head, and thus are using `<` instead of `<=`
@@ -627,7 +618,7 @@ const makePullQueueSet = Effect.gen(function* () {
     });
     const offer = (item) => Effect.gen(function* () {
         const seqNumStr = EventSequenceNumber.Client.toString(item.leaderHead);
-        if (cachedPayloads.has(seqNumStr)) {
+        if (cachedPayloads.has(seqNumStr) === true) {
             cachedPayloads.get(seqNumStr).push(item.payload);
         }
         else {
@@ -656,24 +647,63 @@ const validatePushBatch = (batch, pushHead) => Effect.gen(function* () {
     if (batch.length === 0) {
         return;
     }
-    // Example: session A already enqueued e1…e6 while session B (same client, different
-    // session) still believes the head is e1 and submits [e2, e7, e8]. The numbers look
-    // monotonic from B’s perspective, but we must reject and force B to rebase locally
-    // so the leader never regresses.
+    // Defensive check: callers should already provide a strictly increasing sequence
+    // of event numbers.
     for (let i = 1; i < batch.length; i++) {
-        if (EventSequenceNumber.Client.isGreaterThanOrEqual(batch[i - 1].seqNum, batch[i].seqNum)) {
-            return yield* LeaderAheadError.make({
-                minimumExpectedNum: batch[i - 1].seqNum,
-                providedNum: batch[i].seqNum,
+        if (EventSequenceNumber.Client.isGreaterThanOrEqual(batch[i - 1].seqNum, batch[i].seqNum) === true) {
+            return yield* NonMonotonicBatchError.make({
+                precedingSeqNum: batch[i - 1].seqNum,
+                violatingSeqNum: batch[i].seqNum,
+                violationIndex: i,
+                sessionId: batch[i].sessionId,
             });
         }
     }
-    // Make sure smallest sequence number is > pushHead
-    if (EventSequenceNumber.Client.isGreaterThanOrEqual(pushHead, batch[0].seqNum)) {
+    // Reject stale batches whose first event is at or behind the leader's push head.
+    if (EventSequenceNumber.Client.isGreaterThanOrEqual(pushHead, batch[0].seqNum) === true) {
         return yield* LeaderAheadError.make({
             minimumExpectedNum: pushHead,
             providedNum: batch[0].seqNum,
+            sessionId: batch[0].sessionId,
         });
     }
 });
+/**
+ * Handles a BackendIdMismatchError based on the configured behavior.
+ * This occurs when the sync backend has been reset and has a new identity.
+ */
+const handleBackendIdMismatch = Effect.fn('@livestore/common:LeaderSyncProcessor:handleBackendIdMismatch')(function* ({ error, onBackendIdMismatch, shutdownChannel, }) {
+    const { dbEventlog, dbState } = yield* LeaderThreadCtx;
+    if (onBackendIdMismatch === 'reset') {
+        yield* Effect.logWarning('Sync backend identity changed (backend was reset). Clearing local storage and shutting down.', error);
+        // Clear local databases so the client can start fresh on next boot
+        yield* clearLocalDatabases({ dbEventlog, dbState });
+        // Send shutdown signal with special reason
+        yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'backend-id-mismatch' })).pipe(Effect.orDie);
+        return yield* Effect.die(error);
+    }
+    if (onBackendIdMismatch === 'shutdown') {
+        yield* Effect.logWarning('Sync backend identity changed (backend was reset). Shutting down without clearing local storage.', error);
+        yield* shutdownChannel.send(error).pipe(Effect.orDie);
+        return yield* Effect.die(error);
+    }
+    // ignore mode
+    if (LS_DEV === true) {
+        yield* Effect.logDebug('Ignoring BackendIdMismatchError (sync backend was reset but client continues with stale data)', error);
+    }
+});
+/**
+ * Clears local databases (eventlog and state) so the client can start fresh on next boot.
+ * This is used when the sync backend identity has changed (i.e. backend was reset).
+ */
+const clearLocalDatabases = ({ dbEventlog, dbState }) => Effect.sync(() => {
+    // Clear eventlog tables
+    dbEventlog.execute(sql `DELETE FROM ${EVENTLOG_META_TABLE}`);
+    dbEventlog.execute(sql `DELETE FROM ${SYNC_STATUS_TABLE}`);
+    // Drop all state tables - they'll be recreated on next boot
+    const tables = dbState.select(sql `SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`);
+    for (const { name } of tables) {
+        dbState.execute(`DROP TABLE IF EXISTS "${name}"`);
+    }
+});
 //# sourceMappingURL=LeaderSyncProcessor.js.map