@livestore/common 0.4.0-dev.1 → 0.4.0-dev.11

package/src/devtools/devtools-messages-common.ts

@@ -2,14 +2,7 @@ import { Schema } from '@livestore/utils/effect'
 
 import { liveStoreVersion as pkgVersion } from '../version.ts'
 
-export const NetworkStatus = Schema.Struct({
-  isConnected: Schema.Boolean,
-  timestampMs: Schema.Number,
-  /** Whether the network status devtools latch is closed. Used to simulate network disconnection. */
-  latchClosed: Schema.Boolean,
-})
-
-export type NetworkStatus = typeof NetworkStatus.Type
+export { NetworkStatus } from '../sync/sync-backend.ts'
 
 export const requestId = Schema.String
 export const clientId = Schema.String

package/src/errors.ts

@@ -1,4 +1,6 @@
-import { Effect, Schema, Stream } from '@livestore/utils/effect'
+import { Cause, Effect, Layer, Schema, Stream } from '@livestore/utils/effect'
+
+import * as LiveStoreEvent from './schema/LiveStoreEvent.ts'
 
 export class UnexpectedError extends Schema.TaggedError<UnexpectedError>()('LiveStore.UnexpectedError', {
   cause: Schema.Defect,
@@ -11,15 +13,30 @@ export class UnexpectedError extends Schema.TaggedError<UnexpectedError>()('Live
       Effect.catchAllDefect((cause) => new UnexpectedError({ cause })),
     )
 
+  static mapToUnexpectedErrorLayer = <A, E, R>(layer: Layer.Layer<A, E, R>) =>
+    layer.pipe(
+      Layer.catchAllCause((cause) =>
+        Cause.isFailType(cause) && Schema.is(UnexpectedError)(cause.error)
+          ? Layer.fail(cause.error)
+          : Layer.fail(new UnexpectedError({ cause: cause })),
+      ),
+    )
+
   static mapToUnexpectedErrorStream = <A, E, R>(stream: Stream.Stream<A, E, R>) =>
     stream.pipe(
       Stream.mapError((cause) => (Schema.is(UnexpectedError)(cause) ? cause : new UnexpectedError({ cause }))),
     )
 }
 
-export class SyncError extends Schema.TaggedError<SyncError>()('LiveStore.SyncError', {
-  cause: Schema.Defect,
-}) {}
+export class MaterializerHashMismatchError extends Schema.TaggedError<MaterializerHashMismatchError>()(
+  'LiveStore.MaterializerHashMismatchError',
+  {
+    eventName: Schema.String,
+    note: Schema.optionalWith(Schema.String, {
+      default: () => 'Please make sure your event materializer is a pure function without side effects.',
+    }),
+  },
+) {}
 
 export class IntentionalShutdownCause extends Schema.TaggedError<IntentionalShutdownCause>()(
   'LiveStore.IntentionalShutdownCause',
@@ -47,3 +64,15 @@ export class SqliteError extends Schema.TaggedError<SqliteError>()('LiveStore.Sq
   cause: Schema.Defect,
   note: Schema.optional(Schema.String),
 }) {}
+
+export class UnknownEventError extends Schema.TaggedError<UnknownEventError>()('LiveStore.UnknownEventError', {
+  event: LiveStoreEvent.AnyEncoded.pipe(Schema.pick('name', 'args', 'seqNum', 'clientId', 'sessionId')),
+  reason: Schema.Literal('event-definition-missing', 'materializer-missing'),
+  operation: Schema.String,
+  note: Schema.optional(Schema.String),
+}) {}
+
+export class MaterializeError extends Schema.TaggedError<MaterializeError>()('LiveStore.MaterializeError', {
+  cause: Schema.Union(MaterializerHashMismatchError, SqliteError, UnknownEventError),
+  note: Schema.optional(Schema.String),
+}) {}

package/src/leader-thread/LeaderSyncProcessor.ts

@@ -2,27 +2,39 @@ import { casesHandled, isNotUndefined, LS_DEV, shouldNeverHappen, TRACE_VERBOSE
 import type { HttpClient, Runtime, Scope, Tracer } from '@livestore/utils/effect'
 import {
   BucketQueue,
+  Cause,
   Deferred,
+  Duration,
   Effect,
   Exit,
   FiberHandle,
+  Layer,
   Option,
   OtelTracer,
-  pipe,
   Queue,
   ReadonlyArray,
+  Schedule,
   Stream,
   Subscribable,
   SubscriptionRef,
 } from '@livestore/utils/effect'
 import type * as otel from '@opentelemetry/api'
-
-import type { SqliteDb } from '../adapter-types.ts'
-import { SyncError, UnexpectedError } from '../adapter-types.ts'
+import {
+  type IntentionalShutdownCause,
+  type MaterializeError,
+  type SqliteDb,
+  UnexpectedError,
+} from '../adapter-types.ts'
 import { makeMaterializerHash } from '../materializer-helper.ts'
 import type { LiveStoreSchema } from '../schema/mod.ts'
-import { EventSequenceNumber, getEventDef, LiveStoreEvent, SystemTables } from '../schema/mod.ts'
-import { LeaderAheadError } from '../sync/sync.ts'
+import { EventSequenceNumber, LiveStoreEvent, resolveEventDef, SystemTables } from '../schema/mod.ts'
+import {
+  type InvalidPullError,
+  type InvalidPushError,
+  type IsOfflineError,
+  LeaderAheadError,
+  type SyncBackend,
+} from '../sync/sync.ts'
 import * as SyncState from '../sync/syncstate.ts'
 import { sql } from '../util.ts'
 import * as Eventlog from './eventlog.ts'
@@ -71,6 +83,7 @@ export const makeLeaderSyncProcessor = ({
   initialBlockingSyncContext,
   initialSyncState,
   onError,
+  livePull,
   params,
   testing,
 }: {
@@ -90,6 +103,11 @@ export const makeLeaderSyncProcessor = ({
      */
     backendPushBatchSize?: number
   }
+  /**
+   * Whether the sync backend should reactively pull new events from the sync backend
+   * When `false`, the sync processor will only do an initial pull
+   */
+  livePull: boolean
   testing: {
     delays?: {
       localPushProcessing?: Effect.Effect<void>
@@ -103,10 +121,8 @@ export const makeLeaderSyncProcessor = ({
 
     const syncStateSref = yield* SubscriptionRef.make<SyncState.SyncState | undefined>(undefined)
 
-    const isClientEvent = (eventEncoded: LiveStoreEvent.EncodedWithMeta) => {
-      const { eventDef } = getEventDef(schema, eventEncoded.name)
-      return eventDef.options.clientOnly
-    }
+    const isClientEvent = (eventEncoded: LiveStoreEvent.EncodedWithMeta) =>
+      schema.eventsDefsMap.get(eventEncoded.name)?.options.clientOnly ?? false
 
     const connectedClientSessionPullQueues = yield* makePullQueueSet
 
@@ -180,14 +196,32 @@ export const makeLeaderSyncProcessor = ({
         const syncState = yield* syncStateSref
         if (syncState === undefined) return shouldNeverHappen('Not initialized')
 
-        const { eventDef } = getEventDef(schema, name)
+        const resolution = yield* resolveEventDef(schema, {
+          operation: '@livestore/common:LeaderSyncProcessor:pushPartial',
+          event: {
+            name,
+            args,
+            clientId,
+            sessionId,
+            seqNum: syncState.localHead,
+          },
+        }).pipe(UnexpectedError.mapToUnexpectedError)
+
+        if (resolution._tag === 'unknown') {
+          // Ignore partial pushes for unrecognised events – they are still
+          // persisted server-side once a schema update ships.
+          return
+        }
 
         const eventEncoded = new LiveStoreEvent.EncodedWithMeta({
           name,
           args,
           clientId,
           sessionId,
-          ...EventSequenceNumber.nextPair({ seqNum: syncState.localHead, isClient: eventDef.options.clientOnly }),
+          ...EventSequenceNumber.nextPair({
+            seqNum: syncState.localHead,
+            isClient: resolution.eventDef.options.clientOnly,
+          }),
         })
 
         yield* push([eventEncoded])
@@ -215,8 +249,8 @@ export const makeLeaderSyncProcessor = ({
         const globalPendingEvents = initialSyncState.pending
           // Don't sync clientOnly events
           .filter((eventEncoded) => {
-            const { eventDef } = getEventDef(schema, eventEncoded.name)
-            return eventDef.options.clientOnly === false
+            const eventDef = schema.eventsDefsMap.get(eventEncoded.name)
+            return eventDef === undefined ? true : eventDef.options.clientOnly === false
           })
 
         if (globalPendingEvents.length > 0) {
@@ -224,12 +258,31 @@ export const makeLeaderSyncProcessor = ({
         }
       }
 
-      const shutdownOnError = (cause: unknown) =>
+      const maybeShutdownOnError = (
+        cause: Cause.Cause<
+          | UnexpectedError
+          | IntentionalShutdownCause
+          | IsOfflineError
+          | InvalidPushError
+          | InvalidPullError
+          | MaterializeError
+        >,
+      ) =>
         Effect.gen(function* () {
-          if (onError === 'shutdown') {
-            yield* shutdownChannel.send(UnexpectedError.make({ cause }))
-            yield* Effect.die(cause)
+          if (onError === 'ignore') {
+            if (LS_DEV) {
+              yield* Effect.logDebug(
+                `Ignoring sync error (${cause._tag === 'Fail' ? cause.error._tag : cause._tag})`,
+                Cause.pretty(cause),
+              )
+            }
+            return
           }
+
+          const errorToSend = Cause.isFailType(cause) ? cause.error : UnexpectedError.make({ cause })
+          yield* shutdownChannel.send(errorToSend).pipe(Effect.orDie)
+
+          return yield* Effect.die(cause)
         })
 
       yield* backgroundApplyLocalPushes({
@@ -246,20 +299,19 @@ export const makeLeaderSyncProcessor = ({
         testing: {
           delay: testing?.delays?.localPushProcessing,
         },
-      }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped)
+      }).pipe(Effect.catchAllCause(maybeShutdownOnError), Effect.forkScoped)
 
-      const backendPushingFiberHandle = yield* FiberHandle.make()
+      const backendPushingFiberHandle = yield* FiberHandle.make<void, never>()
       const backendPushingEffect = backgroundBackendPushing({
         syncBackendPushQueue,
         otelSpan,
         devtoolsLatch: ctxRef.current?.devtoolsLatch,
         backendPushBatchSize,
-      }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError))
+      }).pipe(Effect.catchAllCause(maybeShutdownOnError))
 
       yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect)
 
       yield* backgroundBackendPulling({
-        initialBackendHead: initialSyncState.upstreamHead.global,
         isClientEvent,
         restartBackendPushing: (filteredRebasedPending) =>
           Effect.gen(function* () {
@@ -276,13 +328,24 @@ export const makeLeaderSyncProcessor = ({
         syncStateSref,
         localPushesLatch,
         pullLatch,
+        livePull,
         dbState,
         otelSpan,
         initialBlockingSyncContext,
         devtoolsLatch: ctxRef.current?.devtoolsLatch,
         connectedClientSessionPullQueues,
         advancePushHead,
-      }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped)
+      }).pipe(
+        Effect.retry({
+          // We want to retry pulling if we've lost connection to the sync backend
+          while: (cause) => cause._tag === 'IsOfflineError',
+        }),
+        Effect.catchAllCause(maybeShutdownOnError),
+        // Needed to avoid `Fiber terminated with an unhandled error` logs which seem to happen because of the `Effect.retry` above.
+        // This might be a bug in Effect. Only seems to happen in the browser.
+        Effect.provide(Layer.setUnhandledErrorLogLevel(Option.none())),
+        Effect.forkScoped,
+      )
 
       return { initialLeaderHead: initialSyncState.localHead }
     }).pipe(Effect.withSpanScoped('@livestore/common:LeaderSyncProcessor:boot'))
@@ -379,19 +442,44 @@ const backgroundApplyLocalPushes = ({
 
       // Since the rebase generation might have changed since enqueuing, we need to filter out items with older generation
       // It's important that we filter after we got localPushesLatch, otherwise we might filter with the old generation
-      const [newEvents, deferreds] = pipe(
+      const [droppedItems, filteredItems] = ReadonlyArray.partition(
         batchItems,
-        ReadonlyArray.filter(([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration === currentRebaseGeneration),
-        ReadonlyArray.unzip,
+        ([eventEncoded]) => eventEncoded.seqNum.rebaseGeneration >= currentRebaseGeneration,
       )
 
-      if (newEvents.length === 0) {
-        // console.log('dropping old-gen batch', currentLocalPushGenerationRef.current)
-        // Allow the backend pulling to start
+      if (droppedItems.length > 0) {
+        otelSpan?.addEvent(`push:drop-old-generation`, {
+          droppedCount: droppedItems.length,
+          currentRebaseGeneration,
+        })
+
+        /**
+         * Dropped pushes may still have a deferred awaiting completion.
+         * Fail it so the caller learns the leader advanced and resubmits with the updated generation.
+         */
+        yield* Effect.forEach(
+          droppedItems.filter(
+            (item): item is [LiveStoreEvent.EncodedWithMeta, Deferred.Deferred<void, LeaderAheadError>] =>
+              item[1] !== undefined,
+          ),
+          ([eventEncoded, deferred]) =>
+            Deferred.fail(
+              deferred,
+              LeaderAheadError.make({
+                minimumExpectedNum: syncState.localHead,
+                providedNum: eventEncoded.seqNum,
+              }),
+            ),
+        )
+      }
+
+      if (filteredItems.length === 0) {
         yield* pullLatch.open
         continue
       }
 
+      const [newEvents, deferreds] = ReadonlyArray.unzip(filteredItems)
+
       const mergeResult = SyncState.merge({
         syncState,
         payload: { _tag: 'local-push', newEvents },
@@ -405,7 +493,7 @@ const backgroundApplyLocalPushes = ({
             batchSize: newEvents.length,
             newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
           })
-          return yield* new SyncError({ cause: mergeResult.message })
+          return yield* new UnexpectedError({ cause: mergeResult.message })
         }
         case 'rebase': {
           return shouldNeverHappen('The leader thread should never have to rebase due to a local push')
@@ -476,8 +564,8 @@ const backgroundApplyLocalPushes = ({
 
       // Don't sync clientOnly events
       const filteredBatch = mergeResult.newEvents.filter((eventEncoded) => {
-        const { eventDef } = getEventDef(schema, eventEncoded.name)
-        return eventDef.options.clientOnly === false
+        const eventDef = schema.eventsDefsMap.get(eventEncoded.name)
+        return eventDef === undefined ? true : eventDef.options.clientOnly === false
       })
 
       yield* BucketQueue.offerAll(syncBackendPushQueue, filteredBatch)
@@ -496,7 +584,7 @@ type MaterializeEventsBatch = (_: {
    * Indexes are aligned with `batchItems`
    */
   deferreds: ReadonlyArray<Deferred.Deferred<void, LeaderAheadError> | undefined> | undefined
-}) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
+}) => Effect.Effect<void, MaterializeError, LeaderThreadCtx>
 
 // TODO how to handle errors gracefully
 const materializeEventsBatch: MaterializeEventsBatch = ({ batchItems, deferreds }) =>
@@ -536,24 +624,22 @@ const materializeEventsBatch: MaterializeEventsBatch = ({ batchItems, deferreds
       attributes: { batchSize: batchItems.length },
     }),
     Effect.tapCauseLogPretty,
-    UnexpectedError.mapToUnexpectedError,
   )
 
 const backgroundBackendPulling = ({
-  initialBackendHead,
   isClientEvent,
   restartBackendPushing,
   otelSpan,
   dbState,
   syncStateSref,
   localPushesLatch,
+  livePull,
   pullLatch,
   devtoolsLatch,
   initialBlockingSyncContext,
   connectedClientSessionPullQueues,
   advancePushHead,
 }: {
-  initialBackendHead: EventSequenceNumber.GlobalEventSequenceNumber
   isClientEvent: (eventEncoded: LiveStoreEvent.EncodedWithMeta) => boolean
   restartBackendPushing: (
     filteredRebasedPending: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>,
@@ -563,6 +649,7 @@ const backgroundBackendPulling = ({
   dbState: SqliteDb
   localPushesLatch: Effect.Latch
   pullLatch: Effect.Latch
+  livePull: boolean
   devtoolsLatch: Effect.Latch | undefined
   initialBlockingSyncContext: InitialBlockingSyncContext
   connectedClientSessionPullQueues: PullQueueSet
@@ -573,7 +660,7 @@ const backgroundBackendPulling = ({
 
     if (syncBackend === undefined) return
 
-    const onNewPullChunk = (newEvents: LiveStoreEvent.EncodedWithMeta[], remaining: number) =>
+    const onNewPullChunk = (newEvents: LiveStoreEvent.EncodedWithMeta[], pageInfo: SyncBackend.PullResPageInfo) =>
       Effect.gen(function* () {
         if (newEvents.length === 0) return
 
@@ -605,7 +692,7 @@ const backgroundBackendPulling = ({
             newEventsCount: newEvents.length,
             newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
           })
-          return yield* new SyncError({ cause: mergeResult.message })
+          return yield* new UnexpectedError({ cause: mergeResult.message })
         }
 
         const newBackendHead = newEvents.at(-1)!.seqNum
@@ -621,8 +708,8 @@ const backgroundBackendPulling = ({
           })
 
           const globalRebasedPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
-            const { eventDef } = getEventDef(schema, event.name)
-            return eventDef.options.clientOnly === false
+            const eventDef = schema.eventsDefsMap.get(event.name)
+            return eventDef === undefined ? true : eventDef.options.clientOnly === false
           })
           yield* restartBackendPushing(globalRebasedPendingEvents)
 
@@ -644,6 +731,13 @@ const backgroundBackendPulling = ({
             mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
           })
 
+          // Ensure push fiber is active after advance by restarting with current pending (non-client) events
+          const globalPendingEvents = mergeResult.newSyncState.pending.filter((event) => {
+            const eventDef = schema.eventsDefsMap.get(event.name)
+            return eventDef === undefined ? true : eventDef.options.clientOnly === false
+          })
+          yield* restartBackendPushing(globalPendingEvents)
+
           yield* connectedClientSessionPullQueues.offer({
             payload: SyncState.payloadFromMergeResult(mergeResult),
             leaderHead: mergeResult.newSyncState.localHead,
@@ -657,7 +751,7 @@ const backgroundBackendPulling = ({
                 EventSequenceNumber.isEqual(event.seqNum, confirmedEvent.seqNum),
               ),
             )
-            yield* Eventlog.updateSyncMetadata(confirmedNewEvents)
+            yield* Eventlog.updateSyncMetadata(confirmedNewEvents).pipe(UnexpectedError.mapToUnexpectedError)
           }
         }
 
@@ -671,18 +765,20 @@ const backgroundBackendPulling = ({
         yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState)
 
         // Allow local pushes to be processed again
-        if (remaining === 0) {
+        if (pageInfo._tag === 'NoMore') {
           yield* localPushesLatch.open
         }
       })
 
-    const cursorInfo = yield* Eventlog.getSyncBackendCursorInfo({ remoteHead: initialBackendHead })
+    const syncState = yield* syncStateSref
+    if (syncState === undefined) return shouldNeverHappen('Not initialized')
+    const cursorInfo = yield* Eventlog.getSyncBackendCursorInfo({ remoteHead: syncState.upstreamHead.global })
 
     const hashMaterializerResult = makeMaterializerHash({ schema, dbState })
 
-    yield* syncBackend.pull(cursorInfo).pipe(
+    yield* syncBackend.pull(cursorInfo, { live: livePull }).pipe(
       // TODO only take from queue while connected
-      Stream.tap(({ batch, remaining }) =>
+      Stream.tap(({ batch, pageInfo }) =>
         Effect.gen(function* () {
           // yield* Effect.spanEvent('batch', {
           //   attributes: {
@@ -690,12 +786,10 @@ const backgroundBackendPulling = ({
           //     batch: TRACE_VERBOSE ? batch : undefined,
           //   },
           // })
-
           // NOTE we only want to take process events when the sync backend is connected
           // (e.g. needed for simulating being offline)
           // TODO remove when there's a better way to handle this in stream above
           yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
-
           yield* onNewPullChunk(
             batch.map((_) =>
               LiveStoreEvent.EncodedWithMeta.fromGlobal(_.eventEncoded, {
@@ -706,15 +800,17 @@ const backgroundBackendPulling = ({
                 materializerHashSession: Option.none(),
               }),
             ),
-            remaining,
+            pageInfo,
           )
-
-          yield* initialBlockingSyncContext.update({ processed: batch.length, remaining })
+          yield* initialBlockingSyncContext.update({ processed: batch.length, pageInfo })
         }),
       ),
       Stream.runDrain,
       Effect.interruptible,
     )
+
+    // Should only ever happen when livePull is false
+    yield* Effect.logDebug('backend-pulling finished', { livePull })
   }).pipe(Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pulling'))
 
 const backgroundBackendPushing = ({
@@ -748,17 +844,53 @@ const backgroundBackendPushing = ({
         batch: TRACE_VERBOSE ? JSON.stringify(queueItems) : undefined,
       })
 
-      // TODO handle push errors (should only happen during concurrent pull+push)
-      const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either)
+      // Push with declarative retry/backoff using Effect schedules
+      // - Exponential backoff starting at 1s and doubling (1s, 2s, 4s, 8s, 16s, 30s ...)
+      // - Delay clamped at 30s (continues retrying at 30s)
+      // - Resets automatically after successful push
+      // TODO(metrics): expose counters/gauges for retry attempts and queue health via devtools/metrics
+
+      // Only retry for transient UnexpectedError cases
+      const isRetryable = (err: InvalidPushError | IsOfflineError) =>
+        err._tag === 'InvalidPushError' && err.cause._tag === 'LiveStore.UnexpectedError'
+
+      // Input: InvalidPushError | IsOfflineError, Output: Duration
+      const retrySchedule: Schedule.Schedule<Duration.DurationInput, InvalidPushError | IsOfflineError> =
+        Schedule.exponential(Duration.seconds(1)).pipe(
+          Schedule.andThenEither(Schedule.spaced(Duration.seconds(30))), // clamp at 30 second intervals
+          Schedule.compose(Schedule.elapsed),
+          Schedule.whileInput(isRetryable),
+        )
+
+      yield* Effect.gen(function* () {
+        const iteration = yield* Schedule.CurrentIterationMetadata
 
-      if (pushResult._tag === 'Left') {
-        if (LS_DEV) {
-          yield* Effect.logDebug('handled backend-push-error', { error: pushResult.left.toString() })
+        const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either)
+
+        const retries = iteration.recurrence
+        if (retries > 0 && pushResult._tag === 'Right') {
+          otelSpan?.addEvent('backend-push-retry-success', { retries, batchSize: queueItems.length })
         }
-        otelSpan?.addEvent('backend-push-error', { error: pushResult.left.toString() })
-        // wait for interrupt caused by background pulling which will then restart pushing
-        return yield* Effect.never
-      }
+
+        if (pushResult._tag === 'Left') {
+          otelSpan?.addEvent('backend-push-error', {
+            error: pushResult.left.toString(),
+            retries,
+            batchSize: queueItems.length,
+          })
+          const error = pushResult.left
+          if (
+            error._tag === 'IsOfflineError' ||
+            (error._tag === 'InvalidPushError' && error.cause._tag === 'ServerAheadError')
+          ) {
+            // It's a core part of the sync protocol that the sync backend will emit a new pull chunk alongside the ServerAheadError
+            yield* Effect.logDebug('handled backend-push-error (waiting for interupt caused by pull)', { error })
+            return yield* Effect.never
+          }
+
+          return yield* error
+        }
+      }).pipe(Effect.retry(retrySchedule))
     }
   }).pipe(Effect.interruptible, Effect.withSpan('@livestore/common:LeaderSyncProcessor:backend-pushing'))
 
@@ -890,6 +1022,11 @@ const makePullQueueSet = Effect.gen(function* () {
   }
 })
 
+/**
+ * Validate a client-provided batch before it is admitted to the leader queue.
+ * Ensures the numbers form a strictly increasing chain and that the first
+ * event sits ahead of the current push head.
+ */
 const validatePushBatch = (
   batch: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>,
   pushHead: EventSequenceNumber.EventSequenceNumber,
@@ -899,12 +1036,16 @@ const validatePushBatch = (
       return
     }
 
-    // Make sure batch is monotonically increasing
+    // Example: session A already enqueued e1…e6 while session B (same client, different
+    // session) still believes the head is e1 and submits [e2, e7, e8]. The numbers look
+    // monotonic from B’s perspective, but we must reject and force B to rebase locally
+    // so the leader never regresses.
     for (let i = 1; i < batch.length; i++) {
       if (EventSequenceNumber.isGreaterThanOrEqual(batch[i - 1]!.seqNum, batch[i]!.seqNum)) {
-        shouldNeverHappen(
-          `Events must be ordered in monotonically ascending order by eventNum. Received: [${batch.map((e) => EventSequenceNumber.toString(e.seqNum)).join(', ')}]`,
-        )
+        return yield* LeaderAheadError.make({
+          minimumExpectedNum: batch[i - 1]!.seqNum,
+          providedNum: batch[i]!.seqNum,
+        })
       }
     }
 

package/src/leader-thread/eventlog.ts

@@ -123,6 +123,14 @@ export const getBackendHeadFromDb = (dbEventlog: SqliteDb): EventSequenceNumber.
 export const updateBackendHead = (dbEventlog: SqliteDb, head: EventSequenceNumber.EventSequenceNumber) =>
   dbEventlog.execute(sql`UPDATE ${SYNC_STATUS_TABLE} SET head = ${head.global}`)
 
+export const getBackendIdFromDb = (dbEventlog: SqliteDb): Option.Option<string> =>
+  Option.fromNullable(
+    dbEventlog.select<{ backendId: string | null }>(sql`select backendId from ${SYNC_STATUS_TABLE}`)[0]?.backendId,
+  )
+
+export const updateBackendId = (dbEventlog: SqliteDb, backendId: string) =>
+  dbEventlog.execute(sql`UPDATE ${SYNC_STATUS_TABLE} SET backendId = '${backendId}'`)
+
 export const insertIntoEventlog = (
   eventEncoded: LiveStoreEvent.EncodedWithMeta,
   dbEventlog: SqliteDb,
@@ -141,7 +149,7 @@ export const insertIntoEventlog = (
 
       if (parentEventExists === false) {
         shouldNeverHappen(
-          `Parent mutation ${eventEncoded.parentSeqNum.global},${eventEncoded.parentSeqNum.client} does not exist`,
+          `Parent event ${eventEncoded.parentSeqNum.global},${eventEncoded.parentSeqNum.client} does not exist in eventlog`,
         )
       }
     }
@@ -213,11 +221,7 @@ export const getSyncBackendCursorInfo = ({
     ).pipe(Effect.andThen(Schema.decode(EventlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie)
 
     return Option.some({
-      cursor: {
-        global: remoteHead,
-        client: EventSequenceNumber.clientDefault,
-        rebaseGeneration: EventSequenceNumber.rebaseGenerationDefault,
-      },
+      eventSequenceNumber: remoteHead,
       metadata: syncMetadataOption,
     }) satisfies InitialSyncInfo
   }).pipe(Effect.withSpan('@livestore/common:eventlog:getSyncBackendCursorInfo', { attributes: { remoteHead } }))

package/src/leader-thread/leader-worker-devtools.ts

@@ -262,7 +262,7 @@ const listenToDevtools = ({
 
               if (syncBackend !== undefined) {
                 // TODO consider piggybacking on the existing leader-thread sync-pulling
-                yield* syncBackend.pull(Option.none()).pipe(
+                yield* syncBackend.pull(Option.none(), { live: true }).pipe(
                   Stream.map((_) => _.batch),
                   Stream.flattenIterables,
                   Stream.tap(({ eventEncoded, metadata }) =>
@@ -319,7 +319,11 @@ const listenToDevtools = ({
                   Stream.tap(([isConnected, { latchClosed }]) =>
                     sendMessage(
                       Devtools.Leader.NetworkStatusRes.make({
-                        networkStatus: { isConnected, timestampMs: Date.now(), latchClosed },
+                        networkStatus: {
+                          isConnected,
+                          timestampMs: Date.now(),
+                          devtools: { latchClosed },
+                        },
                         subscriptionId,
                         ...reqPayload,
                         requestId: nanoid(10),