@livestore/common 0.3.0-dev.16 → 0.3.0-dev.18

package/src/leader-thread/LeaderSyncProcessor.ts

@@ -1,4 +1,4 @@
-import { isNotUndefined, LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
+import { casesHandled, isNotUndefined, LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
 import type { HttpClient, Scope, Tracer } from '@livestore/utils/effect'
 import {
   BucketQueue,
@@ -21,13 +21,14 @@ import { UnexpectedError } from '../adapter-types.js'
 import type { LiveStoreSchema, SessionChangesetMetaRow } from '../schema/mod.js'
 import {
   EventId,
+  getMutationDef,
   MUTATION_LOG_META_TABLE,
   MutationEvent,
   mutationLogMetaTable,
   SESSION_CHANGESET_META_TABLE,
 } from '../schema/mod.js'
 import { updateRows } from '../sql-queries/index.js'
-import { InvalidPushError } from '../sync/sync.js'
+import { LeaderAheadError } from '../sync/sync.js'
 import * as SyncState from '../sync/syncstate.js'
 import { sql } from '../util.js'
 import { makeApplyMutation } from './apply-mutation.js'
@@ -36,9 +37,12 @@ import { getBackendHeadFromDb, getClientHeadFromDb, getMutationEventsSince, upda
 import type { InitialBlockingSyncContext, InitialSyncInfo, LeaderSyncProcessor } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
-type PushQueueItem = [
+export const BACKEND_PUSH_BATCH_SIZE = 50
+
+type LocalPushQueueItem = [
   mutationEvent: MutationEvent.EncodedWithMeta,
-  deferred: Deferred.Deferred<void, InvalidPushError> | undefined,
+  deferred: Deferred.Deferred<void, LeaderAheadError> | undefined,
+  generation: number,
 ]
 
 /**
@@ -68,12 +72,14 @@ export const makeLeaderSyncProcessor = ({
   schema,
   dbMissing,
   dbMutationLog,
+  clientId,
   initialBlockingSyncContext,
 }: {
   schema: LiveStoreSchema
   /** Only used to know whether we can safely query dbMutationLog during setup execution */
   dbMissing: boolean
   dbMutationLog: SqliteDb
+  clientId: string
   initialBlockingSyncContext: InitialBlockingSyncContext
 }): Effect.Effect<LeaderSyncProcessor, UnexpectedError, Scope.Scope> =>
   Effect.gen(function* () {
@@ -82,10 +88,17 @@ export const makeLeaderSyncProcessor = ({
     const syncStateSref = yield* SubscriptionRef.make<SyncState.SyncState | undefined>(undefined)
 
     const isLocalEvent = (mutationEventEncoded: MutationEvent.EncodedWithMeta) => {
-      const mutationDef = schema.mutations.get(mutationEventEncoded.mutation)!
+      const mutationDef = getMutationDef(schema, mutationEventEncoded.mutation)
       return mutationDef.options.clientOnly
     }
 
+    /**
+     * Tracks generations of queued local push events.
+     * If a batch is rejected, all subsequent push queue items with the same generation are also rejected,
+     * even if they would be valid on their own.
+     */
+    const currentLocalPushGenerationRef = { current: 0 }
+
     // This context depends on data from `boot`, we should find a better implementation to avoid this ref indirection.
     const ctxRef = {
       current: undefined as
@@ -97,7 +110,7 @@ export const makeLeaderSyncProcessor = ({
           },
     }
 
-    const localPushesQueue = yield* BucketQueue.make<PushQueueItem>()
+    const localPushesQueue = yield* BucketQueue.make<LocalPushQueueItem>()
     const localPushesLatch = yield* Effect.makeLatch(true)
     const pullLatch = yield* Effect.makeLatch(true)
 
@@ -106,20 +119,36 @@ export const makeLeaderSyncProcessor = ({
         // TODO validate batch
         if (newEvents.length === 0) return
 
+        // if (options.generation < currentLocalPushGenerationRef.current) {
+        //   debugger
+        //   // We can safely drop this batch as it's from a previous push generation
+        //   return
+        // }
+
+        if (clientId === 'client-b') {
+          // console.log(
+          //   'push from client session',
+          //   newEvents.map((item) => item.toJSON()),
+          // )
+        }
+
         const waitForProcessing = options?.waitForProcessing ?? false
+        const generation = currentLocalPushGenerationRef.current
 
         if (waitForProcessing) {
-          const deferreds = yield* Effect.forEach(newEvents, () => Deferred.make<void, InvalidPushError>())
+          const deferreds = yield* Effect.forEach(newEvents, () => Deferred.make<void, LeaderAheadError>())
 
           const items = newEvents.map(
-            (mutationEventEncoded, i) => [mutationEventEncoded, deferreds[i]] as PushQueueItem,
+            (mutationEventEncoded, i) => [mutationEventEncoded, deferreds[i], generation] as LocalPushQueueItem,
           )
 
           yield* BucketQueue.offerAll(localPushesQueue, items)
 
           yield* Effect.all(deferreds)
         } else {
-          const items = newEvents.map((mutationEventEncoded) => [mutationEventEncoded, undefined] as PushQueueItem)
+          const items = newEvents.map(
+            (mutationEventEncoded) => [mutationEventEncoded, undefined, generation] as LocalPushQueueItem,
+          )
           yield* BucketQueue.offerAll(localPushesQueue, items)
         }
       }).pipe(
@@ -141,9 +170,7 @@ export const makeLeaderSyncProcessor = ({
         const syncState = yield* syncStateSref
         if (syncState === undefined) return shouldNeverHappen('Not initialized')
 
-        const mutationDef =
-          schema.mutations.get(partialMutationEvent.mutation) ??
-          shouldNeverHappen(`Unknown mutation: ${partialMutationEvent.mutation}`)
+        const mutationDef = getMutationDef(schema, partialMutationEvent.mutation)
 
         const mutationEventEncoded = new MutationEvent.EncodedWithMeta({
           ...partialMutationEvent,
@@ -153,14 +180,14 @@ export const makeLeaderSyncProcessor = ({
         })
 
         yield* push([mutationEventEncoded])
-      }).pipe(Effect.catchTag('InvalidPushError', Effect.orDie))
+      }).pipe(Effect.catchTag('LeaderAheadError', Effect.orDie))
 
     // Starts various background loops
     const boot: LeaderSyncProcessor['boot'] = ({ dbReady }) =>
       Effect.gen(function* () {
         const span = yield* Effect.currentSpan.pipe(Effect.orDie)
         const otelSpan = yield* OtelTracer.currentOtelSpan.pipe(Effect.catchAll(() => Effect.succeed(undefined)))
-        const { devtools } = yield* LeaderThreadCtx
+        const { devtools, shutdownChannel } = yield* LeaderThreadCtx
 
         ctxRef.current = {
           otelSpan,
@@ -198,13 +225,19 @@ export const makeLeaderSyncProcessor = ({
           const filteredBatch = pendingMutationEvents
             // Don't sync clientOnly mutations
             .filter((mutationEventEncoded) => {
-              const mutationDef = schema.mutations.get(mutationEventEncoded.mutation)!
+              const mutationDef = getMutationDef(schema, mutationEventEncoded.mutation)
               return mutationDef.options.clientOnly === false
             })
 
           yield* BucketQueue.offerAll(syncBackendQueue, filteredBatch)
         }
 
+        const shutdownOnError = (cause: unknown) =>
+          Effect.gen(function* () {
+            yield* shutdownChannel.send(UnexpectedError.make({ cause }))
+            yield* Effect.die(cause)
+          })
+
         yield* backgroundApplyLocalPushes({
           localPushesLatch,
           localPushesQueue,
@@ -214,7 +247,8 @@ export const makeLeaderSyncProcessor = ({
           schema,
           isLocalEvent,
           otelSpan,
-        }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
+          currentLocalPushGenerationRef,
+        }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped)
 
         const backendPushingFiberHandle = yield* FiberHandle.make()
 
@@ -225,7 +259,7 @@ export const makeLeaderSyncProcessor = ({
             syncBackendQueue,
             otelSpan,
             devtoolsLatch: ctxRef.current?.devtoolsLatch,
-          }).pipe(Effect.tapCauseLogPretty),
+          }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError)),
         )
 
         yield* backgroundBackendPulling({
@@ -249,7 +283,7 @@ export const makeLeaderSyncProcessor = ({
                   syncBackendQueue,
                   otelSpan,
                   devtoolsLatch: ctxRef.current?.devtoolsLatch,
-                }).pipe(Effect.tapCauseLogPretty),
+                }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError)),
               )
             }),
           syncStateSref,
@@ -258,7 +292,7 @@ export const makeLeaderSyncProcessor = ({
           otelSpan,
           initialBlockingSyncContext,
           devtoolsLatch: ctxRef.current?.devtoolsLatch,
-        }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
+        }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped)
 
         return { initialLeaderHead: initialLocalHead }
       }).pipe(Effect.withSpanScoped('@livestore/common:leader-thread:syncing'))
@@ -287,32 +321,48 @@ const backgroundApplyLocalPushes = ({
   schema,
   isLocalEvent,
   otelSpan,
+  currentLocalPushGenerationRef,
 }: {
   pullLatch: Effect.Latch
   localPushesLatch: Effect.Latch
-  localPushesQueue: BucketQueue.BucketQueue<PushQueueItem>
+  localPushesQueue: BucketQueue.BucketQueue<LocalPushQueueItem>
   syncStateSref: SubscriptionRef.SubscriptionRef<SyncState.SyncState | undefined>
   syncBackendQueue: BucketQueue.BucketQueue<MutationEvent.EncodedWithMeta>
   schema: LiveStoreSchema
   isLocalEvent: (mutationEventEncoded: MutationEvent.EncodedWithMeta) => boolean
   otelSpan: otel.Span | undefined
+  currentLocalPushGenerationRef: { current: number }
 }) =>
   Effect.gen(function* () {
-    const { connectedClientSessionPullQueues } = yield* LeaderThreadCtx
+    const { connectedClientSessionPullQueues, clientId } = yield* LeaderThreadCtx
 
     const applyMutationItems = yield* makeApplyMutationItems
 
     while (true) {
       // TODO make batch size configurable
       const batchItems = yield* BucketQueue.takeBetween(localPushesQueue, 1, 10)
-      const [newEvents, deferreds] = ReadonlyArray.unzip(batchItems)
 
       // Wait for the backend pulling to finish
       yield* localPushesLatch.await
 
-      // Prevent the backend pulling from starting until this local push is finished
+      // Prevent backend pull processing until this local push is finished
       yield* pullLatch.close
 
+      // Since the generation might have changed since enqueuing, we need to filter out items with older generation
+      // It's important that we filter after we got localPushesLatch, otherwise we might filter with the old generation
+      const filteredBatchItems = batchItems
+        .filter(([_1, _2, generation]) => generation === currentLocalPushGenerationRef.current)
+        .map(([mutationEventEncoded, deferred]) => [mutationEventEncoded, deferred] as const)
+
+      if (filteredBatchItems.length === 0) {
+        // console.log('dropping old-gen batch', currentLocalPushGenerationRef.current)
+        // Allow the backend pulling to start
+        yield* pullLatch.open
+        continue
+      }
+
+      const [newEvents, deferreds] = ReadonlyArray.unzip(filteredBatchItems)
+
       const syncState = yield* syncStateSref
       if (syncState === undefined) return shouldNeverHappen('Not initialized')
 
@@ -323,42 +373,82 @@ const backgroundApplyLocalPushes = ({
         isEqualEvent: MutationEvent.isEqualEncoded,
       })
 
-      if (updateResult._tag === 'rebase') {
-        return shouldNeverHappen('The leader thread should never have to rebase due to a local push')
-      } else if (updateResult._tag === 'reject') {
-        otelSpan?.addEvent('local-push:reject', {
-          batchSize: newEvents.length,
-          updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
-        })
+      switch (updateResult._tag) {
+        case 'unexpected-error': {
+          otelSpan?.addEvent('local-push:unexpected-error', {
+            batchSize: newEvents.length,
+            newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
+          })
+          return yield* Effect.fail(updateResult.cause)
+        }
+        case 'rebase': {
+          return shouldNeverHappen('The leader thread should never have to rebase due to a local push')
+        }
+        case 'reject': {
+          otelSpan?.addEvent('local-push:reject', {
+            batchSize: newEvents.length,
+            updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
+          })
+
+          /*
+
+          TODO: how to test this?
+          */
+          currentLocalPushGenerationRef.current++
+
+          const nextGeneration = currentLocalPushGenerationRef.current
 
-        const providedId = newEvents.at(0)!.id
-        const remainingEvents = yield* BucketQueue.takeAll(localPushesQueue)
-        const allDeferreds = [...deferreds, ...remainingEvents.map(([_, deferred]) => deferred)].filter(isNotUndefined)
-        yield* Effect.forEach(allDeferreds, (deferred) =>
-          Deferred.fail(
-            deferred,
-            InvalidPushError.make({
-              // TODO improve error handling so it differentiates between a push being rejected
-              // because of itself or because of another push
-              reason: {
-                _tag: 'LeaderAhead',
+          const providedId = newEvents.at(0)!.id
+          // All subsequent pushes with same generation should be rejected as well
+          // We're also handling the case where the localPushQueue already contains events
+          // from the next generation which we preserve in the queue
+          const remainingEventsMatchingGeneration = yield* BucketQueue.takeSplitWhere(
+            localPushesQueue,
+            (item) => item[2] >= nextGeneration,
+          )
+
+          if ((yield* BucketQueue.size(localPushesQueue)) > 0) {
+            console.log('localPushesQueue is not empty', yield* BucketQueue.size(localPushesQueue))
+            debugger
+          }
+
+          const allDeferredsToReject = [
+            ...deferreds,
+            ...remainingEventsMatchingGeneration.map(([_, deferred]) => deferred),
+          ].filter(isNotUndefined)
+
+          yield* Effect.forEach(allDeferredsToReject, (deferred) =>
+            Deferred.fail(
+              deferred,
+              LeaderAheadError.make({
                 minimumExpectedId: updateResult.expectedMinimumId,
                 providedId,
-              },
-            }),
-          ),
-        )
+                // nextGeneration,
+              }),
+            ),
+          )
 
-        // Allow the backend pulling to start
-        yield* pullLatch.open
+          // Allow the backend pulling to start
+          yield* pullLatch.open
 
-        // In this case we're skipping state update and down/upstream processing
-        // We've cleared the local push queue and are now waiting for new local pushes / backend pulls
-        continue
+          // In this case we're skipping state update and down/upstream processing
+          // We've cleared the local push queue and are now waiting for new local pushes / backend pulls
+          continue
+        }
+        case 'advance': {
+          break
+        }
+        default: {
+          casesHandled(updateResult)
+        }
       }
 
       yield* SubscriptionRef.set(syncStateSref, updateResult.newSyncState)
 
+      if (clientId === 'client-b') {
+        // yield* Effect.log('offer upstream-advance due to local-push')
+        // debugger
+      }
       yield* connectedClientSessionPullQueues.offer({
         payload: { _tag: 'upstream-advance', newEvents: updateResult.newEvents },
         remaining: 0,
@@ -371,7 +461,7 @@ const backgroundApplyLocalPushes = ({
 
       // Don't sync clientOnly mutations
       const filteredBatch = updateResult.newEvents.filter((mutationEventEncoded) => {
-        const mutationDef = schema.mutations.get(mutationEventEncoded.mutation)!
+        const mutationDef = getMutationDef(schema, mutationEventEncoded.mutation)
         return mutationDef.options.clientOnly === false
       })
 
@@ -387,7 +477,7 @@ const backgroundApplyLocalPushes = ({
 type ApplyMutationItems = (_: {
   batchItems: ReadonlyArray<MutationEvent.EncodedWithMeta>
   /** Indexes are aligned with `batchItems` */
-  deferreds: ReadonlyArray<Deferred.Deferred<void, InvalidPushError> | undefined> | undefined
+  deferreds: ReadonlyArray<Deferred.Deferred<void, LeaderAheadError> | undefined> | undefined
 }) => Effect.Effect<void, UnexpectedError>
 
 // TODO how to handle errors gracefully
@@ -466,6 +556,7 @@ const backgroundBackendPulling = ({
       dbMutationLog,
       connectedClientSessionPullQueues,
       schema,
+      clientId,
     } = yield* LeaderThreadCtx
 
     if (syncBackend === undefined) return
@@ -503,6 +594,12 @@ const backgroundBackendPulling = ({
 
         if (updateResult._tag === 'reject') {
           return shouldNeverHappen('The leader thread should never reject upstream advances')
+        } else if (updateResult._tag === 'unexpected-error') {
+          otelSpan?.addEvent('backend-pull:unexpected-error', {
+            newEventsCount: newEvents.length,
+            newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
+          })
+          return yield* Effect.fail(updateResult.cause)
         }
 
         const newBackendHead = newEvents.at(-1)!.id
@@ -518,7 +615,7 @@ const backgroundBackendPulling = ({
           })
 
           const filteredRebasedPending = updateResult.newSyncState.pending.filter((mutationEvent) => {
-            const mutationDef = schema.mutations.get(mutationEvent.mutation)!
+            const mutationDef = getMutationDef(schema, mutationEvent.mutation)
             return mutationDef.options.clientOnly === false
           })
           yield* restartBackendPushing(filteredRebasedPending)
@@ -542,6 +639,9 @@ const backgroundBackendPulling = ({
             updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
           })
 
+          if (clientId === 'client-b') {
+            // yield* Effect.log('offer upstream-advance due to pull')
+          }
           yield* connectedClientSessionPullQueues.offer({
             payload: { _tag: 'upstream-advance', newEvents: updateResult.newEvents, trimRollbackUntil },
             remaining,
@@ -617,15 +717,23 @@ const rollback = ({
       }
     }
 
-    // Delete the changeset rows
-    db.execute(
-      sql`DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdsToRollback.map((id) => `(${id.global}, ${id.client})`).join(', ')})`,
+    const eventIdPairChunks = ReadonlyArray.chunksOf(100)(
+      eventIdsToRollback.map((id) => `(${id.global}, ${id.client})`),
     )
 
+    // Delete the changeset rows
+    for (const eventIdPairChunk of eventIdPairChunks) {
+      db.execute(
+        sql`DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdPairChunk.join(', ')})`,
+      )
+    }
+
     // Delete the mutation log rows
-    dbMutationLog.execute(
-      sql`DELETE FROM ${MUTATION_LOG_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdsToRollback.map((id) => `(${id.global}, ${id.client})`).join(', ')})`,
-    )
+    for (const eventIdPairChunk of eventIdPairChunks) {
+      dbMutationLog.execute(
+        sql`DELETE FROM ${MUTATION_LOG_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdPairChunk.join(', ')})`,
+      )
+    }
   }).pipe(
     Effect.withSpan('@livestore/common:leader-thread:syncing:rollback', {
       attributes: { count: eventIdsToRollback.length },
@@ -675,7 +783,7 @@ const backgroundBackendPushing = ({
       yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
 
       // TODO make batch size configurable
-      const queueItems = yield* BucketQueue.takeBetween(syncBackendQueue, 1, 50)
+      const queueItems = yield* BucketQueue.takeBetween(syncBackendQueue, 1, BACKEND_PUSH_BATCH_SIZE)
 
       yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
 
@@ -693,7 +801,7 @@ const backgroundBackendPushing = ({
 
       if (pushResult._tag === 'Left') {
         if (LS_DEV) {
-          yield* Effect.logDebug('backend-push-error', { error: pushResult.left.toString() })
+          yield* Effect.logDebug('handled backend-push-error', { error: pushResult.left.toString() })
         }
         otelSpan?.addEvent('backend-push-error', { error: pushResult.left.toString() })
         // wait for interrupt caused by background pulling which will then restart pushing

package/src/leader-thread/apply-mutation.ts

@@ -1,10 +1,12 @@
-import { memoizeByRef, shouldNeverHappen } from '@livestore/utils'
+import { LS_DEV, memoizeByRef, shouldNeverHappen } from '@livestore/utils'
 import type { Scope } from '@livestore/utils/effect'
 import { Effect, Option, Schema } from '@livestore/utils/effect'
 
-import type { SqliteDb, SqliteError, UnexpectedError } from '../index.js'
+import type { PreparedBindValues, SqliteDb, SqliteError, UnexpectedError } from '../index.js'
 import { getExecArgsFromMutation } from '../mutation.js'
 import {
+  EventId,
+  getMutationDef,
   type LiveStoreSchema,
   MUTATION_LOG_META_TABLE,
   type MutationEvent,
@@ -33,7 +35,7 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
       // TODO Running `Schema.hash` can be a bottleneck for larger schemas. There is an opportunity to run this
       // at build time and lookup the pre-computed hash at runtime.
       // Also see https://github.com/Effect-TS/effect/issues/2719
-      [...leaderThreadCtx.schema.mutations.entries()].map(([k, v]) => [k, Schema.hash(v.schema)] as const),
+      [...leaderThreadCtx.schema.mutations.map.entries()].map(([k, v]) => [k, Schema.hash(v.schema)] as const),
     )
 
     return (mutationEventEncoded, options) =>
@@ -42,7 +44,7 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
         const skipMutationLog = options?.skipMutationLog ?? false
 
         const mutationName = mutationEventEncoded.mutation
-        const mutationDef = schema.mutations.get(mutationName) ?? shouldNeverHappen(`Unknown mutation: ${mutationName}`)
+        const mutationDef = getMutationDef(schema, mutationName)
 
         const execArgsArr = getExecArgsFromMutation({
           mutationDef,
@@ -127,6 +129,20 @@ const insertIntoMutationLog = (
     const mutationDefSchemaHash =
       mutationDefSchemaHashMap.get(mutationName) ?? shouldNeverHappen(`Unknown mutation: ${mutationName}`)
 
+    if (LS_DEV && mutationEventEncoded.parentId.global !== EventId.ROOT.global) {
+      const parentMutationExists =
+        dbMutationLog.select<{ count: number }>(
+          `SELECT COUNT(*) as count FROM ${MUTATION_LOG_META_TABLE} WHERE idGlobal = ? AND idClient = ?`,
+          [mutationEventEncoded.parentId.global, mutationEventEncoded.parentId.client] as any as PreparedBindValues,
+        )[0]!.count === 1
+
+      if (parentMutationExists === false) {
+        shouldNeverHappen(
+          `Parent mutation ${mutationEventEncoded.parentId.global},${mutationEventEncoded.parentId.client} does not exist`,
+        )
+      }
+    }
+
     // TODO use prepared statements
     yield* execSql(
       dbMutationLog,
@@ -160,7 +176,7 @@ const makeShouldExcludeMutationFromLog = memoizeByRef((schema: LiveStoreSchema)
   return (mutationName: string, mutationEventEncoded: MutationEvent.AnyEncoded): boolean => {
     if (mutationLogExclude.has(mutationName)) return true
 
-    const mutationDef = schema.mutations.get(mutationName) ?? shouldNeverHappen(`Unknown mutation: ${mutationName}`)
+    const mutationDef = getMutationDef(schema, mutationName)
     const execArgsArr = getExecArgsFromMutation({
       mutationDef,
       mutationEvent: { decoded: undefined, encoded: mutationEventEncoded },

package/src/leader-thread/make-leader-thread-layer.ts

@@ -65,6 +65,7 @@ export const makeLeaderThreadLayer = ({
       dbMissing,
       dbMutationLog,
       initialBlockingSyncContext,
+      clientId,
     })
 
     const extraIncomingMessagesQueue = yield* Queue.unbounded<Devtools.Leader.MessageToApp>().pipe(

package/src/leader-thread/pull-queue-set.ts

@@ -2,7 +2,7 @@ import { Effect, Queue } from '@livestore/utils/effect'
 
 import * as MutationEvent from '../schema/MutationEvent.js'
 import { getMutationEventsSince } from './mutationlog.js'
-import { type PullQueueItem, type PullQueueSet } from './types.js'
+import { LeaderThreadCtx, type PullQueueItem, type PullQueueSet } from './types.js'
 
 export const makePullQueueSet = Effect.gen(function* () {
   const set = new Set<Queue.Queue<PullQueueItem>>()
@@ -46,6 +46,15 @@ export const makePullQueueSet = Effect.gen(function* () {
         return
       }
 
+      const { clientId } = yield* LeaderThreadCtx
+      if (clientId === 'client-b') {
+        // console.log(
+        //   'offer',
+        //   item.payload._tag,
+        //   item.payload.newEvents.map((_) => _.toJSON()),
+        // )
+      }
+
       for (const queue of set) {
         yield* Queue.offer(queue, item)
       }

package/src/leader-thread/shutdown-channel.ts

@@ -1,11 +1,9 @@
 import type { WebChannel } from '@livestore/utils/effect'
 import { Schema } from '@livestore/utils/effect'
 
-import { IntentionalShutdownCause } from '../index.js'
+import { IntentionalShutdownCause, UnexpectedError } from '../index.js'
 
-export class DedicatedWorkerDisconnectBroadcast extends Schema.TaggedStruct('DedicatedWorkerDisconnectBroadcast', {}) {}
-
-export class All extends Schema.Union(IntentionalShutdownCause, DedicatedWorkerDisconnectBroadcast) {}
+export class All extends Schema.Union(IntentionalShutdownCause, UnexpectedError) {}
 
 /**
  * Used internally by an adapter to shutdown gracefully.

package/src/leader-thread/types.ts

@@ -14,7 +14,7 @@ import { Context, Schema } from '@livestore/utils/effect'
 import type {
   BootStatus,
   Devtools,
-  InvalidPushError,
+  LeaderAheadError,
   MakeSqliteDb,
   MigrationsReport,
   PersistenceInfo,
@@ -126,13 +126,19 @@ export interface LeaderSyncProcessor {
     /** `batch` needs to follow the same rules as `batch` in `SyncBackend.push` */
     batch: ReadonlyArray<MutationEvent.EncodedWithMeta>,
     options?: {
+      /**
+       * This generation number is used to automatically reject subsequent pushes
+       * of a previously rejected push from a client session. This might occur in
+       * certain concurrent scenarios.
+       */
+      // generation: number
       /**
        * If true, the effect will only finish when the local push has been processed (i.e. succeeded or was rejected).
        * @default false
        */
       waitForProcessing?: boolean
     },
-  ) => Effect.Effect<void, InvalidPushError>
+  ) => Effect.Effect<void, LeaderAheadError>
 
   pushPartial: (args: {
     mutationEvent: MutationEvent.PartialAnyEncoded

package/src/rehydrate-from-mutationlog.ts

@@ -4,7 +4,7 @@ import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
 import { type MigrationOptionsFromMutationLog, type SqliteDb, UnexpectedError } from './adapter-types.js'
 import { makeApplyMutation } from './leader-thread/apply-mutation.js'
 import type { LiveStoreSchema, MutationDef, MutationEvent, MutationLogMetaRow } from './schema/mod.js'
-import { EventId, MUTATION_LOG_META_TABLE } from './schema/mod.js'
+import { EventId, getMutationDef, MUTATION_LOG_META_TABLE } from './schema/mod.js'
 import type { PreparedBindValues } from './util.js'
 import { sql } from './util.js'
 
@@ -33,7 +33,7 @@ export const rehydrateFromMutationLog = ({
 
     const processMutation = (row: MutationLogMetaRow) =>
       Effect.gen(function* () {
-        const mutationDef = schema.mutations.get(row.mutation) ?? shouldNeverHappen(`Unknown mutation ${row.mutation}`)
+        const mutationDef = getMutationDef(schema, row.mutation)
 
         if (migrationOptions.excludeMutations?.has(row.mutation) === true) return
 

package/src/schema/EventId.ts

@@ -33,6 +33,22 @@ export const compare = (a: EventId, b: EventId) => {
   return a.client - b.client
 }
 
+/**
+ * Convert an event id to a string representation.
+ */
+export const toString = (id: EventId) => `(${id.global},${id.client})`
+
+/**
+ * Convert a string representation of an event id to an event id.
+ */
+export const fromString = (str: string): EventId => {
+  const [global, client] = str.slice(1, -1).split(',').map(Number)
+  if (global === undefined || client === undefined) {
+    throw new Error('Invalid event id string')
+  }
+  return { global, client } as EventId
+}
+
 export const isEqual = (a: EventId, b: EventId) => a.global === b.global && a.client === b.client
 
 export type EventIdPair = { id: EventId; parentId: EventId }