@livestore/common 0.4.0-dev.9 → 0.5.0-dev.0

package/src/sync/ClientSessionSyncProcessor.ts

@@ -1,26 +1,35 @@
 /// <reference lib="dom" />
-import { LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
+import { LS_DEV, TRACE_VERBOSE } from '@livestore/utils'
 import {
-  BucketQueue,
+  Cause,
+  Deferred,
   Effect,
   Exit,
+  Filter,
   FiberHandle,
   Option,
   Queue,
-  type Runtime,
   Schema,
+  Semaphore,
   type Scope,
   Stream,
   Subscribable,
+  TxQueue,
 } from '@livestore/utils/effect'
-import type * as otel from '@opentelemetry/api'
 
-import { type ClientSession, type MaterializeError, UnexpectedError } from '../adapter-types.ts'
-import * as EventSequenceNumber from '../schema/EventSequenceNumber.ts'
-import * as LiveStoreEvent from '../schema/LiveStoreEvent.ts'
-import { getEventDef, type LiveStoreSchema } from '../schema/mod.ts'
+import type { ClientSession } from '../adapter-types.ts'
+import { MaterializeError } from '../errors.ts'
+import { isRejectedPushError } from '../leader-thread/RejectedPushError.ts'
+import * as EventSequenceNumber from '../schema/EventSequenceNumber/mod.ts'
+import * as LiveStoreEvent from '../schema/LiveStoreEvent/mod.ts'
+import type { LiveStoreSchema } from '../schema/mod.ts'
+import { resolveSessionIdSymbolInEventArgs } from '../session-id-symbol.ts'
+import * as StateHead from '../StateHead.ts'
 import * as SyncState from './syncstate.ts'
 
+/** Serialize value to JSON string for trace attributes */
+const jsonStringify = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown))
+
 /**
  * Rebase behaviour:
  * - We continously pull events from the leader and apply them to the local store.
@@ -36,23 +45,20 @@ import * as SyncState from './syncstate.ts'
  * - The leader sync processor pulls regular LiveStore events, while the session sync processor pulls SyncState.PayloadUpstream items
  * - The session sync processor has no downstream nodes.
  */
-export const makeClientSessionSyncProcessor = ({
+export const makeClientSessionSyncProcessor = Effect.fn('makeClientSessionSyncProcessor')(function* ({
   schema,
   clientSession,
-  runtime,
   materializeEvent,
   rollback,
   refreshTables,
-  span,
   params,
   confirmUnsavedChanges,
 }: {
   schema: LiveStoreSchema
   clientSession: ClientSession
-  runtime: Runtime.Runtime<Scope.Scope>
   materializeEvent: (
-    eventDecoded: LiveStoreEvent.AnyDecoded,
-    options: { withChangeset: boolean; materializerHashLeader: Option.Option<number> },
+    eventEncoded: LiveStoreEvent.Client.EncodedWithMeta,
+    options: { materializerHashLeader: Option.Option<number> },
   ) => Effect.Effect<
     {
       writeTables: Set<string>
@@ -66,23 +72,27 @@ export const makeClientSessionSyncProcessor = ({
   >
   rollback: (changeset: Uint8Array<ArrayBuffer>) => void
   refreshTables: (tables: Set<string>) => void
-  span: otel.Span
   params: {
     leaderPushBatchSize: number
-    simulation?: ClientSessionSyncProcessorSimulationParams
+    /**
+     * Test-only deterministic barriers, awaited at fixed points of the rebase critical section.
+     * A test can park the pull fiber at a chosen point, inject a concurrent operation
+     * (a synchronous `push` or a `shutdown`), then release the barrier — without relying on
+     * virtual-time scheduling. Unset in production, where each lookup resolves to `Effect.void`.
+     */
+    rebaseBarriers?: Partial<Record<RebaseBarrierPoint, Effect.Effect<void>>>
   }
   /**
    * Currently only used in the web adapter:
    * If true, registers a beforeunload event listener to confirm unsaved changes.
    */
   confirmUnsavedChanges: boolean
-}): ClientSessionSyncProcessor => {
-  const eventSchema = LiveStoreEvent.makeEventDefSchemaMemo(schema)
+}) {
+  const stateHead = yield* StateHead.StateHead
+  const eventSchema = LiveStoreEvent.Client.makeSchemaMemo(schema)
 
-  const simSleep = <TKey extends keyof ClientSessionSyncProcessorSimulationParams>(
-    key: TKey,
-    key2: keyof ClientSessionSyncProcessorSimulationParams[TKey],
-  ) => Effect.sleep((params.simulation?.[key]?.[key2] ?? 0) as number)
+  const rebaseBarrier = (point: RebaseBarrierPoint): Effect.Effect<void> =>
+    params.rebaseBarriers?.[point] ?? Effect.void
 
   const syncStateRef = {
     // The initial state is identical to the leader's initial state
@@ -95,98 +105,50 @@ export const makeClientSessionSyncProcessor = ({
   }
 
   /** Only used for debugging / observability / testing, it's not relied upon for correctness of the sync processor. */
-  const syncStateUpdateQueue = Queue.unbounded<SyncState.SyncState>().pipe(Effect.runSync)
-  const isClientEvent = (eventEncoded: LiveStoreEvent.EncodedWithMeta) =>
-    getEventDef(schema, eventEncoded.name).eventDef.options.clientOnly
+  const syncStateUpdateQueue = yield* Queue.unbounded<SyncState.SyncState>()
+  const isClientOnlyEvent = (eventEncoded: LiveStoreEvent.Client.EncodedWithMeta) =>
+    schema.eventsDefsMap.get(eventEncoded.name)?.options.clientOnly ?? false
 
   /** We're queuing push requests to reduce the number of messages sent to the leader by batching them */
-  const leaderPushQueue = BucketQueue.make<LiveStoreEvent.EncodedWithMeta>().pipe(Effect.runSync)
-
-  const push: ClientSessionSyncProcessor['push'] = Effect.fn('client-session-sync-processor:push')(function* (batch) {
-    // TODO validate batch
-
-    let baseEventSequenceNumber = syncStateRef.current.localHead
-    const encodedEventDefs = batch.map(({ name, args }) => {
-      const eventDef = getEventDef(schema, name)
-      const nextNumPair = EventSequenceNumber.nextPair({
-        seqNum: baseEventSequenceNumber,
-        isClient: eventDef.eventDef.options.clientOnly,
-      })
-      baseEventSequenceNumber = nextNumPair.seqNum
-      return new LiveStoreEvent.EncodedWithMeta(
-        Schema.encodeUnknownSync(eventSchema)({
-          name,
-          args,
-          ...nextNumPair,
-          clientId: clientSession.clientId,
-          sessionId: clientSession.sessionId,
-        }),
-      )
-    })
-
-    const mergeResult = SyncState.merge({
-      syncState: syncStateRef.current,
-      payload: { _tag: 'local-push', newEvents: encodedEventDefs },
-      isClientEvent,
-      isEqualEvent: LiveStoreEvent.isEqualEncoded,
-    })
-
-    yield* Effect.annotateCurrentSpan({
-      batchSize: encodedEventDefs.length,
-      mergeResultTag: mergeResult._tag,
-      eventCounts: encodedEventDefs.reduce<Record<string, number>>((acc, event) => {
-        acc[event.name] = (acc[event.name] ?? 0) + 1
-        return acc
-      }, {}),
-      ...(TRACE_VERBOSE && { mergeResult: JSON.stringify(mergeResult) }),
-    })
-
-    if (mergeResult._tag === 'unexpected-error') {
-      return shouldNeverHappen('Unexpected error in client-session-sync-processor', mergeResult.message)
-    }
-
-    if (mergeResult._tag !== 'advance') {
-      return shouldNeverHappen(`Expected advance, got ${mergeResult._tag}`)
-    }
-
-    syncStateRef.current = mergeResult.newSyncState
-    yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
-
-    // Materialize events to state
-    const writeTables = new Set<string>()
-    for (const event of mergeResult.newEvents) {
-      // TODO avoid encoding and decoding here again
-      const decodedEventDef = Schema.decodeSync(eventSchema)(event)
-      const {
-        writeTables: newWriteTables,
-        sessionChangeset,
-        materializerHash,
-      } = yield* materializeEvent(decodedEventDef, {
-        withChangeset: true,
-        materializerHashLeader: Option.none(),
-      })
-      for (const table of newWriteTables) {
-        writeTables.add(table)
+  const leaderPushQueue = yield* TxQueue.unbounded<LiveStoreEvent.Client.EncodedWithMeta, Cause.Done>()
+  /**
+   * Prevents pull reconciliation, push-rejection handling, and shutdown from running concurrently.
+   * These transitions inspect or update the pending events, leader push queue, and rejection state,
+   * so each must observe the others either fully before or fully after—not midway through bookkeeping.
+   *
+   * Regular local commits do not acquire this mutex; they remain synchronous and are incorporated
+   * when `reconcileLeaderPushQueue` reads the current pending events transactionally.
+   */
+  const pullReconciliationMutex = yield* Semaphore.make(1)
+  const shutdownDone = yield* Deferred.make<void>()
+  const drainStartedSignal = yield* Deferred.make<void>()
+  const rejectionObserved = yield* Deferred.make<void>()
+  let shutdownStarted = false
+  let terminalPushCause: Cause.Cause<never> | undefined
+  let unresolvedRejection:
+    | {
+        readonly error: Error
+        readonly events: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>
       }
-      event.meta.sessionChangeset = sessionChangeset
-      event.meta.materializerHashSession = materializerHash
-    }
-
-    // Trigger push to leader
-    // console.debug('pushToLeader', encodedEventDefs.length, ...encodedEventDefs.map((_) => _.toJSON()))
-    yield* BucketQueue.offerAll(leaderPushQueue, encodedEventDefs)
-
-    return { writeTables }
-  })
-
-  const debugInfo = {
-    rebaseCount: 0,
-    advanceCount: 0,
-    rejectCount: 0,
-  }
+    | undefined
+  let leaderPushingFiberHandle: FiberHandle.FiberHandle<void, never> | undefined
+  let pullingFiberHandle: FiberHandle.FiberHandle<void, never> | undefined
+
+  /** Rebuild the leader push queue from the current pending events without racing synchronous commits. */
+  const reconcileLeaderPushQueue = Effect.tx(
+    Effect.gen(function* () {
+      const livePending = syncStateRef.current.pending
+      yield* TxQueue.clear(leaderPushQueue)
+      yield* TxQueue.offerAll(leaderPushQueue, livePending)
+    }),
+  )
 
   const boot: ClientSessionSyncProcessor['boot'] = Effect.gen(function* () {
-    if (confirmUnsavedChanges && typeof window !== 'undefined' && typeof window.addEventListener === 'function') {
+    if (
+      confirmUnsavedChanges === true &&
+      typeof window !== 'undefined' &&
+      typeof window.addEventListener === 'function'
+    ) {
       const onBeforeUnload = (event: BeforeUnloadEvent) => {
         if (syncStateRef.current.pending.length > 0) {
           // Trigger the default browser dialog
@@ -200,70 +162,117 @@ export const makeClientSessionSyncProcessor = ({
       )
     }
 
-    const leaderPushingFiberHandle = yield* FiberHandle.make()
+    const leaderPushingHandle = yield* FiberHandle.make<void, never>()
+    const pullingHandle = yield* FiberHandle.make<void, never>()
+    leaderPushingFiberHandle = leaderPushingHandle
+    pullingFiberHandle = pullingHandle
+
+    const backgroundLeaderPushing: Effect.Effect<void> = Effect.gen(function* () {
+      while (true) {
+        const batch = yield* TxQueue.takeBetween(leaderPushQueue, 1, params.leaderPushBatchSize).pipe(
+          Effect.catchIf(Cause.isDone, () => Effect.void),
+        )
+        if (batch === undefined) return
+
+        yield* clientSession.leaderThread.events.push(batch).pipe(
+          Effect.catchIf(isRejectedPushError, (error) =>
+            // A pull carrying the leader's corrective history can complete before or after this
+            // rejection response. Use the same mutex as pull reconciliation so this decision uses
+            // one stable view of pending events and cannot install a stale rejection fence after recovery.
+            pullReconciliationMutex
+              .withPermits(1)(
+                Effect.gen(function* () {
+                  debugInfo.rejectCount++
+                  if (shutdownStarted === true) return yield* Effect.die(error)
+
+                  // A concurrent pull may have already confirmed or rebased this batch while the
+                  // push response was in flight. In that case, rebuild the queue from the reconciled
+                  // pending events instead of creating a fence that no future pull could release.
+                  if (isRejectedBatchRecovered(batch, syncStateRef.current.pending) === true) {
+                    yield* reconcileLeaderPushQueue
+                    yield* Deferred.succeed(rejectionObserved, undefined)
+                    return false
+                  }
+
+                  unresolvedRejection = { error, events: batch }
+                  yield* TxQueue.clear(leaderPushQueue)
+                  yield* Deferred.succeed(rejectionObserved, undefined)
+                  return true
+                }),
+              )
+              .pipe(
+                Effect.flatMap((shouldFence) =>
+                  // Local commits remain synchronous and continue accumulating in pending/the FIFO,
+                  // but this sole drain worker stays parked until pull reconciliation reseeds it.
+                  shouldFence === true ? Effect.never : Effect.void,
+                ),
+              ),
+          ),
+        )
+      }
+    }).pipe(
+      Effect.interruptible,
+      Effect.tapCauseLogPretty,
+      Effect.catchCause((cause) => {
+        if (Cause.hasInterruptsOnly(cause) === true) return Effect.void
 
-    const backgroundLeaderPushing = Effect.gen(function* () {
-      const batch = yield* BucketQueue.takeBetween(leaderPushQueue, 1, params.leaderPushBatchSize)
-      yield* clientSession.leaderThread.events.push(batch).pipe(
-        Effect.catchTag('LeaderAheadError', () => {
-          debugInfo.rejectCount++
-          return BucketQueue.clear(leaderPushQueue)
-        }),
-      )
-    }).pipe(Effect.forever, Effect.interruptible, Effect.tapCauseLogPretty)
+        terminalPushCause ??= cause
+        return shutdownStarted === true
+          ? Effect.void
+          : clientSession.shutdown(Exit.failCause(cause)).pipe(Effect.forkDetach, Effect.asVoid)
+      }),
+    )
 
-    yield* FiberHandle.run(leaderPushingFiberHandle, backgroundLeaderPushing)
+    yield* FiberHandle.run(leaderPushingHandle, backgroundLeaderPushing)
 
     // NOTE We need to lazily call `.pull` as we want the cursor to be updated
-    yield* Stream.suspend(() =>
+    const backgroundPulling = Stream.suspend(() =>
       clientSession.leaderThread.events.pull({ cursor: syncStateRef.current.upstreamHead }),
     ).pipe(
+      Stream.tap(() =>
+        clientSession.devtools.enabled === true ? clientSession.devtools.pullLatch.await : Effect.void,
+      ),
       Stream.tap(({ payload }) =>
         Effect.gen(function* () {
           // yield* Effect.logDebug('ClientSessionSyncProcessor:pull', payload)
 
-          if (clientSession.devtools.enabled) {
-            yield* clientSession.devtools.pullLatch.await
-          }
-
-          const mergeResult = SyncState.merge({
+          const rejectionAtPullStart = unresolvedRejection
+          const mergeResult = yield* SyncState.merge({
             syncState: syncStateRef.current,
             payload,
-            isClientEvent,
-            isEqualEvent: LiveStoreEvent.isEqualEncoded,
-          })
-
-          if (mergeResult._tag === 'unexpected-error') {
-            return yield* new UnexpectedError({ cause: mergeResult.message })
-          } else if (mergeResult._tag === 'reject') {
-            return shouldNeverHappen('Unexpected reject in client-session-sync-processor', mergeResult)
-          }
+            isClientOnlyEvent,
+            isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
+          }).pipe(
+            Effect.filterOrElse(
+              (r) => r._tag !== 'reject',
+              () => Effect.die(new Error('Unexpected reject in client-session-sync-processor')),
+            ),
+          )
 
           syncStateRef.current = mergeResult.newSyncState
 
+          const recoveredRejection =
+            rejectionAtPullStart !== undefined &&
+            unresolvedRejection === rejectionAtPullStart &&
+            isRejectedBatchRecovered(rejectionAtPullStart.events, mergeResult.newSyncState.pending) === true
+
           if (mergeResult._tag === 'rebase') {
-            span.addEvent('merge:pull:rebase', {
+            yield* Effect.spanEvent('merge:pull:rebase', {
               payloadTag: payload._tag,
-              payload: TRACE_VERBOSE ? JSON.stringify(payload) : undefined,
+              ...(TRACE_VERBOSE === true ? { payload: jsonStringify(payload) } : {}),
               newEventsCount: mergeResult.newEvents.length,
               rollbackCount: mergeResult.rollbackEvents.length,
-              res: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
+              ...(TRACE_VERBOSE === true ? { res: jsonStringify(mergeResult) } : {}),
             })
 
             debugInfo.rebaseCount++
 
-            if (SIMULATION_ENABLED) yield* simSleep('pull', '1_before_leader_push_fiber_interrupt')
-
-            yield* FiberHandle.clear(leaderPushingFiberHandle)
-
-            if (SIMULATION_ENABLED) yield* simSleep('pull', '2_before_leader_push_queue_clear')
+            // Barrier: before we interrupt the in-flight leader-push worker.
+            yield* rebaseBarrier('before_leader_push_fiber_interrupt')
 
-            // Reset the leader push queue since we're rebasing and will push again
-            yield* BucketQueue.clear(leaderPushQueue)
+            yield* FiberHandle.clear(leaderPushingHandle)
 
-            if (SIMULATION_ENABLED) yield* simSleep('pull', '3_before_rebase_rollback')
-
-            if (LS_DEV) {
+            if (LS_DEV === true) {
               yield* Effect.logDebug(
                 'merge:pull:rebase: rollback',
                 mergeResult.rollbackEvents.length,
@@ -271,6 +280,8 @@ export const makeClientSessionSyncProcessor = ({
               )
             }
 
+            // Roll back the optimistic session changesets for the events this rebase discards.
+            // (Independent of the push queue below; order relative to the queue reconcile is irrelevant.)
             for (let i = mergeResult.rollbackEvents.length - 1; i >= 0; i--) {
               const event = mergeResult.rollbackEvents[i]!
               if (event.meta.sessionChangeset._tag !== 'no-op' && event.meta.sessionChangeset._tag !== 'unset') {
@@ -278,41 +289,68 @@ export const makeClientSessionSyncProcessor = ({
                 event.meta.sessionChangeset = { _tag: 'unset' }
               }
             }
+            if (mergeResult.rollbackEvents.length > 0) {
+              yield* stateHead
+                .set(mergeResult.rollbackEvents[0]!.parentSeqNum)
+                .pipe(Effect.mapError((cause) => MaterializeError.make({ cause })))
+            }
 
-            if (SIMULATION_ENABLED) yield* simSleep('pull', '4_before_leader_push_queue_offer')
-
-            yield* BucketQueue.offerAll(leaderPushQueue, mergeResult.newSyncState.pending)
-
-            if (SIMULATION_ENABLED) yield* simSleep('pull', '5_before_leader_push_fiber_run')
-
-            yield* FiberHandle.run(leaderPushingFiberHandle, backgroundLeaderPushing)
+            // Barrier: before the atomic queue reconciliation (the "discard + re-offer" step).
+            // A `push` admitted here appends its event to `syncStateRef.current.pending` AND to
+            // `leaderPushQueue`; the reconciliation below re-reads the current pending events, so
+            // that event is preserved rather than removed by the queue clear.
+            yield* rebaseBarrier('before_queue_reconcile')
+
+            // Atomic queue reconciliation. `push` runs via `Effect.runSyncWith` (a synchronous run,
+            // per the store's fully-synchronous commit contract), so it executes as an indivisible
+            // unit that can only interleave in THIS fiber's async gaps — never inside a synchronous
+            // stretch. By reading the current pending events, clearing, and re-offering with no async
+            // park between them, this block is atomic w.r.t. `push`. This replaces the blocking
+            // `pullReconciliationMutex` permit that previously forced `push` to wait: queue rebuilding
+            // and synchronous commits cannot interleave, without suspending the commit path.
+            //
+            // We re-read the current `syncStateRef.current.pending` instead of the stale
+            // `mergeResult.newSyncState.pending` snapshot captured at merge time, so any event a
+            // concurrent push appended during the async steps above (fiber interrupt / rollback /
+            // barriers) is included. `Effect.tx` commits the clear+offer as one transaction.
+            yield* reconcileLeaderPushQueue
+
+            // Barrier: before restarting the leader-push worker.
+            yield* rebaseBarrier('before_leader_push_fiber_run')
+
+            if (recoveredRejection === true) unresolvedRejection = undefined
+            yield* FiberHandle.run(leaderPushingHandle, backgroundLeaderPushing)
           } else {
-            span.addEvent('merge:pull:advance', {
+            yield* Effect.spanEvent('merge:pull:advance', {
               payloadTag: payload._tag,
-              payload: TRACE_VERBOSE ? JSON.stringify(payload) : undefined,
+              ...(TRACE_VERBOSE === true ? { payload: jsonStringify(payload) } : {}),
               newEventsCount: mergeResult.newEvents.length,
-              res: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
+              ...(TRACE_VERBOSE === true ? { res: jsonStringify(mergeResult) } : {}),
             })
 
             debugInfo.advanceCount++
+
+            if (recoveredRejection === true) {
+              yield* FiberHandle.clear(leaderPushingHandle)
+              yield* reconcileLeaderPushQueue
+              unresolvedRejection = undefined
+              yield* FiberHandle.run(leaderPushingHandle, backgroundLeaderPushing)
+            }
           }
 
           if (mergeResult.newEvents.length === 0) {
             // If there are no new events, we need to update the sync state as well
-            yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
+            yield* Queue.offer(syncStateUpdateQueue, mergeResult.newSyncState)
             return
           }
 
           const writeTables = new Set<string>()
           for (const event of mergeResult.newEvents) {
-            // TODO apply changeset if available (will require tracking of write tables as well)
-            const decodedEventDef = Schema.decodeSync(eventSchema)(event)
             const {
               writeTables: newWriteTables,
               sessionChangeset,
               materializerHash,
-            } = yield* materializeEvent(decodedEventDef, {
-              withChangeset: true,
+            } = yield* materializeEvent(event, {
               materializerHashLeader: event.meta.materializerHashLeader,
             })
             for (const table of newWriteTables) {
@@ -326,10 +364,11 @@ export const makeClientSessionSyncProcessor = ({
           refreshTables(writeTables)
 
           // We're only triggering the sync state update after all events have been materialized
-          yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
+          yield* Queue.offer(syncStateUpdateQueue, mergeResult.newSyncState)
         }).pipe(
+          pullReconciliationMutex.withPermits(1),
           Effect.tapCauseLogPretty,
-          Effect.catchAllCause((cause) => clientSession.shutdown(Exit.failCause(cause))),
+          Effect.catchCause((cause) => clientSession.shutdown(Exit.failCause(cause))),
         ),
       ),
       Stream.runDrain,
@@ -337,49 +376,210 @@ export const makeClientSessionSyncProcessor = ({
       Effect.interruptible,
       Effect.withSpan('client-session-sync-processor:pull'),
       Effect.tapCauseLogPretty,
-      Effect.forkScoped,
     )
+    yield* FiberHandle.run(pullingHandle, backgroundPulling)
+  }).pipe(Effect.withSpan('client-session-sync-processor:boot'))
+
+  const runShutdown = Effect.fn('client-session-sync-processor:shutdown')(function* (
+    exit: Exit.Exit<unknown, unknown>,
+  ) {
+    if (Exit.isFailure(exit) === true) {
+      if (pullingFiberHandle !== undefined) yield* FiberHandle.clear(pullingFiberHandle)
+      yield* pullReconciliationMutex.withPermits(1)(TxQueue.end(leaderPushQueue))
+      if (leaderPushingFiberHandle !== undefined) yield* FiberHandle.clear(leaderPushingFiberHandle)
+      return
+    }
+
+    yield* pullReconciliationMutex.withPermits(1)(
+      Effect.gen(function* () {
+        if (pullingFiberHandle !== undefined) yield* FiberHandle.clear(pullingFiberHandle)
+        yield* TxQueue.end(leaderPushQueue)
+        yield* Deferred.succeed(drainStartedSignal, undefined)
+      }),
+    )
+    if (leaderPushingFiberHandle !== undefined) {
+      if (unresolvedRejection === undefined) {
+        yield* FiberHandle.awaitEmpty(leaderPushingFiberHandle)
+      } else {
+        yield* FiberHandle.clear(leaderPushingFiberHandle)
+      }
+    }
+    if (terminalPushCause !== undefined) return yield* Effect.failCause(terminalPushCause)
+    if (unresolvedRejection !== undefined) return yield* Effect.die(unresolvedRejection.error)
   })
 
+  const shutdown: ClientSessionSyncProcessor['shutdown'] = (exit) =>
+    Effect.suspend(() => {
+      if (shutdownStarted === true) return Deferred.await(shutdownDone)
+      shutdownStarted = true
+      return runShutdown(exit).pipe(
+        Effect.exit,
+        Effect.tap((shutdownExit) => Deferred.done(shutdownDone, shutdownExit)),
+        Effect.forkDetach,
+        Effect.andThen(Deferred.await(shutdownDone)),
+      )
+    })
+
+  const encodeEvents: ClientSessionSyncProcessor['encodeEvents'] = Effect.fn(
+    'client-session-sync-processor:encode-events',
+  )(function* (events) {
+    let baseEventSequenceNumber = syncStateRef.current.localHead
+    return yield* Effect.forEach(events, ({ name, args }) =>
+      Effect.gen(function* () {
+        const eventDef = yield* Effect.fromNullishOr(schema.eventsDefsMap.get(name)).pipe(Effect.orDieDebugger)
+        const nextNumPair = EventSequenceNumber.Client.nextPair({
+          seqNum: baseEventSequenceNumber,
+          isClientOnly: eventDef.options.clientOnly,
+          rebaseGeneration: baseEventSequenceNumber.rebaseGeneration,
+        })
+        baseEventSequenceNumber = nextNumPair.seqNum
+        // Encoding known-valid domain data: an encode failure is an invariant violation (a defect),
+        // so `Effect.orDie` is the correct modeling — it keeps the typed error channel narrow.
+        const encoded = yield* Schema.encodeUnknownEffect(eventSchema)({
+          name,
+          // Client-document events expose SessionIdSymbol as an input placeholder, but encoded events are persisted
+          // and replayed by concrete id. Resolve during schema encoding so commit never mutates the caller's event.
+          args: resolveSessionIdSymbolInEventArgs(args, clientSession.sessionId),
+          ...nextNumPair,
+          clientId: clientSession.clientId,
+          sessionId: clientSession.sessionId,
+        }).pipe(Effect.orDie)
+        return new LiveStoreEvent.Client.EncodedWithMeta(encoded)
+      }),
+    )
+  })
+
+  const materializeEvents: ClientSessionSyncProcessor['materializeEvents'] = Effect.fn(
+    'client-session-sync-processor:materialize-events',
+  )(function* (events) {
+    const writeTables = new Set<string>()
+    for (const event of events) {
+      const {
+        writeTables: newWriteTables,
+        sessionChangeset,
+        materializerHash,
+      } = yield* materializeEvent(event, {
+        materializerHashLeader: Option.none(),
+      })
+      for (const table of newWriteTables) {
+        writeTables.add(table)
+      }
+      event.meta.sessionChangeset = sessionChangeset
+      event.meta.materializerHashSession = materializerHash
+    }
+    return { writeTables }
+  })
+
+  const push: ClientSessionSyncProcessor['push'] = Effect.fn('client-session-sync-processor:push')(
+    function* (encodedEvents) {
+      if (shutdownStarted === true) {
+        return yield* Effect.die(
+          new Error('Cannot push events after the client session sync processor starts shutting down'),
+        )
+      }
+
+      const mergeResult = yield* SyncState.merge({
+        syncState: syncStateRef.current,
+        payload: { _tag: 'local-push', newEvents: encodedEvents },
+        isClientOnlyEvent,
+        isEqualEvent: LiveStoreEvent.Client.isEqualEncoded,
+      }).pipe(
+        Effect.filterMapOrElse(Filter.tagged<typeof SyncState.MergeResult.Type>()('advance'), () =>
+          Effect.die(new Error('Expected advance from local-push merge')),
+        ),
+      )
+
+      yield* Effect.annotateCurrentSpan({
+        batchSize: encodedEvents.length,
+        mergeResultTag: mergeResult._tag,
+        eventCounts: encodedEvents.reduce<Record<string, number>>((acc, event) => {
+          acc[event.name] = (acc[event.name] ?? 0) + 1
+          return acc
+        }, {}),
+        ...(TRACE_VERBOSE === true ? { mergeResult: jsonStringify(mergeResult) } : {}),
+      })
+
+      syncStateRef.current = mergeResult.newSyncState
+      yield* Queue.offer(syncStateUpdateQueue, mergeResult.newSyncState)
+      const rejectedEvents = yield* TxQueue.offerAll(leaderPushQueue, mergeResult.newEvents)
+      if (rejectedEvents.length > 0) {
+        return yield* Effect.die(new Error('Leader push queue closed while accepting events'))
+      }
+    },
+  )
+
+  const debugInfo = {
+    rebaseCount: 0,
+    advanceCount: 0,
+    rejectCount: 0,
+  }
+
   return {
-    push,
     boot,
+    shutdown,
+    encodeEvents,
+    materializeEvents,
+    push,
     syncState: Subscribable.make({
-      get: Effect.gen(function* () {
-        const syncState = syncStateRef.current
-        if (syncStateRef === undefined) return shouldNeverHappen('Not initialized')
-        return syncState
-      }),
+      get: Effect.sync(() => syncStateRef.current),
       changes: Stream.fromQueue(syncStateUpdateQueue),
     }),
     debug: {
+      awaitDrainStarted: Deferred.await(drainStartedSignal),
+      awaitRejection: Deferred.await(rejectionObserved),
       print: () =>
         Effect.gen(function* () {
           console.log('debugInfo', debugInfo)
           console.log('syncState', syncStateRef.current)
-          const pushQueueSize = yield* BucketQueue.size(leaderPushQueue)
-          console.log('pushQueueSize', pushQueueSize)
-          const pushQueueItems = yield* BucketQueue.peekAll(leaderPushQueue)
+          const pushQueueItems = yield* snapshotTxQueue(leaderPushQueue)
+          console.log('pushQueueSize', pushQueueItems.length)
           console.log(
             'pushQueueItems',
             pushQueueItems.map((_) => _.toJSON()),
           )
-        }).pipe(Effect.provide(runtime), Effect.runSync),
+        }).pipe(Effect.runSync),
       debugInfo: () => debugInfo,
     },
   } satisfies ClientSessionSyncProcessor
-}
+})
+
+const snapshotTxQueue = <A, E>(queue: TxQueue.TxQueue<A, E>): Effect.Effect<ReadonlyArray<A>, E> =>
+  Effect.tx(
+    Effect.gen(function* () {
+      if ((yield* TxQueue.isOpen(queue)) === false) return []
+
+      const items = yield* TxQueue.clear(queue)
+      yield* TxQueue.offerAll(queue, items)
+      return items
+    }),
+  )
+
+const isRejectedBatchRecovered = (
+  rejectedEvents: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>,
+  pendingEvents: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>,
+): boolean =>
+  rejectedEvents.every(
+    (rejectedEvent) =>
+      pendingEvents.some((pendingEvent) => LiveStoreEvent.Client.isEqualEncoded(pendingEvent, rejectedEvent)) === false,
+  )
 
 export interface ClientSessionSyncProcessor {
-  push: (
-    batch: ReadonlyArray<LiveStoreEvent.PartialAnyDecoded>,
+  boot: Effect.Effect<void, never, Scope.Scope>
+  shutdown: (exit: Exit.Exit<unknown, unknown>) => Effect.Effect<void>
+  encodeEvents: (
+    events: ReadonlyArray<LiveStoreEvent.Input.Decoded>,
+  ) => Effect.Effect<ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>>
+  push: (events: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>) => Effect.Effect<void>
+  materializeEvents: (
+    events: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>,
   ) => Effect.Effect<{ writeTables: Set<string> }, MaterializeError>
-  boot: Effect.Effect<void, UnexpectedError, Scope.Scope>
   /**
    * Only used for debugging / observability.
    */
   syncState: Subscribable.Subscribable<SyncState.SyncState>
   debug: {
+    awaitDrainStarted: Effect.Effect<void>
+    awaitRejection: Effect.Effect<void>
     print: () => void
     debugInfo: () => {
       rebaseCount: number
@@ -388,17 +588,11 @@ export interface ClientSessionSyncProcessor {
   }
 }
 
-// TODO turn this into a build-time "macro" so all simulation snippets are removed for production builds
-const SIMULATION_ENABLED = true
-
-// Warning: High values for the simulation params can lead to very long test runs since those get multiplied with the number of events
-export const ClientSessionSyncProcessorSimulationParams = Schema.Struct({
-  pull: Schema.Struct({
-    '1_before_leader_push_fiber_interrupt': Schema.Int.pipe(Schema.between(0, 15)),
-    '2_before_leader_push_queue_clear': Schema.Int.pipe(Schema.between(0, 15)),
-    '3_before_rebase_rollback': Schema.Int.pipe(Schema.between(0, 15)),
-    '4_before_leader_push_queue_offer': Schema.Int.pipe(Schema.between(0, 15)),
-    '5_before_leader_push_fiber_run': Schema.Int.pipe(Schema.between(0, 15)),
-  }),
-})
-type ClientSessionSyncProcessorSimulationParams = typeof ClientSessionSyncProcessorSimulationParams.Type
+/**
+ * Injection points inside the rebase critical section where a test-only barrier may be awaited.
+ * Named for the step they precede so the deterministic no-loss tests can target the exact window.
+ */
+export type RebaseBarrierPoint =
+  | 'before_leader_push_fiber_interrupt'
+  | 'before_queue_reconcile'
+  | 'before_leader_push_fiber_run'