@livestore/common 0.4.0-dev.2 → 0.4.0-dev.5

package/src/leader-thread/LeaderSyncProcessor.ts

@@ -2,10 +2,12 @@ import { casesHandled, isNotUndefined, LS_DEV, shouldNeverHappen, TRACE_VERBOSE
 import type { HttpClient, Runtime, Scope, Tracer } from '@livestore/utils/effect'
 import {
   BucketQueue,
+  Cause,
   Deferred,
   Effect,
   Exit,
   FiberHandle,
+  Layer,
   Option,
   OtelTracer,
   pipe,
@@ -16,13 +18,22 @@ import {
   SubscriptionRef,
 } from '@livestore/utils/effect'
 import type * as otel from '@opentelemetry/api'
-
-import type { SqliteDb } from '../adapter-types.ts'
-import { SyncError, UnexpectedError } from '../adapter-types.ts'
+import {
+  type IntentionalShutdownCause,
+  type MaterializeError,
+  type SqliteDb,
+  UnexpectedError,
+} from '../adapter-types.ts'
 import { makeMaterializerHash } from '../materializer-helper.ts'
 import type { LiveStoreSchema } from '../schema/mod.ts'
 import { EventSequenceNumber, getEventDef, LiveStoreEvent, SystemTables } from '../schema/mod.ts'
-import { LeaderAheadError } from '../sync/sync.ts'
+import {
+  type InvalidPullError,
+  type InvalidPushError,
+  type IsOfflineError,
+  LeaderAheadError,
+  type SyncBackend,
+} from '../sync/sync.ts'
 import * as SyncState from '../sync/syncstate.ts'
 import { sql } from '../util.ts'
 import * as Eventlog from './eventlog.ts'
@@ -71,6 +82,7 @@ export const makeLeaderSyncProcessor = ({
   initialBlockingSyncContext,
   initialSyncState,
   onError,
+  livePull,
   params,
   testing,
 }: {
@@ -90,6 +102,8 @@ export const makeLeaderSyncProcessor = ({
      */
     backendPushBatchSize?: number
   }
+  /** * Whether the sync backend should reactively pull new events from the sync backend */
+  livePull: boolean
   testing: {
     delays?: {
       localPushProcessing?: Effect.Effect<void>
@@ -224,12 +238,31 @@ export const makeLeaderSyncProcessor = ({
         }
       }
 
-      const shutdownOnError = (cause: unknown) =>
+      const maybeShutdownOnError = (
+        cause: Cause.Cause<
+          | UnexpectedError
+          | IntentionalShutdownCause
+          | IsOfflineError
+          | InvalidPushError
+          | InvalidPullError
+          | MaterializeError
+        >,
+      ) =>
         Effect.gen(function* () {
-          if (onError === 'shutdown') {
-            yield* shutdownChannel.send(UnexpectedError.make({ cause }))
-            yield* Effect.die(cause)
+          if (onError === 'ignore') {
+            if (LS_DEV) {
+              yield* Effect.logDebug(
+                `Ignoring sync error (${cause._tag === 'Fail' ? cause.error._tag : cause._tag})`,
+                Cause.pretty(cause),
+              )
+            }
+            return
           }
+
+          const errorToSend = Cause.isFailType(cause) ? cause.error : UnexpectedError.make({ cause })
+          yield* shutdownChannel.send(errorToSend).pipe(Effect.orDie)
+
+          return yield* Effect.die(cause)
         })
 
       yield* backgroundApplyLocalPushes({
@@ -246,20 +279,19 @@ export const makeLeaderSyncProcessor = ({
         testing: {
           delay: testing?.delays?.localPushProcessing,
         },
-      }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped)
+      }).pipe(Effect.catchAllCause(maybeShutdownOnError), Effect.forkScoped)
 
-      const backendPushingFiberHandle = yield* FiberHandle.make()
+      const backendPushingFiberHandle = yield* FiberHandle.make<undefined, never>()
       const backendPushingEffect = backgroundBackendPushing({
         syncBackendPushQueue,
         otelSpan,
         devtoolsLatch: ctxRef.current?.devtoolsLatch,
         backendPushBatchSize,
-      }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError))
+      }).pipe(Effect.catchAllCause(maybeShutdownOnError))
 
       yield* FiberHandle.run(backendPushingFiberHandle, backendPushingEffect)
 
       yield* backgroundBackendPulling({
-        initialBackendHead: initialSyncState.upstreamHead.global,
         isClientEvent,
         restartBackendPushing: (filteredRebasedPending) =>
           Effect.gen(function* () {
@@ -276,13 +308,24 @@ export const makeLeaderSyncProcessor = ({
         syncStateSref,
         localPushesLatch,
         pullLatch,
+        livePull,
         dbState,
         otelSpan,
         initialBlockingSyncContext,
         devtoolsLatch: ctxRef.current?.devtoolsLatch,
         connectedClientSessionPullQueues,
         advancePushHead,
-      }).pipe(Effect.tapCauseLogPretty, Effect.catchAllCause(shutdownOnError), Effect.forkScoped)
+      }).pipe(
+        Effect.retry({
+          // We want to retry pulling if we've lost connection to the sync backend
+          while: (cause) => cause._tag === 'IsOfflineError',
+        }),
+        Effect.catchAllCause(maybeShutdownOnError),
+        // Needed to avoid `Fiber terminated with an unhandled error` logs which seem to happen because of the `Effect.retry` above.
+        // This might be a bug in Effect. Only seems to happen in the browser.
+        Effect.provide(Layer.setUnhandledErrorLogLevel(Option.none())),
+        Effect.forkScoped,
+      )
 
       return { initialLeaderHead: initialSyncState.localHead }
     }).pipe(Effect.withSpanScoped('@livestore/common:LeaderSyncProcessor:boot'))
@@ -405,7 +448,7 @@ const backgroundApplyLocalPushes = ({
             batchSize: newEvents.length,
             newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
           })
-          return yield* new SyncError({ cause: mergeResult.message })
+          return yield* new UnexpectedError({ cause: mergeResult.message })
         }
         case 'rebase': {
           return shouldNeverHappen('The leader thread should never have to rebase due to a local push')
@@ -496,7 +539,7 @@ type MaterializeEventsBatch = (_: {
    * Indexes are aligned with `batchItems`
    */
   deferreds: ReadonlyArray<Deferred.Deferred<void, LeaderAheadError> | undefined> | undefined
-}) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
+}) => Effect.Effect<void, MaterializeError, LeaderThreadCtx>
 
 // TODO how to handle errors gracefully
 const materializeEventsBatch: MaterializeEventsBatch = ({ batchItems, deferreds }) =>
@@ -536,24 +579,22 @@ const materializeEventsBatch: MaterializeEventsBatch = ({ batchItems, deferreds
       attributes: { batchSize: batchItems.length },
     }),
     Effect.tapCauseLogPretty,
-    UnexpectedError.mapToUnexpectedError,
   )
 
 const backgroundBackendPulling = ({
-  initialBackendHead,
   isClientEvent,
   restartBackendPushing,
   otelSpan,
   dbState,
   syncStateSref,
   localPushesLatch,
+  livePull,
   pullLatch,
   devtoolsLatch,
   initialBlockingSyncContext,
   connectedClientSessionPullQueues,
   advancePushHead,
 }: {
-  initialBackendHead: EventSequenceNumber.GlobalEventSequenceNumber
   isClientEvent: (eventEncoded: LiveStoreEvent.EncodedWithMeta) => boolean
   restartBackendPushing: (
     filteredRebasedPending: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>,
@@ -563,6 +604,7 @@ const backgroundBackendPulling = ({
   dbState: SqliteDb
   localPushesLatch: Effect.Latch
   pullLatch: Effect.Latch
+  livePull: boolean
   devtoolsLatch: Effect.Latch | undefined
   initialBlockingSyncContext: InitialBlockingSyncContext
   connectedClientSessionPullQueues: PullQueueSet
@@ -573,7 +615,7 @@ const backgroundBackendPulling = ({
 
     if (syncBackend === undefined) return
 
-    const onNewPullChunk = (newEvents: LiveStoreEvent.EncodedWithMeta[], remaining: number) =>
+    const onNewPullChunk = (newEvents: LiveStoreEvent.EncodedWithMeta[], pageInfo: SyncBackend.PullResPageInfo) =>
       Effect.gen(function* () {
         if (newEvents.length === 0) return
 
@@ -605,7 +647,7 @@ const backgroundBackendPulling = ({
             newEventsCount: newEvents.length,
             newEvents: TRACE_VERBOSE ? JSON.stringify(newEvents) : undefined,
           })
-          return yield* new SyncError({ cause: mergeResult.message })
+          return yield* new UnexpectedError({ cause: mergeResult.message })
         }
 
         const newBackendHead = newEvents.at(-1)!.seqNum
@@ -657,7 +699,7 @@ const backgroundBackendPulling = ({
                 EventSequenceNumber.isEqual(event.seqNum, confirmedEvent.seqNum),
               ),
             )
-            yield* Eventlog.updateSyncMetadata(confirmedNewEvents)
+            yield* Eventlog.updateSyncMetadata(confirmedNewEvents).pipe(UnexpectedError.mapToUnexpectedError)
           }
         }
 
@@ -671,18 +713,20 @@ const backgroundBackendPulling = ({
         yield* SubscriptionRef.set(syncStateSref, mergeResult.newSyncState)
 
         // Allow local pushes to be processed again
-        if (remaining === 0) {
+        if (pageInfo._tag === 'NoMore') {
           yield* localPushesLatch.open
         }
       })
 
-    const cursorInfo = yield* Eventlog.getSyncBackendCursorInfo({ remoteHead: initialBackendHead })
+    const syncState = yield* syncStateSref
+    if (syncState === undefined) return shouldNeverHappen('Not initialized')
+    const cursorInfo = yield* Eventlog.getSyncBackendCursorInfo({ remoteHead: syncState.upstreamHead.global })
 
     const hashMaterializerResult = makeMaterializerHash({ schema, dbState })
 
-    yield* syncBackend.pull(cursorInfo).pipe(
+    yield* syncBackend.pull(cursorInfo, { live: livePull }).pipe(
       // TODO only take from queue while connected
-      Stream.tap(({ batch, remaining }) =>
+      Stream.tap(({ batch, pageInfo }) =>
         Effect.gen(function* () {
           // yield* Effect.spanEvent('batch', {
           //   attributes: {
@@ -690,12 +734,10 @@ const backgroundBackendPulling = ({
           //     batch: TRACE_VERBOSE ? batch : undefined,
           //   },
           // })
-
           // NOTE we only want to take process events when the sync backend is connected
           // (e.g. needed for simulating being offline)
           // TODO remove when there's a better way to handle this in stream above
           yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
-
           yield* onNewPullChunk(
             batch.map((_) =>
               LiveStoreEvent.EncodedWithMeta.fromGlobal(_.eventEncoded, {
@@ -706,10 +748,9 @@ const backgroundBackendPulling = ({
                 materializerHashSession: Option.none(),
               }),
             ),
-            remaining,
+            pageInfo,
           )
-
-          yield* initialBlockingSyncContext.update({ processed: batch.length, remaining })
+          yield* initialBlockingSyncContext.update({ processed: batch.length, pageInfo })
         }),
       ),
       Stream.runDrain,
@@ -752,6 +793,14 @@ const backgroundBackendPushing = ({
       const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either)
 
       if (pushResult._tag === 'Left') {
+        if (
+          pushResult.left._tag === 'InvalidPushError' &&
+          // server ahead errors are gracefully handled
+          pushResult.left.cause._tag !== 'ServerAheadError'
+        ) {
+          return yield* pushResult.left
+        }
+
         if (LS_DEV) {
           yield* Effect.logDebug('handled backend-push-error', { error: pushResult.left.toString() })
         }

package/src/leader-thread/eventlog.ts

@@ -123,6 +123,14 @@ export const getBackendHeadFromDb = (dbEventlog: SqliteDb): EventSequenceNumber.
 export const updateBackendHead = (dbEventlog: SqliteDb, head: EventSequenceNumber.EventSequenceNumber) =>
   dbEventlog.execute(sql`UPDATE ${SYNC_STATUS_TABLE} SET head = ${head.global}`)
 
+export const getBackendIdFromDb = (dbEventlog: SqliteDb): Option.Option<string> =>
+  Option.fromNullable(
+    dbEventlog.select<{ backendId: string | null }>(sql`select backendId from ${SYNC_STATUS_TABLE}`)[0]?.backendId,
+  )
+
+export const updateBackendId = (dbEventlog: SqliteDb, backendId: string) =>
+  dbEventlog.execute(sql`UPDATE ${SYNC_STATUS_TABLE} SET backendId = '${backendId}'`)
+
 export const insertIntoEventlog = (
   eventEncoded: LiveStoreEvent.EncodedWithMeta,
   dbEventlog: SqliteDb,
@@ -213,11 +221,7 @@ export const getSyncBackendCursorInfo = ({
     ).pipe(Effect.andThen(Schema.decode(EventlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie)
 
     return Option.some({
-      cursor: {
-        global: remoteHead,
-        client: EventSequenceNumber.clientDefault,
-        rebaseGeneration: EventSequenceNumber.rebaseGenerationDefault,
-      },
+      eventSequenceNumber: remoteHead,
       metadata: syncMetadataOption,
     }) satisfies InitialSyncInfo
   }).pipe(Effect.withSpan('@livestore/common:eventlog:getSyncBackendCursorInfo', { attributes: { remoteHead } }))

package/src/leader-thread/leader-worker-devtools.ts

@@ -262,7 +262,7 @@ const listenToDevtools = ({
 
               if (syncBackend !== undefined) {
                 // TODO consider piggybacking on the existing leader-thread sync-pulling
-                yield* syncBackend.pull(Option.none()).pipe(
+                yield* syncBackend.pull(Option.none(), { live: true }).pipe(
                   Stream.map((_) => _.batch),
                   Stream.flattenIterables,
                   Stream.tap(({ eventEncoded, metadata }) =>

package/src/leader-thread/make-leader-thread-layer.ts

@@ -1,9 +1,15 @@
 import { shouldNeverHappen } from '@livestore/utils'
 import type { HttpClient, Schema, Scope } from '@livestore/utils/effect'
-import { Deferred, Effect, Layer, Queue, SubscriptionRef } from '@livestore/utils/effect'
-
-import type { BootStatus, MakeSqliteDb, SqliteDb, SqliteError } from '../adapter-types.ts'
-import { UnexpectedError } from '../adapter-types.ts'
+import { Deferred, Effect, KeyValueStore, Layer, PlatformError, Queue, SubscriptionRef } from '@livestore/utils/effect'
+import {
+  type BootStatus,
+  type MakeSqliteDb,
+  type MaterializerHashMismatchError,
+  type SqliteDb,
+  type SqliteError,
+  UnexpectedError,
+} from '../adapter-types.ts'
+import type { MigrationsReport } from '../defs.ts'
 import type * as Devtools from '../devtools/mod.ts'
 import type { LiveStoreSchema } from '../schema/mod.ts'
 import { EventSequenceNumber, LiveStoreEvent, SystemTables } from '../schema/mod.ts'
@@ -71,10 +77,45 @@ export const makeLeaderThreadLayer = ({
     // Either happens on initial boot or if schema changes
     const dbStateMissing = !hasStateTables(dbState)
 
+    yield* Eventlog.initEventlogDb(dbEventlog)
+
     const syncBackend =
       syncOptions?.backend === undefined
         ? undefined
-        : yield* syncOptions.backend({ storeId, clientId, payload: syncPayload })
+        : yield* syncOptions.backend({ storeId, clientId, payload: syncPayload }).pipe(
+            Effect.provide(
+              Layer.succeed(
+                KeyValueStore.KeyValueStore,
+                KeyValueStore.makeStringOnly({
+                  get: (_key) =>
+                    Effect.sync(() => Eventlog.getBackendIdFromDb(dbEventlog)).pipe(
+                      Effect.catchAllDefect((cause) =>
+                        PlatformError.BadArgument.make({
+                          method: 'getBackendIdFromDb',
+                          description: 'Failed to get backendId',
+                          module: 'KeyValueStore',
+                          cause,
+                        }),
+                      ),
+                    ),
+                  set: (_key, value) =>
+                    Effect.sync(() => Eventlog.updateBackendId(dbEventlog, value)).pipe(
+                      Effect.catchAllDefect((cause) =>
+                        PlatformError.BadArgument.make({
+                          method: 'updateBackendId',
+                          module: 'KeyValueStore',
+                          description: 'Failed to update backendId',
+                          cause,
+                        }),
+                      ),
+                    ),
+                  clear: Effect.dieMessage(`Not implemented. Should never be used.`),
+                  remove: () => Effect.dieMessage(`Not implemented. Should never be used.`),
+                  size: Effect.dieMessage(`Not implemented. Should never be used.`),
+                }),
+              ),
+            ),
+          )
 
     if (syncBackend !== undefined) {
       // We're already connecting to the sync backend concurrently
@@ -86,12 +127,21 @@ export const makeLeaderThreadLayer = ({
       bootStatusQueue,
     })
 
+    const materializeEvent = yield* makeMaterializeEvent({ schema, dbState, dbEventlog })
+
+    // Recreate state database if needed BEFORE creating sync processor
+    // This ensures all system tables exist before any queries are made
+    const { migrationsReport } = dbStateMissing
+      ? yield* recreateDb({ dbState, dbEventlog, schema, bootStatusQueue, materializeEvent })
+      : { migrationsReport: { migrations: [] } }
+
     const syncProcessor = yield* makeLeaderSyncProcessor({
       schema,
       dbState,
       initialSyncState: getInitialSyncState({ dbEventlog, dbState, dbEventlogMissing }),
       initialBlockingSyncContext,
       onError: syncOptions?.onSyncError ?? 'ignore',
+      livePull: syncOptions?.livePull ?? true,
       params: {
         localPushBatchSize: params?.localPushBatchSize,
         backendPushBatchSize: params?.backendPushBatchSize,
@@ -113,8 +163,6 @@ export const makeLeaderThreadLayer = ({
         }
       : { enabled: false as const }
 
-    const materializeEvent = yield* makeMaterializeEvent({ schema, dbState, dbEventlog })
-
     const ctx = {
       schema,
       bootStatusQueue,
@@ -141,7 +189,7 @@ export const makeLeaderThreadLayer = ({
     const layer = Layer.succeed(LeaderThreadCtx, ctx)
 
     ctx.initialState = yield* bootLeaderThread({
-      dbStateMissing,
+      migrationsReport,
       initialBlockingSyncContext,
       devtoolsOptions,
     }).pipe(Effect.provide(layer))
@@ -244,12 +292,12 @@ const makeInitialBlockingSyncContext = ({
 
     return {
       blockingDeferred,
-      update: ({ processed, remaining }) =>
+      update: ({ processed, pageInfo }) =>
         Effect.gen(function* () {
           if (ctx.isDone === true) return
 
-          if (ctx.total === -1) {
-            ctx.total = remaining + processed
+          if (ctx.total === -1 && pageInfo._tag === 'MoreKnown') {
+            ctx.total = pageInfo.remaining + processed
           }
 
           ctx.processedEvents += processed
@@ -258,7 +306,7 @@ const makeInitialBlockingSyncContext = ({
             progress: { done: ctx.processedEvents, total: ctx.total },
           })
 
-          if (remaining === 0 && blockingDeferred !== undefined) {
+          if (pageInfo._tag === 'NoMore' && blockingDeferred !== undefined) {
             yield* Deferred.succeed(blockingDeferred, void 0)
             ctx.isDone = true
           }
@@ -271,26 +319,20 @@ const makeInitialBlockingSyncContext = ({
  * It also starts various background processes (e.g. syncing)
  */
 const bootLeaderThread = ({
-  dbStateMissing,
+  migrationsReport,
   initialBlockingSyncContext,
   devtoolsOptions,
 }: {
-  dbStateMissing: boolean
+  migrationsReport: MigrationsReport
   initialBlockingSyncContext: InitialBlockingSyncContext
   devtoolsOptions: DevtoolsOptions
 }): Effect.Effect<
   LeaderThreadCtx['Type']['initialState'],
-  UnexpectedError | SqliteError | IsOfflineError | InvalidPullError,
+  UnexpectedError | SqliteError | IsOfflineError | InvalidPullError | MaterializerHashMismatchError,
   LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
 > =>
   Effect.gen(function* () {
-    const { dbEventlog, bootStatusQueue, syncProcessor, schema, materializeEvent, dbState } = yield* LeaderThreadCtx
-
-    yield* Eventlog.initEventlogDb(dbEventlog)
-
-    const { migrationsReport } = dbStateMissing
-      ? yield* recreateDb({ dbState, dbEventlog, schema, bootStatusQueue, materializeEvent })
-      : { migrationsReport: { migrations: [] } }
+    const { bootStatusQueue, syncProcessor } = yield* LeaderThreadCtx
 
     // NOTE the sync processor depends on the dbs being initialized properly
     const { initialLeaderHead } = yield* syncProcessor.boot

package/src/leader-thread/materialize-event.ts

@@ -1,7 +1,7 @@
 import { isDevEnv, LS_DEV, shouldNeverHappen } from '@livestore/utils'
 import { Effect, Option, ReadonlyArray, Schema } from '@livestore/utils/effect'
 
-import { type SqliteDb, UnexpectedError } from '../adapter-types.ts'
+import { MaterializeError, MaterializerHashMismatchError, type SqliteDb } from '../adapter-types.ts'
 import { getExecStatementsFromMaterializer, hashMaterializerResults } from '../materializer-helper.ts'
 import type { LiveStoreSchema } from '../schema/mod.ts'
 import { EventSequenceNumber, getEventDef, SystemTables } from '../schema/mod.ts'
@@ -11,6 +11,7 @@ import { execSql, execSqlPrepared } from './connection.ts'
 import * as Eventlog from './eventlog.ts'
 import type { MaterializeEvent } from './types.ts'
 
+// TODO refactor `makeMaterializeEvent` to not return an Effect for the constructor as it's not needed
 export const makeMaterializeEvent = ({
   schema,
   dbState,
@@ -19,7 +20,7 @@ export const makeMaterializeEvent = ({
   schema: LiveStoreSchema
   dbState: SqliteDb
   dbEventlog: SqliteDb
-}): Effect.Effect<MaterializeEvent, UnexpectedError> =>
+}): Effect.Effect<MaterializeEvent> =>
   Effect.gen(function* () {
     const eventDefSchemaHashMap = new Map(
       // TODO Running `Schema.hash` can be a bottleneck for larger schemas. There is an opportunity to run this
@@ -49,10 +50,7 @@ export const makeMaterializeEvent = ({
           eventEncoded.meta.materializerHashSession._tag === 'Some' &&
           eventEncoded.meta.materializerHashSession.value !== materializerHash.value
         ) {
-          yield* UnexpectedError.make({
-            cause: `Materializer hash mismatch detected for event "${eventEncoded.name}".`,
-            note: `Please make sure your event materializer is a pure function without side effects.`,
-          })
+          return yield* MaterializerHashMismatchError.make({ eventName: eventEncoded.name })
         }
 
         // NOTE we might want to bring this back if we want to debug no-op events
@@ -126,6 +124,7 @@ export const makeMaterializeEvent = ({
           hash: materializerHash,
         }
       }).pipe(
+        Effect.mapError((cause) => MaterializeError.make({ cause })),
         Effect.withSpan(`@livestore/common:leader-thread:materializeEvent`, {
           attributes: {
             eventName: eventEncoded.name,

package/src/leader-thread/recreate-db.ts

@@ -2,8 +2,16 @@ import { casesHandled } from '@livestore/utils'
 import { Effect, Queue } from '@livestore/utils/effect'
 
 import type { MigrationsReport } from '../defs.ts'
-import type { BootStatus, MigrationHooks, SqliteDb, SqliteError } from '../index.ts'
-import { migrateDb, rematerializeFromEventlog, UnexpectedError } from '../index.ts'
+import {
+  type BootStatus,
+  type MaterializeError,
+  type MigrationHooks,
+  migrateDb,
+  rematerializeFromEventlog,
+  type SqliteDb,
+  type SqliteError,
+  UnexpectedError,
+} from '../index.ts'
 import type { LiveStoreSchema } from '../schema/mod.ts'
 import { configureConnection } from './connection.ts'
 import type { MaterializeEvent } from './types.ts'
@@ -20,7 +28,7 @@ export const recreateDb = ({
   schema: LiveStoreSchema
   bootStatusQueue: Queue.Queue<BootStatus>
   materializeEvent: MaterializeEvent
-}): Effect.Effect<{ migrationsReport: MigrationsReport }, UnexpectedError | SqliteError> =>
+}): Effect.Effect<{ migrationsReport: MigrationsReport }, UnexpectedError | MaterializeError | SqliteError> =>
   Effect.gen(function* () {
     const migrationOptions = schema.state.sqlite.migrations
     let migrationsReport: MigrationsReport

package/src/leader-thread/shutdown-channel.ts

@@ -1,9 +1,23 @@
 import type { WebChannel } from '@livestore/utils/effect'
 import { Schema } from '@livestore/utils/effect'
 
-import { IntentionalShutdownCause, SyncError, UnexpectedError } from '../index.ts'
+import {
+  IntentionalShutdownCause,
+  InvalidPullError,
+  InvalidPushError,
+  IsOfflineError,
+  MaterializeError,
+  UnexpectedError,
+} from '../index.ts'
 
-export class All extends Schema.Union(IntentionalShutdownCause, UnexpectedError, SyncError) {}
+export class All extends Schema.Union(
+  IntentionalShutdownCause,
+  UnexpectedError,
+  IsOfflineError,
+  InvalidPushError,
+  InvalidPullError,
+  MaterializeError,
+) {}
 
 /**
  * Used internally by an adapter to shutdown gracefully.

package/src/leader-thread/types.ts

@@ -13,7 +13,7 @@ import { Context, Schema } from '@livestore/utils/effect'
 import type { MeshNode } from '@livestore/webmesh'
 
 import type { MigrationsReport } from '../defs.ts'
-import type { SqliteError } from '../errors.ts'
+import type { MaterializeError } from '../errors.ts'
 import type {
   BootStatus,
   Devtools,
@@ -43,7 +43,7 @@ export const InitialSyncOptions = Schema.Union(InitialSyncOptionsSkip, InitialSy
 export type InitialSyncOptions = typeof InitialSyncOptions.Type
 
 export type InitialSyncInfo = Option.Option<{
-  cursor: EventSequenceNumber.EventSequenceNumber
+  eventSequenceNumber: EventSequenceNumber.GlobalEventSequenceNumber
   metadata: Option.Option<Schema.JsonValue>
 }>
 
@@ -98,7 +98,7 @@ export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
     shutdownChannel: ShutdownChannel
     eventSchema: LiveStoreEvent.ForEventDefRecord<any>
     devtools: DevtoolsContext
-    syncBackend: SyncBackend | undefined
+    syncBackend: SyncBackend.SyncBackend | undefined
     syncProcessor: LeaderSyncProcessor
     materializeEvent: MaterializeEvent
     initialState: {
@@ -125,12 +125,12 @@ export type MaterializeEvent = (
     sessionChangeset: { _tag: 'sessionChangeset'; data: Uint8Array<ArrayBuffer>; debug: any } | { _tag: 'no-op' }
     hash: Option.Option<number>
   },
-  SqliteError | UnexpectedError
+  MaterializeError
 >
 
 export type InitialBlockingSyncContext = {
   blockingDeferred: Deferred.Deferred<void> | undefined
-  update: (_: { remaining: number; processed: number }) => Effect.Effect<void>
+  update: (_: { pageInfo: SyncBackend.PullResPageInfo; processed: number }) => Effect.Effect<void>
 }
 
 export interface LeaderSyncProcessor {

package/src/materializer-helper.ts

@@ -31,8 +31,13 @@ export const getExecStatementsFromMaterializer = ({
   bindValues: PreparedBindValues
   writeTables: ReadonlySet<string> | undefined
 }> => {
-  const eventArgsDecoded =
-    event.decoded === undefined ? Schema.decodeUnknownSync(eventDef.schema)(event.encoded!.args) : event.decoded.args
+  const eventDecoded =
+    event.decoded === undefined
+      ? {
+          ...event.encoded!,
+          args: Schema.decodeUnknownSync(eventDef.schema)(event.encoded!.args),
+        }
+      : event.decoded
 
   const eventArgsEncoded = isNil(event.decoded?.args)
     ? undefined
@@ -58,11 +63,12 @@ export const getExecStatementsFromMaterializer = ({
   }
 
   const statementResults = fromMaterializerResult(
-    materializer(eventArgsDecoded, {
+    materializer(eventDecoded.args, {
       eventDef,
       query,
       // TODO properly implement this
       currentFacts: new Map(),
+      event: eventDecoded,
     }),
   )
 

package/src/schema/EventDef.ts

@@ -4,6 +4,7 @@ import { Schema } from '@livestore/utils/effect'
 
 import type { BindValues } from '../sql-queries/sql-queries.ts'
 import type { ParamsObject } from '../util.ts'
+import type * as LiveStoreEvent from './LiveStoreEvent.ts'
 import type { QueryBuilder } from './state/sqlite/query-builder/mod.ts'
 
 export type EventDefMap = {
@@ -191,6 +192,8 @@ export type Materializer<TEventDef extends EventDef.AnyWithoutFn = EventDef.AnyW
     eventDef: TEventDef
     /** Can be used to query the current state */
     query: MaterializerContextQuery
+    /** The full LiveStore event with clientId, sessionId, etc. */
+    event: LiveStoreEvent.AnyDecoded
   },
 ) => SingleOrReadonlyArray<MaterializerResult>