@livestore/common 0.4.0-dev.2 → 0.4.0-dev.21

package/src/leader-thread/stream-events.ts

@@ -0,0 +1,201 @@
+import type { Subscribable } from '@livestore/utils/effect'
+import { Chunk, Effect, Option, Queue, Stream } from '@livestore/utils/effect'
+import { EventSequenceNumber, type LiveStoreEvent } from '../schema/mod.ts'
+import type * as SyncState from '../sync/syncstate.ts'
+import * as Eventlog from './eventlog.ts'
+import type { LeaderSqliteDb, StreamEventsOptions } from './types.ts'
+
+/**
+ * Streams events for leader-thread adapters.
+ *
+ * Provides a continuous stream from the eventlog as the upstream head advances.
+ * When an until event is passed in the stream finalizes upon reaching it.
+ *
+ * The batch size is set to 100 by default as this was meassured to provide the
+ * best performance and 1000 as the upper limit.
+ *
+ * Adapters that call this helper:
+ * - `packages/@livestore/adapter-web/src/in-memory/in-memory-adapter.ts`
+ * - `packages/@livestore/adapter-web/src/web-worker/leader-worker/make-leader-worker.ts`
+ * - `packages/@livestore/adapter-node/src/client-session/adapter.ts`
+ * - `packages/@livestore/adapter-node/src/make-leader-worker.ts`
+ * - `packages/@livestore/adapter-cloudflare/src/make-adapter.ts`
+ *
+ * Each caller resolves dependencies inside the leader scope before invoking this helper,
+ * so the stream stays environment-agnostic and does not leak `LeaderThreadCtx` into runtime
+ * entry points such as `Store.eventsStream`.
+ *
+ * Test files:
+ * Unit: `tests/package-common/src/leader-thread/stream-events.test.ts`
+ * Integration: `packages/@livestore/livestore/src/store/store-eventstream.test.ts`
+ * Performance: `tests/perf-eventlog/tests/suites/event-streaming.test.ts`
+ *
+ * Optimization explorations
+ *
+ * In order to alleviate the occurence of many small queries when the syncState
+ * is sequentially progressing quickly we have explored some time-based batching
+ * approaches. It remains to be determined if and when the added complexity of
+ * these approaches are worth the benefit. They come with some drawbacks such as
+ * degraded time to first event or general performance degredation for larger
+ * query steps. These aspects can likely be mitigated with some more work but
+ * that is best assessed when we have a final implementation of event streaming
+ * with support for session and leader level streams.
+ *
+ * Fetch plans into a Sink
+ * https://gist.github.com/slashv/f1223689f2d1171d2eeb60a2823f4c7c
+ *
+ * Fetch plans into sink and decompose into windows
+ * https://gist.github.com/slashv/a8f55f50121c080937f42e44b4039ac8
+ *
+ * Mailbox and Latch approach (suggestion by Tim Smart)
+ * https://gist.github.com/slashv/d6b12395c85415bf0d3363372a1636c3
+ */
+export const streamEventsWithSyncState = ({
+  dbEventlog,
+  syncState,
+  options,
+}: {
+  dbEventlog: LeaderSqliteDb
+  syncState: Subscribable.Subscribable<SyncState.SyncState>
+  options: StreamEventsOptions
+}): Stream.Stream<LiveStoreEvent.Client.Encoded> => {
+  const initialCursor = options.since ?? EventSequenceNumber.Client.ROOT
+  const batchSize = options.batchSize ?? 100
+
+  return Stream.unwrapScoped(
+    Effect.gen(function* () {
+      /**
+       * Single-element Queue allows suspending the event stream until head
+       * advances because Queue.take is a suspending effect. SubscriptionRef in
+       * comparrison lacks a primitive for suspending a stream until a new value
+       * is set and would require polling.
+       *
+       * The use of a sliding Queue here is useful since it ensures only the
+       * lastest head from syncState is the one present on the queue without the
+       * need for manual substitution.
+       */
+      const headQueue = yield* Queue.sliding<EventSequenceNumber.Client.Composite>(1)
+
+      /**
+       * We run a separate fiber which listens to changes in syncState and
+       * offer the latest head to the headQueue. Keeping track of the previous
+       * value is done to prevent syncState changes unrelated to the
+       * upstreamHead triggering empty queries.
+       *
+       * When we implement support for leader and session level streams
+       * this will need to be adapted to support the relevant value from
+       * syncState that we are interested in tracking.
+       */
+      let prevGlobalHead = -1
+      yield* syncState.changes.pipe(
+        Stream.map((state) => state.upstreamHead),
+        Stream.filter((head) => {
+          if (head.global > prevGlobalHead) {
+            prevGlobalHead = head.global
+            return true
+          }
+          return false
+        }),
+        Stream.runForEach((head) => Queue.offer(headQueue, head)),
+        Effect.forkScoped,
+      )
+
+      return Stream.paginateChunkEffect(
+        { cursor: initialCursor, head: EventSequenceNumber.Client.ROOT },
+        ({ cursor, head }) =>
+          Effect.gen(function* () {
+            /**
+             * Early check guards agains:
+             * since === until : Prevent empty query
+             * since > until : Incorrectly inverted interval
+             */
+            if (options.until && EventSequenceNumber.Client.isGreaterThanOrEqual(cursor, options.until)) {
+              return [Chunk.empty(), Option.none()]
+            }
+
+            /**
+             * There are two scenarios where we take the next head from the headQueue:
+             *
+             * 1. We need to wait for the head to advance
+             * The Stream suspends until a new head is available on the headQueue
+             *
+             * 2. Head has advanced during itteration
+             * While itterating towards the lastest head taken from the headQueue
+             * in increments of batchSize it's possible the head could have
+             * advanced. This leads to a suboptimal amount of queries. Therefor we
+             * check if the headQueue is full which tells us that there's a new
+             * head available to take. Example:
+             *
+             * batchSize: 2
+             *
+             * --> head at: e3
+             * First query: e0 -> e2 (two events)
+             * --> head advances to: e4
+             * Second query: e2 -> e3 (one event but we could have taken 2)
+             * --> Take the new head of e4
+             * Third query: e3 -> e4 (unnecessary third query)
+             *
+             *
+             * To define the target, which will be used as the temporary until
+             * marker for the eventlog query, we select the lowest of three possible values:
+             *
+             * hardStop: A user supplied until marker
+             * current cursor + batchSize: A batchSize step towards the latest head from headQueue
+             * nextHead: The latest head from headQueue
+             */
+            const waitForHead = EventSequenceNumber.Client.isGreaterThanOrEqual(cursor, head)
+            const maybeHead = waitForHead
+              ? yield* Queue.take(headQueue).pipe(Effect.map(Option.some))
+              : yield* Queue.poll(headQueue)
+            const nextHead = Option.getOrElse(maybeHead, () => head)
+            const hardStop = options.until?.global ?? Number.POSITIVE_INFINITY
+            const target = EventSequenceNumber.Client.Composite.make({
+              global: Math.min(hardStop, cursor.global + batchSize, nextHead.global),
+              client: EventSequenceNumber.Client.DEFAULT,
+            })
+
+            /**
+             * Eventlog.getEventsFromEventlog returns a Chunk from each
+             * query which is what we emit at each itteration.
+             */
+            const chunk = yield* Eventlog.getEventsFromEventlog({
+              dbEventlog,
+              options: {
+                ...options,
+                since: cursor,
+                until: target,
+              },
+            })
+
+            /**
+             * We construct the state for the following itteration of the stream
+             * loop by setting the current target as the since cursor and pass
+             * along the latest head.
+             *
+             * If we have the reached the user supplied until marker we signal the
+             * finalization of the stream by passing Option.none() instead.
+             */
+            const reachedUntil =
+              options.until !== undefined && EventSequenceNumber.Client.isGreaterThanOrEqual(target, options.until)
+
+            const nextState: Option.Option<{
+              cursor: EventSequenceNumber.Client.Composite
+              head: EventSequenceNumber.Client.Composite
+            }> = reachedUntil ? Option.none() : Option.some({ cursor: target, head: nextHead })
+
+            const spanAttributes = {
+              'livestore.streamEvents.cursor.global': cursor.global,
+              'livestore.streamEvents.target.global': target.global,
+              'livestore.streamEvents.batchSize': batchSize,
+              'livestore.streamEvents.waitedForHead': waitForHead,
+            }
+
+            return yield* Effect.succeed<[Chunk.Chunk<LiveStoreEvent.Client.Encoded>, typeof nextState]>([
+              chunk,
+              nextState,
+            ]).pipe(Effect.withSpan('@livestore/common:streamEvents:segment', { attributes: spanAttributes }))
+          }),
+      )
+    }),
+  )
+}

package/src/leader-thread/types.ts

@@ -13,7 +13,7 @@ import { Context, Schema } from '@livestore/utils/effect'
 import type { MeshNode } from '@livestore/webmesh'
 
 import type { MigrationsReport } from '../defs.ts'
-import type { SqliteError } from '../errors.ts'
+import type { MaterializeError } from '../errors.ts'
 import type {
   BootStatus,
   Devtools,
@@ -22,9 +22,9 @@ import type {
   PersistenceInfo,
   SqliteDb,
   SyncBackend,
-  UnexpectedError,
+  UnknownError,
 } from '../index.ts'
-import type { EventSequenceNumber, LiveStoreEvent, LiveStoreSchema } from '../schema/mod.ts'
+import { EventSequenceNumber, type LiveStoreEvent, type LiveStoreSchema } from '../schema/mod.ts'
 import type * as SyncState from '../sync/syncstate.ts'
 import type { ShutdownChannel } from './shutdown-channel.ts'
 
@@ -43,7 +43,7 @@ export const InitialSyncOptions = Schema.Union(InitialSyncOptionsSkip, InitialSy
 export type InitialSyncOptions = typeof InitialSyncOptions.Type
 
 export type InitialSyncInfo = Option.Option<{
-  cursor: EventSequenceNumber.EventSequenceNumber
+  eventSequenceNumber: EventSequenceNumber.Global.Type
   metadata: Option.Option<Schema.JsonValue>
 }>
 
@@ -66,7 +66,7 @@ export type DevtoolsOptions =
           persistenceInfo: PersistenceInfoPair
           mode: 'proxy' | 'direct'
         },
-        UnexpectedError,
+        UnknownError,
         Scope.Scope | HttpClient.HttpClient | LeaderThreadCtx
       >
     }
@@ -96,13 +96,13 @@ export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
     // TODO we should find a more elegant way to handle cases which need this ref for their implementation
     shutdownStateSubRef: SubscriptionRef.SubscriptionRef<ShutdownState>
     shutdownChannel: ShutdownChannel
-    eventSchema: LiveStoreEvent.ForEventDefRecord<any>
+    eventSchema: LiveStoreEvent.ForEventDef.ForRecord<any>
     devtools: DevtoolsContext
-    syncBackend: SyncBackend | undefined
+    syncBackend: SyncBackend.SyncBackend | undefined
     syncProcessor: LeaderSyncProcessor
     materializeEvent: MaterializeEvent
     initialState: {
-      leaderHead: EventSequenceNumber.EventSequenceNumber
+      leaderHead: EventSequenceNumber.Client.Composite
       migrationsReport: MigrationsReport
     }
     /**
@@ -111,11 +111,12 @@ export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
      * This is currently separated from `.devtools` as it also needs to work when devtools are disabled
      */
     extraIncomingMessagesQueue: Queue.Queue<Devtools.Leader.MessageToApp>
+    networkStatus: Subscribable.Subscribable<SyncBackend.NetworkStatus>
   }
 >() {}
 
 export type MaterializeEvent = (
-  eventEncoded: LiveStoreEvent.EncodedWithMeta,
+  eventEncoded: LiveStoreEvent.Client.EncodedWithMeta,
   options?: {
     /** Needed for rematerializeFromEventlog */
     skipEventlog?: boolean
@@ -125,31 +126,80 @@ export type MaterializeEvent = (
     sessionChangeset: { _tag: 'sessionChangeset'; data: Uint8Array<ArrayBuffer>; debug: any } | { _tag: 'no-op' }
     hash: Option.Option<number>
   },
-  SqliteError | UnexpectedError
+  MaterializeError
 >
 
 export type InitialBlockingSyncContext = {
   blockingDeferred: Deferred.Deferred<void> | undefined
-  update: (_: { remaining: number; processed: number }) => Effect.Effect<void>
+  update: (_: { pageInfo: SyncBackend.PullResPageInfo; processed: number }) => Effect.Effect<void>
+}
+
+export const STREAM_EVENTS_BATCH_SIZE_DEFAULT = 100
+export const STREAM_EVENTS_BATCH_SIZE_MAX = 1_000
+
+export const StreamEventsOptionsFields = {
+  since: Schema.optional(EventSequenceNumber.Client.Composite),
+  until: Schema.optional(EventSequenceNumber.Client.Composite),
+  filter: Schema.optional(Schema.Array(Schema.String)),
+  clientIds: Schema.optional(Schema.Array(Schema.String)),
+  sessionIds: Schema.optional(Schema.Array(Schema.String)),
+  batchSize: Schema.optional(Schema.Int.pipe(Schema.between(1, STREAM_EVENTS_BATCH_SIZE_MAX))),
+  includeClientOnly: Schema.optional(Schema.Boolean),
+} as const
+
+export const StreamEventsOptionsSchema = Schema.Struct(StreamEventsOptionsFields)
+
+export interface StreamEventsOptions {
+  /**
+   * Only include events after this logical timestamp (exclusive).
+   * Defaults to `EventSequenceNumber.Client.ROOT` when omitted.
+   */
+  since?: EventSequenceNumber.Client.Composite
+  /**
+   * Only include events up to this logical timestamp (inclusive).
+   */
+  until?: EventSequenceNumber.Client.Composite
+  /**
+   * Only include events of the given names.
+   */
+  filter?: ReadonlyArray<string>
+  /**
+   * Only include events from specific client identifiers.
+   */
+  clientIds?: ReadonlyArray<string>
+  /**
+   * Only include events from specific session identifiers.
+   */
+  sessionIds?: ReadonlyArray<string>
+  /**
+   * Number of events to fetch in each batch when streaming from the eventlog.
+   * Defaults to 100.
+   */
+  batchSize?: number
+  /**
+   * Include client-only events (i.e. events with a positive client sequence number).
+   */
+  includeClientOnly?: boolean
 }
 
 export interface LeaderSyncProcessor {
   /** Used by client sessions to subscribe to upstream sync state changes */
   pull: (args: {
-    cursor: EventSequenceNumber.EventSequenceNumber
-  }) => Stream.Stream<{ payload: typeof SyncState.PayloadUpstream.Type }, UnexpectedError>
+    cursor: EventSequenceNumber.Client.Composite
+  }) => Stream.Stream<{ payload: typeof SyncState.PayloadUpstream.Type }, UnknownError>
   /** The `pullQueue` API can be used instead of `pull` when more convenient */
   pullQueue: (args: {
-    cursor: EventSequenceNumber.EventSequenceNumber
-  }) => Effect.Effect<Queue.Queue<{ payload: typeof SyncState.PayloadUpstream.Type }>, UnexpectedError, Scope.Scope>
+    cursor: EventSequenceNumber.Client.Composite
+  }) => Effect.Effect<Queue.Queue<{ payload: typeof SyncState.PayloadUpstream.Type }>, UnknownError, Scope.Scope>
 
   /** Used by client sessions to push events to the leader thread */
   push: (
     /** `batch` needs to follow the same rules as `batch` in `SyncBackend.push` */
-    batch: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>,
+    batch: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>,
     options?: {
       /**
        * If true, the effect will only finish when the local push has been processed (i.e. succeeded or was rejected).
+       * `true` doesn't mean the events have been pushed to the sync backend.
        * @default false
        */
       waitForProcessing?: boolean
@@ -158,14 +208,14 @@ export interface LeaderSyncProcessor {
 
   /** Currently only used by devtools which don't provide their own event numbers */
   pushPartial: (args: {
-    event: LiveStoreEvent.PartialAnyEncoded
+    event: LiveStoreEvent.Input.Encoded
     clientId: string
     sessionId: string
-  }) => Effect.Effect<void, UnexpectedError>
+  }) => Effect.Effect<void, UnknownError>
 
   boot: Effect.Effect<
-    { initialLeaderHead: EventSequenceNumber.EventSequenceNumber },
-    UnexpectedError,
+    { initialLeaderHead: EventSequenceNumber.Client.Composite },
+    UnknownError,
     LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
   >
   syncState: Subscribable.Subscribable<SyncState.SyncState>

package/src/logging.ts

@@ -0,0 +1,62 @@
+import { isDevEnv } from '@livestore/utils'
+import { Effect, type Layer, Logger, LogLevel } from '@livestore/utils/effect'
+
+/**
+ * Optional Effect logger configuration that LiveStore entry points accept.
+ *
+ * When provided, `logger` replaces the default pretty logger for the runtime.
+ * Use `logLevel` to control verbosity. Set to `LogLevel.None` to disable logging
+ * entirely while keeping the same logger implementation.
+ */
+export type WithLoggerOptions = {
+  /** Optional Effect logger layer to control logging output. */
+  logger?: Layer.Layer<never> | undefined
+  /** Optional minimum log level for the runtime. */
+  logLevel?: LogLevel.LogLevel | undefined
+}
+
+/**
+ * Common defaults for resolving a logger configuration.
+ * - `threadName` is used by the default pretty logger when `logger` is not provided.
+ * - `mode` selects pretty logger mode (e.g. 'browser' for web workers).
+ * - `defaultLogLevel` is used when `logLevel` is not provided.
+ */
+export type LoggerDefaults = {
+  threadName?: string
+  mode?: 'tty' | 'browser'
+  defaultLogLevel?: LogLevel.LogLevel
+  /** Optional default logger layer to use when `config.logger` is not provided. */
+  defaultLogger?: Layer.Layer<never>
+}
+
+/**
+ * Resolve the logger layer to provide to the Effect runtime.
+ */
+export const resolveLoggerLayer = (config?: WithLoggerOptions, defaults?: LoggerDefaults): Layer.Layer<never> => {
+  if (config?.logger) return config.logger
+  if (defaults?.defaultLogger) return defaults.defaultLogger
+  const threadName = defaults?.threadName ?? 'livestore'
+  const mode = defaults?.mode
+  return Logger.prettyWithThread(threadName, mode ? { mode } : {})
+}
+
+/**
+ * Resolve the minimum log level, falling back to `defaults.defaultLogLevel` then `LogLevel.Debug`.
+ */
+export const resolveLogLevel = (config?: WithLoggerOptions, defaults?: LoggerDefaults): LogLevel.LogLevel => {
+  if (config?.logLevel !== undefined) return config.logLevel
+  if (defaults?.defaultLogLevel !== undefined) return defaults.defaultLogLevel
+  return isDevEnv() ? LogLevel.Debug : LogLevel.Info
+}
+
+/**
+ * Wrap an effect by applying the resolved minimum log level and providing the resolved logger layer.
+ */
+export const withLoggerConfig = <TEnv, TError, TOut>(
+  config?: WithLoggerOptions,
+  defaults?: LoggerDefaults,
+): ((effect: Effect.Effect<TOut, TError, TEnv>) => Effect.Effect<TOut, TError, TEnv>) => {
+  const level = resolveLogLevel(config, defaults)
+  const layer = resolveLoggerLayer(config, defaults)
+  return (effect) => effect.pipe(Logger.withMinimumLogLevel(level), Effect.provide(layer))
+}

package/src/make-client-session.ts

@@ -8,7 +8,7 @@ import type {
   ClientSessionLeaderThreadProxy,
   LockStatus,
   SqliteDb,
-  UnexpectedError,
+  UnknownError,
 } from './adapter-types.ts'
 import * as Devtools from './devtools/mod.ts'
 import { liveStoreVersion } from './version.ts'
@@ -33,6 +33,7 @@ export const makeClientSession = <R>({
   webmeshMode,
   registerBeforeUnload,
   debugInstanceId,
+  origin,
 }: AdapterArgs & {
   clientId: string
   sessionId: string
@@ -43,9 +44,11 @@ export const makeClientSession = <R>({
   connectWebmeshNode: (args: {
     webmeshNode: Webmesh.MeshNode
     sessionInfo: Devtools.SessionInfo.SessionInfo
-  }) => Effect.Effect<void, UnexpectedError, Scope.Scope | R>
+  }) => Effect.Effect<void, UnknownError, Scope.Scope | R>
   webmeshMode: 'direct' | 'proxy'
   registerBeforeUnload: (onBeforeUnload: () => void) => () => void
+  /** Browser origin of the client session; used for origin-scoped DevTools mesh channels */
+  origin: string | undefined
 }): Effect.Effect<ClientSession, never, Scope.Scope | R> =>
   Effect.gen(function* () {
     const devtools: ClientSession['devtools'] = devtoolsEnabled
@@ -67,11 +70,14 @@ export const makeClientSession = <R>({
           sessionId,
           schemaAlias,
           isLeader,
+          origin,
         })
 
         yield* connectWebmeshNode({ webmeshNode, sessionInfo })
 
-        const sessionInfoBroadcastChannel = yield* Devtools.makeSessionInfoBroadcastChannel(webmeshNode)
+        const sessionInfoBroadcastChannel = yield* Devtools.makeSessionInfoBroadcastChannel(webmeshNode, {
+          origin,
+        })
 
         yield* Devtools.SessionInfo.provideSessionInfo({
           webChannel: sessionInfoBroadcastChannel,

package/src/materializer-helper.ts

@@ -3,9 +3,9 @@ import { Hash, Option, Schema } from '@livestore/utils/effect'
 
 import type { SqliteDb } from './adapter-types.ts'
 import { SessionIdSymbol } from './adapter-types.ts'
-import type { EventDef, Materializer, MaterializerContextQuery, MaterializerResult } from './schema/EventDef.ts'
-import type * as LiveStoreEvent from './schema/LiveStoreEvent.ts'
-import { getEventDef, type LiveStoreSchema } from './schema/schema.ts'
+import type { EventDef, Materializer, MaterializerContextQuery, MaterializerResult } from './schema/EventDef/mod.ts'
+import type * as LiveStoreEvent from './schema/LiveStoreEvent/mod.ts'
+import type { LiveStoreSchema } from './schema/schema.ts'
 import type { QueryBuilder } from './schema/state/sqlite/query-builder/api.ts'
 import { isQueryBuilder } from './schema/state/sqlite/query-builder/api.ts'
 import { getResultSchema } from './schema/state/sqlite/query-builder/impl.ts'
@@ -24,15 +24,20 @@ export const getExecStatementsFromMaterializer = ({
   dbState: SqliteDb
   /** Both encoded and decoded events are supported to reduce the number of times we need to decode/encode */
   event:
-    | { decoded: LiveStoreEvent.AnyDecoded; encoded: undefined }
-    | { decoded: undefined; encoded: LiveStoreEvent.AnyEncoded }
+    | { decoded: LiveStoreEvent.Client.Decoded; encoded: undefined }
+    | { decoded: undefined; encoded: LiveStoreEvent.Client.Encoded }
 }): ReadonlyArray<{
   statementSql: string
   bindValues: PreparedBindValues
   writeTables: ReadonlySet<string> | undefined
 }> => {
-  const eventArgsDecoded =
-    event.decoded === undefined ? Schema.decodeUnknownSync(eventDef.schema)(event.encoded!.args) : event.decoded.args
+  const eventDecoded =
+    event.decoded === undefined
+      ? {
+          ...event.encoded!,
+          args: Schema.decodeUnknownSync(eventDef.schema)(event.encoded!.args),
+        }
+      : event.decoded
 
   const eventArgsEncoded = isNil(event.decoded?.args)
     ? undefined
@@ -58,11 +63,12 @@ export const getExecStatementsFromMaterializer = ({
   }
 
   const statementResults = fromMaterializerResult(
-    materializer(eventArgsDecoded, {
+    materializer(eventDecoded.args, {
       eventDef,
       query,
       // TODO properly implement this
       currentFacts: new Map(),
+      event: eventDecoded,
     }),
   )
 
@@ -79,9 +85,20 @@ export const getExecStatementsFromMaterializer = ({
 
 export const makeMaterializerHash =
   ({ schema, dbState }: { schema: LiveStoreSchema; dbState: SqliteDb }) =>
-  (event: LiveStoreEvent.AnyEncoded): Option.Option<number> => {
+  (event: LiveStoreEvent.Client.Encoded): Option.Option<number> => {
     if (isDevEnv()) {
-      const { eventDef, materializer } = getEventDef(schema, event.name)
+      // Hashing is only needed during dev-mode diagnostics. Skip work entirely for
+      // unknown events (no definition/materializer) so we do not introduce noisy
+      // warnings while still returning `Option.none()` to disable hash checks.
+      const eventDef = schema.eventsDefsMap.get(event.name)
+      const materializer = schema.state.materializers.get(event.name)
+      if (eventDef === undefined || materializer === undefined) {
+        return Option.none()
+      }
+      // For known events we replay the materializer with the encoded payload and
+      // hash the resulting SQL statements. This lets us cheaply detect
+      // side-effects or logic drift between leader/client materializers without
+      // mutating the underlying state.
       const materializerResults = getExecStatementsFromMaterializer({
         eventDef,
         materializer,

package/src/otel.ts

@@ -2,6 +2,16 @@ import { makeNoopTracer } from '@livestore/utils'
 import { Effect, identity, Layer, OtelTracer } from '@livestore/utils/effect'
 import * as otel from '@opentelemetry/api'
 
+export const OtelLiveDummy: Layer.Layer<OtelTracer.OtelTracer> = Layer.suspend(() => {
+  const OtelTracerLive = Layer.succeed(OtelTracer.OtelTracer, makeNoopTracer())
+
+  const TracingLive = Layer.unwrapEffect(Effect.map(OtelTracer.make, Layer.setTracer)).pipe(
+    Layer.provideMerge(OtelTracerLive),
+  )
+
+  return TracingLive
+})
+
 export const provideOtel =
   ({ otelTracer, parentSpanContext }: { otelTracer?: otel.Tracer; parentSpanContext?: otel.Context }) =>
   <A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, E, Exclude<R, OtelTracer.OtelTracer>> => {

package/src/rematerialize-from-eventlog.ts

@@ -1,10 +1,10 @@
 import { memoizeByRef } from '@livestore/utils'
 import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
 
-import { type SqliteDb, UnexpectedError } from './adapter-types.ts'
+import { type SqliteDb, UnknownError } from './adapter-types.ts'
 import type { MaterializeEvent } from './leader-thread/mod.ts'
 import type { EventDef, LiveStoreSchema } from './schema/mod.ts'
-import { EventSequenceNumber, getEventDef, LiveStoreEvent, SystemTables } from './schema/mod.ts'
+import { EventSequenceNumber, LiveStoreEvent, resolveEventDef, SystemTables } from './schema/mod.ts'
 import type { PreparedBindValues } from './util.ts'
 import { sql } from './util.ts'
 
@@ -31,20 +31,47 @@ export const rematerializeFromEventlog = ({
 
     const processEvent = (row: SystemTables.EventlogMetaRow) =>
       Effect.gen(function* () {
-        const eventDef = getEventDef(schema, row.name)
+        const args = JSON.parse(row.argsJson)
+        const eventEncoded = LiveStoreEvent.Client.EncodedWithMeta.make({
+          name: row.name,
+          args,
+          seqNum: {
+            global: row.seqNumGlobal,
+            client: row.seqNumClient,
+            rebaseGeneration: row.seqNumRebaseGeneration,
+          },
+          parentSeqNum: {
+            global: row.parentSeqNumGlobal,
+            client: row.parentSeqNumClient,
+            rebaseGeneration: row.parentSeqNumRebaseGeneration,
+          },
+          clientId: row.clientId,
+          sessionId: row.sessionId,
+        })
+
+        const resolution = yield* resolveEventDef(schema, {
+          operation: '@livestore/common:rematerializeFromEventlog:processEvent',
+          event: eventEncoded,
+        }).pipe(UnknownError.mapToUnknownError)
+
+        if (resolution._tag === 'unknown') {
+          // Old snapshots can contain newer events. Skip until the runtime has
+          // been updated; the event stays in the log for future replays.
+          return
+        }
 
-        if (hashEventDef(eventDef.eventDef) !== row.schemaHash) {
+        const { eventDef } = resolution
+
+        if (hashEventDef(eventDef) !== row.schemaHash) {
           yield* Effect.logWarning(
             `Schema hash mismatch for event definition ${row.name}. Trying to materialize event anyway.`,
           )
         }
 
-        const args = JSON.parse(row.argsJson)
-
         // Checking whether the schema has changed in an incompatible way
-        yield* Schema.decodeUnknown(eventDef.eventDef.schema)(args).pipe(
+        yield* Schema.decodeUnknown(eventDef.schema)(args).pipe(
           Effect.mapError((cause) =>
-            UnexpectedError.make({
+            UnknownError.make({
               cause,
               note: `\
 There was an error during rematerializing from the eventlog while decoding
@@ -55,23 +82,6 @@ This likely means the schema has changed in an incompatible way.
           ),
         )
 
-        const eventEncoded = LiveStoreEvent.EncodedWithMeta.make({
-          seqNum: {
-            global: row.seqNumGlobal,
-            client: row.seqNumClient,
-            rebaseGeneration: row.seqNumRebaseGeneration,
-          },
-          parentSeqNum: {
-            global: row.parentSeqNumGlobal,
-            client: row.parentSeqNumClient,
-            rebaseGeneration: row.parentSeqNumRebaseGeneration,
-          },
-          name: row.name,
-          args,
-          clientId: row.clientId,
-          sessionId: row.sessionId,
-        })
-
         yield* materializeEvent(eventEncoded, { skipEventlog: true })
       }).pipe(Effect.withSpan(`@livestore/common:rematerializeFromEventlog:processEvent`))
 
@@ -96,9 +106,9 @@ LIMIT ${CHUNK_SIZE}
       const lastId = Chunk.isChunk(item)
         ? Chunk.last(item).pipe(
             Option.map((_) => ({ global: _.seqNumGlobal, client: _.seqNumClient })),
-            Option.getOrElse(() => EventSequenceNumber.ROOT),
+            Option.getOrElse(() => EventSequenceNumber.Client.ROOT),
           )
-        : EventSequenceNumber.ROOT
+        : EventSequenceNumber.Client.ROOT
       const nextItem = Chunk.fromIterable(
         stmt.select<SystemTables.EventlogMetaRow>({
           $seqNumGlobal: lastId?.global,