@livestore/common 0.4.0-dev.9 → 0.5.0-dev.0

package/src/leader-thread/shutdown-channel.ts

@@ -1,23 +1,8 @@
-import type { WebChannel } from '@livestore/utils/effect'
-import { Schema } from '@livestore/utils/effect'
+import { type WebChannel, Schema } from '@livestore/utils/effect'
 
-import {
-  IntentionalShutdownCause,
-  InvalidPullError,
-  InvalidPushError,
-  IsOfflineError,
-  MaterializeError,
-  UnexpectedError,
-} from '../index.ts'
+import { BackendIdMismatchError, IntentionalShutdownCause, MaterializeError, UnknownError } from '../index.ts'
 
-export class All extends Schema.Union(
-  IntentionalShutdownCause,
-  UnexpectedError,
-  IsOfflineError,
-  InvalidPushError,
-  InvalidPullError,
-  MaterializeError,
-) {}
+export const All = Schema.Union([IntentionalShutdownCause, UnknownError, BackendIdMismatchError, MaterializeError])
 
 /**
  * Used internally by an adapter to shutdown gracefully.

package/src/leader-thread/stream-events.ts

@@ -0,0 +1,209 @@
+import { type Subscribable, Effect, Option, Queue, Stream } from '@livestore/utils/effect'
+
+import { EventSequenceNumber, type LiveStoreEvent } from '../schema/mod.ts'
+import type * as SyncState from '../sync/syncstate.ts'
+import * as Eventlog from './eventlog.ts'
+import type { LeaderSqliteDb, StreamEventsOptions } from './types.ts'
+
+/**
+ * Streams events for leader-thread adapters.
+ *
+ * Provides a continuous stream from the eventlog as the upstream head advances.
+ * When an until event is passed in the stream finalizes upon reaching it.
+ *
+ * The batch size is set to 100 by default as this was meassured to provide the
+ * best performance and 1000 as the upper limit.
+ *
+ * Adapters that call this helper include:
+ * - `packages/@livestore/adapter-web/src/in-memory/in-memory-adapter.ts`
+ * - `packages/@livestore/adapter-web/src/web-worker/leader-worker/make-leader-worker.ts`
+ * - `packages/@livestore/adapter-cloudflare/src/make-adapter.ts`
+ * - external leader-thread adapters
+ *
+ * Each caller resolves dependencies inside the leader scope before invoking this helper,
+ * so the stream stays environment-agnostic and does not leak `LeaderThreadCtx` into runtime
+ * entry points such as `Store.eventsStream`.
+ *
+ * Test files:
+ * Integration: `packages/@livestore/livestore/src/store/store-eventstream.test.ts`
+ * Performance: `tests/perf-eventlog/tests/suites/event-streaming.test.ts`
+ *
+ * Optimization explorations
+ *
+ * In order to alleviate the occurence of many small queries when the syncState
+ * is sequentially progressing quickly we have explored some time-based batching
+ * approaches. It remains to be determined if and when the added complexity of
+ * these approaches are worth the benefit. They come with some drawbacks such as
+ * degraded time to first event or general performance degredation for larger
+ * query steps. These aspects can likely be mitigated with some more work but
+ * that is best assessed when we have a final implementation of event streaming
+ * with support for session and leader level streams.
+ *
+ * Fetch plans into a Sink
+ * https://gist.github.com/slashv/f1223689f2d1171d2eeb60a2823f4c7c
+ *
+ * Fetch plans into sink and decompose into windows
+ * https://gist.github.com/slashv/a8f55f50121c080937f42e44b4039ac8
+ *
+ * Queue and Latch approach (suggestion by Tim Smart)
+ * https://gist.github.com/slashv/d6b12395c85415bf0d3363372a1636c3
+ */
+export const streamEventsWithSyncState = ({
+  dbEventlog,
+  syncState,
+  options,
+}: {
+  dbEventlog: LeaderSqliteDb
+  syncState: Subscribable.Subscribable<SyncState.SyncState>
+  options: StreamEventsOptions
+}): Stream.Stream<LiveStoreEvent.Client.Encoded> => {
+  const initialCursor = options.since ?? EventSequenceNumber.Client.ROOT
+  const batchSize = options.batchSize ?? 100
+
+  return Stream.unwrap(
+    Effect.gen(function* () {
+      /**
+       * Single-element Queue allows suspending the event stream until head
+       * advances because Queue.take is a suspending effect. SubscriptionRef in
+       * comparrison lacks a primitive for suspending a stream until a new value
+       * is set and would require polling.
+       *
+       * The use of a sliding Queue here is useful since it ensures only the
+       * lastest head from syncState is the one present on the queue without the
+       * need for manual substitution.
+       */
+      const headQueue = yield* Queue.sliding<EventSequenceNumber.Client.Composite>(1)
+
+      /**
+       * We run a separate fiber which listens to changes in syncState and
+       * offer the latest head to the headQueue. Keeping track of the previous
+       * value is done to prevent syncState changes unrelated to the
+       * upstreamHead triggering empty queries.
+       *
+       * When we implement support for leader and session level streams
+       * this will need to be adapted to support the relevant value from
+       * syncState that we are interested in tracking.
+       */
+      let prevGlobalHead = -1
+      yield* syncState.changes.pipe(
+        Stream.map((state) => state.upstreamHead),
+        Stream.filter((head) => {
+          if (head.global > prevGlobalHead) {
+            prevGlobalHead = head.global
+            return true
+          }
+          return false
+        }),
+        Stream.runForEach((head) => Queue.offer(headQueue, head)),
+        Effect.forkScoped,
+      )
+
+      return Stream.paginate({ cursor: initialCursor, head: EventSequenceNumber.Client.ROOT }, ({ cursor, head }) =>
+        Effect.gen(function* () {
+          /**
+           * Early check guards agains:
+           * since === until : Prevent empty query
+           * since > until : Incorrectly inverted interval
+           */
+          if (
+            options.until !== undefined &&
+            EventSequenceNumber.Client.isGreaterThanOrEqual(cursor, options.until) === true
+          ) {
+            const done: readonly [
+              ReadonlyArray<LiveStoreEvent.Client.Encoded>,
+              Option.Option<{
+                cursor: EventSequenceNumber.Client.Composite
+                head: EventSequenceNumber.Client.Composite
+              }>,
+            ] = [[], Option.none()]
+            return done
+          }
+
+          /**
+           * There are two scenarios where we take the next head from the headQueue:
+           *
+           * 1. We need to wait for the head to advance
+           * The Stream suspends until a new head is available on the headQueue
+           *
+           * 2. Head has advanced during itteration
+           * While itterating towards the lastest head taken from the headQueue
+           * in increments of batchSize it's possible the head could have
+           * advanced. This leads to a suboptimal amount of queries. Therefor we
+           * check if the headQueue is full which tells us that there's a new
+           * head available to take. Example:
+           *
+           * batchSize: 2
+           *
+           * --> head at: e3
+           * First query: e0 -> e2 (two events)
+           * --> head advances to: e4
+           * Second query: e2 -> e3 (one event but we could have taken 2)
+           * --> Take the new head of e4
+           * Third query: e3 -> e4 (unnecessary third query)
+           *
+           *
+           * To define the target, which will be used as the temporary until
+           * marker for the eventlog query, we select the lowest of three possible values:
+           *
+           * hardStop: A user supplied until marker
+           * current cursor + batchSize: A batchSize step towards the latest head from headQueue
+           * nextHead: The latest head from headQueue
+           */
+          const waitForHead = EventSequenceNumber.Client.isGreaterThanOrEqual(cursor, head)
+          const maybeHead =
+            waitForHead === true
+              ? yield* Queue.take(headQueue).pipe(Effect.map(Option.some))
+              : yield* Queue.poll(headQueue)
+          const nextHead = Option.getOrElse(maybeHead, () => head)
+          const hardStop = options.until?.global ?? Number.POSITIVE_INFINITY
+          const target = EventSequenceNumber.Client.Composite.make({
+            global: Math.min(hardStop, cursor.global + batchSize, nextHead.global),
+            client: EventSequenceNumber.Client.DEFAULT,
+          })
+
+          /**
+           * Eventlog.getEventsFromEventlog returns a batch from each query
+           * which is what we emit at each itteration.
+           */
+          const events = yield* Eventlog.getEventsFromEventlog({
+            dbEventlog,
+            options: {
+              ...options,
+              since: cursor,
+              until: target,
+            },
+          })
+
+          /**
+           * We construct the state for the following itteration of the stream
+           * loop by setting the current target as the since cursor and pass
+           * along the latest head.
+           *
+           * If we have the reached the user supplied until marker we signal the
+           * finalization of the stream by passing Option.none() instead.
+           */
+          const reachedUntil =
+            options.until !== undefined && EventSequenceNumber.Client.isGreaterThanOrEqual(target, options.until)
+
+          const nextState: Option.Option<{
+            cursor: EventSequenceNumber.Client.Composite
+            head: EventSequenceNumber.Client.Composite
+          }> = reachedUntil === true ? Option.none() : Option.some({ cursor: target, head: nextHead })
+
+          const spanAttributes = {
+            'livestore.streamEvents.cursor.global': cursor.global,
+            'livestore.streamEvents.target.global': target.global,
+            'livestore.streamEvents.batchSize': batchSize,
+            'livestore.streamEvents.waitedForHead': waitForHead,
+          }
+
+          const result: readonly [ReadonlyArray<LiveStoreEvent.Client.Encoded>, typeof nextState] = [events, nextState]
+
+          return yield* Effect.succeed(result).pipe(
+            Effect.withSpan('@livestore/common:streamEvents:segment', { attributes: spanAttributes }),
+          )
+        }),
+      )
+    }),
+  )
+}

package/src/leader-thread/types.ts

@@ -1,15 +1,17 @@
-import type {
-  Deferred,
-  Effect,
-  HttpClient,
-  Option,
-  Queue,
-  Scope,
-  Stream,
-  Subscribable,
-  SubscriptionRef,
+import {
+  type Deferred,
+  type Effect,
+  type HttpClient,
+  type Latch,
+  type Option,
+  type Queue,
+  type Scope,
+  type Stream,
+  type Subscribable,
+  type SubscriptionRef,
+  Context,
+  Schema,
 } from '@livestore/utils/effect'
-import { Context, Schema } from '@livestore/utils/effect'
 import type { MeshNode } from '@livestore/webmesh'
 
 import type { MigrationsReport } from '../defs.ts'
@@ -17,15 +19,15 @@ import type { MaterializeError } from '../errors.ts'
 import type {
   BootStatus,
   Devtools,
-  LeaderAheadError,
   MakeSqliteDb,
   PersistenceInfo,
   SqliteDb,
   SyncBackend,
-  UnexpectedError,
+  UnknownError,
 } from '../index.ts'
-import type { EventSequenceNumber, LiveStoreEvent, LiveStoreSchema } from '../schema/mod.ts'
+import { EventSequenceNumber, type LiveStoreEvent, type LiveStoreSchema } from '../schema/mod.ts'
 import type * as SyncState from '../sync/syncstate.ts'
+import type * as LeaderSyncProcessor from './LeaderSyncProcessor.ts'
 import type { ShutdownChannel } from './shutdown-channel.ts'
 
 export type ShutdownState = 'running' | 'shutting-down'
@@ -34,17 +36,17 @@ export const InitialSyncOptionsSkip = Schema.TaggedStruct('Skip', {})
 export type InitialSyncOptionsSkip = typeof InitialSyncOptionsSkip.Type
 
 export const InitialSyncOptionsBlocking = Schema.TaggedStruct('Blocking', {
-  timeout: Schema.Union(Schema.DurationFromMillis, Schema.Number),
+  timeout: Schema.Union([Schema.DurationFromMillis, Schema.Finite]),
 })
 
 export type InitialSyncOptionsBlocking = typeof InitialSyncOptionsBlocking.Type
 
-export const InitialSyncOptions = Schema.Union(InitialSyncOptionsSkip, InitialSyncOptionsBlocking)
+export const InitialSyncOptions = Schema.Union([InitialSyncOptionsSkip, InitialSyncOptionsBlocking])
 export type InitialSyncOptions = typeof InitialSyncOptions.Type
 
 export type InitialSyncInfo = Option.Option<{
-  eventSequenceNumber: EventSequenceNumber.GlobalEventSequenceNumber
-  metadata: Option.Option<Schema.JsonValue>
+  eventSequenceNumber: EventSequenceNumber.Global.Type
+  metadata: Option.Option<Schema.Json>
 }>
 
 // export type InitialSetup =
@@ -66,7 +68,7 @@ export type DevtoolsOptions =
           persistenceInfo: PersistenceInfoPair
           mode: 'proxy' | 'direct'
         },
-        UnexpectedError,
+        UnknownError,
         Scope.Scope | HttpClient.HttpClient | LeaderThreadCtx
       >
     }
@@ -74,16 +76,16 @@ export type DevtoolsOptions =
 export type DevtoolsContext =
   | {
       enabled: true
-      // syncBackendPullLatch: Effect.Latch
-      // syncBackendPushLatch: Effect.Latch
-      syncBackendLatch: Effect.Latch
+      // syncBackendPullLatch: Latch.Latch
+      // syncBackendPushLatch: Latch.Latch
+      syncBackendLatch: Latch.Latch
       syncBackendLatchState: SubscriptionRef.SubscriptionRef<{ latchClosed: boolean }>
     }
   | {
       enabled: false
     }
 
-export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
+export class LeaderThreadCtx extends Context.Service<
   LeaderThreadCtx,
   {
     schema: LiveStoreSchema
@@ -96,13 +98,13 @@ export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
     // TODO we should find a more elegant way to handle cases which need this ref for their implementation
     shutdownStateSubRef: SubscriptionRef.SubscriptionRef<ShutdownState>
     shutdownChannel: ShutdownChannel
-    eventSchema: LiveStoreEvent.ForEventDefRecord<any>
+    eventSchema: LiveStoreEvent.ForEventDef.ForRecord<any>
     devtools: DevtoolsContext
     syncBackend: SyncBackend.SyncBackend | undefined
-    syncProcessor: LeaderSyncProcessor
+    syncProcessor: LeaderSyncProcessor.Service
     materializeEvent: MaterializeEvent
     initialState: {
-      leaderHead: EventSequenceNumber.EventSequenceNumber
+      leaderHead: EventSequenceNumber.Client.Composite
       migrationsReport: MigrationsReport
     }
     /**
@@ -111,11 +113,12 @@ export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
      * This is currently separated from `.devtools` as it also needs to work when devtools are disabled
      */
     extraIncomingMessagesQueue: Queue.Queue<Devtools.Leader.MessageToApp>
+    networkStatus: Subscribable.Subscribable<SyncBackend.NetworkStatus>
   }
->() {}
+>()('LeaderThreadCtx') {}
 
 export type MaterializeEvent = (
-  eventEncoded: LiveStoreEvent.EncodedWithMeta,
+  eventEncoded: LiveStoreEvent.Client.EncodedWithMeta,
   options?: {
     /** Needed for rematerializeFromEventlog */
     skipEventlog?: boolean
@@ -133,41 +136,50 @@ export type InitialBlockingSyncContext = {
   update: (_: { pageInfo: SyncBackend.PullResPageInfo; processed: number }) => Effect.Effect<void>
 }
 
-export interface LeaderSyncProcessor {
-  /** Used by client sessions to subscribe to upstream sync state changes */
-  pull: (args: {
-    cursor: EventSequenceNumber.EventSequenceNumber
-  }) => Stream.Stream<{ payload: typeof SyncState.PayloadUpstream.Type }, UnexpectedError>
-  /** The `pullQueue` API can be used instead of `pull` when more convenient */
-  pullQueue: (args: {
-    cursor: EventSequenceNumber.EventSequenceNumber
-  }) => Effect.Effect<Queue.Queue<{ payload: typeof SyncState.PayloadUpstream.Type }>, UnexpectedError, Scope.Scope>
-
-  /** Used by client sessions to push events to the leader thread */
-  push: (
-    /** `batch` needs to follow the same rules as `batch` in `SyncBackend.push` */
-    batch: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>,
-    options?: {
-      /**
-       * If true, the effect will only finish when the local push has been processed (i.e. succeeded or was rejected).
-       * `true` doesn't mean the events have been pushed to the sync backend.
-       * @default false
-       */
-      waitForProcessing?: boolean
-    },
-  ) => Effect.Effect<void, LeaderAheadError>
-
-  /** Currently only used by devtools which don't provide their own event numbers */
-  pushPartial: (args: {
-    event: LiveStoreEvent.PartialAnyEncoded
-    clientId: string
-    sessionId: string
-  }) => Effect.Effect<void, UnexpectedError>
-
-  boot: Effect.Effect<
-    { initialLeaderHead: EventSequenceNumber.EventSequenceNumber },
-    UnexpectedError,
-    LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
-  >
-  syncState: Subscribable.Subscribable<SyncState.SyncState>
+export const STREAM_EVENTS_BATCH_SIZE_DEFAULT = 100
+export const STREAM_EVENTS_BATCH_SIZE_MAX = 1_000
+
+export const StreamEventsOptionsFields = {
+  since: Schema.optional(EventSequenceNumber.Client.Composite),
+  until: Schema.optional(EventSequenceNumber.Client.Composite),
+  filter: Schema.optional(Schema.Array(Schema.String)),
+  clientIds: Schema.optional(Schema.Array(Schema.String)),
+  sessionIds: Schema.optional(Schema.Array(Schema.String)),
+  batchSize: Schema.optional(Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: STREAM_EVENTS_BATCH_SIZE_MAX }))),
+  includeClientOnly: Schema.optional(Schema.Boolean),
+} as const
+
+export const StreamEventsOptionsSchema = Schema.Struct(StreamEventsOptionsFields)
+
+export interface StreamEventsOptions {
+  /**
+   * Only include events after this logical timestamp (exclusive).
+   * Defaults to `EventSequenceNumber.Client.ROOT` when omitted.
+   */
+  since?: EventSequenceNumber.Client.Composite
+  /**
+   * Only include events up to this logical timestamp (inclusive).
+   */
+  until?: EventSequenceNumber.Client.Composite
+  /**
+   * Only include events of the given names.
+   */
+  filter?: ReadonlyArray<string>
+  /**
+   * Only include events from specific client identifiers.
+   */
+  clientIds?: ReadonlyArray<string>
+  /**
+   * Only include events from specific session identifiers.
+   */
+  sessionIds?: ReadonlyArray<string>
+  /**
+   * Number of events to fetch in each batch when streaming from the eventlog.
+   * Defaults to 100.
+   */
+  batchSize?: number
+  /**
+   * Include client-only events (i.e. events with a positive client sequence number).
+   */
+  includeClientOnly?: boolean
 }

package/src/logging.ts

@@ -0,0 +1,15 @@
+import type { Layer, LogLevel } from '@livestore/utils/effect'
+
+/**
+ * Optional Effect logger configuration accepted by LiveStore entry points.
+ *
+ * When provided, `logger` replaces the default logger for the runtime.
+ * Use `logLevel` to control verbosity. Set to `"None"` to disable logging
+ * entirely while keeping the same logger implementation.
+ */
+export type LoggerOptions = {
+  /** Optional Effect logger layer to control logging output. */
+  logger?: Layer.Layer<never> | undefined
+  /** Optional minimum log level for the runtime. */
+  logLevel?: LogLevel.LogLevel | undefined
+}

package/src/make-client-session.ts

@@ -1,5 +1,4 @@
-import type { Scope, SubscriptionRef } from '@livestore/utils/effect'
-import { Effect, Stream } from '@livestore/utils/effect'
+import { type Scope, type SubscriptionRef, Effect, Latch, Stream } from '@livestore/utils/effect'
 import * as Webmesh from '@livestore/webmesh'
 
 import type {
@@ -8,7 +7,7 @@ import type {
   ClientSessionLeaderThreadProxy,
   LockStatus,
   SqliteDb,
-  UnexpectedError,
+  UnknownError,
 } from './adapter-types.ts'
 import * as Devtools from './devtools/mod.ts'
 import { liveStoreVersion } from './version.ts'
@@ -33,6 +32,7 @@ export const makeClientSession = <R>({
   webmeshMode,
   registerBeforeUnload,
   debugInstanceId,
+  origin,
 }: AdapterArgs & {
   clientId: string
   sessionId: string
@@ -43,16 +43,19 @@ export const makeClientSession = <R>({
   connectWebmeshNode: (args: {
     webmeshNode: Webmesh.MeshNode
     sessionInfo: Devtools.SessionInfo.SessionInfo
-  }) => Effect.Effect<void, UnexpectedError, Scope.Scope | R>
+  }) => Effect.Effect<void, UnknownError, Scope.Scope | R>
   webmeshMode: 'direct' | 'proxy'
   registerBeforeUnload: (onBeforeUnload: () => void) => () => void
+  /** Browser origin of the client session; used for origin-scoped DevTools mesh channels */
+  origin: string | undefined
 }): Effect.Effect<ClientSession, never, Scope.Scope | R> =>
   Effect.gen(function* () {
-    const devtools: ClientSession['devtools'] = devtoolsEnabled
-      ? { enabled: true, pullLatch: yield* Effect.makeLatch(true), pushLatch: yield* Effect.makeLatch(true) }
-      : { enabled: false }
+    const devtools: ClientSession['devtools'] =
+      devtoolsEnabled === true
+        ? { enabled: true, pullLatch: yield* Latch.make(true), pushLatch: yield* Latch.make(true) }
+        : { enabled: false }
 
-    if (devtoolsEnabled) {
+    if (devtoolsEnabled === true) {
       yield* Effect.gen(function* () {
         const webmeshNode = yield* Webmesh.makeMeshNode(
           Devtools.makeNodeName.client.session({ storeId, clientId, sessionId }),
@@ -67,11 +70,14 @@ export const makeClientSession = <R>({
           sessionId,
           schemaAlias,
           isLeader,
+          origin,
         })
 
         yield* connectWebmeshNode({ webmeshNode, sessionInfo })
 
-        const sessionInfoBroadcastChannel = yield* Devtools.makeSessionInfoBroadcastChannel(webmeshNode)
+        const sessionInfoBroadcastChannel = yield* Devtools.makeSessionInfoBroadcastChannel(webmeshNode, {
+          origin,
+        })
 
         yield* Devtools.SessionInfo.provideSessionInfo({
           webChannel: sessionInfoBroadcastChannel,
@@ -110,7 +116,7 @@ export const makeClientSession = <R>({
                 yield* connectDevtoolsToStore(clientSessionDevtoolsChannel)
               },
               Effect.tapCauseLogPretty,
-              Effect.forkScoped,
+              Effect.forkScoped(),
             ),
           ),
           Stream.runDrain,