@livestore/common 0.4.0-dev.20 → 0.4.0-dev.21

package/src/leader-thread/stream-events.ts

@@ -0,0 +1,201 @@
+import type { Subscribable } from '@livestore/utils/effect'
+import { Chunk, Effect, Option, Queue, Stream } from '@livestore/utils/effect'
+import { EventSequenceNumber, type LiveStoreEvent } from '../schema/mod.ts'
+import type * as SyncState from '../sync/syncstate.ts'
+import * as Eventlog from './eventlog.ts'
+import type { LeaderSqliteDb, StreamEventsOptions } from './types.ts'
+
+/**
+ * Streams events for leader-thread adapters.
+ *
+ * Provides a continuous stream from the eventlog as the upstream head advances.
+ * When an until event is passed in the stream finalizes upon reaching it.
+ *
+ * The batch size is set to 100 by default as this was meassured to provide the
+ * best performance and 1000 as the upper limit.
+ *
+ * Adapters that call this helper:
+ * - `packages/@livestore/adapter-web/src/in-memory/in-memory-adapter.ts`
+ * - `packages/@livestore/adapter-web/src/web-worker/leader-worker/make-leader-worker.ts`
+ * - `packages/@livestore/adapter-node/src/client-session/adapter.ts`
+ * - `packages/@livestore/adapter-node/src/make-leader-worker.ts`
+ * - `packages/@livestore/adapter-cloudflare/src/make-adapter.ts`
+ *
+ * Each caller resolves dependencies inside the leader scope before invoking this helper,
+ * so the stream stays environment-agnostic and does not leak `LeaderThreadCtx` into runtime
+ * entry points such as `Store.eventsStream`.
+ *
+ * Test files:
+ * Unit: `tests/package-common/src/leader-thread/stream-events.test.ts`
+ * Integration: `packages/@livestore/livestore/src/store/store-eventstream.test.ts`
+ * Performance: `tests/perf-eventlog/tests/suites/event-streaming.test.ts`
+ *
+ * Optimization explorations
+ *
+ * In order to alleviate the occurence of many small queries when the syncState
+ * is sequentially progressing quickly we have explored some time-based batching
+ * approaches. It remains to be determined if and when the added complexity of
+ * these approaches are worth the benefit. They come with some drawbacks such as
+ * degraded time to first event or general performance degredation for larger
+ * query steps. These aspects can likely be mitigated with some more work but
+ * that is best assessed when we have a final implementation of event streaming
+ * with support for session and leader level streams.
+ *
+ * Fetch plans into a Sink
+ * https://gist.github.com/slashv/f1223689f2d1171d2eeb60a2823f4c7c
+ *
+ * Fetch plans into sink and decompose into windows
+ * https://gist.github.com/slashv/a8f55f50121c080937f42e44b4039ac8
+ *
+ * Mailbox and Latch approach (suggestion by Tim Smart)
+ * https://gist.github.com/slashv/d6b12395c85415bf0d3363372a1636c3
+ */
+export const streamEventsWithSyncState = ({
+  dbEventlog,
+  syncState,
+  options,
+}: {
+  dbEventlog: LeaderSqliteDb
+  syncState: Subscribable.Subscribable<SyncState.SyncState>
+  options: StreamEventsOptions
+}): Stream.Stream<LiveStoreEvent.Client.Encoded> => {
+  const initialCursor = options.since ?? EventSequenceNumber.Client.ROOT
+  const batchSize = options.batchSize ?? 100
+
+  return Stream.unwrapScoped(
+    Effect.gen(function* () {
+      /**
+       * Single-element Queue allows suspending the event stream until head
+       * advances because Queue.take is a suspending effect. SubscriptionRef in
+       * comparrison lacks a primitive for suspending a stream until a new value
+       * is set and would require polling.
+       *
+       * The use of a sliding Queue here is useful since it ensures only the
+       * lastest head from syncState is the one present on the queue without the
+       * need for manual substitution.
+       */
+      const headQueue = yield* Queue.sliding<EventSequenceNumber.Client.Composite>(1)
+
+      /**
+       * We run a separate fiber which listens to changes in syncState and
+       * offer the latest head to the headQueue. Keeping track of the previous
+       * value is done to prevent syncState changes unrelated to the
+       * upstreamHead triggering empty queries.
+       *
+       * When we implement support for leader and session level streams
+       * this will need to be adapted to support the relevant value from
+       * syncState that we are interested in tracking.
+       */
+      let prevGlobalHead = -1
+      yield* syncState.changes.pipe(
+        Stream.map((state) => state.upstreamHead),
+        Stream.filter((head) => {
+          if (head.global > prevGlobalHead) {
+            prevGlobalHead = head.global
+            return true
+          }
+          return false
+        }),
+        Stream.runForEach((head) => Queue.offer(headQueue, head)),
+        Effect.forkScoped,
+      )
+
+      return Stream.paginateChunkEffect(
+        { cursor: initialCursor, head: EventSequenceNumber.Client.ROOT },
+        ({ cursor, head }) =>
+          Effect.gen(function* () {
+            /**
+             * Early check guards agains:
+             * since === until : Prevent empty query
+             * since > until : Incorrectly inverted interval
+             */
+            if (options.until && EventSequenceNumber.Client.isGreaterThanOrEqual(cursor, options.until)) {
+              return [Chunk.empty(), Option.none()]
+            }
+
+            /**
+             * There are two scenarios where we take the next head from the headQueue:
+             *
+             * 1. We need to wait for the head to advance
+             * The Stream suspends until a new head is available on the headQueue
+             *
+             * 2. Head has advanced during itteration
+             * While itterating towards the lastest head taken from the headQueue
+             * in increments of batchSize it's possible the head could have
+             * advanced. This leads to a suboptimal amount of queries. Therefor we
+             * check if the headQueue is full which tells us that there's a new
+             * head available to take. Example:
+             *
+             * batchSize: 2
+             *
+             * --> head at: e3
+             * First query: e0 -> e2 (two events)
+             * --> head advances to: e4
+             * Second query: e2 -> e3 (one event but we could have taken 2)
+             * --> Take the new head of e4
+             * Third query: e3 -> e4 (unnecessary third query)
+             *
+             *
+             * To define the target, which will be used as the temporary until
+             * marker for the eventlog query, we select the lowest of three possible values:
+             *
+             * hardStop: A user supplied until marker
+             * current cursor + batchSize: A batchSize step towards the latest head from headQueue
+             * nextHead: The latest head from headQueue
+             */
+            const waitForHead = EventSequenceNumber.Client.isGreaterThanOrEqual(cursor, head)
+            const maybeHead = waitForHead
+              ? yield* Queue.take(headQueue).pipe(Effect.map(Option.some))
+              : yield* Queue.poll(headQueue)
+            const nextHead = Option.getOrElse(maybeHead, () => head)
+            const hardStop = options.until?.global ?? Number.POSITIVE_INFINITY
+            const target = EventSequenceNumber.Client.Composite.make({
+              global: Math.min(hardStop, cursor.global + batchSize, nextHead.global),
+              client: EventSequenceNumber.Client.DEFAULT,
+            })
+
+            /**
+             * Eventlog.getEventsFromEventlog returns a Chunk from each
+             * query which is what we emit at each itteration.
+             */
+            const chunk = yield* Eventlog.getEventsFromEventlog({
+              dbEventlog,
+              options: {
+                ...options,
+                since: cursor,
+                until: target,
+              },
+            })
+
+            /**
+             * We construct the state for the following itteration of the stream
+             * loop by setting the current target as the since cursor and pass
+             * along the latest head.
+             *
+             * If we have the reached the user supplied until marker we signal the
+             * finalization of the stream by passing Option.none() instead.
+             */
+            const reachedUntil =
+              options.until !== undefined && EventSequenceNumber.Client.isGreaterThanOrEqual(target, options.until)
+
+            const nextState: Option.Option<{
+              cursor: EventSequenceNumber.Client.Composite
+              head: EventSequenceNumber.Client.Composite
+            }> = reachedUntil ? Option.none() : Option.some({ cursor: target, head: nextHead })
+
+            const spanAttributes = {
+              'livestore.streamEvents.cursor.global': cursor.global,
+              'livestore.streamEvents.target.global': target.global,
+              'livestore.streamEvents.batchSize': batchSize,
+              'livestore.streamEvents.waitedForHead': waitForHead,
+            }
+
+            return yield* Effect.succeed<[Chunk.Chunk<LiveStoreEvent.Client.Encoded>, typeof nextState]>([
+              chunk,
+              nextState,
+            ]).pipe(Effect.withSpan('@livestore/common:streamEvents:segment', { attributes: spanAttributes }))
+          }),
+      )
+    }),
+  )
+}

package/src/leader-thread/types.ts

@@ -24,7 +24,7 @@ import type {
   SyncBackend,
   UnknownError,
 } from '../index.ts'
-import type { EventSequenceNumber, LiveStoreEvent, LiveStoreSchema } from '../schema/mod.ts'
+import { EventSequenceNumber, type LiveStoreEvent, type LiveStoreSchema } from '../schema/mod.ts'
 import type * as SyncState from '../sync/syncstate.ts'
 import type { ShutdownChannel } from './shutdown-channel.ts'
 
@@ -134,6 +134,54 @@ export type InitialBlockingSyncContext = {
   update: (_: { pageInfo: SyncBackend.PullResPageInfo; processed: number }) => Effect.Effect<void>
 }
 
+export const STREAM_EVENTS_BATCH_SIZE_DEFAULT = 100
+export const STREAM_EVENTS_BATCH_SIZE_MAX = 1_000
+
+export const StreamEventsOptionsFields = {
+  since: Schema.optional(EventSequenceNumber.Client.Composite),
+  until: Schema.optional(EventSequenceNumber.Client.Composite),
+  filter: Schema.optional(Schema.Array(Schema.String)),
+  clientIds: Schema.optional(Schema.Array(Schema.String)),
+  sessionIds: Schema.optional(Schema.Array(Schema.String)),
+  batchSize: Schema.optional(Schema.Int.pipe(Schema.between(1, STREAM_EVENTS_BATCH_SIZE_MAX))),
+  includeClientOnly: Schema.optional(Schema.Boolean),
+} as const
+
+export const StreamEventsOptionsSchema = Schema.Struct(StreamEventsOptionsFields)
+
+export interface StreamEventsOptions {
+  /**
+   * Only include events after this logical timestamp (exclusive).
+   * Defaults to `EventSequenceNumber.Client.ROOT` when omitted.
+   */
+  since?: EventSequenceNumber.Client.Composite
+  /**
+   * Only include events up to this logical timestamp (inclusive).
+   */
+  until?: EventSequenceNumber.Client.Composite
+  /**
+   * Only include events of the given names.
+   */
+  filter?: ReadonlyArray<string>
+  /**
+   * Only include events from specific client identifiers.
+   */
+  clientIds?: ReadonlyArray<string>
+  /**
+   * Only include events from specific session identifiers.
+   */
+  sessionIds?: ReadonlyArray<string>
+  /**
+   * Number of events to fetch in each batch when streaming from the eventlog.
+   * Defaults to 100.
+   */
+  batchSize?: number
+  /**
+   * Include client-only events (i.e. events with a positive client sequence number).
+   */
+  includeClientOnly?: boolean
+}
+
 export interface LeaderSyncProcessor {
   /** Used by client sessions to subscribe to upstream sync state changes */
   pull: (args: {

package/src/otel.ts

@@ -2,6 +2,16 @@ import { makeNoopTracer } from '@livestore/utils'
 import { Effect, identity, Layer, OtelTracer } from '@livestore/utils/effect'
 import * as otel from '@opentelemetry/api'
 
+export const OtelLiveDummy: Layer.Layer<OtelTracer.OtelTracer> = Layer.suspend(() => {
+  const OtelTracerLive = Layer.succeed(OtelTracer.OtelTracer, makeNoopTracer())
+
+  const TracingLive = Layer.unwrapEffect(Effect.map(OtelTracer.make, Layer.setTracer)).pipe(
+    Layer.provideMerge(OtelTracerLive),
+  )
+
+  return TracingLive
+})
+
 export const provideOtel =
   ({ otelTracer, parentSpanContext }: { otelTracer?: otel.Tracer; parentSpanContext?: otel.Context }) =>
   <A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, E, Exclude<R, OtelTracer.OtelTracer>> => {

package/src/schema/state/sqlite/client-document-def.ts

@@ -513,27 +513,17 @@ export namespace ClientDocumentTableDef {
     }
   }
 
-  export type GetOptions<TTableDef extends TraitAny> = TTableDef extends ClientDocumentTableDef.Trait<
-    any,
-    any,
-    any,
-    infer TOptions
-  >
-    ? TOptions
-    : never
+  export type GetOptions<TTableDef extends TraitAny> =
+    TTableDef extends ClientDocumentTableDef.Trait<any, any, any, infer TOptions> ? TOptions : never
 
   export type TraitAny = Trait<any, any, any, any>
 
-  export type DefaultIdType<TTableDef extends TraitAny> = TTableDef extends ClientDocumentTableDef.Trait<
-    any,
-    any,
-    any,
-    infer TOptions
-  >
-    ? TOptions['default']['id'] extends SessionIdSymbol | string
-      ? TOptions['default']['id']
+  export type DefaultIdType<TTableDef extends TraitAny> =
+    TTableDef extends ClientDocumentTableDef.Trait<any, any, any, infer TOptions>
+      ? TOptions['default']['id'] extends SessionIdSymbol | string
+        ? TOptions['default']['id']
+        : never
       : never
-    : never
 
   export type SetEventDefLike<
     TName extends string,

package/src/schema/state/sqlite/column-def.test.ts

@@ -144,11 +144,10 @@ describe('getColumnDefForSchema', () => {
 
     it('should map tagged unions to json column', () => {
       const ResultSchema = Schema.Union(
-        Schema.Struct({
-          _tag: Schema.Literal('success'),
+        Schema.TaggedStruct('success', {
           value: Schema.String,
         }),
-        Schema.Struct({ _tag: Schema.Literal('error'), error: Schema.String }),
+        Schema.TaggedStruct('error', { error: Schema.String }),
       )
 
       const columnDef = State.SQLite.getColumnDefForSchema(ResultSchema)

package/src/schema/state/sqlite/db-schema/dsl/mod.ts

@@ -20,13 +20,12 @@ export type DbSchemaInput = Record<string, TableDefinition<any, any>> | Readonly
  * - array: we use the table name of each array item (= table definition) as the object key
  * - object: we discard the keys of the input object and use the table name of each object value (= table definition) as the new object key
  */
-export type DbSchemaFromInputSchema<TSchemaInput extends DbSchemaInput> = TSchemaInput extends ReadonlyArray<
-  TableDefinition<any, any>
->
-  ? { [K in TSchemaInput[number] as K['name']]: K }
-  : TSchemaInput extends Record<string, TableDefinition<any, any>>
-    ? { [K in keyof TSchemaInput as TSchemaInput[K]['name']]: TSchemaInput[K] }
-    : never
+export type DbSchemaFromInputSchema<TSchemaInput extends DbSchemaInput> =
+  TSchemaInput extends ReadonlyArray<TableDefinition<any, any>>
+    ? { [K in TSchemaInput[number] as K['name']]: K }
+    : TSchemaInput extends Record<string, TableDefinition<any, any>>
+      ? { [K in keyof TSchemaInput as TSchemaInput[K]['name']]: TSchemaInput[K] }
+      : never
 
 // TODO ensure via runtime check (possibly even via type-level check) that all index names are unique
 export const makeDbSchema = <TDbSchemaInput extends DbSchemaInput>(
@@ -116,12 +115,8 @@ export type TableDefinition<TName extends string, TColumns extends Columns> = {
 
 export type Columns = Record<string, ColumnDefinition<any, any>>
 
-export type IsSingleColumn<TColumns extends Columns | ColumnDefinition<any, any>> = TColumns extends ColumnDefinition<
-  any,
-  any
->
-  ? true
-  : false
+export type IsSingleColumn<TColumns extends Columns | ColumnDefinition<any, any>> =
+  TColumns extends ColumnDefinition<any, any> ? true : false
 
 /**
  * NOTE this is only needed to avoid a TS limitation where `StructSchemaForColumns` in the default case
@@ -214,9 +209,8 @@ export namespace FromColumns {
 
   export type RequiredInsertColumnNames<TColumns extends Columns> = keyof RequiredInsertColumns<TColumns>
 
-  export type RequiresInsertValues<TColumns extends Columns> = RequiredInsertColumnNames<TColumns> extends never
-    ? false
-    : true
+  export type RequiresInsertValues<TColumns extends Columns> =
+    RequiredInsertColumnNames<TColumns> extends never ? false : true
 
   export type InsertRowDecoded<TColumns extends Columns> = Types.Simplify<
     Pick<RowDecodedAll<TColumns>, RequiredInsertColumnNames<TColumns>> &

package/src/schema/state/sqlite/query-builder/api.ts

@@ -242,7 +242,12 @@ export namespace QueryBuilder {
       ): QueryBuilder<TResult, TTableDef, TWithout | 'row' | 'select'>
       <TColName extends keyof TTableDef['sqliteDef']['columns']>(
         col: TColName,
-        op: QueryBuilder.WhereOps,
+        op: QueryBuilder.WhereOps.MultiValue,
+        value: ReadonlyArray<TTableDef['sqliteDef']['columns'][TColName]['schema']['Type']>,
+      ): QueryBuilder<TResult, TTableDef, TWithout | 'row' | 'select'>
+      <TColName extends keyof TTableDef['sqliteDef']['columns']>(
+        col: TColName,
+        op: QueryBuilder.WhereOps.SingleValue,
         value: TTableDef['sqliteDef']['columns'][TColName]['schema']['Type'],
       ): QueryBuilder<TResult, TTableDef, TWithout | 'row' | 'select'>
     }

package/src/schema/state/sqlite/table-def.ts

@@ -381,14 +381,15 @@ export declare namespace SchemaToColumns {
   export type ColumnDefForType<TEncoded, TType> = SqliteDsl.ColumnDefinition<TEncoded, TType>
 
   // Create columns type from schema Type and Encoded
-  export type FromTypes<TType, TEncoded> = TEncoded extends Record<string, any>
-    ? {
-        [K in keyof TEncoded]-?: ColumnDefForType<
-          TEncoded[K],
-          TType extends Record<string, any> ? (K extends keyof TType ? TType[K] : TEncoded[K]) : TEncoded[K]
-        >
-      }
-    : SqliteDsl.Columns
+  export type FromTypes<TType, TEncoded> =
+    TEncoded extends Record<string, any>
+      ? {
+          [K in keyof TEncoded]-?: ColumnDefForType<
+            TEncoded[K],
+            TType extends Record<string, any> ? (K extends keyof TType ? TType[K] : TEncoded[K]) : TEncoded[K]
+          >
+        }
+      : SqliteDsl.Columns
 }
 
 export declare namespace TableDefInput {

package/src/version.ts

@@ -2,7 +2,7 @@
 // import packageJson from '../package.json' with { type: 'json' }
 // export const liveStoreVersion = packageJson.version
 
-export const liveStoreVersion = '0.4.0-dev.20' as const
+export const liveStoreVersion = '0.4.0-dev.21' as const
 
 /**
  * CRITICAL: Increment this version whenever you modify client-side EVENTLOG table schemas.