@livestore/livestore 0.4.0-dev.9 → 0.4.0

package/src/store/store-eventstream.test.ts

@@ -0,0 +1,114 @@
+import { expect } from 'vitest'
+
+import { makeInMemoryAdapter } from '@livestore/adapter-web'
+import type { MockSyncBackend } from '@livestore/common'
+import { type ClientSessionLeaderThreadProxy, makeMockSyncBackend, type UnknownError } from '@livestore/common'
+import type { LiveStoreEvent, LiveStoreSchema } from '@livestore/common/schema'
+import { EventFactory } from '@livestore/common/testing'
+import type { ShutdownDeferred, Store } from '@livestore/livestore'
+import { createStore, makeShutdownDeferred } from '@livestore/livestore'
+import { omitUndefineds } from '@livestore/utils'
+import { Vitest } from '@livestore/utils-dev/node-vitest'
+import type { OtelTracer, Scope } from '@livestore/utils/effect'
+import { Context, Effect, FetchHttpClient, Layer, Logger, LogLevel, Queue, Stream } from '@livestore/utils/effect'
+import { nanoid } from '@livestore/utils/nanoid'
+import { PlatformNode } from '@livestore/utils/node'
+
+import { events, schema } from '../utils/tests/fixture.ts'
+
+const withTestCtx = Vitest.makeWithTestCtx({
+  makeLayer: () =>
+    Layer.mergeAll(
+      TestContextLive,
+      PlatformNode.NodeFileSystem.layer,
+      FetchHttpClient.layer,
+      Logger.minimumLogLevel(LogLevel.Debug),
+    ),
+})
+
+/**
+ * The purpose of this test is a store integration test for event streaming.
+ * Main test covering event streaming logic itself is located in:
+ * tests/package-common/src/leader-thread/stream-events.test.ts
+ */
+Vitest.describe('Store events API', () => {
+  Vitest.scopedLive('should resume when reconnected to sync backend', (test) =>
+    Effect.gen(function* () {
+      const { makeStore, mockSyncBackend } = yield* TestContext
+      const store = yield* makeStore()
+      yield* mockSyncBackend.connect
+
+      const eventFactory = EventFactory.makeFactory(events)({
+        client: EventFactory.clientIdentity('other-client', 'static-session-id'),
+      })
+
+      // Queue is used in order to allow analyzing the stream in stages
+      const eventsQueue = yield* Queue.unbounded<LiveStoreEvent.Client.ForSchema<typeof schema>>()
+
+      yield* store.eventsStream().pipe(
+        Stream.tap((event) => Queue.offer(eventsQueue, event)),
+        Stream.runDrain,
+        Effect.forkScoped,
+      )
+
+      store.commit(eventFactory.todoCreated.next({ id: '1', text: 't1', completed: false }))
+      const initialEvent = yield* Queue.take(eventsQueue)
+      expect(initialEvent.name).toEqual('todo.created')
+      expect(initialEvent.args).toMatchObject({ id: '1' })
+
+      yield* mockSyncBackend.disconnect
+      store.commit(eventFactory.todoCreated.next({ id: '2', text: 't2', completed: false }))
+      const maybeWhileDisconnected = yield* Queue.take(eventsQueue).pipe(Effect.timeout('250 millis'), Effect.option)
+      expect(maybeWhileDisconnected._tag).toEqual('None')
+
+      yield* mockSyncBackend.connect
+      const resumedEvent = yield* Queue.take(eventsQueue)
+      expect(resumedEvent.name).toEqual('todo.created')
+      expect(resumedEvent.args).toMatchObject({ id: '2' })
+    }).pipe(withTestCtx(test)),
+  )
+})
+
+class TestContext extends Context.Tag('TestContext')<
+  TestContext,
+  {
+    makeStore: (args?: {
+      boot?: (store: Store) => void
+      testing?: {
+        overrides?: {
+          clientSession?: {
+            leaderThreadProxy?: (
+              original: ClientSessionLeaderThreadProxy.ClientSessionLeaderThreadProxy,
+            ) => Partial<ClientSessionLeaderThreadProxy.ClientSessionLeaderThreadProxy>
+          }
+        }
+      }
+    }) => Effect.Effect<Store, UnknownError, Scope.Scope | OtelTracer.OtelTracer>
+    mockSyncBackend: MockSyncBackend
+    shutdownDeferred: ShutdownDeferred
+  }
+>() {}
+
+const TestContextLive = Layer.scoped(
+  TestContext,
+  Effect.gen(function* () {
+    const mockSyncBackend = yield* makeMockSyncBackend()
+    const shutdownDeferred = yield* makeShutdownDeferred
+
+    const makeStore: typeof TestContext.Service.makeStore = (args) => {
+      const adapter = makeInMemoryAdapter({
+        sync: { backend: () => mockSyncBackend.makeSyncBackend, onSyncError: 'shutdown' },
+        ...omitUndefineds({ testing: args?.testing }),
+      })
+      return createStore({
+        schema: schema as LiveStoreSchema,
+        adapter,
+        storeId: nanoid(),
+        shutdownDeferred,
+        ...omitUndefineds({ boot: args?.boot }),
+      })
+    }
+
+    return { makeStore, mockSyncBackend, shutdownDeferred }
+  }),
+)

package/src/store/store-types.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, it } from 'vitest'
+
+import type { QueryBuilder } from '@livestore/common'
+import { QueryBuilderTypeId } from '@livestore/common'
+import { Schema } from '@livestore/utils/effect'
+
+import { TypeId } from '../live-queries/base-class.ts'
+import { queryDb, signal } from '../live-queries/mod.ts'
+import { isQueryable } from './store-types.ts'
+
+const makeQueryBuilder = (): QueryBuilder<any, any, any> =>
+  ({
+    [QueryBuilderTypeId]: QueryBuilderTypeId,
+    ResultType: null,
+    asSql: () => ({ query: 'select 1', bindValues: [], usedTables: new Set<string>() }),
+    toString: () => 'select 1',
+  }) as unknown as QueryBuilder<any, any, any>
+
+describe('isQueryable', () => {
+  it('identifies live query definitions', () => {
+    const def = queryDb({
+      query: 'select 1 as value',
+      schema: Schema.Array(Schema.Struct({ value: Schema.Number })),
+    })
+
+    expect(isQueryable(def)).toBe(true)
+  })
+
+  it('identifies signal definitions', () => {
+    const sig = signal(0, { label: 'count' })
+
+    expect(isQueryable(sig)).toBe(true)
+  })
+
+  it('identifies live query instances', () => {
+    const liveQueryLike = { [TypeId]: TypeId } as const
+
+    expect(isQueryable(liveQueryLike)).toBe(true)
+  })
+
+  it('identifies query builders', () => {
+    const qb = makeQueryBuilder()
+
+    expect(isQueryable(qb)).toBe(true)
+  })
+
+  it('rejects unrelated values', () => {
+    expect(isQueryable(null)).toBe(false)
+    expect(isQueryable(undefined)).toBe(false)
+    expect(isQueryable({})).toBe(false)
+  })
+})

package/src/store/store-types.ts

@@ -1,46 +1,79 @@
-import type {
-  ClientSession,
-  ClientSessionSyncProcessorSimulationParams,
-  IntentionalShutdownCause,
-  InvalidPullError,
-  IsOfflineError,
-  MaterializeError,
-  StoreInterrupted,
-  SyncError,
-  UnexpectedError,
-} from '@livestore/common'
-import type { EventSequenceNumber, LiveStoreEvent, LiveStoreSchema } from '@livestore/common/schema'
-import type { Effect, Runtime, Scope } from '@livestore/utils/effect'
-import { Deferred } from '@livestore/utils/effect'
 import type * as otel from '@opentelemetry/api'
 
-import type { DebugRefreshReasonBase } from '../reactive.ts'
+import {
+  type ClientSession,
+  type ClientSessionSyncProcessor,
+  type ClientSessionSyncProcessorSimulationParams,
+  type IntentionalShutdownCause,
+  isQueryBuilder,
+  type MaterializeError,
+  type QueryBuilder,
+  type StoreInterrupted,
+  type BackendIdMismatchError,
+  type UnknownError,
+} from '@livestore/common'
+import type { StreamEventsOptions } from '@livestore/common/leader-thread'
+import type { LiveStoreEvent, LiveStoreSchema } from '@livestore/common/schema'
+import type { Effect, Runtime, Schema, Scope } from '@livestore/utils/effect'
+import { Deferred, Predicate } from '@livestore/utils/effect'
+
+import type {
+  LiveQuery,
+  LiveQueryDef,
+  ReactivityGraph,
+  ReactivityGraphContext,
+  SignalDef,
+} from '../live-queries/base-class.ts'
+import { TypeId } from '../live-queries/base-class.ts'
+import type { DebugRefreshReasonBase, Ref } from '../reactive.ts'
+import type { SqliteDbWrapper } from '../SqliteDbWrapper.ts'
+import type { ReferenceCountedSet } from '../utils/data-structures.ts'
 import type { StackInfo } from '../utils/stack-info.ts'
 import type { Store } from './store.ts'
 
-export type LiveStoreContext =
-  | LiveStoreContextRunning
+/**
+ * Union type representing the possible states of a LiveStore context.
+ *
+ * Used by framework integrations (React, Solid, etc.) to track Store lifecycle:
+ * - `running`: Store is active and ready for queries/commits
+ * - `error`: Store failed during boot or operation
+ * - `shutdown`: Store was intentionally shut down or interrupted
+ *
+ * @typeParam TSchema - The LiveStore schema type. Defaults to `LiveStoreSchema.Any`.
+ */
+export type LiveStoreContext<TSchema extends LiveStoreSchema = LiveStoreSchema.Any> =
+  | LiveStoreContextRunning<TSchema>
   | {
       stage: 'error'
-      error: UnexpectedError | unknown
+      error: unknown
     }
   | {
       stage: 'shutdown'
-      cause: IntentionalShutdownCause | StoreInterrupted | SyncError
+      cause: IntentionalShutdownCause | StoreInterrupted | UnknownError
     }
 
 export type ShutdownDeferred = Deferred.Deferred<
   IntentionalShutdownCause,
-  UnexpectedError | SyncError | StoreInterrupted | MaterializeError | InvalidPullError | IsOfflineError
+  UnknownError | StoreInterrupted | MaterializeError | BackendIdMismatchError
 >
 export const makeShutdownDeferred: Effect.Effect<ShutdownDeferred> = Deferred.make<
   IntentionalShutdownCause,
-  UnexpectedError | SyncError | StoreInterrupted | MaterializeError | InvalidPullError | IsOfflineError
+  UnknownError | StoreInterrupted | MaterializeError | BackendIdMismatchError
 >()
 
-export type LiveStoreContextRunning = {
+/**
+ * Context state when the Store is active and ready for use.
+ *
+ * This is the normal operating state where you can query data, commit events,
+ * and subscribe to changes.
+ *
+ * @typeParam TSchema - The LiveStore schema type. Defaults to `LiveStoreSchema.Any`
+ *   for backwards compatibility, but prefer providing the concrete schema type
+ *   for full type safety.
+ */
+export type LiveStoreContextRunning<TSchema extends LiveStoreSchema = LiveStoreSchema.Any> = {
   stage: 'running'
-  store: Store
+  store: Store<TSchema>
 }
 
 export type OtelOptions = {
@@ -48,7 +81,108 @@ export type OtelOptions = {
   rootSpanContext: otel.Context
 }
 
-export type StoreOptions<TSchema extends LiveStoreSchema = LiveStoreSchema.Any, TContext = {}> = {
+export const StoreInternalsSymbol = Symbol.for('livestore.StoreInternals')
+export type StoreInternalsSymbol = typeof StoreInternalsSymbol
+
+/**
+ * Opaque bag containing the Store's implementation details.
+ *
+ * Not part of the public API — shapes and semantics may change without notice.
+ * Access only from within the @livestore/livestore package (and Devtools) via
+ * `StoreInternalsSymbol` to avoid accidental coupling in application code.
+ */
+export type StoreInternals = {
+  /**
+   * Runtime event schema used for encoding/decoding events.
+   *
+   * Exposed primarily for Devtools (e.g. databrowser) to validate ad‑hoc
+   * event payloads. Application code should not depend on it directly.
+   */
+  readonly eventSchema: Schema.Schema<LiveStoreEvent.Client.Decoded, LiveStoreEvent.Client.Encoded>
+
+  /**
+   * The active client session backing this Store. Provides access to the
+   * leader thread, network status, and shutdown signaling.
+   *
+   * Do not close or mutate directly — use `store.shutdown(...)`.
+   */
+  readonly clientSession: ClientSession
+
+  /**
+   * Wrapper around the local SQLite state database. Centralizes query
+   * planning, caching, and change tracking used by reads and materializers.
+   */
+  readonly sqliteDbWrapper: SqliteDbWrapper
+
+  /**
+   * Effect runtime and scope used to fork background fibers for the Store.
+   *
+   * - `runtime` executes effects from imperative Store APIs.
+   * - `lifetimeScope` owns forked fibers; closed during Store shutdown.
+   */
+  readonly effectContext: {
+    /** Effect runtime to run Store effects with proper environment. */
+    readonly runtime: Runtime.Runtime<Scope.Scope>
+    /** Scope that owns all long‑lived fibers spawned by the Store. */
+    readonly lifetimeScope: Scope.Scope
+  }
+
+  /**
+   * OpenTelemetry primitives used for instrumentation of commits, queries,
+   * and Store boot lifecycle.
+   */
+  readonly otel: StoreOtel
+
+  /**
+   * The Store's reactive graph instance used to model dependencies and
+   * propagate updates. Provides APIs to create refs/thunks/effects and to
+   * subscribe to refresh cycles.
+   */
+  readonly reactivityGraph: ReactivityGraph
+
+  /**
+   * Per‑table reactive refs used to broadcast invalidations when materializers
+   * write to tables. Values are always `null`; equality is intentionally
+   * `false` to force recomputation.
+   *
+   * Keys are SQLite table names (user tables; some system tables may be
+   * intentionally excluded from refresh).
+   */
+  readonly tableRefs: Readonly<Record<string, Ref<null, ReactivityGraphContext, RefreshReason>>>
+
+  /**
+   * Set of currently subscribed LiveQuery instances (reference‑counted).
+   * Used for Devtools and diagnostics.
+   */
+  readonly activeQueries: ReferenceCountedSet<LiveQuery<any>>
+
+  /**
+   * Client‑session sync processor orchestrating push/pull and materialization
+   * of events into local state.
+   */
+  readonly syncProcessor: ClientSessionSyncProcessor
+
+  /**
+   * Starts background fibers for sync and observation. Must be run exactly
+   * once per Store instance. Scoped; installs finalizers to end spans and
+   * detach reactive refs.
+   */
+  readonly boot: Effect.Effect<void, UnknownError, Scope.Scope>
+
+  /**
+   * Tracks whether the Store has been shut down. When true, mutating APIs
+   * should reject via `checkShutdown`.
+   */
+  isShutdown: boolean
+}
+
+/**
+ * Parameters for constructing a Store instance.
+ *
+ * @internal This type is used by the Store constructor and is not part of the public API.
+ * For creating stores, use `createStore()` or `StoreRegistry` instead.
+ */
+export type StoreConstructorParams<TSchema extends LiveStoreSchema = LiveStoreSchema.Any, TContext = {}> = {
   clientSession: ClientSession
   schema: TSchema
   storeId: string
@@ -62,6 +196,7 @@ export type StoreOptions<TSchema extends LiveStoreSchema = LiveStoreSchema.Any,
   batchUpdates: (runUpdates: () => void) => void
   params: {
     leaderPushBatchSize: number
+    eventQueryBatchSize?: number
     simulation?: {
       clientSessionSyncProcessor: typeof ClientSessionSyncProcessorSimulationParams.Type
     }
@@ -69,12 +204,18 @@ export type StoreOptions<TSchema extends LiveStoreSchema = LiveStoreSchema.Any,
   __runningInDevtools: boolean
 }
 
+/**
+ * Tagged union describing why a reactive refresh occurred.
+ *
+ * Used internally for debugging and devtools to trace the cause of query re-evaluations.
+ * Each variant includes context about what triggered the refresh.
+ */
 export type RefreshReason =
   | DebugRefreshReasonBase
   | {
       _tag: 'commit'
       /** The events that were applied */
-      events: ReadonlyArray<LiveStoreEvent.AnyDecoded | LiveStoreEvent.PartialAnyDecoded>
+      events: ReadonlyArray<LiveStoreEvent.Client.Decoded | LiveStoreEvent.Input.Decoded>
 
       /** The tables that were written to by the event */
       writeTables: ReadonlyArray<string>
@@ -90,10 +231,19 @@ export type RefreshReason =
   | { _tag: 'subscribe.update'; label?: string }
   | { _tag: 'manual'; label?: string }
 
+/**
+ * Debug information captured for each query execution.
+ *
+ * Used by devtools and performance monitoring to track query behavior.
+ */
 export type QueryDebugInfo = {
+  /** Query type discriminator ('db', 'computed', etc.) */
   _tag: string
+  /** Human-readable query label */
   label: string
+  /** SQL query string or computed function representation */
   query: string
+  /** Execution time in milliseconds */
   durationMs: number
 }
 
@@ -111,27 +261,197 @@ export type StoreCommitOptions = {
   otelContext?: otel.Context
 }
 
-export type StoreEventsOptions<TSchema extends LiveStoreSchema> = {
-  /**
-   * By default only new events are returned.
-   * Use this to get all events from a specific point in time.
-   */
-  cursor?: EventSequenceNumber.EventSequenceNumber
+/**
+ * filter: Narrowed to the store's event types
+ * includeClientOnly: Omitted from public API until supported
+ */
+export type StoreEventsOptions<TSchema extends LiveStoreSchema> = Omit<
+  StreamEventsOptions,
+  'filter' | 'includeClientOnly'
+> & {
   /**
-   * Only include events of the given names
+   * Only include events of the given names.
    * @default undefined (include all)
    */
   filter?: ReadonlyArray<keyof TSchema['_EventDefMapType']>
+}
+
+/**
+ * Function returned by `store.subscribe()` to stop receiving updates.
+ *
+ * Call this to unsubscribe from a query and release the associated resources.
+ *
+ * @example
+ * ```ts
+ * const unsubscribe = store.subscribe(todos$, (todos) => console.log(todos))
+ * // Later...
+ * unsubscribe()
+ * ```
+ */
+export type Unsubscribe = () => void
+
+/**
+ * Options for `store.subscribe()`.
+ *
+ * @typeParam TResult - The result type of the subscribed query
+ */
+export type SubscribeOptions<TResult> = {
+  /** Callback invoked when the subscription is established (receives the live query instance) */
+  onSubscribe?: (query$: LiveQuery<TResult>) => void
+  /** Callback invoked when the subscription is terminated */
+  onUnsubsubscribe?: () => void
+  /** Label for debugging and devtools */
+  label?: string
+  /** If true, skips invoking the callback for the initial value */
+  skipInitialRun?: boolean
+  /** OpenTelemetry context for tracing */
+  otelContext?: otel.Context
+  /** Stack trace info for debugging subscription origins */
+  stackInfo?: StackInfo
+}
+
+/** All query definitions or instances the store can execute or subscribe to. */
+export type Queryable<TResult> =
+  | LiveQueryDef<TResult>
+  | SignalDef<TResult>
+  | LiveQuery<TResult>
+  | QueryBuilder<TResult, any, any>
+
+/**
+ * Helper types for `Queryable`.
+ *
+ * Provides type-level utilities to work with `Queryable` values.
+ */
+export namespace Queryable {
+  /**
+   * Extracts the result type from a `Queryable`.
+   *
+   * Example:
+   * - `Queryable.Result<LiveQueryDef<number>>` → `number`
+   * - `Queryable.Result<SignalDef<string>>` → `string`
+   * - `Queryable.Result<LiveQuery<{ id: string }>>` → `{ id: string }`
+   * - `Queryable.Result<LiveQueryDef<A> | SignalDef<B>>` → `A | B`
+   */
+  export type Result<TQueryable extends Queryable<any>> = TQueryable extends Queryable<infer TResult> ? TResult : never
+}
+
+/**
+ * Type guard that checks if a value is a query or signal definition.
+ *
+ * Use this to distinguish between definitions (blueprints) and instances (live queries).
+ * Definitions are created by `queryDb()`, `computed()`, and `signal()`.
+ *
+ * @example
+ * ```ts
+ * const todos$ = queryDb(tables.todos.all())
+ *
+ * if (isLiveQueryDef(todos$)) {
+ *   console.log('This is a definition:', todos$.label)
+ * }
+ * ```
+ */
+export const isLiveQueryDef = (value: unknown): value is LiveQueryDef<any> | SignalDef<any> => {
+  if (typeof value !== 'object' || value === null) {
+    return false
+  }
+
+  if (!('_tag' in value)) {
+    return false
+  }
+
+  const tag = (value as LiveQueryDef<any> | SignalDef<any>)._tag
+  if (tag !== 'def' && tag !== 'signal-def') {
+    return false
+  }
+
+  const candidate = value as LiveQueryDef<any>
+  if (typeof candidate.make !== 'function') {
+    // The store calls make() to turn the definition into a live query instance.
+    return false
+  }
+
+  if (typeof candidate.hash !== 'string' || typeof candidate.label !== 'string') {
+    // Both identifiers must be present so the store can cache and log the query.
+    return false
+  }
+
+  return true
+}
+
+/**
+ * Type guard that checks if a value is a live query instance.
+ *
+ * Live query instances are stateful objects bound to a Store's reactivity graph.
+ * They're created internally when you use a definition with `store.query()` or `store.subscribe()`.
+ *
+ * @example
+ * ```ts
+ * const [, , , query$] = useClientDocument(tables.uiState)
+ *
+ * if (isLiveQueryInstance(query$)) {
+ *   console.log('Execution count:', query$.runs)
+ * }
+ * ```
+ */
+export const isLiveQueryInstance = (value: unknown): value is LiveQuery<any> => Predicate.hasProperty(value, TypeId)
+
+/**
+ * Type guard that checks if a value can be used with `store.query()` or `store.subscribe()`.
+ *
+ * Queryable values include:
+ * - Query definitions (`LiveQueryDef` from `queryDb()`, `computed()`)
+ * - Signal definitions (`SignalDef` from `signal()`)
+ * - Live query instances (`LiveQuery`)
+ * - Query builders (e.g., `tables.todos.where(...)`)
+ *
+ * @example
+ * ```ts
+ * const handleQuery = (input: unknown) => {
+ *   if (isQueryable(input)) {
+ *     return store.query(input)
+ *   }
+ *   throw new Error('Not a valid query')
+ * }
+ * ```
+ */
+export const isQueryable = (value: unknown): value is Queryable<unknown> =>
+  isQueryBuilder(value) || isLiveQueryInstance(value) || isLiveQueryDef(value)
+
+/**
+ * Represents the current synchronization status of the store.
+ *
+ * This provides visibility into the sync state between the client session
+ * and the leader thread, allowing applications to show sync indicators
+ * or determine backend health.
+ *
+ * @example
+ * ```ts
+ * const status = store.syncStatus()
+ * if (status.isSynced) {
+ *   console.log('All changes synced')
+ * } else {
+ *   console.log(`${status.pendingCount} events pending sync`)
+ * }
+ * ```
+ */
+export type SyncStatus = {
+  /**
+   * The local head sequence number (most recent event in the client session).
+   * Represented as a string in the format "e{global}.{client}" (e.g., "e5.2").
+   */
+  localHead: string
   /**
-   * Whether to include client-only events or only return synced events
-   * @default true
+   * The upstream head sequence number (what the leader thread has confirmed).
+   * Represented as a string in the format "e{global}" (e.g., "e3").
    */
-  includeClientOnly?: boolean
+  upstreamHead: string
   /**
-   * Exclude own events that have not been pushed to the sync backend yet
-   * @default false
+   * Number of events pending synchronization to the leader thread.
    */
-  excludeUnpushed?: boolean
+  pendingCount: number
+  /**
+   * Whether the client session is fully synced with the leader thread.
+   * True when there are no pending events (pendingCount === 0).
+   */
+  isSynced: boolean
 }
-
-export type Unsubscribe = () => void