@livestore/livestore 0.4.0-dev.9 → 0.5.0-dev.0

package/src/store/create-store.ts

@@ -1,17 +1,18 @@
+import * as otel from '@opentelemetry/api'
+
 import {
   type Adapter,
+  type BackendIdMismatchError,
   type BootStatus,
   type ClientSession,
   type ClientSessionDevtoolsChannel,
-  type ClientSessionSyncProcessorSimulationParams,
   type IntentionalShutdownCause,
-  type InvalidPullError,
-  type IsOfflineError,
   type MaterializeError,
   type MigrationsReport,
   provideOtel,
-  type SyncError,
-  UnexpectedError,
+  type ServerAheadError,
+  UnknownError,
+  type LogConfig,
 } from '@livestore/common'
 import type { LiveStoreSchema } from '@livestore/common/schema'
 import { isDevEnv, LS_DEV, omitUndefineds } from '@livestore/utils'
@@ -23,47 +24,76 @@ import {
   Fiber,
   identity,
   Layer,
-  Logger,
-  LogLevel,
   OtelTracer,
   Queue,
-  Runtime,
-  type Schema,
+  References,
+  Schema,
   Scope,
   TaskTracing,
 } from '@livestore/utils/effect'
 import { nanoid } from '@livestore/utils/nanoid'
-import * as otel from '@opentelemetry/api'
 
 import { connectDevtoolsToStore } from './devtools.ts'
-import { Store } from './store.ts'
 import type {
   LiveStoreContextRunning as LiveStoreContextRunning_,
   OtelOptions,
   ShutdownDeferred,
 } from './store-types.ts'
-
-export const DEFAULT_PARAMS = {
-  leaderPushBatchSize: 100,
+import { StoreInternalsSymbol } from './store-types.ts'
+import { STORE_DEFAULT_PARAMS, Store } from './store.ts'
+
+/**
+ * Hard upper bound (ms) for the detached shutdown drain. A dead/unresponsive leader must not keep
+ * the drain (and therefore the lifetime scope) alive forever; after this bound the scope is
+ * force-closed. Kept comfortably above the 1s caller-side soft wait so that a still-progressing
+ * in-flight leader push is allowed to finish rather than being interrupted.
+ */
+const SHUTDOWN_DRAIN_HARD_TIMEOUT_MS = 30_000
+
+declare global {
+  /** Store instances for console debugging */
+  var __debugLiveStore: Record<string, Store<any, any>> | undefined
 }
 
-export class LiveStoreContextRunning extends Context.Tag('@livestore/livestore/effect/LiveStoreContextRunning')<
-  LiveStoreContextRunning,
-  LiveStoreContextRunning_
->() {
+/**
+ * @deprecated Use `makeStoreContext()` from `@livestore/livestore/effect` instead.
+ * This service doesn't preserve schema types. See the Effect integration docs for migration.
+ *
+ * @example Migration
+ * ```ts
+ * // Before (untyped)
+ * import { LiveStoreContextRunning } from '@livestore/livestore/effect'
+ * const { store } = yield* LiveStoreContextRunning
+ *
+ * // After (typed)
+ * import { makeStoreContext } from '@livestore/livestore/effect'
+ * const AppStore = makeStoreContext<typeof schema>()('app')
+ * const { store } = yield* AppStore.Tag
+ * ```
+ */
+export class LiveStoreContextRunning extends Context.Service<LiveStoreContextRunning, LiveStoreContextRunning_>()(
+  '@livestore/livestore/effect/LiveStoreContextRunning',
+) {
   static fromDeferred = Effect.gen(function* () {
     const deferred = yield* DeferredStoreContext
-    const ctx = yield* deferred
-    return Layer.succeed(LiveStoreContextRunning, ctx)
-  }).pipe(Layer.unwrapScoped)
+    const ctx = yield* Deferred.await(deferred)
+    return Layer.succeed(LiveStoreContextRunning, LiveStoreContextRunning.of(ctx))
+  }).pipe(Layer.unwrap)
 }
 
-export class DeferredStoreContext extends Context.Tag('@livestore/livestore/effect/DeferredStoreContext')<
+/**
+ * @deprecated Use `StoreContext.DeferredTag` from `makeStoreContext()` instead.
+ */
+export class DeferredStoreContext extends Context.Service<
   DeferredStoreContext,
-  Deferred.Deferred<LiveStoreContextRunning['Type'], UnexpectedError>
->() {}
-
-export type LiveStoreContextProps<TSchema extends LiveStoreSchema, TContext = {}> = {
+  Deferred.Deferred<LiveStoreContextRunning['Service'], UnknownError>
+>()('@livestore/livestore/effect/DeferredStoreContext') {}
+
+export type LiveStoreContextProps<
+  TSchema extends LiveStoreSchema,
+  TContext = {},
+  TSyncPayloadSchema extends Schema.Codec<Schema.Json, Schema.Json> = typeof Schema.Json,
+> = {
   schema: TSchema
   /**
    * The `storeId` can be used to isolate multiple stores from each other.
@@ -88,12 +118,48 @@ export type LiveStoreContextProps<TSchema extends LiveStoreSchema, TContext = {}
   disableDevtools?: boolean | 'auto'
   onBootStatus?: (status: BootStatus) => void
   batchUpdates: (run: () => void) => void
+  /**
+   * Schema describing the shape of the sync payload and used to encode it.
+   *
+   * - If omitted, `Schema.Json` is used (no additional typing/validation).
+   * - Prefer exporting a schema from your app (e.g. `export const SyncPayload = Schema.Struct({ authToken: Schema.String })`)
+   *   and pass it here to get end-to-end type safety and validation.
+   */
+  syncPayloadSchema?: TSyncPayloadSchema
+  /**
+   * Payload that is sent to the sync backend when connecting
+   *
+   * - Its TypeScript type is inferred from `syncPayloadSchema` (i.e. `typeof SyncPayload.Type`).
+   * - At runtime this value is encoded with `syncPayloadSchema` before being handed to the adapter.
+   *
+   * Example:
+   *   const SyncPayload = Schema.Struct({ authToken: Schema.String })
+   *   useStore({ ..., syncPayloadSchema: SyncPayload, syncPayload: { authToken: '...' } })
+   */
+  syncPayload?: TSyncPayloadSchema['Type']
 }
 
-export interface CreateStoreOptions<TSchema extends LiveStoreSchema, TContext = {}> {
+export interface CreateStoreOptions<
+  TSchema extends LiveStoreSchema,
+  TContext = {},
+  TSyncPayloadSchema extends Schema.Codec<Schema.Json, Schema.Json> = typeof Schema.Json,
+>
+  extends LogConfig.LoggerOptions {
+  /** The LiveStore schema defining tables, events, and materializers. */
   schema: TSchema
+  /** Adapter used for data storage and synchronization. */
   adapter: Adapter
+  /**
+   * Unique identifier for the Store instance, stable for its lifetime.
+   *
+   * - **Valid characters**: Only alphanumeric characters, underscores (`_`), and hyphens (`-`)
+   *   are allowed. Must match `/^[a-zA-Z0-9_-]+$/`.
+   * - **Globally unique**: Use globally unique IDs (e.g., nanoid) to prevent collisions across stores.
+   * - **Use namespaces**: Prefix to avoid collisions and for easier identification when debugging
+   *   (e.g., `app-root`, `workspace-abc123`, `issue-456`)
+   */
   storeId: string
+  /** User-defined context that will be attached to the created Store (e.g. for dependency injection). */
   context?: TContext
   boot?: (
     store: Store<TSchema, TContext>,
@@ -102,6 +168,19 @@ export interface CreateStoreOptions<TSchema extends LiveStoreSchema, TContext =
       parentSpan: otel.Span
     },
   ) => Effect.SyncOrPromiseOrEffect<void, unknown, OtelTracer.OtelTracer | LiveStoreContextRunning>
+  onBootStatus?: (status: BootStatus) => void
+  /**
+   * Needed in React so LiveStore can apply multiple events in a single render.
+   *
+   * @example
+   * ```ts
+   * // With React DOM
+   * import { unstable_batchedUpdates as batchUpdates } from 'react-dom'
+   *
+   * // With React Native
+   * import { unstable_batchedUpdates as batchUpdates } from 'react-native'
+   * ```
+   */
   batchUpdates?: (run: () => void) => void
   /**
    * Whether to disable devtools.
@@ -109,7 +188,6 @@ export interface CreateStoreOptions<TSchema extends LiveStoreSchema, TContext =
    * @default 'auto'
    */
   disableDevtools?: boolean | 'auto'
-  onBootStatus?: (status: BootStatus) => void
   shutdownDeferred?: ShutdownDeferred
   /**
    * Currently only used in the web adapter:
@@ -119,42 +197,65 @@ export interface CreateStoreOptions<TSchema extends LiveStoreSchema, TContext =
    */
   confirmUnsavedChanges?: boolean
   /**
-   * Payload that will be passed to the sync backend when connecting
+   * Schema describing the shape of the sync payload and used to encode it.
+   *
+   * - If omitted, `Schema.Json` is used (no additional typing/validation).
+   * - Prefer exporting a schema from your app (e.g. `export const SyncPayload = Schema.Struct({ authToken: Schema.String })`)
+   *   and pass it here to get end-to-end type safety and validation.
+   */
+  syncPayloadSchema?: TSyncPayloadSchema
+  /**
+   * Payload that is sent to the sync backend when connecting
+   *
+   * - Its TypeScript type is inferred from `syncPayloadSchema` (i.e. `typeof SyncPayload.Type`).
+   * - At runtime this value is encoded with `syncPayloadSchema` and carried through the adapter
+   *   to the backend where it can be decoded with the same schema.
    *
    * @default undefined
    */
-  syncPayload?: Schema.JsonValue
+  syncPayload?: TSyncPayloadSchema['Type']
+  /** Options provided to the Store constructor. */
   params?: {
+    /** Max events pushed to the leader per write batch. */
     leaderPushBatchSize?: number
-    simulation?: {
-      clientSessionSyncProcessor: typeof ClientSessionSyncProcessorSimulationParams.Type
-    }
+    /** Chunk size used when the stream replays confirmed events. */
+    eventQueryBatchSize?: number
   }
   debug?: {
     instanceId?: string
   }
 }
 
+export type CreateStoreOptionsPromise<
+  TSchema extends LiveStoreSchema = LiveStoreSchema.Any,
+  TContext = {},
+  TSyncPayloadSchema extends Schema.Codec<Schema.Json, Schema.Json> = typeof Schema.Json,
+> = CreateStoreOptions<TSchema, TContext, TSyncPayloadSchema> & {
+  signal?: AbortSignal
+  otelOptions?: Partial<OtelOptions>
+}
+
 /** Create a new LiveStore Store */
-export const createStorePromise = async <TSchema extends LiveStoreSchema = LiveStoreSchema.Any, TContext = {}>({
+export const createStorePromise = async <
+  TSchema extends LiveStoreSchema = LiveStoreSchema.Any,
+  TContext = {},
+  TSyncPayloadSchema extends Schema.Codec<Schema.Json, Schema.Json> = typeof Schema.Json,
+>({
   signal,
   otelOptions,
   ...options
-}: CreateStoreOptions<TSchema, TContext> & {
-  signal?: AbortSignal
-  otelOptions?: Partial<OtelOptions>
-}): Promise<Store<TSchema, TContext>> =>
+}: CreateStoreOptionsPromise<TSchema, TContext, TSyncPayloadSchema>): Promise<Store<TSchema, TContext>> =>
   Effect.gen(function* () {
     const scope = yield* Scope.make()
-    const runtime = yield* Effect.runtime()
+    const services = yield* Effect.context()
 
     if (signal !== undefined) {
       signal.addEventListener('abort', () => {
-        Scope.close(scope, Exit.void).pipe(Effect.tapCauseLogPretty, Runtime.runFork(runtime))
+        Scope.close(scope, Exit.void).pipe(Effect.tapCauseLogPretty, Effect.runForkWith(services))
       })
     }
 
-    return yield* createStore({ ...options }).pipe(Scope.extend(scope))
+    return yield* createStore({ ...options }).pipe(Scope.provide(scope))
   }).pipe(
     Effect.withSpan('createStore', {
       attributes: { storeId: options.storeId, disableDevtools: options.disableDevtools },
@@ -162,12 +263,20 @@ export const createStorePromise = async <TSchema extends LiveStoreSchema = LiveS
     provideOtel(omitUndefineds({ parentSpanContext: otelOptions?.rootSpanContext, otelTracer: otelOptions?.tracer })),
     Effect.tapCauseLogPretty,
     Effect.annotateLogs({ thread: 'window' }),
-    Effect.provide(Logger.prettyWithThread('window')),
-    Logger.withMinimumLogLevel(LogLevel.Debug),
+    Effect.provide(
+      Layer.mergeAll(
+        options.logger ?? Layer.empty,
+        Layer.succeed(References.MinimumLogLevel, options.logLevel ?? (isDevEnv() === true ? 'Debug' : 'Info')),
+      ),
+    ),
     Effect.runPromise,
   )
 
-export const createStore = <TSchema extends LiveStoreSchema = LiveStoreSchema.Any, TContext = {}>({
+export const createStore = <
+  TSchema extends LiveStoreSchema = LiveStoreSchema.Any,
+  TContext = {},
+  TSyncPayloadSchema extends Schema.Codec<Schema.Json, Schema.Json> = typeof Schema.Json,
+>({
   schema,
   adapter,
   storeId,
@@ -181,9 +290,10 @@ export const createStore = <TSchema extends LiveStoreSchema = LiveStoreSchema.An
   debug,
   confirmUnsavedChanges = true,
   syncPayload,
-}: CreateStoreOptions<TSchema, TContext>): Effect.Effect<
+  syncPayloadSchema,
+}: CreateStoreOptions<TSchema, TContext, TSyncPayloadSchema>): Effect.Effect<
   Store<TSchema, TContext>,
-  UnexpectedError,
+  UnknownError,
   Scope.Scope | OtelTracer.OtelTracer
 > =>
   Effect.gen(function* () {
@@ -194,17 +304,20 @@ export const createStore = <TSchema extends LiveStoreSchema = LiveStoreSchema.An
     yield* Effect.addFinalizer((_) => Scope.close(lifetimeScope, _))
 
     const debugInstanceId = debug?.instanceId ?? nanoid(10)
+    const resolvedSyncPayloadSchema = (syncPayloadSchema ?? Schema.Json) as TSyncPayloadSchema
 
     return yield* Effect.gen(function* () {
       const span = yield* OtelTracer.currentOtelSpan.pipe(Effect.orDie)
       const otelRootSpanContext = otel.trace.setSpan(otel.context.active(), span)
       const otelTracer = yield* OtelTracer.OtelTracer
 
-      const bootStatusQueue = yield* Queue.unbounded<BootStatus>().pipe(Effect.acquireRelease(Queue.shutdown))
+      const bootStatusQueue = yield* Effect.acquireRelease(Queue.unbounded<BootStatus>(), Queue.shutdown)
 
       yield* Queue.take(bootStatusQueue).pipe(
         Effect.tapSync((status) => onBootStatus?.(status)),
-        Effect.tap((status) => (status.stage === 'done' ? Queue.shutdown(bootStatusQueue) : Effect.void)),
+        Effect.tap((status) =>
+          status.stage === 'done' ? Queue.shutdown(bootStatusQueue).pipe(Effect.asVoid) : Effect.void,
+        ),
         Effect.forever,
         Effect.tapCauseLogPretty,
         Effect.forkScoped,
@@ -214,35 +327,52 @@ export const createStore = <TSchema extends LiveStoreSchema = LiveStoreSchema.An
 
       const connectDevtoolsToStore_ = (storeDevtoolsChannel: ClientSessionDevtoolsChannel) =>
         Effect.gen(function* () {
-          const store = yield* storeDeferred
+          const store = yield* Deferred.await(storeDeferred)
           yield* connectDevtoolsToStore({ storeDevtoolsChannel, store })
         })
 
-      const runtime = yield* Effect.runtime<Scope.Scope>()
+      const services = yield* Effect.context<Scope.Scope>()
+      let shutdownSyncProcessor: ((exit: Exit.Exit<unknown, unknown>) => Effect.Effect<void>) | undefined
 
       const shutdown = (
-        exit: Exit.Exit<
-          IntentionalShutdownCause,
-          UnexpectedError | MaterializeError | SyncError | InvalidPullError | IsOfflineError
-        >,
+        exit: Exit.Exit<IntentionalShutdownCause, UnknownError | MaterializeError | BackendIdMismatchError>,
       ) =>
         Effect.gen(function* () {
-          yield* Scope.close(lifetimeScope, exit).pipe(
+          // Hard outer bound on the DETACHED teardown: the processor drain `awaitEmpty`s the
+          // leader-push worker, which never completes if the leader is dead/unresponsive. Without a
+          // bound the drain would block forever, so `Scope.close(lifetimeScope)` (in `ensuring`)
+          // would never run and the lifetime scope + its resources would leak indefinitely (a later
+          // `createStore`/registry dispose on the same `storeId` would observe a never-closed store).
+          // Drain up to the bound, then force-close the scope regardless. The bound exceeds the
+          // caller-side wait below so an in-flight (but progressing) push is not cut short.
+          const closeFiber = yield* (shutdownSyncProcessor?.(exit) ?? Effect.void).pipe(
+            Effect.timeout(SHUTDOWN_DRAIN_HARD_TIMEOUT_MS),
+            Effect.catchTag('TimeoutError', () =>
+              Effect.logError(
+                `@livestore/livestore:shutdown: drain exceeded hard bound of ${SHUTDOWN_DRAIN_HARD_TIMEOUT_MS}ms; forcing scope close`,
+              ),
+            ),
+            Effect.ensuring(Scope.close(lifetimeScope, exit)),
+            Effect.forkDetach,
+          )
+          // Caller-side soft wait: stop blocking the shutdown() caller after 1s without cancelling
+          // the detached teardown above (which remains bounded by SHUTDOWN_DRAIN_HARD_TIMEOUT_MS).
+          yield* Fiber.join(closeFiber).pipe(
             Effect.logWarnIfTakesLongerThan({ label: '@livestore/livestore:shutdown', duration: 500 }),
             Effect.timeout(1000),
-            Effect.catchTag('TimeoutException', () =>
+            Effect.catchTag('TimeoutError', () =>
               Effect.logError('@livestore/livestore:shutdown: Timed out after 1 second'),
             ),
           )
 
-          if (shutdownDeferred) {
+          if (shutdownDeferred !== undefined) {
             yield* Deferred.done(shutdownDeferred, exit)
           }
 
           yield* Effect.logDebug('LiveStore shutdown complete')
         }).pipe(
           Effect.withSpan('@livestore/livestore:shutdown'),
-          Effect.provide(runtime),
+          Effect.provide(services),
           Effect.tapCauseLogPretty,
           // Given that the shutdown flow might also interrupt the effect that is calling the shutdown,
           // we want to detach the shutdown effect so it's not interrupted by itself
@@ -250,6 +380,11 @@ export const createStore = <TSchema extends LiveStoreSchema = LiveStoreSchema.An
           Fiber.join,
         )
 
+      const syncPayloadEncoded =
+        syncPayload === undefined
+          ? undefined
+          : yield* Schema.encodeEffect(resolvedSyncPayloadSchema)(syncPayload).pipe(UnknownError.mapToUnknownError)
+
       const clientSession: ClientSession = yield* adapter({
         schema,
         storeId,
@@ -258,10 +393,11 @@ export const createStore = <TSchema extends LiveStoreSchema = LiveStoreSchema.An
         shutdown,
         connectDevtoolsToStore: connectDevtoolsToStore_,
         debugInstanceId,
-        syncPayload,
+        syncPayloadSchema: resolvedSyncPayloadSchema,
+        syncPayloadEncoded,
       }).pipe(Effect.withPerformanceMeasure('livestore:makeAdapter'), Effect.withSpan('createStore:makeAdapter'))
 
-      if (LS_DEV && clientSession.leaderThread.initialState.migrationsReport.migrations.length > 0) {
+      if (LS_DEV === true && clientSession.leaderThread.initialState.migrationsReport.migrations.length > 0) {
         yield* Effect.logDebug(
           '[@livestore/livestore:createStore] migrationsReport',
           ...clientSession.leaderThread.initialState.migrationsReport.migrations.map((m) =>
@@ -277,51 +413,68 @@ export const createStore = <TSchema extends LiveStoreSchema = LiveStoreSchema.An
         schema,
         context,
         otelOptions: { tracer: otelTracer, rootSpanContext: otelRootSpanContext },
-        effectContext: { lifetimeScope, runtime },
+        effectContext: { lifetimeScope, services },
         // TODO find a better way to detect if we're running LiveStore in the LiveStore devtools
         // But for now this is a good enough approximation with little downsides
-        __runningInDevtools: getDevtoolsEnabled(disableDevtools) === false,
+        __runningInDevtools: !getDevtoolsEnabled(disableDevtools),
         confirmUnsavedChanges,
         // NOTE during boot we're not yet executing events in a batched context
         // but only set the provided `batchUpdates` function after boot
         batchUpdates: (run) => run(),
         storeId,
         params: {
-          leaderPushBatchSize: params?.leaderPushBatchSize ?? DEFAULT_PARAMS.leaderPushBatchSize,
-          ...omitUndefineds({ simulation: params?.simulation }),
+          leaderPushBatchSize: params?.leaderPushBatchSize ?? STORE_DEFAULT_PARAMS.leaderPushBatchSize,
+          eventQueryBatchSize: params?.eventQueryBatchSize ?? STORE_DEFAULT_PARAMS.eventQueryBatchSize,
         },
       })
+      shutdownSyncProcessor = store[StoreInternalsSymbol].syncProcessor.shutdown
 
       // Starts background fibers (syncing, event processing, etc) for store
-      yield* store.boot
+      yield* store[StoreInternalsSymbol].boot
 
       if (boot !== undefined) {
         // TODO also incorporate `boot` function progress into `bootStatusQueue`
-        yield* Effect.tryAll(() =>
+        // @effect-diagnostics-next-line anyUnknownInErrorContext:off -- user boot errors are immediately normalized to LiveStore UnknownError
+        yield* Effect.trySyncOrPromiseOrEffect(() =>
           boot(store, { migrationsReport: clientSession.leaderThread.initialState.migrationsReport, parentSpan: span }),
         ).pipe(
-          UnexpectedError.mapToUnexpectedError,
-          Effect.provide(Layer.succeed(LiveStoreContextRunning, { stage: 'running', store: store as any as Store })),
+          UnknownError.mapToUnknownError,
+          Effect.provide(
+            Layer.succeed(
+              LiveStoreContextRunning,
+              LiveStoreContextRunning.of({ stage: 'running', store: store as any as Store }),
+            ),
+          ),
           Effect.withSpan('createStore:boot'),
         )
       }
 
       // NOTE it's important to yield here to allow the forked Effect in the store constructor to run
-      yield* Effect.yieldNow()
+      yield* Effect.yieldNow
 
       if (batchUpdates !== undefined) {
         // Replacing the default batchUpdates function with the provided one after boot
-        store.reactivityGraph.context!.effectsWrapper = batchUpdates
+        store[StoreInternalsSymbol].reactivityGraph.context!.effectsWrapper = batchUpdates
       }
 
       yield* Deferred.succeed(storeDeferred, store as any as Store)
 
+      // Expose store on globalThis for console debugging
+      globalThis.__debugLiveStore ??= {}
+      globalThis.__debugLiveStore[storeId] = store
+
+      yield* Effect.addFinalizer(() =>
+        Effect.sync(() => {
+          delete globalThis.__debugLiveStore?.[storeId]
+        }),
+      )
+
       return store
     }).pipe(
       Effect.withSpan('createStore', { attributes: { debugInstanceId, storeId } }),
       Effect.annotateLogs({ debugInstanceId, storeId }),
-      LS_DEV ? TaskTracing.withAsyncTaggingTracing((name) => (console as any).createTask(name)) : identity,
-      Scope.extend(lifetimeScope),
+      LS_DEV === true ? TaskTracing.withAsyncTaggingTracing((name) => (console as any).createTask(name)) : identity,
+      Scope.provide(lifetimeScope),
     )
   })
 
@@ -329,8 +482,8 @@ const validateStoreId = (storeId: string) =>
   Effect.gen(function* () {
     const validChars = /^[a-zA-Z0-9_-]+$/
 
-    if (!validChars.test(storeId)) {
-      return yield* UnexpectedError.make({
+    if (validChars.test(storeId) === false) {
+      return yield* UnknownError.make({
         cause: `Invalid storeId: ${storeId}. Only alphanumeric characters, underscores, and hyphens are allowed.`,
         payload: { storeId },
       })