@livestore/common 0.4.0-dev.8 → 0.4.0

package/src/leader-thread/make-leader-thread-layer.ts

@@ -1,19 +1,30 @@
 import { omitUndefineds, shouldNeverHappen } from '@livestore/utils'
-import type { HttpClient, Schema, Scope } from '@livestore/utils/effect'
-import { Deferred, Effect, KeyValueStore, Layer, PlatformError, Queue, SubscriptionRef } from '@livestore/utils/effect'
+import type { HttpClient, Scope } from '@livestore/utils/effect'
+import {
+  Deferred,
+  Effect,
+  KeyValueStore,
+  Layer,
+  PlatformError,
+  Queue,
+  Schema,
+  Stream,
+  Subscribable,
+  SubscriptionRef,
+} from '@livestore/utils/effect'
+
 import {
   type BootStatus,
   type MakeSqliteDb,
   type MaterializerHashMismatchError,
   type SqliteDb,
-  type SqliteError,
-  UnexpectedError,
+  UnknownError,
 } from '../adapter-types.ts'
 import type { MigrationsReport } from '../defs.ts'
 import type * as Devtools from '../devtools/mod.ts'
 import type { LiveStoreSchema } from '../schema/mod.ts'
 import { EventSequenceNumber, LiveStoreEvent, SystemTables } from '../schema/mod.ts'
-import type { InvalidPullError, IsOfflineError, SyncOptions } from '../sync/sync.ts'
+import type { SyncBackend, SyncOptions } from '../sync/sync.ts'
 import { SyncState } from '../sync/syncstate.ts'
 import { sql } from '../util.ts'
 import * as Eventlog from './eventlog.ts'
@@ -23,6 +34,7 @@ import { makeMaterializeEvent } from './materialize-event.ts'
 import { recreateDb } from './recreate-db.ts'
 import type { ShutdownChannel } from './shutdown-channel.ts'
 import type {
+  DevtoolsContext,
   DevtoolsOptions,
   InitialBlockingSyncContext,
   InitialSyncOptions,
@@ -33,7 +45,8 @@ import { LeaderThreadCtx } from './types.ts'
 
 export interface MakeLeaderThreadLayerParams {
   storeId: string
-  syncPayload: Schema.JsonValue | undefined
+  syncPayloadSchema: Schema.Schema<any> | undefined
+  syncPayloadEncoded: Schema.JsonValue | undefined
   clientId: string
   schema: LiveStoreSchema
   makeSqliteDb: MakeSqliteDb
@@ -42,6 +55,8 @@ export interface MakeLeaderThreadLayerParams {
   dbEventlog: LeaderSqliteDb
   devtoolsOptions: DevtoolsOptions
   shutdownChannel: ShutdownChannel
+  /** Boot warning to emit (e.g., OPFS unavailable in private browsing) */
+  bootWarning?: BootStatus
   params?: {
     localPushBatchSize?: number
     backendPushBatchSize?: number
@@ -59,19 +74,29 @@ export const makeLeaderThreadLayer = ({
   schema,
   storeId,
   clientId,
-  syncPayload,
+  syncPayloadSchema = Schema.JsonValue,
+  syncPayloadEncoded,
   makeSqliteDb,
   syncOptions,
   dbState,
   dbEventlog,
   devtoolsOptions,
   shutdownChannel,
+  bootWarning,
   params,
   testing,
-}: MakeLeaderThreadLayerParams): Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient> =>
+}: MakeLeaderThreadLayerParams): Layer.Layer<LeaderThreadCtx, UnknownError, Scope.Scope | HttpClient.HttpClient> =>
   Effect.gen(function* () {
+    const syncPayloadDecoded =
+      syncPayloadEncoded === undefined ? undefined : yield* Schema.decodeUnknown(syncPayloadSchema)(syncPayloadEncoded)
+
     const bootStatusQueue = yield* Queue.unbounded<BootStatus>().pipe(Effect.acquireRelease(Queue.shutdown))
 
+    // Emit boot warning if present (e.g., OPFS unavailable in private browsing)
+    if (bootWarning !== undefined) {
+      yield* Queue.offer(bootStatusQueue, bootWarning)
+    }
+
     const dbEventlogMissing = !hasEventlogTables(dbEventlog)
 
     // Either happens on initial boot or if schema changes
@@ -82,7 +107,7 @@ export const makeLeaderThreadLayer = ({
     const syncBackend =
       syncOptions?.backend === undefined
         ? undefined
-        : yield* syncOptions.backend({ storeId, clientId, payload: syncPayload }).pipe(
+        : yield* syncOptions.backend({ storeId, clientId, payload: syncPayloadDecoded }).pipe(
             Effect.provide(
               Layer.succeed(
                 KeyValueStore.KeyValueStore,
@@ -131,9 +156,10 @@ export const makeLeaderThreadLayer = ({
 
     // Recreate state database if needed BEFORE creating sync processor
     // This ensures all system tables exist before any queries are made
-    const { migrationsReport } = dbStateMissing
-      ? yield* recreateDb({ dbState, dbEventlog, schema, bootStatusQueue, materializeEvent })
-      : { migrationsReport: { migrations: [] } }
+    const { migrationsReport } =
+      dbStateMissing === true
+        ? yield* recreateDb({ dbState, dbEventlog, schema, bootStatusQueue, materializeEvent })
+        : { migrationsReport: { migrations: [] } }
 
     const syncProcessor = yield* makeLeaderSyncProcessor({
       schema,
@@ -141,6 +167,7 @@ export const makeLeaderThreadLayer = ({
       initialSyncState: getInitialSyncState({ dbEventlog, dbState, dbEventlogMissing }),
       initialBlockingSyncContext,
       onError: syncOptions?.onSyncError ?? 'ignore',
+      onBackendIdMismatch: syncOptions?.onBackendIdMismatch ?? 'reset',
       livePull: syncOptions?.livePull ?? true,
       params: {
         ...omitUndefineds({
@@ -157,13 +184,16 @@ export const makeLeaderThreadLayer = ({
       Effect.acquireRelease(Queue.shutdown),
     )
 
-    const devtoolsContext = devtoolsOptions.enabled
-      ? {
-          enabled: true as const,
-          syncBackendLatch: yield* Effect.makeLatch(true),
-          syncBackendLatchState: yield* SubscriptionRef.make<{ latchClosed: boolean }>({ latchClosed: false }),
-        }
-      : { enabled: false as const }
+    const devtoolsContext =
+      devtoolsOptions.enabled === true
+        ? {
+            enabled: true as const,
+            syncBackendLatch: yield* Effect.makeLatch(true),
+            syncBackendLatchState: yield* SubscriptionRef.make<{ latchClosed: boolean }>({ latchClosed: false }),
+          }
+        : { enabled: false as const }
+
+    const networkStatus = yield* makeNetworkStatusSubscribable({ syncBackend, devtoolsContext })
 
     const ctx = {
       schema,
@@ -173,7 +203,7 @@ export const makeLeaderThreadLayer = ({
       dbState,
       dbEventlog,
       makeSqliteDb,
-      eventSchema: LiveStoreEvent.makeEventDefSchema(schema),
+      eventSchema: LiveStoreEvent.Client.makeSchema(schema),
       shutdownStateSubRef: yield* SubscriptionRef.make<ShutdownState>('running'),
       shutdownChannel,
       syncBackend,
@@ -181,6 +211,7 @@ export const makeLeaderThreadLayer = ({
       materializeEvent,
       extraIncomingMessagesQueue,
       devtools: devtoolsContext,
+      networkStatus,
       // State will be set during `bootLeaderThread`
       initialState: {} as any as LeaderThreadCtx['Type']['initialState'],
     } satisfies typeof LeaderThreadCtx.Service
@@ -200,7 +231,7 @@ export const makeLeaderThreadLayer = ({
   }).pipe(
     Effect.withSpan('@livestore/common:leader-thread:boot'),
     Effect.withSpanScoped('@livestore/common:leader-thread'),
-    UnexpectedError.mapToUnexpectedError,
+    UnknownError.mapToUnknownError,
     Effect.tapCauseLogPretty,
     Layer.unwrapScoped,
   )
@@ -219,7 +250,7 @@ const hasStateTables = (db: SqliteDb) => {
 
 const isSubsetOf = (a: Set<string>, b: Set<string>): boolean => {
   for (const item of a) {
-    if (!b.has(item)) {
+    if (b.has(item) === false) {
       return false
     }
   }
@@ -236,11 +267,11 @@ const getInitialSyncState = ({
   dbState: SqliteDb
   dbEventlogMissing: boolean
 }) => {
-  const initialBackendHead = dbEventlogMissing
-    ? EventSequenceNumber.ROOT.global
-    : Eventlog.getBackendHeadFromDb(dbEventlog)
+  const initialBackendHead =
+    dbEventlogMissing === true ? EventSequenceNumber.Client.ROOT.global : Eventlog.getBackendHeadFromDb(dbEventlog)
 
-  const initialLocalHead = dbEventlogMissing ? EventSequenceNumber.ROOT : Eventlog.getClientHeadFromDb(dbEventlog)
+  const initialLocalHead =
+    dbEventlogMissing === true ? EventSequenceNumber.Client.ROOT : Eventlog.getClientHeadFromDb(dbEventlog)
 
   if (initialBackendHead > initialLocalHead.global) {
     return shouldNeverHappen(
@@ -252,20 +283,21 @@ const getInitialSyncState = ({
     localHead: initialLocalHead,
     upstreamHead: {
       global: initialBackendHead,
-      client: EventSequenceNumber.clientDefault,
-      rebaseGeneration: EventSequenceNumber.rebaseGenerationDefault,
+      client: EventSequenceNumber.Client.DEFAULT,
+      rebaseGeneration: EventSequenceNumber.Client.REBASE_GENERATION_DEFAULT,
     },
-    pending: dbEventlogMissing
-      ? []
-      : Eventlog.getEventsSince({
-          dbEventlog,
-          dbState,
-          since: {
-            global: initialBackendHead,
-            client: EventSequenceNumber.clientDefault,
-            rebaseGeneration: initialLocalHead.rebaseGeneration,
-          },
-        }),
+    pending:
+      dbEventlogMissing === true
+        ? []
+        : Eventlog.getEventsSince({
+            dbEventlog,
+            dbState,
+            since: {
+              global: initialBackendHead,
+              client: EventSequenceNumber.Client.DEFAULT,
+              rebaseGeneration: initialLocalHead.rebaseGeneration,
+            },
+          }),
   })
 }
 
@@ -330,7 +362,7 @@ const bootLeaderThread = ({
   devtoolsOptions: DevtoolsOptions
 }): Effect.Effect<
   LeaderThreadCtx['Type']['initialState'],
-  UnexpectedError | SqliteError | IsOfflineError | InvalidPullError | MaterializerHashMismatchError,
+  UnknownError | MaterializerHashMismatchError,
   LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
 > =>
   Effect.gen(function* () {
@@ -357,3 +389,56 @@ const bootLeaderThread = ({
 
     return { migrationsReport, leaderHead: initialLeaderHead }
   })
+
+/** @internal */
+export const makeNetworkStatusSubscribable = ({
+  syncBackend,
+  devtoolsContext,
+}: {
+  syncBackend: SyncBackend.SyncBackend | undefined
+  devtoolsContext: DevtoolsContext
+}): Effect.Effect<Subscribable.Subscribable<SyncBackend.NetworkStatus>, never, Scope.Scope> =>
+  Effect.gen(function* () {
+    const initialIsConnected = syncBackend !== undefined ? yield* SubscriptionRef.get(syncBackend.isConnected) : false
+    const initialLatchClosed =
+      devtoolsContext.enabled === true
+        ? (yield* SubscriptionRef.get(devtoolsContext.syncBackendLatchState)).latchClosed
+        : false
+
+    const networkStatusRef = yield* SubscriptionRef.make<SyncBackend.NetworkStatus>({
+      isConnected: initialIsConnected,
+      timestampMs: Date.now(),
+      devtools: { latchClosed: initialLatchClosed },
+    })
+
+    const updateNetworkStatus = (patch: { isConnected?: boolean; latchClosed?: boolean }) =>
+      SubscriptionRef.update(networkStatusRef, (previous) => ({
+        isConnected: patch.isConnected ?? previous.isConnected,
+        timestampMs: Date.now(),
+        devtools: {
+          latchClosed: patch.latchClosed ?? previous.devtools.latchClosed,
+        },
+      }))
+
+    if (syncBackend !== undefined) {
+      yield* syncBackend.isConnected.changes.pipe(
+        Stream.tap((isConnected) => updateNetworkStatus({ isConnected })),
+        Stream.runDrain,
+        Effect.interruptible,
+        Effect.tapCauseLogPretty,
+        Effect.forkScoped,
+      )
+    }
+
+    if (devtoolsContext.enabled === true) {
+      yield* devtoolsContext.syncBackendLatchState.changes.pipe(
+        Stream.tap(({ latchClosed }) => updateNetworkStatus({ latchClosed })),
+        Stream.runDrain,
+        Effect.interruptible,
+        Effect.tapCauseLogPretty,
+        Effect.forkScoped,
+      )
+    }
+
+    return Subscribable.fromSubscriptionRef(networkStatusRef)
+  })

package/src/leader-thread/materialize-event.ts

@@ -3,8 +3,9 @@ import { Effect, Option, ReadonlyArray, Schema } from '@livestore/utils/effect'
 
 import { MaterializeError, MaterializerHashMismatchError, type SqliteDb } from '../adapter-types.ts'
 import { getExecStatementsFromMaterializer, hashMaterializerResults } from '../materializer-helper.ts'
+import { logDeprecationWarnings } from '../schema/EventDef/deprecated.ts'
 import type { LiveStoreSchema } from '../schema/mod.ts'
-import { EventSequenceNumber, getEventDef, SystemTables } from '../schema/mod.ts'
+import { EventSequenceNumber, resolveEventDef, SystemTables, UNKNOWN_EVENT_SCHEMA_HASH } from '../schema/mod.ts'
 import { insertRow } from '../sql-queries/index.ts'
 import { sql } from '../util.ts'
 import { execSql, execSqlPrepared } from './connection.ts'
@@ -33,8 +34,37 @@ export const makeMaterializeEvent = ({
       Effect.gen(function* () {
         const skipEventlog = options?.skipEventlog ?? false
 
-        const eventName = eventEncoded.name
-        const { eventDef, materializer } = getEventDef(schema, eventName)
+        const resolution = yield* resolveEventDef(schema, {
+          operation: '@livestore/common:leader-thread:materializeEvent',
+          event: eventEncoded,
+        })
+
+        if (resolution._tag === 'unknown') {
+          // Unknown events still enter the eventlog so newer clients can replay
+          // them once they learn the schema. We skip materialization to keep the
+          // local state consistent with the knowledge of the current client.
+          if (skipEventlog === false) {
+            yield* Eventlog.insertIntoEventlog(
+              eventEncoded,
+              dbEventlog,
+              UNKNOWN_EVENT_SCHEMA_HASH,
+              eventEncoded.clientId,
+              eventEncoded.sessionId,
+            )
+          }
+
+          dbState.debug.head = eventEncoded.seqNum
+
+          return {
+            sessionChangeset: { _tag: 'no-op' as const },
+            hash: Option.none(),
+          }
+        }
+
+        const { eventDef, materializer } = resolution
+
+        // Log deprecation warnings for deprecated events/fields
+        yield* logDeprecationWarnings(eventDef, eventEncoded.args as Record<string, unknown>)
 
         const execArgsArr = getExecStatementsFromMaterializer({
           eventDef,
@@ -43,7 +73,7 @@ export const makeMaterializeEvent = ({
           event: { decoded: undefined, encoded: eventEncoded },
         })
 
-        const materializerHash = isDevEnv() ? Option.some(hashMaterializerResults(execArgsArr)) : Option.none()
+        const materializerHash = isDevEnv() === true ? Option.some(hashMaterializerResults(execArgsArr)) : Option.none()
 
         if (
           materializerHash._tag === 'Some' &&
@@ -89,7 +119,7 @@ export const makeMaterializeEvent = ({
               seqNumRebaseGeneration: eventEncoded.seqNum.rebaseGeneration,
               // NOTE the changeset will be empty (i.e. null) for no-op events
               changeset: changeset ?? null,
-              debug: LS_DEV ? execArgsArr : null,
+              debug: LS_DEV === true ? execArgsArr : null,
             },
           }),
         )
@@ -114,11 +144,11 @@ export const makeMaterializeEvent = ({
         }
 
         return {
-          sessionChangeset: changeset
+          sessionChangeset: changeset !== undefined
             ? {
                 _tag: 'sessionChangeset' as const,
                 data: changeset,
-                debug: LS_DEV ? execArgsArr : null,
+                debug: LS_DEV === true ? execArgsArr : null,
               }
             : { _tag: 'no-op' as const },
           hash: materializerHash,
@@ -129,7 +159,7 @@ export const makeMaterializeEvent = ({
           attributes: {
             eventName: eventEncoded.name,
             eventNum: eventEncoded.seqNum,
-            'span.label': `${EventSequenceNumber.toString(eventEncoded.seqNum)} ${eventEncoded.name}`,
+            'span.label': `${EventSequenceNumber.Client.toString(eventEncoded.seqNum)} ${eventEncoded.name}`,
           },
         }),
         // Effect.logDuration('@livestore/common:leader-thread:materializeEvent'),
@@ -143,7 +173,7 @@ export const rollback = ({
 }: {
   dbState: SqliteDb
   dbEventlog: SqliteDb
-  eventNumsToRollback: EventSequenceNumber.EventSequenceNumber[]
+  eventNumsToRollback: EventSequenceNumber.Client.Composite[]
 }) =>
   Effect.gen(function* () {
     const rollbackEvents = dbState
@@ -159,7 +189,7 @@ export const rollback = ({
         changeset: _.changeset,
         debug: _.debug,
       }))
-      .toSorted((a, b) => EventSequenceNumber.compare(a.seqNum, b.seqNum))
+      .toSorted((a, b) => EventSequenceNumber.Client.compare(a.seqNum, b.seqNum))
 
     // Apply changesets in reverse order
     for (let i = rollbackEvents.length - 1; i >= 0; i--) {

package/src/leader-thread/mod.ts

@@ -5,4 +5,5 @@ export * from './make-leader-thread-layer.ts'
 export * from './materialize-event.ts'
 export * from './recreate-db.ts'
 export * as ShutdownChannel from './shutdown-channel.ts'
+export * from './stream-events.ts'
 export * from './types.ts'

package/src/leader-thread/recreate-db.ts

@@ -10,7 +10,7 @@ import {
   rematerializeFromEventlog,
   type SqliteDb,
   type SqliteError,
-  UnexpectedError,
+  UnknownError,
 } from '../index.ts'
 import type { LiveStoreSchema } from '../schema/mod.ts'
 import { configureConnection } from './connection.ts'
@@ -28,10 +28,10 @@ export const recreateDb = ({
   schema: LiveStoreSchema
   bootStatusQueue: Queue.Queue<BootStatus>
   materializeEvent: MaterializeEvent
-}): Effect.Effect<{ migrationsReport: MigrationsReport }, UnexpectedError | MaterializeError | SqliteError> =>
+}): Effect.Effect<{ migrationsReport: MigrationsReport }, UnknownError | MaterializeError | SqliteError> =>
   Effect.gen(function* () {
     const migrationOptions = schema.state.sqlite.migrations
-    let migrationsReport: MigrationsReport
+    let migrationsReport: MigrationsReport = { migrations: [] }
 
     yield* Effect.addFinalizer(
       Effect.fn('recreateDb:finalizer')(function* (ex) {
@@ -48,7 +48,7 @@ export const recreateDb = ({
 
     const initDb = (hooks: Partial<MigrationHooks> | undefined) =>
       Effect.gen(function* () {
-        yield* Effect.tryAll(() => hooks?.init?.(tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
+        yield* Effect.tryAll(() => hooks?.init?.(tmpDb)).pipe(UnknownError.mapToUnknownError)
 
         const migrationsReport = yield* migrateDb({
           db: tmpDb,
@@ -57,7 +57,7 @@ export const recreateDb = ({
             Queue.offer(bootStatusQueue, { stage: 'migrating', progress: { done, total } }),
         })
 
-        yield* Effect.tryAll(() => hooks?.pre?.(tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
+        yield* Effect.tryAll(() => hooks?.pre?.(tmpDb)).pipe(UnknownError.mapToUnknownError)
 
         return { migrationsReport, tmpDb }
       })
@@ -78,7 +78,7 @@ export const recreateDb = ({
             Queue.offer(bootStatusQueue, { stage: 'rehydrating', progress: { done, total } }),
         })
 
-        yield* Effect.tryAll(() => hooks?.post?.(initResult.tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
+        yield* Effect.tryAll(() => hooks?.post?.(initResult.tmpDb)).pipe(UnknownError.mapToUnknownError)
 
         break
       }
@@ -88,7 +88,7 @@ export const recreateDb = ({
         migrationsReport = { migrations: [] }
 
         const newDbData = yield* Effect.tryAll(() => migrationOptions.migrate(oldDbData)).pipe(
-          UnexpectedError.mapToUnexpectedError,
+          UnknownError.mapToUnknownError,
         )
 
         tmpDb.import(newDbData)

package/src/leader-thread/shutdown-channel.ts

@@ -2,20 +2,16 @@ import type { WebChannel } from '@livestore/utils/effect'
 import { Schema } from '@livestore/utils/effect'
 
 import {
+  BackendIdMismatchError,
   IntentionalShutdownCause,
-  InvalidPullError,
-  InvalidPushError,
-  IsOfflineError,
   MaterializeError,
-  UnexpectedError,
+  UnknownError,
 } from '../index.ts'
 
 export class All extends Schema.Union(
   IntentionalShutdownCause,
-  UnexpectedError,
-  IsOfflineError,
-  InvalidPushError,
-  InvalidPullError,
+  UnknownError,
+  BackendIdMismatchError,
   MaterializeError,
 ) {}