@livestore/common 0.4.0 → 0.5.0-dev.1

package/src/leader-thread/make-leader-thread-layer.ts

@@ -1,12 +1,15 @@
 import { omitUndefineds, shouldNeverHappen } from '@livestore/utils'
-import type { HttpClient, Scope } from '@livestore/utils/effect'
 import {
+  type HttpClient,
+  type Scope,
   Deferred,
   Effect,
   KeyValueStore,
+  Latch,
   Layer,
-  PlatformError,
+  Option,
   Queue,
+  ReadonlyArray,
   Schema,
   Stream,
   Subscribable,
@@ -24,14 +27,16 @@ import type { MigrationsReport } from '../defs.ts'
 import type * as Devtools from '../devtools/mod.ts'
 import type { LiveStoreSchema } from '../schema/mod.ts'
 import { EventSequenceNumber, LiveStoreEvent, SystemTables } from '../schema/mod.ts'
+import type * as StateHead from '../StateHead.ts'
 import type { SyncBackend, SyncOptions } from '../sync/sync.ts'
 import { SyncState } from '../sync/syncstate.ts'
 import { sql } from '../util.ts'
+import { configureConnection } from './connection.ts'
 import * as Eventlog from './eventlog.ts'
-import { makeLeaderSyncProcessor } from './LeaderSyncProcessor.ts'
 import { bootDevtools } from './leader-worker-devtools.ts'
+import * as LeaderSyncProcessor from './LeaderSyncProcessor.ts'
 import { makeMaterializeEvent } from './materialize-event.ts'
-import { recreateDb } from './recreate-db.ts'
+import { hasCompletedState, recreateDb } from './recreate-db.ts'
 import type { ShutdownChannel } from './shutdown-channel.ts'
 import type {
   DevtoolsContext,
@@ -45,8 +50,8 @@ import { LeaderThreadCtx } from './types.ts'
 
 export interface MakeLeaderThreadLayerParams {
   storeId: string
-  syncPayloadSchema: Schema.Schema<any> | undefined
-  syncPayloadEncoded: Schema.JsonValue | undefined
+  syncPayloadSchema: Schema.Decoder<Schema.Json> | undefined
+  syncPayloadEncoded: Schema.Json | undefined
   clientId: string
   schema: LiveStoreSchema
   makeSqliteDb: MakeSqliteDb
@@ -60,11 +65,17 @@ export interface MakeLeaderThreadLayerParams {
   params?: {
     localPushBatchSize?: number
     backendPushBatchSize?: number
+    stateRebuildBatchSize?: number
   }
   testing?: {
     syncProcessor?: {
       delays?: {
         localPushProcessing?: Effect.Effect<void>
+        backendPullReconnect?: (fallbackMs: number) => number
+      }
+      hooks?: {
+        localPushAdmitted?: (events: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>) => Effect.Effect<void>
+        backendPullReconnectWait?: (state: { isConnected: boolean; fallbackMs: number }) => Effect.Effect<void>
       }
     }
   }
@@ -74,7 +85,7 @@ export const makeLeaderThreadLayer = ({
   schema,
   storeId,
   clientId,
-  syncPayloadSchema = Schema.JsonValue,
+  syncPayloadSchema = Schema.Json,
   syncPayloadEncoded,
   makeSqliteDb,
   syncOptions,
@@ -85,12 +96,16 @@ export const makeLeaderThreadLayer = ({
   bootWarning,
   params,
   testing,
-}: MakeLeaderThreadLayerParams): Layer.Layer<LeaderThreadCtx, UnknownError, Scope.Scope | HttpClient.HttpClient> =>
+}: MakeLeaderThreadLayerParams): Layer.Layer<
+  LeaderThreadCtx,
+  UnknownError,
+  Scope.Scope | HttpClient.HttpClient | StateHead.StateHead
+> =>
   Effect.gen(function* () {
     const syncPayloadDecoded =
-      syncPayloadEncoded === undefined ? undefined : yield* Schema.decodeUnknown(syncPayloadSchema)(syncPayloadEncoded)
+      syncPayloadEncoded === undefined ? undefined : yield* Schema.decodeEffect(syncPayloadSchema)(syncPayloadEncoded)
 
-    const bootStatusQueue = yield* Queue.unbounded<BootStatus>().pipe(Effect.acquireRelease(Queue.shutdown))
+    const bootStatusQueue = yield* Effect.acquireRelease(Queue.unbounded<BootStatus>(), Queue.shutdown)
 
     // Emit boot warning if present (e.g., OPFS unavailable in private browsing)
     if (bootWarning !== undefined) {
@@ -99,8 +114,20 @@ export const makeLeaderThreadLayer = ({
 
     const dbEventlogMissing = !hasEventlogTables(dbEventlog)
 
-    // Either happens on initial boot or if schema changes
-    const dbStateMissing = !hasStateTables(dbState)
+    const stateNeedsRebuild = !hasCompletedState(dbState)
+
+    if (stateNeedsRebuild === true) {
+      // Import also clears hook-created objects while preserving the open connection.
+      yield* Effect.acquireUseRelease(
+        makeSqliteDb({ _tag: 'in-memory' }),
+        (emptyDb) =>
+          configureConnection(emptyDb, { foreignKeys: false }).pipe(
+            Effect.andThen(Effect.sync(() => dbState.import(emptyDb))),
+            UnknownError.mapToUnknownError,
+          ),
+        (emptyDb) => Effect.sync(() => emptyDb.close()),
+      )
+    }
 
     yield* Eventlog.initEventlogDb(dbEventlog)
 
@@ -114,29 +141,30 @@ export const makeLeaderThreadLayer = ({
                 KeyValueStore.makeStringOnly({
                   get: (_key) =>
                     Effect.sync(() => Eventlog.getBackendIdFromDb(dbEventlog)).pipe(
-                      Effect.catchAllDefect((cause) =>
-                        PlatformError.BadArgument.make({
-                          method: 'getBackendIdFromDb',
-                          description: 'Failed to get backendId',
-                          module: 'KeyValueStore',
-                          cause,
-                        }),
+                      Effect.map(Option.getOrUndefined),
+                      Effect.catchDefect(
+                        (cause) =>
+                          new KeyValueStore.KeyValueStoreError({
+                            method: 'getBackendIdFromDb',
+                            message: 'Failed to get backendId',
+                            cause,
+                          }),
                       ),
                     ),
                   set: (_key, value) =>
                     Effect.sync(() => Eventlog.updateBackendId(dbEventlog, value)).pipe(
-                      Effect.catchAllDefect((cause) =>
-                        PlatformError.BadArgument.make({
-                          method: 'updateBackendId',
-                          module: 'KeyValueStore',
-                          description: 'Failed to update backendId',
-                          cause,
-                        }),
+                      Effect.catchDefect(
+                        (cause) =>
+                          new KeyValueStore.KeyValueStoreError({
+                            method: 'updateBackendId',
+                            message: 'Failed to update backendId',
+                            cause,
+                          }),
                       ),
                     ),
-                  clear: Effect.dieMessage(`Not implemented. Should never be used.`),
-                  remove: () => Effect.dieMessage(`Not implemented. Should never be used.`),
-                  size: Effect.dieMessage(`Not implemented. Should never be used.`),
+                  clear: Effect.die(new Error(`Not implemented. Should never be used.`)),
+                  remove: () => Effect.die(new Error(`Not implemented. Should never be used.`)),
+                  size: Effect.die(new Error(`Not implemented. Should never be used.`)),
                 }),
               ),
             ),
@@ -157,13 +185,39 @@ export const makeLeaderThreadLayer = ({
     // Recreate state database if needed BEFORE creating sync processor
     // This ensures all system tables exist before any queries are made
     const { migrationsReport } =
-      dbStateMissing === true
-        ? yield* recreateDb({ dbState, dbEventlog, schema, bootStatusQueue, materializeEvent })
+      stateNeedsRebuild === true
+        ? yield* recreateDb({
+            dbState,
+            dbEventlog,
+            schema,
+            bootStatusQueue,
+            materializeEvent,
+            ...omitUndefineds({ stateRebuildBatchSize: params?.stateRebuildBatchSize }),
+          })
         : { migrationsReport: { migrations: [] } }
 
-    const syncProcessor = yield* makeLeaderSyncProcessor({
+    const devtoolsContext =
+      devtoolsOptions.enabled === true
+        ? {
+            enabled: true as const,
+            syncBackendLatch: yield* Latch.make(true),
+            syncBackendLatchState: yield* SubscriptionRef.make<{ latchClosed: boolean }>({ latchClosed: false }),
+          }
+        : { enabled: false as const }
+
+    const span = yield* Effect.currentSpan.pipe(Effect.option, Effect.map(Option.getOrUndefined))
+
+    const syncProcessor = yield* LeaderSyncProcessor.make({
       schema,
-      dbState,
+      runtime: {
+        dbState,
+        dbEventlog,
+        materializeEvent,
+        syncBackend,
+        shutdownChannel,
+        devtoolsLatch: devtoolsContext.enabled === true ? devtoolsContext.syncBackendLatch : undefined,
+        span,
+      },
       initialSyncState: getInitialSyncState({ dbEventlog, dbState, dbEventlogMissing }),
       initialBlockingSyncContext,
       onError: syncOptions?.onSyncError ?? 'ignore',
@@ -176,23 +230,15 @@ export const makeLeaderThreadLayer = ({
         }),
       },
       testing: {
-        ...omitUndefineds({ delays: testing?.syncProcessor?.delays }),
+        ...omitUndefineds({ delays: testing?.syncProcessor?.delays, hooks: testing?.syncProcessor?.hooks }),
       },
     })
 
-    const extraIncomingMessagesQueue = yield* Queue.unbounded<Devtools.Leader.MessageToApp>().pipe(
-      Effect.acquireRelease(Queue.shutdown),
+    const extraIncomingMessagesQueue = yield* Effect.acquireRelease(
+      Queue.unbounded<Devtools.Leader.MessageToApp>(),
+      Queue.shutdown,
     )
 
-    const devtoolsContext =
-      devtoolsOptions.enabled === true
-        ? {
-            enabled: true as const,
-            syncBackendLatch: yield* Effect.makeLatch(true),
-            syncBackendLatchState: yield* SubscriptionRef.make<{ latchClosed: boolean }>({ latchClosed: false }),
-          }
-        : { enabled: false as const }
-
     const networkStatus = yield* makeNetworkStatusSubscribable({ syncBackend, devtoolsContext })
 
     const ctx = {
@@ -213,13 +259,13 @@ export const makeLeaderThreadLayer = ({
       devtools: devtoolsContext,
       networkStatus,
       // State will be set during `bootLeaderThread`
-      initialState: {} as any as LeaderThreadCtx['Type']['initialState'],
-    } satisfies typeof LeaderThreadCtx.Service
+      initialState: {} as any as LeaderThreadCtx['Service']['initialState'],
+    } satisfies LeaderThreadCtx['Service']
 
     // @ts-expect-error For debugging purposes
     globalThis.__leaderThreadCtx = ctx
 
-    const layer = Layer.succeed(LeaderThreadCtx, ctx)
+    const layer = Layer.succeed(LeaderThreadCtx, LeaderThreadCtx.of(ctx))
 
     ctx.initialState = yield* bootLeaderThread({
       migrationsReport,
@@ -233,29 +279,12 @@ export const makeLeaderThreadLayer = ({
     Effect.withSpanScoped('@livestore/common:leader-thread'),
     UnknownError.mapToUnknownError,
     Effect.tapCauseLogPretty,
-    Layer.unwrapScoped,
+    Layer.unwrap,
   )
 
 const hasEventlogTables = (db: SqliteDb) => {
   const tableNames = new Set(db.select<{ name: string }>(sql`select name from sqlite_master`).map((_) => _.name))
-  const eventlogTables = new Set(SystemTables.eventlogSystemTables.map((_) => _.sqliteDef.name))
-  return isSubsetOf(eventlogTables, tableNames)
-}
-
-const hasStateTables = (db: SqliteDb) => {
-  const tableNames = new Set(db.select<{ name: string }>(sql`select name from sqlite_master`).map((_) => _.name))
-  const stateTables = new Set(SystemTables.stateSystemTables.map((_) => _.sqliteDef.name))
-  return isSubsetOf(stateTables, tableNames)
-}
-
-const isSubsetOf = (a: Set<string>, b: Set<string>): boolean => {
-  for (const item of a) {
-    if (b.has(item) === false) {
-      return false
-    }
-  }
-
-  return true
+  return ReadonlyArray.every(SystemTables.eventlogSystemTables, (_) => tableNames.has(_.sqliteDef.name))
 }
 
 const getInitialSyncState = ({
@@ -361,7 +390,7 @@ const bootLeaderThread = ({
   initialBlockingSyncContext: InitialBlockingSyncContext
   devtoolsOptions: DevtoolsOptions
 }): Effect.Effect<
-  LeaderThreadCtx['Type']['initialState'],
+  LeaderThreadCtx['Service']['initialState'],
   UnknownError | MaterializerHashMismatchError,
   LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
 > =>
@@ -378,7 +407,7 @@ const bootLeaderThread = ({
         progress: { done: 0, total: -1 },
       })
 
-      yield* initialBlockingSyncContext.blockingDeferred.pipe(
+      yield* Deferred.await(initialBlockingSyncContext.blockingDeferred).pipe(
         Effect.withSpan('@livestore/common:leader-thread:initial-sync-blocking'),
       )
     }
@@ -421,7 +450,7 @@ export const makeNetworkStatusSubscribable = ({
       }))
 
     if (syncBackend !== undefined) {
-      yield* syncBackend.isConnected.changes.pipe(
+      yield* SubscriptionRef.changes(syncBackend.isConnected).pipe(
         Stream.tap((isConnected) => updateNetworkStatus({ isConnected })),
         Stream.runDrain,
         Effect.interruptible,
@@ -431,7 +460,7 @@ export const makeNetworkStatusSubscribable = ({
     }
 
     if (devtoolsContext.enabled === true) {
-      yield* devtoolsContext.syncBackendLatchState.changes.pipe(
+      yield* SubscriptionRef.changes(devtoolsContext.syncBackendLatchState).pipe(
         Stream.tap(({ latchClosed }) => updateNetworkStatus({ latchClosed })),
         Stream.runDrain,
         Effect.interruptible,

package/src/leader-thread/materialize-event.ts

@@ -7,6 +7,7 @@ import { logDeprecationWarnings } from '../schema/EventDef/deprecated.ts'
 import type { LiveStoreSchema } from '../schema/mod.ts'
 import { EventSequenceNumber, resolveEventDef, SystemTables, UNKNOWN_EVENT_SCHEMA_HASH } from '../schema/mod.ts'
 import { insertRow } from '../sql-queries/index.ts'
+import * as StateHead from '../StateHead.ts'
 import { sql } from '../util.ts'
 import { execSql, execSqlPrepared } from './connection.ts'
 import * as Eventlog from './eventlog.ts'
@@ -21,8 +22,9 @@ export const makeMaterializeEvent = ({
   schema: LiveStoreSchema
   dbState: SqliteDb
   dbEventlog: SqliteDb
-}): Effect.Effect<MaterializeEvent> =>
+}): Effect.Effect<MaterializeEvent, never, StateHead.StateHead> =>
   Effect.gen(function* () {
+    const stateHead = yield* StateHead.StateHead
     const eventDefSchemaHashMap = new Map(
       // TODO Running `Schema.hash` can be a bottleneck for larger schemas. There is an opportunity to run this
       // at build time and lookup the pre-computed hash at runtime.
@@ -53,6 +55,7 @@ export const makeMaterializeEvent = ({
             )
           }
 
+          yield* stateHead.set(eventEncoded.seqNum)
           dbState.debug.head = eventEncoded.seqNum
 
           return {
@@ -123,6 +126,7 @@ export const makeMaterializeEvent = ({
             },
           }),
         )
+        yield* stateHead.set(eventEncoded.seqNum)
 
         // console.groupEnd()
 
@@ -144,13 +148,14 @@ export const makeMaterializeEvent = ({
         }
 
         return {
-          sessionChangeset: changeset !== undefined
-            ? {
-                _tag: 'sessionChangeset' as const,
-                data: changeset,
-                debug: LS_DEV === true ? execArgsArr : null,
-              }
-            : { _tag: 'no-op' as const },
+          sessionChangeset:
+            changeset !== undefined
+              ? {
+                  _tag: 'sessionChangeset' as const,
+                  data: changeset,
+                  debug: LS_DEV === true ? execArgsArr : null,
+                }
+              : { _tag: 'no-op' as const },
           hash: materializerHash,
         }
       }).pipe(

package/src/leader-thread/recreate-db.ts

@@ -1,11 +1,9 @@
-import { casesHandled } from '@livestore/utils'
 import { Effect, Queue } from '@livestore/utils/effect'
 
 import type { MigrationsReport } from '../defs.ts'
 import {
   type BootStatus,
   type MaterializeError,
-  type MigrationHooks,
   migrateDb,
   rematerializeFromEventlog,
   type SqliteDb,
@@ -13,8 +11,22 @@ import {
   UnknownError,
 } from '../index.ts'
 import type { LiveStoreSchema } from '../schema/mod.ts'
-import { configureConnection } from './connection.ts'
+import { SystemTables } from '../schema/mod.ts'
+import { configureConnection, execSql } from './connection.ts'
 import type { MaterializeEvent } from './types.ts'
+import { STATE_REBUILD_BATCH_SIZE_DEFAULT } from './types.ts'
+
+export const hasCompletedState = (db: SqliteDb): boolean => {
+  const tableNames = new Set(db.select<{ name: string }>('SELECT name FROM sqlite_master').map((_) => _.name))
+  return (
+    SystemTables.stateSystemTables.every((table) => tableNames.has(table.sqliteDef.name)) &&
+    db.select(`SELECT id FROM ${SystemTables.REBUILD_META_TABLE} WHERE id = 1`).length === 1
+  )
+}
+
+/** Marks a successfully prepared state database as safe to use without rebuilding it. */
+export const markStateAsCompleted = (db: SqliteDb): Effect.Effect<void, SqliteError> =>
+  execSql(db, `INSERT OR IGNORE INTO ${SystemTables.REBUILD_META_TABLE} (id) VALUES (1)`, {})
 
 export const recreateDb = ({
   dbState,
@@ -22,16 +34,17 @@ export const recreateDb = ({
   schema,
   bootStatusQueue,
   materializeEvent,
+  stateRebuildBatchSize = STATE_REBUILD_BATCH_SIZE_DEFAULT,
 }: {
   dbState: SqliteDb
   dbEventlog: SqliteDb
   schema: LiveStoreSchema
   bootStatusQueue: Queue.Queue<BootStatus>
   materializeEvent: MaterializeEvent
+  stateRebuildBatchSize?: number
 }): Effect.Effect<{ migrationsReport: MigrationsReport }, UnknownError | MaterializeError | SqliteError> =>
   Effect.gen(function* () {
-    const migrationOptions = schema.state.sqlite.migrations
-    let migrationsReport: MigrationsReport = { migrations: [] }
+    const hooks = schema.state.sqlite.migrations.hooks
 
     yield* Effect.addFinalizer(
       Effect.fn('recreateDb:finalizer')(function* (ex) {
@@ -39,82 +52,35 @@ export const recreateDb = ({
       }),
     )
 
-    // NOTE to speed up the operations below, we're creating a temporary in-memory database
-    // and later we'll overwrite the persisted database with the new data
-    // TODO bring back this optimization
-    // const tmpDb = yield* makeSqliteDb({ _tag: 'in-memory' })
-    const tmpDb = dbState
-    yield* configureConnection(tmpDb, { foreignKeys: true })
-
-    const initDb = (hooks: Partial<MigrationHooks> | undefined) =>
-      Effect.gen(function* () {
-        yield* Effect.tryAll(() => hooks?.init?.(tmpDb)).pipe(UnknownError.mapToUnknownError)
-
-        const migrationsReport = yield* migrateDb({
-          db: tmpDb,
-          schema,
-          onProgress: ({ done, total }) =>
-            Queue.offer(bootStatusQueue, { stage: 'migrating', progress: { done, total } }),
-        })
-
-        yield* Effect.tryAll(() => hooks?.pre?.(tmpDb)).pipe(UnknownError.mapToUnknownError)
-
-        return { migrationsReport, tmpDb }
-      })
-
-    switch (migrationOptions.strategy) {
-      case 'auto': {
-        const hooks = migrationOptions.hooks
-        const initResult = yield* initDb(hooks)
-
-        migrationsReport = initResult.migrationsReport
-
-        yield* rematerializeFromEventlog({
-          // db: initResult.tmpDb,
-          dbEventlog,
-          schema,
-          materializeEvent,
-          onProgress: ({ done, total }) =>
-            Queue.offer(bootStatusQueue, { stage: 'rehydrating', progress: { done, total } }),
-        })
-
-        yield* Effect.tryAll(() => hooks?.post?.(initResult.tmpDb)).pipe(UnknownError.mapToUnknownError)
-
-        break
-      }
-      case 'manual': {
-        const oldDbData = dbState.export()
-
-        migrationsReport = { migrations: [] }
-
-        const newDbData = yield* Effect.tryAll(() => migrationOptions.migrate(oldDbData)).pipe(
-          UnknownError.mapToUnknownError,
-        )
+    yield* configureConnection(dbState, { foreignKeys: true })
 
-        tmpDb.import(newDbData)
+    // @effect-diagnostics-next-line anyUnknownInErrorContext:off -- user hook errors are immediately normalized to LiveStore UnknownError
+    yield* Effect.trySyncOrPromiseOrEffect(() => hooks?.init?.(dbState)).pipe(UnknownError.mapToUnknownError)
 
-        // TODO validate schema
+    const migrationsReport = yield* migrateDb({
+      db: dbState,
+      schema,
+      onProgress: ({ done, total }) => Queue.offer(bootStatusQueue, { stage: 'migrating', progress: { done, total } }),
+    })
 
-        break
-      }
-      default: {
-        casesHandled(migrationOptions)
-      }
-    }
+    // @effect-diagnostics-next-line anyUnknownInErrorContext:off -- user hook errors are immediately normalized to LiveStore UnknownError
+    yield* Effect.trySyncOrPromiseOrEffect(() => hooks?.pre?.(dbState)).pipe(UnknownError.mapToUnknownError)
 
-    // TODO bring back
-    // Import the temporary in-memory database into the persistent database
-    // yield* Effect.sync(() => db.import(tmpDb)).pipe(
-    //   Effect.withSpan('@livestore/common:leader-thread:recreateDb:import'),
-    // )
+    yield* rematerializeFromEventlog({
+      dbEventlog,
+      dbState,
+      schema,
+      materializeEvent,
+      batchSize: stateRebuildBatchSize,
+      onProgress: ({ done, total }) =>
+        Queue.offer(bootStatusQueue, { stage: 'rehydrating', progress: { done, total } }),
+    })
 
-    // TODO maybe bring back re-using this initial snapshot to avoid calling `.export()` again
-    // We've disabled this for now as it made the code too complex, as we often run syncing right after
-    // so the snapshot is no longer up to date
-    // const snapshotFromTmpDb = tmpDb.export()
+    // @effect-diagnostics-next-line anyUnknownInErrorContext:off -- user hook errors are immediately normalized to LiveStore UnknownError
+    yield* Effect.trySyncOrPromiseOrEffect(() => hooks?.post?.(dbState)).pipe(UnknownError.mapToUnknownError)
 
-    // TODO bring back
-    // tmpDb.close()
+    // Keep this out of finalizers, which also run on failure and interruption.
+    yield* markStateAsCompleted(dbState)
 
     return { migrationsReport }
   }).pipe(

package/src/leader-thread/shutdown-channel.ts

@@ -1,19 +1,8 @@
-import type { WebChannel } from '@livestore/utils/effect'
-import { Schema } from '@livestore/utils/effect'
+import { type WebChannel, Schema } from '@livestore/utils/effect'
 
-import {
-  BackendIdMismatchError,
-  IntentionalShutdownCause,
-  MaterializeError,
-  UnknownError,
-} from '../index.ts'
+import { BackendIdMismatchError, IntentionalShutdownCause, MaterializeError, UnknownError } from '../index.ts'
 
-export class All extends Schema.Union(
-  IntentionalShutdownCause,
-  UnknownError,
-  BackendIdMismatchError,
-  MaterializeError,
-) {}
+export const All = Schema.Union([IntentionalShutdownCause, UnknownError, BackendIdMismatchError, MaterializeError])
 
 /**
  * Used internally by an adapter to shutdown gracefully.