@livestore/common 0.3.0-dev.3 → 0.3.0-dev.30

package/src/leader-thread/make-leader-thread-layer.ts

@@ -1,68 +1,141 @@
-import type { HttpClient, Scope, WebChannel } from '@livestore/utils/effect'
-import { Deferred, Effect, FiberSet, Layer, Queue, SubscriptionRef } from '@livestore/utils/effect'
+import type { HttpClient, Schema, Scope } from '@livestore/utils/effect'
+import { Deferred, Effect, Layer, Queue, SubscriptionRef } from '@livestore/utils/effect'
 
-import type { BootStatus, MakeSynchronousDatabase, SqliteError, SynchronousDatabase } from '../adapter-types.js'
+import type { BootStatus, MakeSqliteDb, MigrationsReport, SqliteError } from '../adapter-types.js'
 import { UnexpectedError } from '../adapter-types.js'
+import type * as Devtools from '../devtools/mod.js'
 import type { LiveStoreSchema } from '../schema/mod.js'
-import { EventId, MutationEvent, mutationLogMetaTable, SYNC_STATUS_TABLE, syncStatusTable } from '../schema/mod.js'
-import { migrateTable } from '../schema-management/migrations.js'
-import type { InvalidPullError, IsOfflineError, SyncBackend } from '../sync/sync.js'
+import { LiveStoreEvent } from '../schema/mod.js'
+import type { InvalidPullError, IsOfflineError, SyncOptions } from '../sync/sync.js'
 import { sql } from '../util.js'
-import { execSql } from './connection.js'
-import { makeLeaderSyncProcessor } from './leader-sync-processor.js'
+import { makeApplyEvent } from './apply-event.js'
+import * as Eventlog from './eventlog.js'
 import { bootDevtools } from './leader-worker-devtools.js'
-import { makePullQueueSet } from './pull-queue-set.js'
+import { makeLeaderSyncProcessor } from './LeaderSyncProcessor.js'
 import { recreateDb } from './recreate-db.js'
-import type { DevtoolsOptions, InitialBlockingSyncContext, InitialSyncOptions, ShutdownState } from './types.js'
+import type { ShutdownChannel } from './shutdown-channel.js'
+import type {
+  DevtoolsOptions,
+  InitialBlockingSyncContext,
+  InitialSyncOptions,
+  LeaderSqliteDb,
+  ShutdownState,
+} from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
+export interface MakeLeaderThreadLayerParams {
+  storeId: string
+  syncPayload: Schema.JsonValue | undefined
+  clientId: string
+  schema: LiveStoreSchema
+  makeSqliteDb: MakeSqliteDb
+  syncOptions: SyncOptions | undefined
+  dbReadModel: LeaderSqliteDb
+  dbEventlog: LeaderSqliteDb
+  devtoolsOptions: DevtoolsOptions
+  shutdownChannel: ShutdownChannel
+  params?: {
+    localPushBatchSize?: number
+    backendPushBatchSize?: number
+  }
+  testing?: {
+    syncProcessor?: {
+      delays?: {
+        localPushProcessing?: Effect.Effect<void>
+      }
+    }
+  }
+}
+
 export const makeLeaderThreadLayer = ({
   schema,
   storeId,
-  originId,
-  makeSyncDb,
-  makeSyncBackend,
-  db,
-  dbLog,
+  clientId,
+  syncPayload,
+  makeSqliteDb,
+  syncOptions,
+  dbReadModel,
+  dbEventlog,
   devtoolsOptions,
-  initialSyncOptions = { _tag: 'Skip' },
-}: {
-  storeId: string
-  originId: string
-  schema: LiveStoreSchema
-  makeSyncDb: MakeSynchronousDatabase
-  makeSyncBackend: Effect.Effect<SyncBackend, UnexpectedError, Scope.Scope> | undefined
-  db: SynchronousDatabase
-  dbLog: SynchronousDatabase
-  devtoolsOptions: DevtoolsOptions
-  initialSyncOptions: InitialSyncOptions | undefined
-}): Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient> =>
+  shutdownChannel,
+  params,
+  testing,
+}: MakeLeaderThreadLayerParams): Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient> =>
   Effect.gen(function* () {
     const bootStatusQueue = yield* Queue.unbounded<BootStatus>().pipe(Effect.acquireRelease(Queue.shutdown))
 
     // TODO do more validation here than just checking the count of tables
     // Either happens on initial boot or if schema changes
-    const dbMissing = db.select<{ count: number }>(sql`select count(*) as count from sqlite_master`)[0]!.count === 0
+    const dbEventlogMissing =
+      dbEventlog.select<{ count: number }>(sql`select count(*) as count from sqlite_master`)[0]!.count === 0
+
+    const dbReadModelMissing =
+      dbReadModel.select<{ count: number }>(sql`select count(*) as count from sqlite_master`)[0]!.count === 0
+
+    const syncBackend =
+      syncOptions?.backend === undefined
+        ? undefined
+        : yield* syncOptions.backend({ storeId, clientId, payload: syncPayload })
+
+    if (syncBackend !== undefined) {
+      // We're already connecting to the sync backend concurrently
+      yield* syncBackend.connect.pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
+    }
+
+    const initialBlockingSyncContext = yield* makeInitialBlockingSyncContext({
+      initialSyncOptions: syncOptions?.initialSyncOptions ?? { _tag: 'Skip' },
+      bootStatusQueue,
+    })
+
+    const syncProcessor = yield* makeLeaderSyncProcessor({
+      schema,
+      dbEventlogMissing,
+      dbEventlog,
+      dbReadModel,
+      dbReadModelMissing,
+      initialBlockingSyncContext,
+      onError: syncOptions?.onSyncError ?? 'ignore',
+      params: {
+        localPushBatchSize: params?.localPushBatchSize,
+        backendPushBatchSize: params?.backendPushBatchSize,
+      },
+      testing: {
+        delays: testing?.syncProcessor?.delays,
+      },
+    })
 
-    const syncBackend = makeSyncBackend === undefined ? undefined : yield* makeSyncBackend
+    const extraIncomingMessagesQueue = yield* Queue.unbounded<Devtools.Leader.MessageToApp>().pipe(
+      Effect.acquireRelease(Queue.shutdown),
+    )
 
-    const initialBlockingSyncContext = yield* makeInitialBlockingSyncContext({ initialSyncOptions, bootStatusQueue })
+    const devtoolsContext = devtoolsOptions.enabled
+      ? {
+          enabled: true as const,
+          syncBackendLatch: yield* Effect.makeLatch(true),
+          syncBackendLatchState: yield* SubscriptionRef.make<{ latchClosed: boolean }>({ latchClosed: false }),
+        }
+      : { enabled: false as const }
 
-    const syncProcessor = yield* makeLeaderSyncProcessor({ schema, dbMissing, dbLog, initialBlockingSyncContext })
+    const applyEvent = yield* makeApplyEvent({ schema, dbReadModel, dbEventlog })
 
     const ctx = {
       schema,
       bootStatusQueue,
       storeId,
-      originId,
-      db,
-      dbLog,
-      makeSyncDb,
-      mutationEventSchema: MutationEvent.makeMutationEventSchema(schema),
+      clientId,
+      dbReadModel,
+      dbEventlog,
+      makeSqliteDb,
+      eventSchema: LiveStoreEvent.makeEventDefSchema(schema),
       shutdownStateSubRef: yield* SubscriptionRef.make<ShutdownState>('running'),
+      shutdownChannel,
       syncBackend,
       syncProcessor,
-      connectedClientSessionPullQueues: yield* makePullQueueSet,
+      applyEvent,
+      extraIncomingMessagesQueue,
+      devtools: devtoolsContext,
+      // State will be set during `bootLeaderThread`
+      initialState: {} as any as LeaderThreadCtx['Type']['initialState'],
     } satisfies typeof LeaderThreadCtx.Service
 
     // @ts-expect-error For debugging purposes
@@ -70,13 +143,18 @@ export const makeLeaderThreadLayer = ({
 
     const layer = Layer.succeed(LeaderThreadCtx, ctx)
 
-    yield* bootLeaderThread({ dbMissing, initialBlockingSyncContext, devtoolsOptions }).pipe(Effect.provide(layer))
+    ctx.initialState = yield* bootLeaderThread({
+      dbReadModelMissing,
+      initialBlockingSyncContext,
+      devtoolsOptions,
+    }).pipe(Effect.provide(layer))
 
     return layer
   }).pipe(
     Effect.withSpan('@livestore/common:leader-thread:boot'),
     Effect.withSpanScoped('@livestore/common:leader-thread'),
     UnexpectedError.mapToUnexpectedError,
+    Effect.tapCauseLogPretty,
     Layer.unwrapScoped,
   )
 
@@ -90,7 +168,7 @@ const makeInitialBlockingSyncContext = ({
   Effect.gen(function* () {
     const ctx = {
       isDone: false,
-      processedMutations: 0,
+      processedEvents: 0,
       total: -1,
     }
 
@@ -113,10 +191,10 @@ const makeInitialBlockingSyncContext = ({
             ctx.total = remaining + processed
           }
 
-          ctx.processedMutations += processed
+          ctx.processedEvents += processed
           yield* Queue.offer(bootStatusQueue, {
             stage: 'syncing',
-            progress: { done: ctx.processedMutations, total: ctx.total },
+            progress: { done: ctx.processedEvents, total: ctx.total },
           })
 
           if (remaining === 0 && blockingDeferred !== undefined) {
@@ -132,61 +210,49 @@ const makeInitialBlockingSyncContext = ({
  * It also starts various background processes (e.g. syncing)
  */
 const bootLeaderThread = ({
-  dbMissing,
+  dbReadModelMissing,
   initialBlockingSyncContext,
   devtoolsOptions,
 }: {
-  dbMissing: boolean
+  dbReadModelMissing: boolean
   initialBlockingSyncContext: InitialBlockingSyncContext
   devtoolsOptions: DevtoolsOptions
 }): Effect.Effect<
-  void,
+  LeaderThreadCtx['Type']['initialState'],
   UnexpectedError | SqliteError | IsOfflineError | InvalidPullError,
   LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
 > =>
   Effect.gen(function* () {
-    const { dbLog, bootStatusQueue, syncProcessor } = yield* LeaderThreadCtx
-
-    yield* migrateTable({
-      db: dbLog,
-      behaviour: 'create-if-not-exists',
-      tableAst: mutationLogMetaTable.sqliteDef.ast,
-      skipMetaTable: true,
-    })
+    const { dbEventlog, bootStatusQueue, syncProcessor } = yield* LeaderThreadCtx
 
-    yield* migrateTable({
-      db: dbLog,
-      behaviour: 'create-if-not-exists',
-      tableAst: syncStatusTable.sqliteDef.ast,
-      skipMetaTable: true,
-    })
+    yield* Eventlog.initEventlogDb(dbEventlog)
 
-    // Create sync status row if it doesn't exist
-    yield* execSql(
-      dbLog,
-      sql`INSERT INTO ${SYNC_STATUS_TABLE} (head)
-          SELECT ${EventId.ROOT.global}
-          WHERE NOT EXISTS (SELECT 1 FROM ${SYNC_STATUS_TABLE})`,
-      {},
-    )
-
-    const dbReady = yield* Deferred.make<void>()
-
-    // We're already starting pulling from the sync backend concurrently but wait until the db is ready before
-    // processing any incoming mutations
-    yield* syncProcessor.boot({ dbReady })
-
-    if (dbMissing) {
-      yield* recreateDb
+    let migrationsReport: MigrationsReport
+    if (dbReadModelMissing) {
+      const recreateResult = yield* recreateDb
+      migrationsReport = recreateResult.migrationsReport
+    } else {
+      migrationsReport = { migrations: [] }
     }
 
-    yield* Deferred.succeed(dbReady, void 0)
+    // NOTE the sync processor depends on the dbs being initialized properly
+    const { initialLeaderHead } = yield* syncProcessor.boot
 
     if (initialBlockingSyncContext.blockingDeferred !== undefined) {
-      yield* initialBlockingSyncContext.blockingDeferred
+      // Provides a syncing status right away before the first pull response comes in
+      yield* Queue.offer(bootStatusQueue, {
+        stage: 'syncing',
+        progress: { done: 0, total: -1 },
+      })
+
+      yield* initialBlockingSyncContext.blockingDeferred.pipe(
+        Effect.withSpan('@livestore/common:leader-thread:initial-sync-blocking'),
+      )
     }
 
     yield* Queue.offer(bootStatusQueue, { stage: 'done' })
 
     yield* bootDevtools(devtoolsOptions).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
+
+    return { migrationsReport, leaderHead: initialLeaderHead }
   })

package/src/leader-thread/mod.ts

@@ -3,4 +3,4 @@ export * from './types.js'
 export * as ShutdownChannel from './shutdown-channel.js'
 export * from './leader-worker-devtools.js'
 export * from './make-leader-thread-layer.js'
-export * from './mutationlog.js'
+export * as Eventlog from './eventlog.js'

package/src/leader-thread/recreate-db.ts

@@ -2,85 +2,92 @@ import { casesHandled } from '@livestore/utils'
 import type { HttpClient } from '@livestore/utils/effect'
 import { Effect, Queue } from '@livestore/utils/effect'
 
-import type { InvalidPullError, IsOfflineError, MigrationHooks, SqliteError } from '../index.js'
-import { initializeSingletonTables, migrateDb, rehydrateFromMutationLog, UnexpectedError } from '../index.js'
+import type { InvalidPullError, IsOfflineError, MigrationHooks, MigrationsReport, SqliteError } from '../index.js'
+import { migrateDb, rehydrateFromEventlog, UnexpectedError } from '../index.js'
 import { configureConnection } from './connection.js'
 import { LeaderThreadCtx } from './types.js'
 
 export const recreateDb: Effect.Effect<
-  void,
+  { migrationsReport: MigrationsReport },
   UnexpectedError | SqliteError | IsOfflineError | InvalidPullError,
   LeaderThreadCtx | HttpClient.HttpClient
 > = Effect.gen(function* () {
-  const { db, dbLog, makeSyncDb, schema, bootStatusQueue } = yield* LeaderThreadCtx
+  const { dbReadModel, dbEventlog, schema, bootStatusQueue, applyEvent } = yield* LeaderThreadCtx
 
   const migrationOptions = schema.migrationOptions
+  let migrationsReport: MigrationsReport
 
   yield* Effect.addFinalizer(
     Effect.fn('recreateDb:finalizer')(function* (ex) {
-      if (ex._tag === 'Failure') db.destroy()
+      if (ex._tag === 'Failure') dbReadModel.destroy()
     }),
   )
 
   // NOTE to speed up the operations below, we're creating a temporary in-memory database
   // and later we'll overwrite the persisted database with the new data
-  const tmpSyncDb = yield* makeSyncDb({ _tag: 'in-memory' })
-  yield* configureConnection(tmpSyncDb, { fkEnabled: true })
+  // TODO bring back this optimization
+  // const tmpDb = yield* makeSqliteDb({ _tag: 'in-memory' })
+  const tmpDb = dbReadModel
+  yield* configureConnection(tmpDb, { foreignKeys: true })
 
   const initDb = (hooks: Partial<MigrationHooks> | undefined) =>
     Effect.gen(function* () {
-      yield* Effect.tryAll(() => hooks?.init?.(tmpSyncDb)).pipe(UnexpectedError.mapToUnexpectedError)
+      yield* Effect.tryAll(() => hooks?.init?.(tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
 
-      yield* migrateDb({
-        db: tmpSyncDb,
+      const migrationsReport = yield* migrateDb({
+        db: tmpDb,
         schema,
         onProgress: ({ done, total }) =>
           Queue.offer(bootStatusQueue, { stage: 'migrating', progress: { done, total } }),
       })
 
-      initializeSingletonTables(schema, tmpSyncDb)
+      yield* Effect.tryAll(() => hooks?.pre?.(tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
 
-      yield* Effect.tryAll(() => hooks?.pre?.(tmpSyncDb)).pipe(UnexpectedError.mapToUnexpectedError)
-
-      return tmpSyncDb
+      return { migrationsReport, tmpDb }
     })
 
   switch (migrationOptions.strategy) {
-    case 'from-mutation-log': {
+    case 'from-eventlog': {
       const hooks = migrationOptions.hooks
-      const tmpSyncDb = yield* initDb(hooks)
+      const initResult = yield* initDb(hooks)
+
+      migrationsReport = initResult.migrationsReport
 
-      yield* rehydrateFromMutationLog({
-        db: tmpSyncDb,
-        logDb: dbLog,
+      yield* rehydrateFromEventlog({
+        // db: initResult.tmpDb,
+        dbEventlog,
         schema,
-        migrationOptions,
+        applyEvent,
         onProgress: ({ done, total }) =>
           Queue.offer(bootStatusQueue, { stage: 'rehydrating', progress: { done, total } }),
       })
 
-      yield* Effect.tryAll(() => hooks?.post?.(tmpSyncDb)).pipe(UnexpectedError.mapToUnexpectedError)
+      yield* Effect.tryAll(() => hooks?.post?.(initResult.tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
 
       break
     }
     case 'hard-reset': {
       const hooks = migrationOptions.hooks
-      const tmpInMemoryDb = yield* initDb(hooks)
+      const initResult = yield* initDb(hooks)
+
+      migrationsReport = initResult.migrationsReport
 
       // The database is migrated but empty now, so nothing else to do
 
-      yield* Effect.tryAll(() => hooks?.post?.(tmpInMemoryDb)).pipe(UnexpectedError.mapToUnexpectedError)
+      yield* Effect.tryAll(() => hooks?.post?.(initResult.tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
 
       break
     }
     case 'manual': {
-      const oldDbData = db.export()
+      const oldDbData = dbReadModel.export()
+
+      migrationsReport = { migrations: [] }
 
       const newDbData = yield* Effect.tryAll(() => migrationOptions.migrate(oldDbData)).pipe(
         UnexpectedError.mapToUnexpectedError,
       )
 
-      tmpSyncDb.import(newDbData)
+      tmpDb.import(newDbData)
 
       // TODO validate schema
 
@@ -91,17 +98,21 @@ export const recreateDb: Effect.Effect<
     }
   }
 
+  // TODO bring back
   // Import the temporary in-memory database into the persistent database
-  yield* Effect.sync(() => db.import(tmpSyncDb)).pipe(
-    Effect.withSpan('@livestore/common:leader-thread:recreateDb:import'),
-  )
+  // yield* Effect.sync(() => db.import(tmpDb)).pipe(
+  //   Effect.withSpan('@livestore/common:leader-thread:recreateDb:import'),
+  // )
 
   // TODO maybe bring back re-using this initial snapshot to avoid calling `.export()` again
   // We've disabled this for now as it made the code too complex, as we often run syncing right after
   // so the snapshot is no longer up to date
-  // const snapshotFromTmpDb = tmpSyncDb.export()
+  // const snapshotFromTmpDb = tmpDb.export()
+
+  // TODO bring back
+  // tmpDb.close()
 
-  tmpSyncDb.close()
+  return { migrationsReport }
 }).pipe(
   Effect.scoped, // NOTE we're closing the scope here so finalizers are called when the effect is done
   Effect.withSpan('@livestore/common:leader-thread:recreateDb'),

package/src/leader-thread/shutdown-channel.ts

@@ -1,11 +1,9 @@
 import type { WebChannel } from '@livestore/utils/effect'
 import { Schema } from '@livestore/utils/effect'
 
-import { IntentionalShutdownCause } from '../index.js'
+import { IntentionalShutdownCause, UnexpectedError } from '../index.js'
 
-export class DedicatedWorkerDisconnectBroadcast extends Schema.TaggedStruct('DedicatedWorkerDisconnectBroadcast', {}) {}
-
-export class All extends Schema.Union(IntentionalShutdownCause, DedicatedWorkerDisconnectBroadcast) {}
+export class All extends Schema.Union(IntentionalShutdownCause, UnexpectedError) {}
 
 /**
  * Used internally by an adapter to shutdown gracefully.