@livestore/common 0.3.0-dev.2 → 0.3.0-dev.22

package/src/leader-thread/make-leader-thread-layer.ts

@@ -1,68 +1,104 @@
-import type { HttpClient, Scope, WebChannel } from '@livestore/utils/effect'
-import { Deferred, Effect, FiberSet, Layer, Queue, SubscriptionRef } from '@livestore/utils/effect'
+import type { HttpClient, Scope } from '@livestore/utils/effect'
+import { Deferred, Effect, Layer, Queue, SubscriptionRef } from '@livestore/utils/effect'
 
-import type { BootStatus, MakeSynchronousDatabase, SqliteError, SynchronousDatabase } from '../adapter-types.js'
+import type { BootStatus, MakeSqliteDb, MigrationsReport, SqliteError } from '../adapter-types.js'
 import { UnexpectedError } from '../adapter-types.js'
+import type * as Devtools from '../devtools/mod.js'
 import type { LiveStoreSchema } from '../schema/mod.js'
 import { EventId, MutationEvent, mutationLogMetaTable, SYNC_STATUS_TABLE, syncStatusTable } from '../schema/mod.js'
 import { migrateTable } from '../schema-management/migrations.js'
-import type { InvalidPullError, IsOfflineError, SyncBackend } from '../sync/sync.js'
+import type { InvalidPullError, IsOfflineError, SyncOptions } from '../sync/sync.js'
 import { sql } from '../util.js'
 import { execSql } from './connection.js'
-import { makeLeaderSyncProcessor } from './leader-sync-processor.js'
 import { bootDevtools } from './leader-worker-devtools.js'
+import { makeLeaderSyncProcessor } from './LeaderSyncProcessor.js'
 import { makePullQueueSet } from './pull-queue-set.js'
 import { recreateDb } from './recreate-db.js'
-import type { DevtoolsOptions, InitialBlockingSyncContext, InitialSyncOptions, ShutdownState } from './types.js'
+import type { ShutdownChannel } from './shutdown-channel.js'
+import type {
+  DevtoolsOptions,
+  InitialBlockingSyncContext,
+  InitialSyncOptions,
+  LeaderSqliteDb,
+  ShutdownState,
+} from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
 export const makeLeaderThreadLayer = ({
   schema,
   storeId,
-  originId,
-  makeSyncDb,
-  makeSyncBackend,
-  db,
-  dbLog,
+  clientId,
+  makeSqliteDb,
+  syncOptions,
+  dbReadModel,
+  dbMutationLog,
   devtoolsOptions,
-  initialSyncOptions = { _tag: 'Skip' },
+  shutdownChannel,
 }: {
   storeId: string
-  originId: string
+  clientId: string
   schema: LiveStoreSchema
-  makeSyncDb: MakeSynchronousDatabase
-  makeSyncBackend: Effect.Effect<SyncBackend, UnexpectedError, Scope.Scope> | undefined
-  db: SynchronousDatabase
-  dbLog: SynchronousDatabase
+  makeSqliteDb: MakeSqliteDb
+  syncOptions: SyncOptions | undefined
+  dbReadModel: LeaderSqliteDb
+  dbMutationLog: LeaderSqliteDb
   devtoolsOptions: DevtoolsOptions
-  initialSyncOptions: InitialSyncOptions | undefined
+  shutdownChannel: ShutdownChannel
 }): Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient> =>
   Effect.gen(function* () {
     const bootStatusQueue = yield* Queue.unbounded<BootStatus>().pipe(Effect.acquireRelease(Queue.shutdown))
 
     // TODO do more validation here than just checking the count of tables
     // Either happens on initial boot or if schema changes
-    const dbMissing = db.select<{ count: number }>(sql`select count(*) as count from sqlite_master`)[0]!.count === 0
+    const dbMissing =
+      dbReadModel.select<{ count: number }>(sql`select count(*) as count from sqlite_master`)[0]!.count === 0
 
-    const syncBackend = makeSyncBackend === undefined ? undefined : yield* makeSyncBackend
+    const syncBackend =
+      syncOptions?.makeBackend === undefined ? undefined : yield* syncOptions.makeBackend({ storeId, clientId })
 
-    const initialBlockingSyncContext = yield* makeInitialBlockingSyncContext({ initialSyncOptions, bootStatusQueue })
+    const initialBlockingSyncContext = yield* makeInitialBlockingSyncContext({
+      initialSyncOptions: syncOptions?.initialSyncOptions ?? { _tag: 'Skip' },
+      bootStatusQueue,
+    })
 
-    const syncProcessor = yield* makeLeaderSyncProcessor({ schema, dbMissing, dbLog, initialBlockingSyncContext })
+    const syncProcessor = yield* makeLeaderSyncProcessor({
+      schema,
+      dbMissing,
+      dbMutationLog,
+      initialBlockingSyncContext,
+      clientId,
+    })
+
+    const extraIncomingMessagesQueue = yield* Queue.unbounded<Devtools.Leader.MessageToApp>().pipe(
+      Effect.acquireRelease(Queue.shutdown),
+    )
+
+    const devtoolsContext = devtoolsOptions.enabled
+      ? {
+          enabled: true as const,
+          syncBackendLatch: yield* Effect.makeLatch(true),
+          syncBackendLatchState: yield* SubscriptionRef.make<{ latchClosed: boolean }>({ latchClosed: false }),
+        }
+      : { enabled: false as const }
 
     const ctx = {
       schema,
       bootStatusQueue,
       storeId,
-      originId,
-      db,
-      dbLog,
-      makeSyncDb,
+      clientId,
+      dbReadModel,
+      dbMutationLog,
+      makeSqliteDb,
       mutationEventSchema: MutationEvent.makeMutationEventSchema(schema),
       shutdownStateSubRef: yield* SubscriptionRef.make<ShutdownState>('running'),
+      shutdownChannel,
       syncBackend,
       syncProcessor,
       connectedClientSessionPullQueues: yield* makePullQueueSet,
+      extraIncomingMessagesQueue,
+      devtools: devtoolsContext,
+      // State will be set during `bootLeaderThread`
+      initialState: {} as any as LeaderThreadCtx['Type']['initialState'],
     } satisfies typeof LeaderThreadCtx.Service
 
     // @ts-expect-error For debugging purposes
@@ -70,7 +106,11 @@ export const makeLeaderThreadLayer = ({
 
     const layer = Layer.succeed(LeaderThreadCtx, ctx)
 
-    yield* bootLeaderThread({ dbMissing, initialBlockingSyncContext, devtoolsOptions }).pipe(Effect.provide(layer))
+    ctx.initialState = yield* bootLeaderThread({
+      dbMissing,
+      initialBlockingSyncContext,
+      devtoolsOptions,
+    }).pipe(Effect.provide(layer))
 
     return layer
   }).pipe(
@@ -140,22 +180,22 @@ const bootLeaderThread = ({
   initialBlockingSyncContext: InitialBlockingSyncContext
   devtoolsOptions: DevtoolsOptions
 }): Effect.Effect<
-  void,
+  LeaderThreadCtx['Type']['initialState'],
   UnexpectedError | SqliteError | IsOfflineError | InvalidPullError,
   LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
 > =>
   Effect.gen(function* () {
-    const { dbLog, bootStatusQueue, syncProcessor } = yield* LeaderThreadCtx
+    const { dbMutationLog, bootStatusQueue, syncProcessor } = yield* LeaderThreadCtx
 
     yield* migrateTable({
-      db: dbLog,
+      db: dbMutationLog,
       behaviour: 'create-if-not-exists',
       tableAst: mutationLogMetaTable.sqliteDef.ast,
       skipMetaTable: true,
     })
 
     yield* migrateTable({
-      db: dbLog,
+      db: dbMutationLog,
       behaviour: 'create-if-not-exists',
       tableAst: syncStatusTable.sqliteDef.ast,
       skipMetaTable: true,
@@ -163,7 +203,7 @@ const bootLeaderThread = ({
 
     // Create sync status row if it doesn't exist
     yield* execSql(
-      dbLog,
+      dbMutationLog,
       sql`INSERT INTO ${SYNC_STATUS_TABLE} (head)
           SELECT ${EventId.ROOT.global}
           WHERE NOT EXISTS (SELECT 1 FROM ${SYNC_STATUS_TABLE})`,
@@ -174,19 +214,33 @@ const bootLeaderThread = ({
 
     // We're already starting pulling from the sync backend concurrently but wait until the db is ready before
     // processing any incoming mutations
-    yield* syncProcessor.boot({ dbReady })
+    const { initialLeaderHead } = yield* syncProcessor.boot({ dbReady })
 
+    let migrationsReport: MigrationsReport
     if (dbMissing) {
-      yield* recreateDb
+      const recreateResult = yield* recreateDb
+      migrationsReport = recreateResult.migrationsReport
+    } else {
+      migrationsReport = { migrations: [] }
     }
 
     yield* Deferred.succeed(dbReady, void 0)
 
     if (initialBlockingSyncContext.blockingDeferred !== undefined) {
-      yield* initialBlockingSyncContext.blockingDeferred
+      // Provides a syncing status right away before the first pull response comes in
+      yield* Queue.offer(bootStatusQueue, {
+        stage: 'syncing',
+        progress: { done: 0, total: -1 },
+      })
+
+      yield* initialBlockingSyncContext.blockingDeferred.pipe(
+        Effect.withSpan('@livestore/common:leader-thread:initial-sync-blocking'),
+      )
     }
 
     yield* Queue.offer(bootStatusQueue, { stage: 'done' })
 
     yield* bootDevtools(devtoolsOptions).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
+
+    return { migrationsReport, leaderHead: initialLeaderHead }
   })

package/src/leader-thread/mutationlog.ts

@@ -1,17 +1,20 @@
 import { Effect, Schema } from '@livestore/utils/effect'
 
-import type { SynchronousDatabase } from '../adapter-types.js'
+import type { SqliteDb } from '../adapter-types.js'
 import * as EventId from '../schema/EventId.js'
+import type * as MutationEvent from '../schema/MutationEvent.js'
 import { MUTATION_LOG_META_TABLE, mutationLogMetaTable, SYNC_STATUS_TABLE } from '../schema/system-tables.js'
 import { prepareBindValues, sql } from '../util.js'
 import { LeaderThreadCtx } from './types.js'
 
-export const getMutationEventsSince = (since: EventId.EventId) =>
+export const getMutationEventsSince = (
+  since: EventId.EventId,
+): Effect.Effect<ReadonlyArray<MutationEvent.AnyEncoded>, never, LeaderThreadCtx> =>
   Effect.gen(function* () {
-    const { dbLog } = yield* LeaderThreadCtx
+    const { dbMutationLog } = yield* LeaderThreadCtx
 
     const query = mutationLogMetaTable.query.where('idGlobal', '>=', since.global).asSql()
-    const pendingMutationEventsRaw = dbLog.select(query.query, prepareBindValues(query.bindValues, query.query))
+    const pendingMutationEventsRaw = dbMutationLog.select(query.query, prepareBindValues(query.bindValues, query.query))
     const pendingMutationEvents = Schema.decodeUnknownSync(mutationLogMetaTable.schema.pipe(Schema.Array))(
       pendingMutationEventsRaw,
     )
@@ -20,23 +23,26 @@ export const getMutationEventsSince = (since: EventId.EventId) =>
       .map((_) => ({
         mutation: _.mutation,
         args: _.argsJson,
-        id: { global: _.idGlobal, local: _.idLocal },
-        parentId: { global: _.parentIdGlobal, local: _.parentIdLocal },
+        id: { global: _.idGlobal, client: _.idClient },
+        parentId: { global: _.parentIdGlobal, client: _.parentIdClient },
+        clientId: _.clientId,
+        sessionId: _.sessionId ?? undefined,
       }))
       .filter((_) => EventId.compare(_.id, since) > 0)
   })
 
-export const getLocalHeadFromDb = (dbLog: SynchronousDatabase) => {
-  const res = dbLog.select<{ idGlobal: number; idLocal: number }>(
-    sql`select idGlobal, idLocal from ${MUTATION_LOG_META_TABLE} order by idGlobal DESC, idLocal DESC limit 1`,
+export const getClientHeadFromDb = (dbMutationLog: SqliteDb): EventId.EventId => {
+  const res = dbMutationLog.select<{ idGlobal: EventId.GlobalEventId; idClient: EventId.ClientEventId }>(
+    sql`select idGlobal, idClient from ${MUTATION_LOG_META_TABLE} order by idGlobal DESC, idClient DESC limit 1`,
   )[0]
 
-  return res ? { global: res.idGlobal, local: res.idLocal } : EventId.ROOT
+  return res ? { global: res.idGlobal, client: res.idClient } : EventId.ROOT
 }
 
-export const getBackendHeadFromDb = (dbLog: SynchronousDatabase) =>
-  dbLog.select<{ head: number }>(sql`select head from ${SYNC_STATUS_TABLE}`)[0]?.head ?? EventId.ROOT.global
+export const getBackendHeadFromDb = (dbMutationLog: SqliteDb): EventId.GlobalEventId =>
+  dbMutationLog.select<{ head: EventId.GlobalEventId }>(sql`select head from ${SYNC_STATUS_TABLE}`)[0]?.head ??
+  EventId.ROOT.global
 
 // TODO use prepared statements
-export const updateBackendHead = (dbLog: SynchronousDatabase, head: EventId.EventId) =>
-  dbLog.execute(sql`UPDATE ${SYNC_STATUS_TABLE} SET head = ${head.global}`)
+export const updateBackendHead = (dbMutationLog: SqliteDb, head: EventId.EventId) =>
+  dbMutationLog.execute(sql`UPDATE ${SYNC_STATUS_TABLE} SET head = ${head.global}`)

package/src/leader-thread/pull-queue-set.ts

@@ -2,7 +2,7 @@ import { Effect, Queue } from '@livestore/utils/effect'
 
 import * as MutationEvent from '../schema/MutationEvent.js'
 import { getMutationEventsSince } from './mutationlog.js'
-import { type PullQueueItem, type PullQueueSet } from './types.js'
+import { LeaderThreadCtx, type PullQueueItem, type PullQueueSet } from './types.js'
 
 export const makePullQueueSet = Effect.gen(function* () {
   const set = new Set<Queue.Queue<PullQueueItem>>()
@@ -46,6 +46,15 @@ export const makePullQueueSet = Effect.gen(function* () {
         return
       }
 
+      const { clientId } = yield* LeaderThreadCtx
+      if (clientId === 'client-b') {
+        // console.log(
+        //   'offer',
+        //   item.payload._tag,
+        //   item.payload.newEvents.map((_) => _.toJSON()),
+        // )
+      }
+
       for (const queue of set) {
         yield* Queue.offer(queue, item)
       }

package/src/leader-thread/recreate-db.ts

@@ -2,85 +2,94 @@ import { casesHandled } from '@livestore/utils'
 import type { HttpClient } from '@livestore/utils/effect'
 import { Effect, Queue } from '@livestore/utils/effect'
 
-import type { InvalidPullError, IsOfflineError, MigrationHooks, SqliteError } from '../index.js'
+import type { InvalidPullError, IsOfflineError, MigrationHooks, MigrationsReport, SqliteError } from '../index.js'
 import { initializeSingletonTables, migrateDb, rehydrateFromMutationLog, UnexpectedError } from '../index.js'
 import { configureConnection } from './connection.js'
 import { LeaderThreadCtx } from './types.js'
 
 export const recreateDb: Effect.Effect<
-  void,
+  { migrationsReport: MigrationsReport },
   UnexpectedError | SqliteError | IsOfflineError | InvalidPullError,
   LeaderThreadCtx | HttpClient.HttpClient
 > = Effect.gen(function* () {
-  const { db, dbLog, makeSyncDb, schema, bootStatusQueue } = yield* LeaderThreadCtx
+  const { dbReadModel, dbMutationLog, schema, bootStatusQueue } = yield* LeaderThreadCtx
 
   const migrationOptions = schema.migrationOptions
+  let migrationsReport: MigrationsReport
 
   yield* Effect.addFinalizer(
     Effect.fn('recreateDb:finalizer')(function* (ex) {
-      if (ex._tag === 'Failure') db.destroy()
+      if (ex._tag === 'Failure') dbReadModel.destroy()
     }),
   )
 
   // NOTE to speed up the operations below, we're creating a temporary in-memory database
   // and later we'll overwrite the persisted database with the new data
-  const tmpSyncDb = yield* makeSyncDb({ _tag: 'in-memory' })
-  yield* configureConnection(tmpSyncDb, { fkEnabled: true })
+  // TODO bring back this optimization
+  // const tmpDb = yield* makeSqliteDb({ _tag: 'in-memory' })
+  const tmpDb = dbReadModel
+  yield* configureConnection(tmpDb, { foreignKeys: true })
 
   const initDb = (hooks: Partial<MigrationHooks> | undefined) =>
     Effect.gen(function* () {
-      yield* Effect.tryAll(() => hooks?.init?.(tmpSyncDb)).pipe(UnexpectedError.mapToUnexpectedError)
+      yield* Effect.tryAll(() => hooks?.init?.(tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
 
-      yield* migrateDb({
-        db: tmpSyncDb,
+      const migrationsReport = yield* migrateDb({
+        db: tmpDb,
         schema,
         onProgress: ({ done, total }) =>
           Queue.offer(bootStatusQueue, { stage: 'migrating', progress: { done, total } }),
       })
 
-      initializeSingletonTables(schema, tmpSyncDb)
+      initializeSingletonTables(schema, tmpDb)
 
-      yield* Effect.tryAll(() => hooks?.pre?.(tmpSyncDb)).pipe(UnexpectedError.mapToUnexpectedError)
+      yield* Effect.tryAll(() => hooks?.pre?.(tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
 
-      return tmpSyncDb
+      return { migrationsReport, tmpDb }
     })
 
   switch (migrationOptions.strategy) {
     case 'from-mutation-log': {
       const hooks = migrationOptions.hooks
-      const tmpSyncDb = yield* initDb(hooks)
+      const initResult = yield* initDb(hooks)
+
+      migrationsReport = initResult.migrationsReport
 
       yield* rehydrateFromMutationLog({
-        db: tmpSyncDb,
-        logDb: dbLog,
+        db: initResult.tmpDb,
+        logDb: dbMutationLog,
         schema,
         migrationOptions,
         onProgress: ({ done, total }) =>
           Queue.offer(bootStatusQueue, { stage: 'rehydrating', progress: { done, total } }),
       })
 
-      yield* Effect.tryAll(() => hooks?.post?.(tmpSyncDb)).pipe(UnexpectedError.mapToUnexpectedError)
+      yield* Effect.tryAll(() => hooks?.post?.(initResult.tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
 
       break
     }
     case 'hard-reset': {
       const hooks = migrationOptions.hooks
-      const tmpInMemoryDb = yield* initDb(hooks)
+      const initResult = yield* initDb(hooks)
+
+      migrationsReport = initResult.migrationsReport
 
       // The database is migrated but empty now, so nothing else to do
 
-      yield* Effect.tryAll(() => hooks?.post?.(tmpInMemoryDb)).pipe(UnexpectedError.mapToUnexpectedError)
+      yield* Effect.tryAll(() => hooks?.post?.(initResult.tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
 
       break
     }
     case 'manual': {
-      const oldDbData = db.export()
+      const oldDbData = dbReadModel.export()
+
+      migrationsReport = { migrations: [] }
 
       const newDbData = yield* Effect.tryAll(() => migrationOptions.migrate(oldDbData)).pipe(
         UnexpectedError.mapToUnexpectedError,
       )
 
-      tmpSyncDb.import(newDbData)
+      tmpDb.import(newDbData)
 
       // TODO validate schema
 
@@ -91,17 +100,21 @@ export const recreateDb: Effect.Effect<
     }
   }
 
+  // TODO bring back
   // Import the temporary in-memory database into the persistent database
-  yield* Effect.sync(() => db.import(tmpSyncDb)).pipe(
-    Effect.withSpan('@livestore/common:leader-thread:recreateDb:import'),
-  )
+  // yield* Effect.sync(() => db.import(tmpDb)).pipe(
+  //   Effect.withSpan('@livestore/common:leader-thread:recreateDb:import'),
+  // )
 
   // TODO maybe bring back re-using this initial snapshot to avoid calling `.export()` again
   // We've disabled this for now as it made the code too complex, as we often run syncing right after
   // so the snapshot is no longer up to date
-  // const snapshotFromTmpDb = tmpSyncDb.export()
+  // const snapshotFromTmpDb = tmpDb.export()
+
+  // TODO bring back
+  // tmpDb.close()
 
-  tmpSyncDb.close()
+  return { migrationsReport }
 }).pipe(
   Effect.scoped, // NOTE we're closing the scope here so finalizers are called when the effect is done
   Effect.withSpan('@livestore/common:leader-thread:recreateDb'),

package/src/leader-thread/shutdown-channel.ts

@@ -1,11 +1,9 @@
 import type { WebChannel } from '@livestore/utils/effect'
 import { Schema } from '@livestore/utils/effect'
 
-import { IntentionalShutdownCause } from '../index.js'
+import { IntentionalShutdownCause, UnexpectedError } from '../index.js'
 
-export class DedicatedWorkerDisconnectBroadcast extends Schema.TaggedStruct('DedicatedWorkerDisconnectBroadcast', {}) {}
-
-export class All extends Schema.Union(IntentionalShutdownCause, DedicatedWorkerDisconnectBroadcast) {}
+export class All extends Schema.Union(IntentionalShutdownCause, UnexpectedError) {}
 
 /**
  * Used internally by an adapter to shutdown gracefully.

package/src/leader-thread/types.ts

@@ -1,11 +1,11 @@
 import type {
   Deferred,
   Effect,
-  Fiber,
   HttpClient,
   Option,
   Queue,
   Scope,
+  Subscribable,
   SubscriptionRef,
   WebChannel,
 } from '@livestore/utils/effect'
@@ -14,31 +14,25 @@ import { Context, Schema } from '@livestore/utils/effect'
 import type {
   BootStatus,
   Devtools,
-  InvalidPushError,
-  MakeSynchronousDatabase,
+  LeaderAheadError,
+  MakeSqliteDb,
+  MigrationsReport,
   PersistenceInfo,
+  SqliteDb,
   SyncBackend,
-  SynchronousDatabase,
   UnexpectedError,
 } from '../index.js'
 import type { EventId, LiveStoreSchema, MutationEvent } from '../schema/mod.js'
-import type { PayloadUpstream, SyncState } from '../sync/syncstate.js'
+import type * as SyncState from '../sync/syncstate.js'
 import type { ShutdownChannel } from './shutdown-channel.js'
 
 export type ShutdownState = 'running' | 'shutting-down'
 
-export class OuterWorkerCtx extends Context.Tag('OuterWorkerCtx')<
-  OuterWorkerCtx,
-  {
-    innerFiber: Fiber.RuntimeFiber<any, any>
-  }
->() {}
-
 export const InitialSyncOptionsSkip = Schema.TaggedStruct('Skip', {})
 export type InitialSyncOptionsSkip = typeof InitialSyncOptionsSkip.Type
 
 export const InitialSyncOptionsBlocking = Schema.TaggedStruct('Blocking', {
-  timeout: Schema.DurationFromMillis,
+  timeout: Schema.Union(Schema.DurationFromMillis, Schema.Number),
 })
 
 export type InitialSyncOptionsBlocking = typeof InitialSyncOptionsBlocking.Type
@@ -55,8 +49,8 @@ export type InitialSyncInfo = Option.Option<{
 //   | { _tag: 'Recreate'; snapshotRef: Ref.Ref<Uint8Array | undefined>; syncInfo: InitialSyncInfo }
 //   | { _tag: 'Reuse'; syncInfo: InitialSyncInfo }
 
-export type LeaderDatabase = SynchronousDatabase<{ dbPointer: number; persistenceInfo: PersistenceInfo }>
-export type PersistenceInfoPair = { db: PersistenceInfo; mutationLog: PersistenceInfo }
+export type LeaderSqliteDb = SqliteDb<{ dbPointer: number; persistenceInfo: PersistenceInfo }>
+export type PersistenceInfoPair = { readModel: PersistenceInfo; mutationLog: PersistenceInfo }
 
 export type DevtoolsOptions =
   | {
@@ -64,10 +58,9 @@ export type DevtoolsOptions =
     }
   | {
       enabled: true
-      makeContext: Effect.Effect<
+      makeBootContext: Effect.Effect<
         {
-          devtoolsWebChannel: WebChannel.WebChannel<Devtools.MessageToAppLeader, Devtools.MessageFromAppLeader>
-          shutdownChannel: ShutdownChannel
+          devtoolsWebChannel: WebChannel.WebChannel<Devtools.Leader.MessageToApp, Devtools.Leader.MessageFromApp>
           persistenceInfo: PersistenceInfoPair
         },
         UnexpectedError,
@@ -75,23 +68,46 @@ export type DevtoolsOptions =
       >
     }
 
+export type DevtoolsContext =
+  | {
+      enabled: true
+      // syncBackendPullLatch: Effect.Latch
+      // syncBackendPushLatch: Effect.Latch
+      syncBackendLatch: Effect.Latch
+      syncBackendLatchState: SubscriptionRef.SubscriptionRef<{ latchClosed: boolean }>
+    }
+  | {
+      enabled: false
+    }
+
 export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
   LeaderThreadCtx,
   {
     schema: LiveStoreSchema
     storeId: string
-    originId: string
-    makeSyncDb: MakeSynchronousDatabase
-    db: LeaderDatabase
-    dbLog: LeaderDatabase
+    clientId: string
+    makeSqliteDb: MakeSqliteDb
+    dbReadModel: LeaderSqliteDb
+    dbMutationLog: LeaderSqliteDb
     bootStatusQueue: Queue.Queue<BootStatus>
     // TODO we should find a more elegant way to handle cases which need this ref for their implementation
     shutdownStateSubRef: SubscriptionRef.SubscriptionRef<ShutdownState>
+    shutdownChannel: ShutdownChannel
     mutationEventSchema: MutationEvent.ForMutationDefRecord<any>
-    // devtools: DevtoolsContext
+    devtools: DevtoolsContext
     syncBackend: SyncBackend | undefined
-    syncProcessor: SyncProcessor
+    syncProcessor: LeaderSyncProcessor
     connectedClientSessionPullQueues: PullQueueSet
+    initialState: {
+      leaderHead: EventId.EventId
+      migrationsReport: MigrationsReport
+    }
+    /**
+     * e.g. used for `store._dev` APIs
+     *
+     * This is currently separated from `.devtools` as it also needs to work when devtools are disabled
+     */
+    extraIncomingMessagesQueue: Queue.Queue<Devtools.Leader.MessageToApp>
   }
 >() {}
 
@@ -101,24 +117,42 @@ export type InitialBlockingSyncContext = {
 }
 
 export type PullQueueItem = {
-  // mutationEvents: ReadonlyArray<MutationEvent.AnyEncoded>
-  // backendHead: number
-  payload: PayloadUpstream
-  // TODO move `remaining` into `PayloadUpstream`
+  payload: SyncState.PayloadUpstream
   remaining: number
 }
 
-export interface SyncProcessor {
+export interface LeaderSyncProcessor {
   push: (
     /** `batch` needs to follow the same rules as `batch` in `SyncBackend.push` */
     batch: ReadonlyArray<MutationEvent.EncodedWithMeta>,
-  ) => Effect.Effect<void, UnexpectedError | InvalidPushError, HttpClient.HttpClient | LeaderThreadCtx>
-
-  pushPartial: (mutationEvent: MutationEvent.PartialAnyEncoded) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
+    options?: {
+      /**
+       * This generation number is used to automatically reject subsequent pushes
+       * of a previously rejected push from a client session. This might occur in
+       * certain concurrent scenarios.
+       */
+      // generation: number
+      /**
+       * If true, the effect will only finish when the local push has been processed (i.e. succeeded or was rejected).
+       * @default false
+       */
+      waitForProcessing?: boolean
+    },
+  ) => Effect.Effect<void, LeaderAheadError>
+
+  pushPartial: (args: {
+    mutationEvent: MutationEvent.PartialAnyEncoded
+    clientId: string
+    sessionId: string | undefined
+  }) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
   boot: (args: {
     dbReady: Deferred.Deferred<void>
-  }) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient>
-  syncState: Effect.Effect<SyncState, UnexpectedError>
+  }) => Effect.Effect<
+    { initialLeaderHead: EventId.EventId },
+    UnexpectedError,
+    LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
+  >
+  syncState: Subscribable.Subscribable<SyncState.SyncState>
 }
 
 export interface PullQueueSet {