@livestore/common 0.3.0-dev.1 → 0.3.0-dev.11

package/src/leader-thread/apply-mutation.ts

@@ -2,21 +2,26 @@ import { memoizeByRef, shouldNeverHappen } from '@livestore/utils'
 import type { Scope } from '@livestore/utils/effect'
 import { Effect, Option, Schema } from '@livestore/utils/effect'
 
-import type { SqliteError, SynchronousDatabase, UnexpectedError } from '../index.js'
+import type { SqliteDb, SqliteError, UnexpectedError } from '../index.js'
+import { getExecArgsFromMutation } from '../mutation.js'
 import {
-  getExecArgsFromMutation,
+  type LiveStoreSchema,
   MUTATION_LOG_META_TABLE,
+  type MutationEvent,
   mutationLogMetaTable,
   SESSION_CHANGESET_META_TABLE,
   sessionChangesetMetaTable,
-} from '../index.js'
-import type { LiveStoreSchema, MutationEvent } from '../schema/mod.js'
+} from '../schema/mod.js'
 import { insertRow } from '../sql-queries/index.js'
 import { execSql, execSqlPrepared } from './connection.js'
 import { LeaderThreadCtx } from './types.js'
 
 export type ApplyMutation = (
   mutationEventEncoded: MutationEvent.AnyEncoded,
+  options?: {
+    /** Needed for rehydrateFromMutationLog */
+    skipMutationLog?: boolean
+  },
 ) => Effect.Effect<void, SqliteError | UnexpectedError>
 
 export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope | LeaderThreadCtx> = Effect.gen(
@@ -31,15 +36,27 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
       [...leaderThreadCtx.schema.mutations.entries()].map(([k, v]) => [k, Schema.hash(v.schema)] as const),
     )
 
-    return (mutationEventEncoded) =>
+    return (mutationEventEncoded, options) =>
       Effect.gen(function* () {
-        const { mutationEventSchema, schema, db, dbLog } = leaderThreadCtx
-        const mutationEventDecoded = Schema.decodeUnknownSync(mutationEventSchema)(mutationEventEncoded)
+        const { schema, dbReadModel: db, dbMutationLog } = leaderThreadCtx
+        const skipMutationLog = options?.skipMutationLog ?? false
 
-        const mutationName = mutationEventDecoded.mutation
+        const mutationName = mutationEventEncoded.mutation
         const mutationDef = schema.mutations.get(mutationName) ?? shouldNeverHappen(`Unknown mutation: ${mutationName}`)
 
-        const execArgsArr = getExecArgsFromMutation({ mutationDef, mutationEventDecoded })
+        const execArgsArr = getExecArgsFromMutation({
+          mutationDef,
+          mutationEvent: { decoded: undefined, encoded: mutationEventEncoded },
+        })
+
+        // NOTE we might want to bring this back if we want to debug no-op mutations
+        // const makeExecuteOptions = (statementSql: string, bindValues: any) => ({
+        //   onRowsChanged: (rowsChanged: number) => {
+        //     if (rowsChanged === 0) {
+        //       console.warn(`Mutation "${mutationDef.name}" did not affect any rows:`, statementSql, bindValues)
+        //     }
+        //   },
+        // })
 
         // console.group('[@livestore/common:leader-thread:applyMutation]', { mutationName })
 
@@ -53,31 +70,29 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
 
         const changeset = session.changeset()
         session.finish()
-        // NOTE for no-op mutations (e.g. if the state didn't change) the changeset will be empty
-        // TODO possibly write a null value instead of omitting the row
-        if (changeset !== undefined && changeset.length > 0) {
-          // TODO use prepared statements
-          yield* execSql(
-            db,
-            ...insertRow({
-              tableName: SESSION_CHANGESET_META_TABLE,
-              columns: sessionChangesetMetaTable.sqliteDef.columns,
-              values: {
-                idGlobal: mutationEventEncoded.id.global,
-                idLocal: mutationEventEncoded.id.local,
-                changeset,
-                debug: execArgsArr,
-              },
-            }),
-          )
-        }
+
+        // TODO use prepared statements
+        yield* execSql(
+          db,
+          ...insertRow({
+            tableName: SESSION_CHANGESET_META_TABLE,
+            columns: sessionChangesetMetaTable.sqliteDef.columns,
+            values: {
+              idGlobal: mutationEventEncoded.id.global,
+              idLocal: mutationEventEncoded.id.local,
+              // NOTE the changeset will be empty (i.e. null) for no-op mutations
+              changeset: changeset ?? null,
+              debug: execArgsArr,
+            },
+          }),
+        )
 
         // console.groupEnd()
 
         // write to mutation_log
-        const excludeFromMutationLog = shouldExcludeMutationFromLog(mutationName, mutationEventDecoded)
-        if (excludeFromMutationLog === false) {
-          yield* insertIntoMutationLog(mutationEventEncoded, dbLog, mutationDefSchemaHashMap)
+        const excludeFromMutationLog = shouldExcludeMutationFromLog(mutationName, mutationEventEncoded)
+        if (skipMutationLog === false && excludeFromMutationLog === false) {
+          yield* insertIntoMutationLog(mutationEventEncoded, dbMutationLog, mutationDefSchemaHashMap)
         } else {
           //   console.debug('[@livestore/common:leader-thread] skipping mutation log write', mutation, statementSql, bindValues)
         }
@@ -86,7 +101,7 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
           attributes: {
             mutationName: mutationEventEncoded.mutation,
             mutationId: mutationEventEncoded.id,
-            'span.label': mutationEventEncoded.mutation,
+            'span.label': `(${mutationEventEncoded.id.global},${mutationEventEncoded.id.local}) ${mutationEventEncoded.mutation}`,
           },
         }),
         // Effect.logDuration('@livestore/common:leader-thread:applyMutation'),
@@ -96,7 +111,7 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
 
 const insertIntoMutationLog = (
   mutationEventEncoded: MutationEvent.AnyEncoded,
-  dbLog: SynchronousDatabase,
+  dbMutationLog: SqliteDb,
   mutationDefSchemaHashMap: Map<string, number>,
 ) =>
   Effect.gen(function* () {
@@ -106,7 +121,7 @@ const insertIntoMutationLog = (
 
     // TODO use prepared statements
     yield* execSql(
-      dbLog,
+      dbMutationLog,
       ...insertRow({
         tableName: MUTATION_LOG_META_TABLE,
         columns: mutationLogMetaTable.sqliteDef.columns,
@@ -132,11 +147,14 @@ const makeShouldExcludeMutationFromLog = memoizeByRef((schema: LiveStoreSchema)
       ? (migrationOptions.excludeMutations ?? new Set(['livestore.RawSql']))
       : new Set(['livestore.RawSql'])
 
-  return (mutationName: string, mutationEventDecoded: MutationEvent.Any): boolean => {
+  return (mutationName: string, mutationEventEncoded: MutationEvent.AnyEncoded): boolean => {
     if (mutationLogExclude.has(mutationName)) return true
 
     const mutationDef = schema.mutations.get(mutationName) ?? shouldNeverHappen(`Unknown mutation: ${mutationName}`)
-    const execArgsArr = getExecArgsFromMutation({ mutationDef, mutationEventDecoded })
+    const execArgsArr = getExecArgsFromMutation({
+      mutationDef,
+      mutationEvent: { decoded: undefined, encoded: mutationEventEncoded },
+    })
 
     return execArgsArr.some((_) => _.statementSql.includes('__livestore'))
   }

package/src/leader-thread/connection.ts

@@ -1,7 +1,7 @@
 // import type { WaSqlite } from '@livestore/sqlite-wasm'
 import { Effect } from '@livestore/utils/effect'
 
-import type { SynchronousDatabase } from '../adapter-types.js'
+import type { SqliteDb } from '../adapter-types.js'
 import { SqliteError } from '../adapter-types.js'
 import type { BindValues } from '../sql-queries/index.js'
 import type { PreparedBindValues } from '../util.js'
@@ -12,9 +12,9 @@ namespace WaSqlite {
   export type SQLiteError = any
 }
 
-export const configureConnection = (syncDb: SynchronousDatabase, { fkEnabled }: { fkEnabled: boolean }) =>
+export const configureConnection = (sqliteDb: SqliteDb, { fkEnabled }: { fkEnabled: boolean }) =>
   execSql(
-    syncDb,
+    sqliteDb,
     sql`
     PRAGMA page_size=8192;
     PRAGMA journal_mode=MEMORY;
@@ -23,10 +23,10 @@ export const configureConnection = (syncDb: SynchronousDatabase, { fkEnabled }:
     {},
   )
 
-export const execSql = (syncDb: SynchronousDatabase, sql: string, bind: BindValues) => {
+export const execSql = (sqliteDb: SqliteDb, sql: string, bind: BindValues) => {
   const bindValues = prepareBindValues(bind, sql)
   return Effect.try({
-    try: () => syncDb.execute(sql, bindValues),
+    try: () => sqliteDb.execute(sql, bindValues),
     catch: (cause) =>
       new SqliteError({ cause, query: { bindValues, sql }, code: (cause as WaSqlite.SQLiteError).code }),
   }).pipe(
@@ -48,9 +48,9 @@ export const execSql = (syncDb: SynchronousDatabase, sql: string, bind: BindValu
 // }
 
 // TODO actually use prepared statements
-export const execSqlPrepared = (syncDb: SynchronousDatabase, sql: string, bindValues: PreparedBindValues) => {
+export const execSqlPrepared = (sqliteDb: SqliteDb, sql: string, bindValues: PreparedBindValues) => {
   return Effect.try({
-    try: () => syncDb.execute(sql, bindValues),
+    try: () => sqliteDb.execute(sql, bindValues),
     catch: (cause) =>
       new SqliteError({ cause, query: { bindValues, sql }, code: (cause as WaSqlite.SQLiteError).code }),
   }).pipe(

package/src/leader-thread/leader-worker-devtools.ts

@@ -1,18 +1,11 @@
-import { Effect, FiberMap, Option, PubSub, Queue, Stream, SubscriptionRef } from '@livestore/utils/effect'
+import { Effect, FiberMap, Option, Stream, SubscriptionRef } from '@livestore/utils/effect'
 
 import { Devtools, IntentionalShutdownCause, liveStoreVersion, UnexpectedError } from '../index.js'
 import { MUTATION_LOG_META_TABLE, SCHEMA_META_TABLE, SCHEMA_MUTATIONS_META_TABLE } from '../schema/mod.js'
-import type { ShutdownChannel } from './shutdown-channel.js'
 import type { DevtoolsOptions, PersistenceInfoPair } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
-type SendMessageToDevtools = (
-  message: Devtools.MessageFromAppLeader,
-  options?: {
-    /** Send message even if not connected (e.g. for initial broadcast messages) */
-    force: boolean
-  },
-) => Effect.Effect<void>
+type SendMessageToDevtools = (message: Devtools.MessageFromAppLeader) => Effect.Effect<void>
 
 // TODO bind scope to the webchannel lifetime
 export const bootDevtools = (options: DevtoolsOptions) =>
@@ -21,44 +14,24 @@ export const bootDevtools = (options: DevtoolsOptions) =>
       return
     }
 
-    const { persistenceInfo, shutdownChannel, devtoolsWebChannel } = yield* options.makeContext
+    const { connectedClientSessionPullQueues, syncProcessor, extraIncomingMessagesQueue } = yield* LeaderThreadCtx
 
-    const isConnected = yield* SubscriptionRef.make(true)
-
-    const incomingMessagesPubSub = yield* PubSub.unbounded<Devtools.MessageToAppLeader>().pipe(
-      Effect.acquireRelease(PubSub.shutdown),
-    )
+    yield* listenToDevtools({
+      incomingMessages: Stream.fromQueue(extraIncomingMessagesQueue),
+      sendMessage: () => Effect.void,
+    }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
 
-    const incomingMessages = Stream.fromPubSub(incomingMessagesPubSub)
+    const { persistenceInfo, devtoolsWebChannel } = yield* options.makeBootContext
 
-    const outgoingMessagesQueue = yield* Queue.unbounded<Devtools.MessageFromAppLeader>().pipe(
-      Effect.acquireRelease(Queue.shutdown),
-    )
+    const sendMessage: SendMessageToDevtools = (message) =>
+      devtoolsWebChannel
+        .send(message)
+        .pipe(
+          Effect.withSpan('@livestore/common:leader-thread:devtools:sendToDevtools'),
+          Effect.interruptible,
+          Effect.ignoreLogged,
+        )
 
-    const devtoolsCoordinatorChannel = devtoolsWebChannel
-    // coordinatorMessagePortOrChannel instanceof MessagePort
-    //   ? yield* WebChannel.messagePortChannel({
-    //       port: coordinatorMessagePortOrChannel,
-    //       schema: { send: Devtools.MessageFromAppLeader, listen: Devtools.MessageToAppLeader },
-    //     })
-    //   : coordinatorMessagePortOrChannel
-
-    const sendMessage: SendMessageToDevtools = (message, options) =>
-      Effect.gen(function* () {
-        if (options?.force === true || (yield* isConnected)) {
-          yield* devtoolsCoordinatorChannel.send(message)
-        } else {
-          yield* Queue.offer(outgoingMessagesQueue, message)
-        }
-      }).pipe(
-        Effect.withSpan('@livestore/common:leader-thread:devtools:sendToDevtools'),
-        Effect.interruptible,
-        Effect.ignoreLogged,
-      )
-
-    // broadcastCallbacks.add((message) => sendMessage(message))
-
-    const { connectedClientSessionPullQueues, syncProcessor } = yield* LeaderThreadCtx
     const { localHead } = yield* syncProcessor.syncState
 
     // TODO close queue when devtools disconnects
@@ -69,13 +42,8 @@ export const bootDevtools = (options: DevtoolsOptions) =>
         Effect.gen(function* () {
           if (msg.payload._tag === 'upstream-advance') {
             for (const mutationEventEncoded of msg.payload.newEvents) {
-              yield* sendMessage(
-                Devtools.MutationBroadcast.make({
-                  mutationEventEncoded,
-
-                  liveStoreVersion,
-                }),
-              )
+              // TODO refactor with push semantics
+              yield* sendMessage(Devtools.MutationBroadcast.make({ mutationEventEncoded, liveStoreVersion }))
             }
           } else {
             yield* Effect.logWarning('TODO implement rebases in devtools')
@@ -86,68 +54,32 @@ export const bootDevtools = (options: DevtoolsOptions) =>
       Effect.forkScoped,
     )
 
-    yield* devtoolsCoordinatorChannel.listen.pipe(
-      Stream.flatten(),
-      // Stream.tapLogWithLabel('@livestore/common:leader-thread:devtools:onPortMessage'),
-      Stream.tap((msg) =>
-        Effect.gen(function* () {
-          // yield* Effect.logDebug(`[@livestore/common:leader-thread:devtools] message from port: ${msg._tag}`, msg)
-          // if (msg._tag === 'LSD.MessagePortForStoreRes') {
-          //   yield* Deferred.succeed(storeMessagePortDeferred, msg.port)
-          // } else {
-          yield* PubSub.publish(incomingMessagesPubSub, msg)
-          // }
-        }),
-      ),
-      Stream.runDrain,
-      Effect.withSpan(`@livestore/common:leader-thread:devtools:onPortMessage`),
-      Effect.ignoreLogged,
-      Effect.forkScoped,
-    )
-
-    // yield* sendMessage(Devtools.AppHostReady.make({ appHostId, liveStoreVersion, isLeader }), { force: true })
-
-    // yield* sendMessage(Devtools.MessagePortForStoreReq.make({ appHostId, liveStoreVersion, requestId: nanoid() }), {
-    //   force: true,
-    // })
-
     yield* listenToDevtools({
-      incomingMessages,
+      incomingMessages: devtoolsWebChannel.listen.pipe(Stream.flatten(), Stream.orDie),
       sendMessage,
-      // isConnected,
-      // disconnect,
-      // storeId,
-      // appHostId,
-      // isLeader,
       persistenceInfo,
-      shutdownChannel,
-    })
+    }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
   }).pipe(Effect.withSpan('@livestore/common:leader-thread:devtools:boot'))
 
 const listenToDevtools = ({
   incomingMessages,
   sendMessage,
-  // isConnected,
-  // disconnect,
-  // appHostId,
-  // storeId,
-  // isLeader,
   persistenceInfo,
-  shutdownChannel,
 }: {
   incomingMessages: Stream.Stream<Devtools.MessageToAppLeader>
   sendMessage: SendMessageToDevtools
-  // isConnected: SubscriptionRef.SubscriptionRef<boolean>
-  // disconnect: Effect.Effect<void>
-  // appHostId: string
-  // storeId: string
-  // isLeader: boolean
-  persistenceInfo: PersistenceInfoPair
-  shutdownChannel: ShutdownChannel
+  persistenceInfo?: PersistenceInfoPair
 }) =>
   Effect.gen(function* () {
-    const innerWorkerCtx = yield* LeaderThreadCtx
-    const { syncBackend, makeSyncDb, db, dbLog, shutdownStateSubRef, syncProcessor } = innerWorkerCtx
+    const {
+      syncBackend,
+      makeSqliteDb,
+      dbReadModel,
+      dbMutationLog,
+      shutdownStateSubRef,
+      shutdownChannel,
+      syncProcessor,
+    } = yield* LeaderThreadCtx
 
     type RequestId = string
     const subscriptionFiberMap = yield* FiberMap.make<RequestId>()
@@ -158,15 +90,6 @@ const listenToDevtools = ({
           // yield* Effect.logDebug('[@livestore/common:leader-thread:devtools] incomingMessage', decodedEvent)
 
           if (decodedEvent._tag === 'LSD.Disconnect') {
-            // yield* SubscriptionRef.set(isConnected, false)
-
-            // yield* disconnect
-
-            // TODO is there a better place for this?
-            // yield* sendMessage(Devtools.AppHostReady.make({ appHostId, liveStoreVersion, isLeader }), {
-            //   force: true,
-            // })
-
             return
           }
 
@@ -179,7 +102,7 @@ const listenToDevtools = ({
               return
             }
             case 'LSD.Leader.SnapshotReq': {
-              const snapshot = db.export()
+              const snapshot = dbReadModel.export()
 
               yield* sendMessage(Devtools.SnapshotRes.make({ snapshot, ...reqPayload }))
 
@@ -191,15 +114,15 @@ const listenToDevtools = ({
               let tableNames: Set<string>
 
               try {
-                const tmpSyncDb = yield* makeSyncDb({ _tag: 'in-memory' })
-                tmpSyncDb.import(data)
-                const tableNameResults = tmpSyncDb.select<{ name: string }>(
+                const tmpDb = yield* makeSqliteDb({ _tag: 'in-memory' })
+                tmpDb.import(data)
+                const tableNameResults = tmpDb.select<{ name: string }>(
                   `select name from sqlite_master where type = 'table'`,
                 )
 
                 tableNames = new Set(tableNameResults.map((_) => _.name))
 
-                tmpSyncDb.close()
+                tmpDb.close()
               } catch (e) {
                 yield* Effect.logError(`Error importing database file`, e)
                 yield* sendMessage(Devtools.LoadDatabaseFileRes.make({ ...reqPayload, status: 'unsupported-file' }))
@@ -210,15 +133,15 @@ const listenToDevtools = ({
               if (tableNames.has(MUTATION_LOG_META_TABLE)) {
                 yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
 
-                dbLog.import(data)
+                dbMutationLog.import(data)
 
-                db.destroy()
+                dbReadModel.destroy()
               } else if (tableNames.has(SCHEMA_META_TABLE) && tableNames.has(SCHEMA_MUTATIONS_META_TABLE)) {
                 yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
 
-                db.import(data)
+                dbReadModel.import(data)
 
-                dbLog.destroy()
+                dbMutationLog.destroy()
               } else {
                 yield* sendMessage(Devtools.LoadDatabaseFileRes.make({ ...reqPayload, status: 'unsupported-database' }))
                 return
@@ -226,7 +149,7 @@ const listenToDevtools = ({
 
               yield* sendMessage(Devtools.LoadDatabaseFileRes.make({ ...reqPayload, status: 'ok' }))
 
-              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' }))
+              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' })) ?? Effect.void
 
               return
             }
@@ -235,26 +158,31 @@ const listenToDevtools = ({
 
               yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
 
-              db.destroy()
+              dbReadModel.destroy()
 
               if (mode === 'all-data') {
-                dbLog.destroy()
+                dbMutationLog.destroy()
               }
 
               yield* sendMessage(Devtools.ResetAllDataRes.make({ ...reqPayload }))
 
-              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-reset' }))
+              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-reset' })) ?? Effect.void
 
               return
             }
             case 'LSD.Leader.DatabaseFileInfoReq': {
+              if (persistenceInfo === undefined) {
+                console.log('[@livestore/common:leader-thread:devtools] persistenceInfo is required for this request')
+                return
+              }
+
               const dbSizeQuery = `SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size();`
-              const dbFileSize = db.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
-              const mutationLogFileSize = dbLog.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
+              const dbFileSize = dbReadModel.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
+              const mutationLogFileSize = dbMutationLog.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
 
               yield* sendMessage(
                 Devtools.DatabaseFileInfoRes.make({
-                  db: { fileSize: dbFileSize, persistenceInfo: persistenceInfo.db },
+                  readModel: { fileSize: dbFileSize, persistenceInfo: persistenceInfo.readModel },
                   mutationLog: { fileSize: mutationLogFileSize, persistenceInfo: persistenceInfo.mutationLog },
                   ...reqPayload,
                 }),
@@ -263,7 +191,7 @@ const listenToDevtools = ({
               return
             }
             case 'LSD.Leader.MutationLogReq': {
-              const mutationLog = dbLog.export()
+              const mutationLog = dbMutationLog.export()
 
               yield* sendMessage(Devtools.MutationLogRes.make({ mutationLog, ...reqPayload }))
 

package/src/leader-thread/make-leader-thread-layer.ts

@@ -1,68 +1,100 @@
-import type { HttpClient, Scope, WebChannel } from '@livestore/utils/effect'
-import { Deferred, Effect, FiberSet, Layer, Queue, SubscriptionRef } from '@livestore/utils/effect'
+import type { HttpClient, Scope } from '@livestore/utils/effect'
+import { Deferred, Effect, Layer, Queue, SubscriptionRef } from '@livestore/utils/effect'
 
-import type { BootStatus, MakeSynchronousDatabase, SqliteError, SynchronousDatabase } from '../adapter-types.js'
+import type { BootStatus, MakeSqliteDb, SqliteError } from '../adapter-types.js'
 import { UnexpectedError } from '../adapter-types.js'
+import type * as Devtools from '../devtools/index.js'
 import type { LiveStoreSchema } from '../schema/mod.js'
 import { EventId, MutationEvent, mutationLogMetaTable, SYNC_STATUS_TABLE, syncStatusTable } from '../schema/mod.js'
 import { migrateTable } from '../schema-management/migrations.js'
-import type { InvalidPullError, IsOfflineError, SyncBackend } from '../sync/sync.js'
+import type { InvalidPullError, IsOfflineError, SyncOptions } from '../sync/sync.js'
 import { sql } from '../util.js'
 import { execSql } from './connection.js'
-import { makeLeaderSyncProcessor } from './leader-sync-processor.js'
 import { bootDevtools } from './leader-worker-devtools.js'
+import { makeLeaderSyncProcessor } from './LeaderSyncProcessor.js'
 import { makePullQueueSet } from './pull-queue-set.js'
 import { recreateDb } from './recreate-db.js'
-import type { DevtoolsOptions, InitialBlockingSyncContext, InitialSyncOptions, ShutdownState } from './types.js'
+import type { ShutdownChannel } from './shutdown-channel.js'
+import type {
+  DevtoolsOptions,
+  InitialBlockingSyncContext,
+  InitialSyncOptions,
+  LeaderSqliteDb,
+  ShutdownState,
+} from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
 export const makeLeaderThreadLayer = ({
   schema,
   storeId,
-  originId,
-  makeSyncDb,
-  makeSyncBackend,
-  db,
-  dbLog,
+  clientId,
+  makeSqliteDb,
+  syncOptions,
+  dbReadModel,
+  dbMutationLog,
   devtoolsOptions,
-  initialSyncOptions = { _tag: 'Skip' },
+  shutdownChannel,
 }: {
   storeId: string
-  originId: string
+  clientId: string
   schema: LiveStoreSchema
-  makeSyncDb: MakeSynchronousDatabase
-  makeSyncBackend: Effect.Effect<SyncBackend, UnexpectedError, Scope.Scope> | undefined
-  db: SynchronousDatabase
-  dbLog: SynchronousDatabase
+  makeSqliteDb: MakeSqliteDb
+  syncOptions: SyncOptions | undefined
+  dbReadModel: LeaderSqliteDb
+  dbMutationLog: LeaderSqliteDb
   devtoolsOptions: DevtoolsOptions
-  initialSyncOptions: InitialSyncOptions | undefined
+  shutdownChannel: ShutdownChannel
 }): Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient> =>
   Effect.gen(function* () {
     const bootStatusQueue = yield* Queue.unbounded<BootStatus>().pipe(Effect.acquireRelease(Queue.shutdown))
 
     // TODO do more validation here than just checking the count of tables
     // Either happens on initial boot or if schema changes
-    const dbMissing = db.select<{ count: number }>(sql`select count(*) as count from sqlite_master`)[0]!.count === 0
+    const dbMissing =
+      dbReadModel.select<{ count: number }>(sql`select count(*) as count from sqlite_master`)[0]!.count === 0
 
-    const syncBackend = makeSyncBackend === undefined ? undefined : yield* makeSyncBackend
+    const syncBackend = syncOptions === undefined ? undefined : yield* syncOptions.makeBackend({ storeId, clientId })
 
-    const initialBlockingSyncContext = yield* makeInitialBlockingSyncContext({ initialSyncOptions, bootStatusQueue })
+    const initialBlockingSyncContext = yield* makeInitialBlockingSyncContext({
+      initialSyncOptions: syncOptions?.initialSyncOptions ?? { _tag: 'Skip' },
+      bootStatusQueue,
+    })
+
+    const syncProcessor = yield* makeLeaderSyncProcessor({
+      schema,
+      dbMissing,
+      dbMutationLog,
+      initialBlockingSyncContext,
+    })
+
+    const extraIncomingMessagesQueue = yield* Queue.unbounded<Devtools.MessageToAppLeader>().pipe(
+      Effect.acquireRelease(Queue.shutdown),
+    )
 
-    const syncProcessor = yield* makeLeaderSyncProcessor({ schema, dbMissing, dbLog, initialBlockingSyncContext })
+    const devtoolsContext = devtoolsOptions.enabled
+      ? {
+          enabled: true as const,
+          syncBackendPullLatch: yield* Effect.makeLatch(true),
+          syncBackendPushLatch: yield* Effect.makeLatch(true),
+        }
+      : { enabled: false as const }
 
     const ctx = {
       schema,
       bootStatusQueue,
       storeId,
-      originId,
-      db,
-      dbLog,
-      makeSyncDb,
+      clientId,
+      dbReadModel,
+      dbMutationLog,
+      makeSqliteDb,
       mutationEventSchema: MutationEvent.makeMutationEventSchema(schema),
       shutdownStateSubRef: yield* SubscriptionRef.make<ShutdownState>('running'),
+      shutdownChannel,
       syncBackend,
       syncProcessor,
       connectedClientSessionPullQueues: yield* makePullQueueSet,
+      extraIncomingMessagesQueue,
+      devtools: devtoolsContext,
     } satisfies typeof LeaderThreadCtx.Service
 
     // @ts-expect-error For debugging purposes
@@ -145,17 +177,17 @@ const bootLeaderThread = ({
   LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
 > =>
   Effect.gen(function* () {
-    const { dbLog, bootStatusQueue, syncProcessor } = yield* LeaderThreadCtx
+    const { dbMutationLog, bootStatusQueue, syncProcessor } = yield* LeaderThreadCtx
 
     yield* migrateTable({
-      db: dbLog,
+      db: dbMutationLog,
       behaviour: 'create-if-not-exists',
       tableAst: mutationLogMetaTable.sqliteDef.ast,
       skipMetaTable: true,
     })
 
     yield* migrateTable({
-      db: dbLog,
+      db: dbMutationLog,
       behaviour: 'create-if-not-exists',
       tableAst: syncStatusTable.sqliteDef.ast,
       skipMetaTable: true,
@@ -163,7 +195,7 @@ const bootLeaderThread = ({
 
     // Create sync status row if it doesn't exist
     yield* execSql(
-      dbLog,
+      dbMutationLog,
       sql`INSERT INTO ${SYNC_STATUS_TABLE} (head)
           SELECT ${EventId.ROOT.global}
           WHERE NOT EXISTS (SELECT 1 FROM ${SYNC_STATUS_TABLE})`,