@livestore/common 0.3.0-dev.0 → 0.3.0-dev.10

package/src/leader-thread/apply-mutation.ts

@@ -3,20 +3,25 @@ import type { Scope } from '@livestore/utils/effect'
 import { Effect, Option, Schema } from '@livestore/utils/effect'
 
 import type { SqliteError, SynchronousDatabase, UnexpectedError } from '../index.js'
+import { getExecArgsFromMutation } from '../mutation.js'
 import {
-  getExecArgsFromMutation,
+  type LiveStoreSchema,
   MUTATION_LOG_META_TABLE,
+  type MutationEvent,
   mutationLogMetaTable,
   SESSION_CHANGESET_META_TABLE,
   sessionChangesetMetaTable,
-} from '../index.js'
-import type { LiveStoreSchema, MutationEvent } from '../schema/mod.js'
+} from '../schema/mod.js'
 import { insertRow } from '../sql-queries/index.js'
 import { execSql, execSqlPrepared } from './connection.js'
 import { LeaderThreadCtx } from './types.js'
 
 export type ApplyMutation = (
   mutationEventEncoded: MutationEvent.AnyEncoded,
+  options?: {
+    /** Needed for rehydrateFromMutationLog */
+    skipMutationLog?: boolean
+  },
 ) => Effect.Effect<void, SqliteError | UnexpectedError>
 
 export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope | LeaderThreadCtx> = Effect.gen(
@@ -31,15 +36,27 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
       [...leaderThreadCtx.schema.mutations.entries()].map(([k, v]) => [k, Schema.hash(v.schema)] as const),
     )
 
-    return (mutationEventEncoded) =>
+    return (mutationEventEncoded, options) =>
       Effect.gen(function* () {
-        const { mutationEventSchema, schema, db, dbLog } = leaderThreadCtx
-        const mutationEventDecoded = Schema.decodeUnknownSync(mutationEventSchema)(mutationEventEncoded)
+        const { schema, db, dbLog } = leaderThreadCtx
+        const skipMutationLog = options?.skipMutationLog ?? false
 
-        const mutationName = mutationEventDecoded.mutation
+        const mutationName = mutationEventEncoded.mutation
         const mutationDef = schema.mutations.get(mutationName) ?? shouldNeverHappen(`Unknown mutation: ${mutationName}`)
 
-        const execArgsArr = getExecArgsFromMutation({ mutationDef, mutationEventDecoded })
+        const execArgsArr = getExecArgsFromMutation({
+          mutationDef,
+          mutationEvent: { decoded: undefined, encoded: mutationEventEncoded },
+        })
+
+        // NOTE we might want to bring this back if we want to debug no-op mutations
+        // const makeExecuteOptions = (statementSql: string, bindValues: any) => ({
+        //   onRowsChanged: (rowsChanged: number) => {
+        //     if (rowsChanged === 0) {
+        //       console.warn(`Mutation "${mutationDef.name}" did not affect any rows:`, statementSql, bindValues)
+        //     }
+        //   },
+        // })
 
         // console.group('[@livestore/common:leader-thread:applyMutation]', { mutationName })
 
@@ -53,30 +70,28 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
 
         const changeset = session.changeset()
         session.finish()
-        // NOTE for no-op mutations (e.g. if the state didn't change) the changeset will be empty
-        // TODO possibly write a null value instead of omitting the row
-        if (changeset !== undefined && changeset.length > 0) {
-          // TODO use prepared statements
-          yield* execSql(
-            db,
-            ...insertRow({
-              tableName: SESSION_CHANGESET_META_TABLE,
-              columns: sessionChangesetMetaTable.sqliteDef.columns,
-              values: {
-                idGlobal: mutationEventEncoded.id.global,
-                idLocal: mutationEventEncoded.id.local,
-                changeset,
-                debug: execArgsArr,
-              },
-            }),
-          )
-        }
+
+        // TODO use prepared statements
+        yield* execSql(
+          db,
+          ...insertRow({
+            tableName: SESSION_CHANGESET_META_TABLE,
+            columns: sessionChangesetMetaTable.sqliteDef.columns,
+            values: {
+              idGlobal: mutationEventEncoded.id.global,
+              idLocal: mutationEventEncoded.id.local,
+              // NOTE the changeset will be empty (i.e. null) for no-op mutations
+              changeset: changeset ?? null,
+              debug: execArgsArr,
+            },
+          }),
+        )
 
         // console.groupEnd()
 
         // write to mutation_log
-        const excludeFromMutationLog = shouldExcludeMutationFromLog(mutationName, mutationEventDecoded)
-        if (excludeFromMutationLog === false) {
+        const excludeFromMutationLog = shouldExcludeMutationFromLog(mutationName, mutationEventEncoded)
+        if (skipMutationLog === false && excludeFromMutationLog === false) {
           yield* insertIntoMutationLog(mutationEventEncoded, dbLog, mutationDefSchemaHashMap)
         } else {
           //   console.debug('[@livestore/common:leader-thread] skipping mutation log write', mutation, statementSql, bindValues)
@@ -86,7 +101,7 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
           attributes: {
             mutationName: mutationEventEncoded.mutation,
             mutationId: mutationEventEncoded.id,
-            'span.label': mutationEventEncoded.mutation,
+            'span.label': `(${mutationEventEncoded.id.global},${mutationEventEncoded.id.local}) ${mutationEventEncoded.mutation}`,
           },
         }),
         // Effect.logDuration('@livestore/common:leader-thread:applyMutation'),
@@ -132,11 +147,14 @@ const makeShouldExcludeMutationFromLog = memoizeByRef((schema: LiveStoreSchema)
       ? (migrationOptions.excludeMutations ?? new Set(['livestore.RawSql']))
       : new Set(['livestore.RawSql'])
 
-  return (mutationName: string, mutationEventDecoded: MutationEvent.Any): boolean => {
+  return (mutationName: string, mutationEventEncoded: MutationEvent.AnyEncoded): boolean => {
     if (mutationLogExclude.has(mutationName)) return true
 
     const mutationDef = schema.mutations.get(mutationName) ?? shouldNeverHappen(`Unknown mutation: ${mutationName}`)
-    const execArgsArr = getExecArgsFromMutation({ mutationDef, mutationEventDecoded })
+    const execArgsArr = getExecArgsFromMutation({
+      mutationDef,
+      mutationEvent: { decoded: undefined, encoded: mutationEventEncoded },
+    })
 
     return execArgsArr.some((_) => _.statementSql.includes('__livestore'))
   }

package/src/leader-thread/leader-worker-devtools.ts

@@ -1,18 +1,11 @@
-import { Effect, FiberMap, Option, PubSub, Queue, Stream, SubscriptionRef } from '@livestore/utils/effect'
+import { Effect, FiberMap, Option, Stream, SubscriptionRef } from '@livestore/utils/effect'
 
 import { Devtools, IntentionalShutdownCause, liveStoreVersion, UnexpectedError } from '../index.js'
 import { MUTATION_LOG_META_TABLE, SCHEMA_META_TABLE, SCHEMA_MUTATIONS_META_TABLE } from '../schema/mod.js'
-import type { ShutdownChannel } from './shutdown-channel.js'
 import type { DevtoolsOptions, PersistenceInfoPair } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
-type SendMessageToDevtools = (
-  message: Devtools.MessageFromAppLeader,
-  options?: {
-    /** Send message even if not connected (e.g. for initial broadcast messages) */
-    force: boolean
-  },
-) => Effect.Effect<void>
+type SendMessageToDevtools = (message: Devtools.MessageFromAppLeader) => Effect.Effect<void>
 
 // TODO bind scope to the webchannel lifetime
 export const bootDevtools = (options: DevtoolsOptions) =>
@@ -21,44 +14,24 @@ export const bootDevtools = (options: DevtoolsOptions) =>
       return
     }
 
-    const { persistenceInfo, shutdownChannel, devtoolsWebChannel } = yield* options.makeContext
+    const { connectedClientSessionPullQueues, syncProcessor, extraIncomingMessagesQueue } = yield* LeaderThreadCtx
 
-    const isConnected = yield* SubscriptionRef.make(true)
-
-    const incomingMessagesPubSub = yield* PubSub.unbounded<Devtools.MessageToAppLeader>().pipe(
-      Effect.acquireRelease(PubSub.shutdown),
-    )
+    yield* listenToDevtools({
+      incomingMessages: Stream.fromQueue(extraIncomingMessagesQueue),
+      sendMessage: () => Effect.void,
+    }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
 
-    const incomingMessages = Stream.fromPubSub(incomingMessagesPubSub)
+    const { persistenceInfo, devtoolsWebChannel } = yield* options.makeContext
 
-    const outgoingMessagesQueue = yield* Queue.unbounded<Devtools.MessageFromAppLeader>().pipe(
-      Effect.acquireRelease(Queue.shutdown),
-    )
+    const sendMessage: SendMessageToDevtools = (message) =>
+      devtoolsWebChannel
+        .send(message)
+        .pipe(
+          Effect.withSpan('@livestore/common:leader-thread:devtools:sendToDevtools'),
+          Effect.interruptible,
+          Effect.ignoreLogged,
+        )
 
-    const devtoolsCoordinatorChannel = devtoolsWebChannel
-    // coordinatorMessagePortOrChannel instanceof MessagePort
-    //   ? yield* WebChannel.messagePortChannel({
-    //       port: coordinatorMessagePortOrChannel,
-    //       schema: { send: Devtools.MessageFromAppLeader, listen: Devtools.MessageToAppLeader },
-    //     })
-    //   : coordinatorMessagePortOrChannel
-
-    const sendMessage: SendMessageToDevtools = (message, options) =>
-      Effect.gen(function* () {
-        if (options?.force === true || (yield* isConnected)) {
-          yield* devtoolsCoordinatorChannel.send(message)
-        } else {
-          yield* Queue.offer(outgoingMessagesQueue, message)
-        }
-      }).pipe(
-        Effect.withSpan('@livestore/common:leader-thread:devtools:sendToDevtools'),
-        Effect.interruptible,
-        Effect.ignoreLogged,
-      )
-
-    // broadcastCallbacks.add((message) => sendMessage(message))
-
-    const { connectedClientSessionPullQueues, syncProcessor } = yield* LeaderThreadCtx
     const { localHead } = yield* syncProcessor.syncState
 
     // TODO close queue when devtools disconnects
@@ -69,13 +42,8 @@ export const bootDevtools = (options: DevtoolsOptions) =>
         Effect.gen(function* () {
           if (msg.payload._tag === 'upstream-advance') {
             for (const mutationEventEncoded of msg.payload.newEvents) {
-              yield* sendMessage(
-                Devtools.MutationBroadcast.make({
-                  mutationEventEncoded,
-
-                  liveStoreVersion,
-                }),
-              )
+              // TODO refactor with push semantics
+              yield* sendMessage(Devtools.MutationBroadcast.make({ mutationEventEncoded, liveStoreVersion }))
             }
           } else {
             yield* Effect.logWarning('TODO implement rebases in devtools')
@@ -86,68 +54,25 @@ export const bootDevtools = (options: DevtoolsOptions) =>
       Effect.forkScoped,
     )
 
-    yield* devtoolsCoordinatorChannel.listen.pipe(
-      Stream.flatten(),
-      // Stream.tapLogWithLabel('@livestore/common:leader-thread:devtools:onPortMessage'),
-      Stream.tap((msg) =>
-        Effect.gen(function* () {
-          // yield* Effect.logDebug(`[@livestore/common:leader-thread:devtools] message from port: ${msg._tag}`, msg)
-          // if (msg._tag === 'LSD.MessagePortForStoreRes') {
-          //   yield* Deferred.succeed(storeMessagePortDeferred, msg.port)
-          // } else {
-          yield* PubSub.publish(incomingMessagesPubSub, msg)
-          // }
-        }),
-      ),
-      Stream.runDrain,
-      Effect.withSpan(`@livestore/common:leader-thread:devtools:onPortMessage`),
-      Effect.ignoreLogged,
-      Effect.forkScoped,
-    )
-
-    // yield* sendMessage(Devtools.AppHostReady.make({ appHostId, liveStoreVersion, isLeader }), { force: true })
-
-    // yield* sendMessage(Devtools.MessagePortForStoreReq.make({ appHostId, liveStoreVersion, requestId: nanoid() }), {
-    //   force: true,
-    // })
-
     yield* listenToDevtools({
-      incomingMessages,
+      incomingMessages: devtoolsWebChannel.listen.pipe(Stream.flatten(), Stream.orDie),
       sendMessage,
-      // isConnected,
-      // disconnect,
-      // storeId,
-      // appHostId,
-      // isLeader,
       persistenceInfo,
-      shutdownChannel,
-    })
+    }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
   }).pipe(Effect.withSpan('@livestore/common:leader-thread:devtools:boot'))
 
 const listenToDevtools = ({
   incomingMessages,
   sendMessage,
-  // isConnected,
-  // disconnect,
-  // appHostId,
-  // storeId,
-  // isLeader,
   persistenceInfo,
-  shutdownChannel,
 }: {
   incomingMessages: Stream.Stream<Devtools.MessageToAppLeader>
   sendMessage: SendMessageToDevtools
-  // isConnected: SubscriptionRef.SubscriptionRef<boolean>
-  // disconnect: Effect.Effect<void>
-  // appHostId: string
-  // storeId: string
-  // isLeader: boolean
-  persistenceInfo: PersistenceInfoPair
-  shutdownChannel: ShutdownChannel
+  persistenceInfo?: PersistenceInfoPair
 }) =>
   Effect.gen(function* () {
-    const innerWorkerCtx = yield* LeaderThreadCtx
-    const { syncBackend, makeSyncDb, db, dbLog, shutdownStateSubRef, syncProcessor } = innerWorkerCtx
+    const { syncBackend, makeSyncDb, db, dbLog, shutdownStateSubRef, shutdownChannel, syncProcessor } =
+      yield* LeaderThreadCtx
 
     type RequestId = string
     const subscriptionFiberMap = yield* FiberMap.make<RequestId>()
@@ -158,15 +83,6 @@ const listenToDevtools = ({
           // yield* Effect.logDebug('[@livestore/common:leader-thread:devtools] incomingMessage', decodedEvent)
 
           if (decodedEvent._tag === 'LSD.Disconnect') {
-            // yield* SubscriptionRef.set(isConnected, false)
-
-            // yield* disconnect
-
-            // TODO is there a better place for this?
-            // yield* sendMessage(Devtools.AppHostReady.make({ appHostId, liveStoreVersion, isLeader }), {
-            //   force: true,
-            // })
-
             return
           }
 
@@ -226,7 +142,7 @@ const listenToDevtools = ({
 
               yield* sendMessage(Devtools.LoadDatabaseFileRes.make({ ...reqPayload, status: 'ok' }))
 
-              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' }))
+              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' })) ?? Effect.void
 
               return
             }
@@ -243,11 +159,16 @@ const listenToDevtools = ({
 
               yield* sendMessage(Devtools.ResetAllDataRes.make({ ...reqPayload }))
 
-              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-reset' }))
+              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-reset' })) ?? Effect.void
 
               return
             }
             case 'LSD.Leader.DatabaseFileInfoReq': {
+              if (persistenceInfo === undefined) {
+                console.log('[@livestore/common:leader-thread:devtools] persistenceInfo is required for this request')
+                return
+              }
+
               const dbSizeQuery = `SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size();`
               const dbFileSize = db.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
               const mutationLogFileSize = dbLog.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size

package/src/leader-thread/make-leader-thread-layer.ts

@@ -1,41 +1,43 @@
-import type { HttpClient, Scope, WebChannel } from '@livestore/utils/effect'
-import { Deferred, Effect, FiberSet, Layer, Queue, SubscriptionRef } from '@livestore/utils/effect'
+import type { HttpClient, Scope } from '@livestore/utils/effect'
+import { Deferred, Effect, Layer, Queue, SubscriptionRef } from '@livestore/utils/effect'
 
 import type { BootStatus, MakeSynchronousDatabase, SqliteError, SynchronousDatabase } from '../adapter-types.js'
 import { UnexpectedError } from '../adapter-types.js'
+import type * as Devtools from '../devtools/index.js'
 import type { LiveStoreSchema } from '../schema/mod.js'
 import { EventId, MutationEvent, mutationLogMetaTable, SYNC_STATUS_TABLE, syncStatusTable } from '../schema/mod.js'
 import { migrateTable } from '../schema-management/migrations.js'
-import type { InvalidPullError, IsOfflineError, SyncBackend } from '../sync/sync.js'
+import type { InvalidPullError, IsOfflineError, SyncOptions } from '../sync/sync.js'
 import { sql } from '../util.js'
 import { execSql } from './connection.js'
-import { makeLeaderSyncProcessor } from './leader-sync-processor.js'
 import { bootDevtools } from './leader-worker-devtools.js'
+import { makeLeaderSyncProcessor } from './LeaderSyncProcessor.js'
 import { makePullQueueSet } from './pull-queue-set.js'
 import { recreateDb } from './recreate-db.js'
+import type { ShutdownChannel } from './shutdown-channel.js'
 import type { DevtoolsOptions, InitialBlockingSyncContext, InitialSyncOptions, ShutdownState } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
 export const makeLeaderThreadLayer = ({
   schema,
   storeId,
-  originId,
+  clientId,
   makeSyncDb,
-  makeSyncBackend,
+  syncOptions,
   db,
   dbLog,
   devtoolsOptions,
-  initialSyncOptions = { _tag: 'Skip' },
+  shutdownChannel,
 }: {
   storeId: string
-  originId: string
+  clientId: string
   schema: LiveStoreSchema
   makeSyncDb: MakeSynchronousDatabase
-  makeSyncBackend: Effect.Effect<SyncBackend, UnexpectedError, Scope.Scope> | undefined
+  syncOptions: SyncOptions | undefined
   db: SynchronousDatabase
   dbLog: SynchronousDatabase
   devtoolsOptions: DevtoolsOptions
-  initialSyncOptions: InitialSyncOptions | undefined
+  shutdownChannel: ShutdownChannel
 }): Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient> =>
   Effect.gen(function* () {
     const bootStatusQueue = yield* Queue.unbounded<BootStatus>().pipe(Effect.acquireRelease(Queue.shutdown))
@@ -44,25 +46,34 @@ export const makeLeaderThreadLayer = ({
     // Either happens on initial boot or if schema changes
     const dbMissing = db.select<{ count: number }>(sql`select count(*) as count from sqlite_master`)[0]!.count === 0
 
-    const syncBackend = makeSyncBackend === undefined ? undefined : yield* makeSyncBackend
+    const syncBackend = syncOptions === undefined ? undefined : yield* syncOptions.makeBackend({ storeId, clientId })
 
-    const initialBlockingSyncContext = yield* makeInitialBlockingSyncContext({ initialSyncOptions, bootStatusQueue })
+    const initialBlockingSyncContext = yield* makeInitialBlockingSyncContext({
+      initialSyncOptions: syncOptions?.initialSyncOptions ?? { _tag: 'Skip' },
+      bootStatusQueue,
+    })
 
     const syncProcessor = yield* makeLeaderSyncProcessor({ schema, dbMissing, dbLog, initialBlockingSyncContext })
 
+    const extraIncomingMessagesQueue = yield* Queue.unbounded<Devtools.MessageToAppLeader>().pipe(
+      Effect.acquireRelease(Queue.shutdown),
+    )
+
     const ctx = {
       schema,
       bootStatusQueue,
       storeId,
-      originId,
+      clientId,
       db,
       dbLog,
       makeSyncDb,
       mutationEventSchema: MutationEvent.makeMutationEventSchema(schema),
       shutdownStateSubRef: yield* SubscriptionRef.make<ShutdownState>('running'),
+      shutdownChannel,
       syncBackend,
       syncProcessor,
       connectedClientSessionPullQueues: yield* makePullQueueSet,
+      extraIncomingMessagesQueue,
     } satisfies typeof LeaderThreadCtx.Service
 
     // @ts-expect-error For debugging purposes

package/src/leader-thread/mutationlog.ts

@@ -2,11 +2,14 @@ import { Effect, Schema } from '@livestore/utils/effect'
 
 import type { SynchronousDatabase } from '../adapter-types.js'
 import * as EventId from '../schema/EventId.js'
+import type * as MutationEvent from '../schema/MutationEvent.js'
 import { MUTATION_LOG_META_TABLE, mutationLogMetaTable, SYNC_STATUS_TABLE } from '../schema/system-tables.js'
 import { prepareBindValues, sql } from '../util.js'
 import { LeaderThreadCtx } from './types.js'
 
-export const getMutationEventsSince = (since: EventId.EventId) =>
+export const getMutationEventsSince = (
+  since: EventId.EventId,
+): Effect.Effect<ReadonlyArray<MutationEvent.AnyEncoded>, never, LeaderThreadCtx> =>
   Effect.gen(function* () {
     const { dbLog } = yield* LeaderThreadCtx
 
@@ -26,16 +29,17 @@ export const getMutationEventsSince = (since: EventId.EventId) =>
       .filter((_) => EventId.compare(_.id, since) > 0)
   })
 
-export const getLocalHeadFromDb = (dbLog: SynchronousDatabase) => {
-  const res = dbLog.select<{ idGlobal: number; idLocal: number }>(
+export const getLocalHeadFromDb = (dbLog: SynchronousDatabase): EventId.EventId => {
+  const res = dbLog.select<{ idGlobal: EventId.GlobalEventId; idLocal: EventId.LocalEventId }>(
     sql`select idGlobal, idLocal from ${MUTATION_LOG_META_TABLE} order by idGlobal DESC, idLocal DESC limit 1`,
   )[0]
 
   return res ? { global: res.idGlobal, local: res.idLocal } : EventId.ROOT
 }
 
-export const getBackendHeadFromDb = (dbLog: SynchronousDatabase) =>
-  dbLog.select<{ head: number }>(sql`select head from ${SYNC_STATUS_TABLE}`)[0]?.head ?? EventId.ROOT.global
+export const getBackendHeadFromDb = (dbLog: SynchronousDatabase): EventId.GlobalEventId =>
+  dbLog.select<{ head: EventId.GlobalEventId }>(sql`select head from ${SYNC_STATUS_TABLE}`)[0]?.head ??
+  EventId.ROOT.global
 
 // TODO use prepared statements
 export const updateBackendHead = (dbLog: SynchronousDatabase, head: EventId.EventId) =>

package/src/leader-thread/recreate-db.ts

@@ -24,7 +24,9 @@ export const recreateDb: Effect.Effect<
 
   // NOTE to speed up the operations below, we're creating a temporary in-memory database
   // and later we'll overwrite the persisted database with the new data
-  const tmpSyncDb = yield* makeSyncDb({ _tag: 'in-memory' })
+  // TODO bring back this optimization
+  // const tmpSyncDb = yield* makeSyncDb({ _tag: 'in-memory' })
+  const tmpSyncDb = db
   yield* configureConnection(tmpSyncDb, { fkEnabled: true })
 
   const initDb = (hooks: Partial<MigrationHooks> | undefined) =>
@@ -91,17 +93,19 @@ export const recreateDb: Effect.Effect<
     }
   }
 
+  // TODO bring back
   // Import the temporary in-memory database into the persistent database
-  yield* Effect.sync(() => db.import(tmpSyncDb)).pipe(
-    Effect.withSpan('@livestore/common:leader-thread:recreateDb:import'),
-  )
+  // yield* Effect.sync(() => db.import(tmpSyncDb)).pipe(
+  //   Effect.withSpan('@livestore/common:leader-thread:recreateDb:import'),
+  // )
 
   // TODO maybe bring back re-using this initial snapshot to avoid calling `.export()` again
   // We've disabled this for now as it made the code too complex, as we often run syncing right after
   // so the snapshot is no longer up to date
   // const snapshotFromTmpDb = tmpSyncDb.export()
 
-  tmpSyncDb.close()
+  // TODO bring back
+  // tmpSyncDb.close()
 }).pipe(
   Effect.scoped, // NOTE we're closing the scope here so finalizers are called when the effect is done
   Effect.withSpan('@livestore/common:leader-thread:recreateDb'),

package/src/leader-thread/types.ts

@@ -6,6 +6,7 @@ import type {
   Option,
   Queue,
   Scope,
+  Subscribable,
   SubscriptionRef,
   WebChannel,
 } from '@livestore/utils/effect'
@@ -22,7 +23,7 @@ import type {
   UnexpectedError,
 } from '../index.js'
 import type { EventId, LiveStoreSchema, MutationEvent } from '../schema/mod.js'
-import type { PayloadUpstream, SyncState } from '../sync/syncstate.js'
+import type * as SyncState from '../sync/syncstate.js'
 import type { ShutdownChannel } from './shutdown-channel.js'
 
 export type ShutdownState = 'running' | 'shutting-down'
@@ -67,7 +68,6 @@ export type DevtoolsOptions =
       makeContext: Effect.Effect<
         {
           devtoolsWebChannel: WebChannel.WebChannel<Devtools.MessageToAppLeader, Devtools.MessageFromAppLeader>
-          shutdownChannel: ShutdownChannel
           persistenceInfo: PersistenceInfoPair
         },
         UnexpectedError,
@@ -80,18 +80,21 @@ export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
   {
     schema: LiveStoreSchema
     storeId: string
-    originId: string
+    clientId: string
     makeSyncDb: MakeSynchronousDatabase
     db: LeaderDatabase
     dbLog: LeaderDatabase
     bootStatusQueue: Queue.Queue<BootStatus>
     // TODO we should find a more elegant way to handle cases which need this ref for their implementation
     shutdownStateSubRef: SubscriptionRef.SubscriptionRef<ShutdownState>
+    shutdownChannel: ShutdownChannel
     mutationEventSchema: MutationEvent.ForMutationDefRecord<any>
     // devtools: DevtoolsContext
     syncBackend: SyncBackend | undefined
-    syncProcessor: SyncProcessor
+    syncProcessor: LeaderSyncProcessor
     connectedClientSessionPullQueues: PullQueueSet
+    /** e.g. used for `store.__dev` APIs */
+    extraIncomingMessagesQueue: Queue.Queue<Devtools.MessageToAppLeader>
   }
 >() {}
 
@@ -101,24 +104,28 @@ export type InitialBlockingSyncContext = {
 }
 
 export type PullQueueItem = {
-  // mutationEvents: ReadonlyArray<MutationEvent.AnyEncoded>
-  // backendHead: number
-  payload: PayloadUpstream
-  // TODO move `remaining` into `PayloadUpstream`
+  payload: SyncState.PayloadUpstream
   remaining: number
 }
 
-export interface SyncProcessor {
+export interface LeaderSyncProcessor {
   push: (
     /** `batch` needs to follow the same rules as `batch` in `SyncBackend.push` */
     batch: ReadonlyArray<MutationEvent.EncodedWithMeta>,
-  ) => Effect.Effect<void, UnexpectedError | InvalidPushError, HttpClient.HttpClient | LeaderThreadCtx>
+    options?: {
+      /**
+       * If true, the effect will only finish when the local push has been processed (i.e. succeeded or was rejected).
+       * @default false
+       */
+      waitForProcessing?: boolean
+    },
+  ) => Effect.Effect<void, InvalidPushError>
 
   pushPartial: (mutationEvent: MutationEvent.PartialAnyEncoded) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
   boot: (args: {
     dbReady: Deferred.Deferred<void>
   }) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient>
-  syncState: Effect.Effect<SyncState, UnexpectedError>
+  syncState: Subscribable.Subscribable<SyncState.SyncState>
 }
 
 export interface PullQueueSet {

package/src/mutation.ts

@@ -8,10 +8,19 @@ import { prepareBindValues } from './util.js'
 
 export const getExecArgsFromMutation = ({
   mutationDef,
-  mutationEventDecoded,
+  mutationEvent,
 }: {
   mutationDef: MutationDef.Any
-  mutationEventDecoded: MutationEvent.Any | MutationEvent.PartialAny
+  /** Both encoded and decoded mutation events are supported to reduce the number of times we need to decode/encode */
+  mutationEvent:
+    | {
+        decoded: MutationEvent.AnyDecoded | MutationEvent.PartialAnyDecoded
+        encoded: undefined
+      }
+    | {
+        decoded: undefined
+        encoded: MutationEvent.AnyEncoded | MutationEvent.PartialAnyEncoded
+      }
 }): ReadonlyArray<{
   statementSql: string
   bindValues: PreparedBindValues
@@ -23,7 +32,9 @@ export const getExecArgsFromMutation = ({
 
   switch (typeof mutationDef.sql) {
     case 'function': {
-      const res = mutationDef.sql(mutationEventDecoded.args)
+      const mutationArgsDecoded =
+        mutationEvent.decoded?.args ?? Schema.decodeUnknownSync(mutationDef.schema)(mutationEvent.encoded!.args)
+      const res = mutationDef.sql(mutationArgsDecoded)
       statementRes = Array.isArray(res) ? res : [res]
       break
     }
@@ -40,10 +51,9 @@ export const getExecArgsFromMutation = ({
   return statementRes.map((statementRes) => {
     const statementSql = typeof statementRes === 'string' ? statementRes : statementRes.sql
 
-    const bindValues =
-      typeof statementRes === 'string'
-        ? Schema.encodeUnknownSync(mutationDef.schema)(mutationEventDecoded.args)
-        : statementRes.bindValues
+    const mutationArgsEncoded =
+      mutationEvent.encoded?.args ?? Schema.encodeUnknownSync(mutationDef.schema)(mutationEvent.decoded!.args)
+    const bindValues = typeof statementRes === 'string' ? mutationArgsEncoded : statementRes.bindValues
 
     const writeTables = typeof statementRes === 'string' ? undefined : statementRes.writeTables