@livestore/common 0.3.0-dev.3 → 0.3.0-dev.5

package/dist/sync/validate-push-payload.d.ts

@@ -1,5 +1,5 @@
 import { Effect } from '@livestore/utils/effect';
-import type { MutationEvent } from '../schema/mod.js';
+import type { EventId, MutationEvent } from '../schema/mod.js';
 import { InvalidPushError } from './sync.js';
-export declare const validatePushPayload: (batch: ReadonlyArray<MutationEvent.AnyEncoded>, currentEventId: number) => Effect.Effect<undefined, InvalidPushError, never>;
+export declare const validatePushPayload: (batch: ReadonlyArray<MutationEvent.AnyEncodedGlobal>, currentEventId: EventId.GlobalEventId) => Effect.Effect<undefined, InvalidPushError, never>;
 //# sourceMappingURL=validate-push-payload.d.ts.map

package/dist/sync/validate-push-payload.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"validate-push-payload.d.ts","sourceRoot":"","sources":["../../src/sync/validate-push-payload.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAEhD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAA;AAG5C,eAAO,MAAM,mBAAmB,UAAW,aAAa,CAAC,aAAa,CAAC,UAAU,CAAC,kBAAkB,MAAM,sDAWtG,CAAA"}
+{"version":3,"file":"validate-push-payload.d.ts","sourceRoot":"","sources":["../../src/sync/validate-push-payload.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAEhD,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAA;AAG5C,eAAO,MAAM,mBAAmB,UACvB,aAAa,CAAC,aAAa,CAAC,gBAAgB,CAAC,kBACpC,OAAO,CAAC,aAAa,sDAYnC,CAAA"}

package/dist/sync/validate-push-payload.js

@@ -2,12 +2,12 @@ import { Effect } from '@livestore/utils/effect';
 import { InvalidPushError } from './sync.js';
 // TODO proper batch validation
 export const validatePushPayload = (batch, currentEventId) => Effect.gen(function* () {
-    if (batch[0].id.global <= currentEventId) {
+    if (batch[0].id <= currentEventId) {
         return yield* InvalidPushError.make({
             reason: {
                 _tag: 'ServerAhead',
                 minimumExpectedId: currentEventId + 1,
-                providedId: batch[0].id.global,
+                providedId: batch[0].id,
             },
         });
     }

package/dist/sync/validate-push-payload.js.map

@@ -1 +1 @@
-{"version":3,"file":"validate-push-payload.js","sourceRoot":"","sources":["../../src/sync/validate-push-payload.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAGhD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAA;AAE5C,+BAA+B;AAC/B,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,KAA8C,EAAE,cAAsB,EAAE,EAAE,CAC5G,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC,MAAM,IAAI,cAAc,EAAE,CAAC;QAC1C,OAAO,KAAK,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC;YAClC,MAAM,EAAE;gBACN,IAAI,EAAE,aAAa;gBACnB,iBAAiB,EAAE,cAAc,GAAG,CAAC;gBACrC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC,MAAM;aAChC;SACF,CAAC,CAAA;IACJ,CAAC;AACH,CAAC,CAAC,CAAA"}
+{"version":3,"file":"validate-push-payload.js","sourceRoot":"","sources":["../../src/sync/validate-push-payload.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAGhD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAA;AAE5C,+BAA+B;AAC/B,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,KAAoD,EACpD,cAAqC,EACrC,EAAE,CACF,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,IAAI,cAAc,EAAE,CAAC;QACnC,OAAO,KAAK,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC;YAClC,MAAM,EAAE;gBACN,IAAI,EAAE,aAAa;gBACnB,iBAAiB,EAAE,cAAc,GAAG,CAAC;gBACrC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE;aACzB;SACF,CAAC,CAAA;IACJ,CAAC;AACH,CAAC,CAAC,CAAA"}

package/dist/version.d.ts

@@ -1,4 +1,4 @@
-export declare const liveStoreVersion: "0.3.0-dev.2";
+export declare const liveStoreVersion: "0.3.0-dev.4";
 /**
  * This version number is incremented whenever the internal storage format changes in a breaking way.
  * Whenever this version changes, LiveStore will start with fresh database files. Old database files are not deleted.

package/dist/version.js

@@ -1,7 +1,7 @@
 // TODO bring back when Expo and Playwright supports `with` imports
 // import packageJson from '../package.json' with { type: 'json' }
 // export const liveStoreVersion = packageJson.version
-export const liveStoreVersion = '0.3.0-dev.2';
+export const liveStoreVersion = '0.3.0-dev.4';
 /**
  * This version number is incremented whenever the internal storage format changes in a breaking way.
  * Whenever this version changes, LiveStore will start with fresh database files. Old database files are not deleted.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livestore/common",
-  "version": "0.3.0-dev.3",
+  "version": "0.3.0-dev.5",
   "type": "module",
   "exports": {
     ".": {
@@ -53,8 +53,8 @@
     "graphology": "0.26.0-alpha1",
     "graphology-dag": "0.4.1",
     "graphology-types": "0.24.7",
-    "@livestore/db-schema": "0.3.0-dev.3",
-    "@livestore/utils": "0.3.0-dev.3"
+    "@livestore/utils": "0.3.0-dev.5",
+    "@livestore/db-schema": "0.3.0-dev.5"
   },
   "devDependencies": {
     "vitest": "^2.1.4"

package/src/adapter-types.ts

@@ -29,8 +29,28 @@ export type SynchronousDatabaseChangeset = {
 export type ClientSession = {
   /** SQLite database with synchronous API running in the same thread (usually in-memory) */
   syncDb: SynchronousDatabase
-  /** The coordinator is responsible for persisting the database, syncing etc */
-  coordinator: Coordinator
+  devtools: { enabled: boolean }
+  clientId: string
+  sessionId: string
+  /** Status info whether current session is leader or not */
+  lockStatus: SubscriptionRef.SubscriptionRef<LockStatus>
+  shutdown: (cause: Cause.Cause<UnexpectedError | IntentionalShutdownCause>) => Effect.Effect<void>
+  /** A proxy API to communicate with the leader thread */
+  leaderThread: ClientSessionLeaderThreadProxy
+}
+
+export type ClientSessionLeaderThreadProxy = {
+  mutations: {
+    pull: Stream.Stream<{ payload: PayloadUpstream; remaining: number }, UnexpectedError>
+    push(batch: ReadonlyArray<MutationEvent.AnyEncoded>): Effect.Effect<void, UnexpectedError | InvalidPushError>
+    initialMutationEventId: EventId
+  }
+  export: Effect.Effect<Uint8Array, UnexpectedError>
+  getMutationLogData: Effect.Effect<Uint8Array, UnexpectedError>
+  getSyncState: Effect.Effect<SyncState, UnexpectedError>
+  networkStatus: SubscriptionRef.SubscriptionRef<NetworkStatus>
+  /** For debugging purposes it can be useful to manually trigger devtools messages (e.g. to reset the database) */
+  sendDevtoolsMessage: (message: Devtools.MessageToAppLeader) => Effect.Effect<void, UnexpectedError>
 }
 
 export type SynchronousDatabase<TReq = any, TMetadata extends TReq = TReq> = {
@@ -103,28 +123,6 @@ export const BootStatus = Schema.Union(
 
 export type BootStatus = typeof BootStatus.Type
 
-// TODO refactor `Coordinator` to embrace more of the "leader semantics"
-export type Coordinator = {
-  devtools: {
-    enabled: boolean
-    // TODO incorporate sessionId and rethink appHostId
-    appHostId: string
-  }
-  sessionId: string
-  // TODO is exposing the lock status really needed (or only relevant for web adapter?)
-  lockStatus: SubscriptionRef.SubscriptionRef<LockStatus>
-  mutations: {
-    pull: Stream.Stream<{ payload: PayloadUpstream; remaining: number }, UnexpectedError>
-    push(batch: ReadonlyArray<MutationEvent.AnyEncoded>): Effect.Effect<void, UnexpectedError | InvalidPushError>
-    initialMutationEventId: EventId
-  }
-  export: Effect.Effect<Uint8Array, UnexpectedError>
-  getMutationLogData: Effect.Effect<Uint8Array, UnexpectedError>
-  getLeaderSyncState: Effect.Effect<SyncState, UnexpectedError>
-  networkStatus: SubscriptionRef.SubscriptionRef<NetworkStatus>
-  shutdown: (cause: Cause.Cause<UnexpectedError | IntentionalShutdownCause>) => Effect.Effect<void>
-}
-
 /**
  * Can be used in queries to refer to the current session id.
  * Will be replaced with the actual session id at runtime

package/src/devtools/devtools-bridge.ts

@@ -6,7 +6,8 @@ export type PrepareDevtoolsBridge = {
   /** Messages coming from the app host (usually responses to requests) */
   responsePubSub: PubSub.PubSub<Devtools.MessageFromAppLeader | Devtools.MessageFromAppClientSession>
   sendToAppHost: (msg: Devtools.MessageToAppLeader | Devtools.MessageToAppClientSession) => Effect.Effect<void>
-  appHostId: string
+  clientId: string
+  sessionId: string
   copyToClipboard: (text: string) => Effect.Effect<void>
   sendEscapeKey?: Effect.Effect<void>
   isLeader: boolean

package/src/devtools/devtools-messages.ts

@@ -7,7 +7,8 @@ import { PreparedBindValues } from '../util.js'
 import { liveStoreVersion as pkgVersion } from '../version.js'
 
 const requestId = Schema.String
-const appHostId = Schema.String
+const clientId = Schema.String
+const sessionId = Schema.String
 const liveStoreVersion = Schema.Literal(pkgVersion)
 
 const LSDMessage = <Tag extends string, Fields extends Schema.Struct.Fields>(tag: Tag, fields: Fields) =>
@@ -23,13 +24,15 @@ const LSDChannelMessage = <Tag extends string, Fields extends Schema.Struct.Fiel
 
 const LSDStoreChannelMessage = <Tag extends string, Fields extends Schema.Struct.Fields>(tag: Tag, fields: Fields) =>
   LSDMessage(tag, {
-    appHostId,
+    clientId,
+    sessionId,
     ...fields,
   })
 
 const LSDStoreReqResMessage = <Tag extends string, Fields extends Schema.Struct.Fields>(tag: Tag, fields: Fields) =>
   LSDMessage(tag, {
-    appHostId,
+    clientId,
+    sessionId,
     requestId,
     ...fields,
   })
@@ -82,12 +85,12 @@ export class DebugInfoRerunQueryRes extends LSDStoreReqResMessage('LSD.DebugInfo
 
 // TODO refactor this to use push/pull semantics
 export class MutationBroadcast extends LSDMessage('LSD.Leader.MutationBroadcast', {
-  mutationEventEncoded: MutationEvent.EncodedAny,
+  mutationEventEncoded: MutationEvent.AnyEncoded,
 }) {}
 
 // TODO refactor this to use push/pull semantics
 export class RunMutationReq extends LSDReqResMessage('LSD.Leader.RunMutationReq', {
-  mutationEventEncoded: MutationEvent.EncodedAny.pipe(Schema.omit('id', 'parentId')),
+  mutationEventEncoded: MutationEvent.AnyEncoded.pipe(Schema.omit('id', 'parentId')),
 }) {}
 
 export class RunMutationRes extends LSDReqResMessage('LSD.Leader.RunMutationRes', {}) {}
@@ -167,7 +170,7 @@ export class SyncingInfoRes extends LSDReqResMessage('LSD.Leader.SyncingInfoRes'
 export class SyncHistorySubscribe extends LSDReqResMessage('LSD.Leader.SyncHistorySubscribe', {}) {}
 export class SyncHistoryUnsubscribe extends LSDReqResMessage('LSD.Leader.SyncHistoryUnsubscribe', {}) {}
 export class SyncHistoryRes extends LSDReqResMessage('LSD.Leader.SyncHistoryRes', {
-  mutationEventEncoded: MutationEvent.EncodedAny,
+  mutationEventEncoded: MutationEvent.AnyEncodedGlobal,
   metadata: Schema.Option(Schema.JsonValue),
 }) {}
 

package/src/leader-thread/apply-mutation.ts

@@ -132,7 +132,7 @@ const makeShouldExcludeMutationFromLog = memoizeByRef((schema: LiveStoreSchema)
       ? (migrationOptions.excludeMutations ?? new Set(['livestore.RawSql']))
       : new Set(['livestore.RawSql'])
 
-  return (mutationName: string, mutationEventDecoded: MutationEvent.Any): boolean => {
+  return (mutationName: string, mutationEventDecoded: MutationEvent.AnyDecoded): boolean => {
     if (mutationLogExclude.has(mutationName)) return true
 
     const mutationDef = schema.mutations.get(mutationName) ?? shouldNeverHappen(`Unknown mutation: ${mutationName}`)

package/src/leader-thread/leader-sync-processor.ts

@@ -9,6 +9,7 @@ import {
   FiberHandle,
   Option,
   OtelTracer,
+  ReadonlyArray,
   Ref,
   Schema,
   Stream,
@@ -102,7 +103,7 @@ export const makeLeaderSyncProcessor = ({
   initialBlockingSyncContext: InitialBlockingSyncContext
 }): Effect.Effect<SyncProcessor, UnexpectedError, Scope.Scope> =>
   Effect.gen(function* () {
-    const syncBackendQueue = yield* BucketQueue.make<MutationEvent.AnyEncoded>()
+    const syncBackendQueue = yield* BucketQueue.make<MutationEvent.EncodedWithMeta>()
 
     const stateRef = yield* Ref.make<ProcessorState>({ _tag: 'init' })
 
@@ -257,13 +258,16 @@ export const makeLeaderSyncProcessor = ({
           )
         }
 
-        const pendingMutationEvents = yield* getMutationEventsSince({ global: initialBackendHead, local: 0 })
+        const pendingMutationEvents = yield* getMutationEventsSince({
+          global: initialBackendHead,
+          local: EventId.localDefault,
+        }).pipe(Effect.map(ReadonlyArray.map((_) => new MutationEvent.EncodedWithMeta(_))))
 
         const initialSyncState = {
-          pending: pendingMutationEvents.map((_) => new MutationEvent.EncodedWithMeta(_)),
+          pending: pendingMutationEvents,
           // On the leader we don't need a rollback tail beyond `pending` items
           rollbackTail: [],
-          upstreamHead: { global: initialBackendHead, local: 0 },
+          upstreamHead: { global: initialBackendHead, local: EventId.localDefault },
           localHead: initialLocalHead,
         } as SyncState.SyncState
 
@@ -410,7 +414,7 @@ const backgroundBackendPulling = ({
   initialBlockingSyncContext,
 }: {
   dbReady: Deferred.Deferred<void>
-  initialBackendHead: number
+  initialBackendHead: EventId.GlobalEventId
   isLocalEvent: (mutationEventEncoded: MutationEvent.EncodedWithMeta) => boolean
   restartBackendPushing: (
     filteredRebasedPending: ReadonlyArray<MutationEvent.EncodedWithMeta>,
@@ -538,7 +542,7 @@ const backgroundBackendPulling = ({
           yield* SubscriptionRef.waitUntil(syncBackend.isConnected, (isConnected) => isConnected === true)
 
           yield* onNewPullChunk(
-            batch.map((_) => new MutationEvent.EncodedWithMeta(_.mutationEventEncoded)),
+            batch.map((_) => MutationEvent.EncodedWithMeta.fromGlobal(_.mutationEventEncoded)),
             remaining,
           )
 
@@ -588,7 +592,7 @@ const rollback = ({
     }),
   )
 
-const getCursorInfo = (remoteHead: number) =>
+const getCursorInfo = (remoteHead: EventId.GlobalEventId) =>
   Effect.gen(function* () {
     const { dbLog } = yield* LeaderThreadCtx
 
@@ -605,7 +609,7 @@ const getCursorInfo = (remoteHead: number) =>
     ).pipe(Effect.andThen(Schema.decode(MutationlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie)
 
     return Option.some({
-      cursor: { global: remoteHead, local: 0 },
+      cursor: { global: remoteHead, local: EventId.localDefault },
       metadata: syncMetadataOption,
     }) satisfies InitialSyncInfo
   }).pipe(Effect.withSpan('@livestore/common:leader-thread:syncing:getCursorInfo', { attributes: { remoteHead } }))
@@ -616,7 +620,7 @@ const backgroundBackendPushing = ({
   span,
 }: {
   dbReady: Deferred.Deferred<void>
-  syncBackendQueue: BucketQueue.BucketQueue<MutationEvent.AnyEncoded>
+  syncBackendQueue: BucketQueue.BucketQueue<MutationEvent.EncodedWithMeta>
   span: otel.Span | undefined
 }) =>
   Effect.gen(function* () {
@@ -639,7 +643,7 @@ const backgroundBackendPushing = ({
       })
 
       // TODO handle push errors (should only happen during concurrent pull+push)
-      const pushResult = yield* syncBackend.push(queueItems).pipe(Effect.either)
+      const pushResult = yield* syncBackend.push(queueItems.map((_) => _.toGlobal())).pipe(Effect.either)
 
       if (pushResult._tag === 'Left') {
         span?.addEvent('backend-push-error', { error: pushResult.left.toString() })

package/src/leader-thread/leader-worker-devtools.ts

@@ -1,18 +1,11 @@
-import { Effect, FiberMap, Option, PubSub, Queue, Stream, SubscriptionRef } from '@livestore/utils/effect'
+import { Effect, FiberMap, Option, Stream, SubscriptionRef } from '@livestore/utils/effect'
 
 import { Devtools, IntentionalShutdownCause, liveStoreVersion, UnexpectedError } from '../index.js'
 import { MUTATION_LOG_META_TABLE, SCHEMA_META_TABLE, SCHEMA_MUTATIONS_META_TABLE } from '../schema/mod.js'
-import type { ShutdownChannel } from './shutdown-channel.js'
 import type { DevtoolsOptions, PersistenceInfoPair } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
-type SendMessageToDevtools = (
-  message: Devtools.MessageFromAppLeader,
-  options?: {
-    /** Send message even if not connected (e.g. for initial broadcast messages) */
-    force: boolean
-  },
-) => Effect.Effect<void>
+type SendMessageToDevtools = (message: Devtools.MessageFromAppLeader) => Effect.Effect<void>
 
 // TODO bind scope to the webchannel lifetime
 export const bootDevtools = (options: DevtoolsOptions) =>
@@ -21,44 +14,24 @@ export const bootDevtools = (options: DevtoolsOptions) =>
       return
     }
 
-    const { persistenceInfo, shutdownChannel, devtoolsWebChannel } = yield* options.makeContext
+    const { connectedClientSessionPullQueues, syncProcessor, extraIncomingMessagesQueue } = yield* LeaderThreadCtx
 
-    const isConnected = yield* SubscriptionRef.make(true)
-
-    const incomingMessagesPubSub = yield* PubSub.unbounded<Devtools.MessageToAppLeader>().pipe(
-      Effect.acquireRelease(PubSub.shutdown),
-    )
+    yield* listenToDevtools({
+      incomingMessages: Stream.fromQueue(extraIncomingMessagesQueue),
+      sendMessage: () => Effect.void,
+    }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
 
-    const incomingMessages = Stream.fromPubSub(incomingMessagesPubSub)
+    const { persistenceInfo, devtoolsWebChannel } = yield* options.makeContext
 
-    const outgoingMessagesQueue = yield* Queue.unbounded<Devtools.MessageFromAppLeader>().pipe(
-      Effect.acquireRelease(Queue.shutdown),
-    )
+    const sendMessage: SendMessageToDevtools = (message) =>
+      devtoolsWebChannel
+        .send(message)
+        .pipe(
+          Effect.withSpan('@livestore/common:leader-thread:devtools:sendToDevtools'),
+          Effect.interruptible,
+          Effect.ignoreLogged,
+        )
 
-    const devtoolsCoordinatorChannel = devtoolsWebChannel
-    // coordinatorMessagePortOrChannel instanceof MessagePort
-    //   ? yield* WebChannel.messagePortChannel({
-    //       port: coordinatorMessagePortOrChannel,
-    //       schema: { send: Devtools.MessageFromAppLeader, listen: Devtools.MessageToAppLeader },
-    //     })
-    //   : coordinatorMessagePortOrChannel
-
-    const sendMessage: SendMessageToDevtools = (message, options) =>
-      Effect.gen(function* () {
-        if (options?.force === true || (yield* isConnected)) {
-          yield* devtoolsCoordinatorChannel.send(message)
-        } else {
-          yield* Queue.offer(outgoingMessagesQueue, message)
-        }
-      }).pipe(
-        Effect.withSpan('@livestore/common:leader-thread:devtools:sendToDevtools'),
-        Effect.interruptible,
-        Effect.ignoreLogged,
-      )
-
-    // broadcastCallbacks.add((message) => sendMessage(message))
-
-    const { connectedClientSessionPullQueues, syncProcessor } = yield* LeaderThreadCtx
     const { localHead } = yield* syncProcessor.syncState
 
     // TODO close queue when devtools disconnects
@@ -69,13 +42,8 @@ export const bootDevtools = (options: DevtoolsOptions) =>
         Effect.gen(function* () {
           if (msg.payload._tag === 'upstream-advance') {
             for (const mutationEventEncoded of msg.payload.newEvents) {
-              yield* sendMessage(
-                Devtools.MutationBroadcast.make({
-                  mutationEventEncoded,
-
-                  liveStoreVersion,
-                }),
-              )
+              // TODO refactor with push semantics
+              yield* sendMessage(Devtools.MutationBroadcast.make({ mutationEventEncoded, liveStoreVersion }))
             }
           } else {
             yield* Effect.logWarning('TODO implement rebases in devtools')
@@ -86,68 +54,25 @@ export const bootDevtools = (options: DevtoolsOptions) =>
       Effect.forkScoped,
     )
 
-    yield* devtoolsCoordinatorChannel.listen.pipe(
-      Stream.flatten(),
-      // Stream.tapLogWithLabel('@livestore/common:leader-thread:devtools:onPortMessage'),
-      Stream.tap((msg) =>
-        Effect.gen(function* () {
-          // yield* Effect.logDebug(`[@livestore/common:leader-thread:devtools] message from port: ${msg._tag}`, msg)
-          // if (msg._tag === 'LSD.MessagePortForStoreRes') {
-          //   yield* Deferred.succeed(storeMessagePortDeferred, msg.port)
-          // } else {
-          yield* PubSub.publish(incomingMessagesPubSub, msg)
-          // }
-        }),
-      ),
-      Stream.runDrain,
-      Effect.withSpan(`@livestore/common:leader-thread:devtools:onPortMessage`),
-      Effect.ignoreLogged,
-      Effect.forkScoped,
-    )
-
-    // yield* sendMessage(Devtools.AppHostReady.make({ appHostId, liveStoreVersion, isLeader }), { force: true })
-
-    // yield* sendMessage(Devtools.MessagePortForStoreReq.make({ appHostId, liveStoreVersion, requestId: nanoid() }), {
-    //   force: true,
-    // })
-
     yield* listenToDevtools({
-      incomingMessages,
+      incomingMessages: devtoolsWebChannel.listen.pipe(Stream.flatten(), Stream.orDie),
       sendMessage,
-      // isConnected,
-      // disconnect,
-      // storeId,
-      // appHostId,
-      // isLeader,
       persistenceInfo,
-      shutdownChannel,
-    })
+    }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
   }).pipe(Effect.withSpan('@livestore/common:leader-thread:devtools:boot'))
 
 const listenToDevtools = ({
   incomingMessages,
   sendMessage,
-  // isConnected,
-  // disconnect,
-  // appHostId,
-  // storeId,
-  // isLeader,
   persistenceInfo,
-  shutdownChannel,
 }: {
   incomingMessages: Stream.Stream<Devtools.MessageToAppLeader>
   sendMessage: SendMessageToDevtools
-  // isConnected: SubscriptionRef.SubscriptionRef<boolean>
-  // disconnect: Effect.Effect<void>
-  // appHostId: string
-  // storeId: string
-  // isLeader: boolean
-  persistenceInfo: PersistenceInfoPair
-  shutdownChannel: ShutdownChannel
+  persistenceInfo?: PersistenceInfoPair
 }) =>
   Effect.gen(function* () {
-    const innerWorkerCtx = yield* LeaderThreadCtx
-    const { syncBackend, makeSyncDb, db, dbLog, shutdownStateSubRef, syncProcessor } = innerWorkerCtx
+    const { syncBackend, makeSyncDb, db, dbLog, shutdownStateSubRef, shutdownChannel, syncProcessor } =
+      yield* LeaderThreadCtx
 
     type RequestId = string
     const subscriptionFiberMap = yield* FiberMap.make<RequestId>()
@@ -158,15 +83,6 @@ const listenToDevtools = ({
           // yield* Effect.logDebug('[@livestore/common:leader-thread:devtools] incomingMessage', decodedEvent)
 
           if (decodedEvent._tag === 'LSD.Disconnect') {
-            // yield* SubscriptionRef.set(isConnected, false)
-
-            // yield* disconnect
-
-            // TODO is there a better place for this?
-            // yield* sendMessage(Devtools.AppHostReady.make({ appHostId, liveStoreVersion, isLeader }), {
-            //   force: true,
-            // })
-
             return
           }
 
@@ -226,7 +142,7 @@ const listenToDevtools = ({
 
               yield* sendMessage(Devtools.LoadDatabaseFileRes.make({ ...reqPayload, status: 'ok' }))
 
-              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' }))
+              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' })) ?? Effect.void
 
               return
             }
@@ -243,11 +159,16 @@ const listenToDevtools = ({
 
               yield* sendMessage(Devtools.ResetAllDataRes.make({ ...reqPayload }))
 
-              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-reset' }))
+              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-reset' })) ?? Effect.void
 
               return
             }
             case 'LSD.Leader.DatabaseFileInfoReq': {
+              if (persistenceInfo === undefined) {
+                console.log('[@livestore/common:leader-thread:devtools] persistenceInfo is required for this request')
+                return
+              }
+
               const dbSizeQuery = `SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size();`
               const dbFileSize = db.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
               const mutationLogFileSize = dbLog.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size

package/src/leader-thread/make-leader-thread-layer.ts

@@ -1,8 +1,9 @@
-import type { HttpClient, Scope, WebChannel } from '@livestore/utils/effect'
-import { Deferred, Effect, FiberSet, Layer, Queue, SubscriptionRef } from '@livestore/utils/effect'
+import type { HttpClient, Scope } from '@livestore/utils/effect'
+import { Deferred, Effect, Layer, Queue, SubscriptionRef } from '@livestore/utils/effect'
 
 import type { BootStatus, MakeSynchronousDatabase, SqliteError, SynchronousDatabase } from '../adapter-types.js'
 import { UnexpectedError } from '../adapter-types.js'
+import type * as Devtools from '../devtools/index.js'
 import type { LiveStoreSchema } from '../schema/mod.js'
 import { EventId, MutationEvent, mutationLogMetaTable, SYNC_STATUS_TABLE, syncStatusTable } from '../schema/mod.js'
 import { migrateTable } from '../schema-management/migrations.js'
@@ -13,22 +14,24 @@ import { makeLeaderSyncProcessor } from './leader-sync-processor.js'
 import { bootDevtools } from './leader-worker-devtools.js'
 import { makePullQueueSet } from './pull-queue-set.js'
 import { recreateDb } from './recreate-db.js'
+import type { ShutdownChannel } from './shutdown-channel.js'
 import type { DevtoolsOptions, InitialBlockingSyncContext, InitialSyncOptions, ShutdownState } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
 export const makeLeaderThreadLayer = ({
   schema,
   storeId,
-  originId,
+  clientId,
   makeSyncDb,
   makeSyncBackend,
   db,
   dbLog,
   devtoolsOptions,
   initialSyncOptions = { _tag: 'Skip' },
+  shutdownChannel,
 }: {
   storeId: string
-  originId: string
+  clientId: string
   schema: LiveStoreSchema
   makeSyncDb: MakeSynchronousDatabase
   makeSyncBackend: Effect.Effect<SyncBackend, UnexpectedError, Scope.Scope> | undefined
@@ -36,6 +39,7 @@ export const makeLeaderThreadLayer = ({
   dbLog: SynchronousDatabase
   devtoolsOptions: DevtoolsOptions
   initialSyncOptions: InitialSyncOptions | undefined
+  shutdownChannel: ShutdownChannel
 }): Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient> =>
   Effect.gen(function* () {
     const bootStatusQueue = yield* Queue.unbounded<BootStatus>().pipe(Effect.acquireRelease(Queue.shutdown))
@@ -50,19 +54,25 @@ export const makeLeaderThreadLayer = ({
 
     const syncProcessor = yield* makeLeaderSyncProcessor({ schema, dbMissing, dbLog, initialBlockingSyncContext })
 
+    const extraIncomingMessagesQueue = yield* Queue.unbounded<Devtools.MessageToAppLeader>().pipe(
+      Effect.acquireRelease(Queue.shutdown),
+    )
+
     const ctx = {
       schema,
       bootStatusQueue,
       storeId,
-      originId,
+      clientId,
       db,
       dbLog,
       makeSyncDb,
       mutationEventSchema: MutationEvent.makeMutationEventSchema(schema),
       shutdownStateSubRef: yield* SubscriptionRef.make<ShutdownState>('running'),
+      shutdownChannel,
       syncBackend,
       syncProcessor,
       connectedClientSessionPullQueues: yield* makePullQueueSet,
+      extraIncomingMessagesQueue,
     } satisfies typeof LeaderThreadCtx.Service
 
     // @ts-expect-error For debugging purposes

package/src/leader-thread/mutationlog.ts

@@ -2,11 +2,14 @@ import { Effect, Schema } from '@livestore/utils/effect'
 
 import type { SynchronousDatabase } from '../adapter-types.js'
 import * as EventId from '../schema/EventId.js'
+import type * as MutationEvent from '../schema/MutationEvent.js'
 import { MUTATION_LOG_META_TABLE, mutationLogMetaTable, SYNC_STATUS_TABLE } from '../schema/system-tables.js'
 import { prepareBindValues, sql } from '../util.js'
 import { LeaderThreadCtx } from './types.js'
 
-export const getMutationEventsSince = (since: EventId.EventId) =>
+export const getMutationEventsSince = (
+  since: EventId.EventId,
+): Effect.Effect<ReadonlyArray<MutationEvent.AnyEncoded>, never, LeaderThreadCtx> =>
   Effect.gen(function* () {
     const { dbLog } = yield* LeaderThreadCtx
 
@@ -26,16 +29,17 @@ export const getMutationEventsSince = (since: EventId.EventId) =>
       .filter((_) => EventId.compare(_.id, since) > 0)
   })
 
-export const getLocalHeadFromDb = (dbLog: SynchronousDatabase) => {
-  const res = dbLog.select<{ idGlobal: number; idLocal: number }>(
+export const getLocalHeadFromDb = (dbLog: SynchronousDatabase): EventId.EventId => {
+  const res = dbLog.select<{ idGlobal: EventId.GlobalEventId; idLocal: EventId.LocalEventId }>(
     sql`select idGlobal, idLocal from ${MUTATION_LOG_META_TABLE} order by idGlobal DESC, idLocal DESC limit 1`,
   )[0]
 
   return res ? { global: res.idGlobal, local: res.idLocal } : EventId.ROOT
 }
 
-export const getBackendHeadFromDb = (dbLog: SynchronousDatabase) =>
-  dbLog.select<{ head: number }>(sql`select head from ${SYNC_STATUS_TABLE}`)[0]?.head ?? EventId.ROOT.global
+export const getBackendHeadFromDb = (dbLog: SynchronousDatabase): EventId.GlobalEventId =>
+  dbLog.select<{ head: EventId.GlobalEventId }>(sql`select head from ${SYNC_STATUS_TABLE}`)[0]?.head ??
+  EventId.ROOT.global
 
 // TODO use prepared statements
 export const updateBackendHead = (dbLog: SynchronousDatabase, head: EventId.EventId) =>