@livestore/common 0.3.0-dev.3 → 0.3.0-dev.31

package/src/leader-thread/leader-worker-devtools.ts

@@ -1,18 +1,12 @@
-import { Effect, FiberMap, Option, PubSub, Queue, Stream, SubscriptionRef } from '@livestore/utils/effect'
+import { Effect, FiberMap, Option, Stream, SubscriptionRef } from '@livestore/utils/effect'
+import { nanoid } from '@livestore/utils/nanoid'
 
 import { Devtools, IntentionalShutdownCause, liveStoreVersion, UnexpectedError } from '../index.js'
-import { MUTATION_LOG_META_TABLE, SCHEMA_META_TABLE, SCHEMA_MUTATIONS_META_TABLE } from '../schema/mod.js'
-import type { ShutdownChannel } from './shutdown-channel.js'
+import { EVENTLOG_META_TABLE, SCHEMA_EVENT_DEFS_META_TABLE, SCHEMA_META_TABLE } from '../schema/mod.js'
 import type { DevtoolsOptions, PersistenceInfoPair } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
-type SendMessageToDevtools = (
-  message: Devtools.MessageFromAppLeader,
-  options?: {
-    /** Send message even if not connected (e.g. for initial broadcast messages) */
-    force: boolean
-  },
-) => Effect.Effect<void>
+type SendMessageToDevtools = (message: Devtools.Leader.MessageFromApp) => Effect.Effect<void>
 
 // TODO bind scope to the webchannel lifetime
 export const bootDevtools = (options: DevtoolsOptions) =>
@@ -21,277 +15,251 @@ export const bootDevtools = (options: DevtoolsOptions) =>
       return
     }
 
-    const { persistenceInfo, shutdownChannel, devtoolsWebChannel } = yield* options.makeContext
+    const { syncProcessor, extraIncomingMessagesQueue } = yield* LeaderThreadCtx
 
-    const isConnected = yield* SubscriptionRef.make(true)
-
-    const incomingMessagesPubSub = yield* PubSub.unbounded<Devtools.MessageToAppLeader>().pipe(
-      Effect.acquireRelease(PubSub.shutdown),
-    )
-
-    const incomingMessages = Stream.fromPubSub(incomingMessagesPubSub)
-
-    const outgoingMessagesQueue = yield* Queue.unbounded<Devtools.MessageFromAppLeader>().pipe(
-      Effect.acquireRelease(Queue.shutdown),
-    )
-
-    const devtoolsCoordinatorChannel = devtoolsWebChannel
-    // coordinatorMessagePortOrChannel instanceof MessagePort
-    //   ? yield* WebChannel.messagePortChannel({
-    //       port: coordinatorMessagePortOrChannel,
-    //       schema: { send: Devtools.MessageFromAppLeader, listen: Devtools.MessageToAppLeader },
-    //     })
-    //   : coordinatorMessagePortOrChannel
-
-    const sendMessage: SendMessageToDevtools = (message, options) =>
-      Effect.gen(function* () {
-        if (options?.force === true || (yield* isConnected)) {
-          yield* devtoolsCoordinatorChannel.send(message)
-        } else {
-          yield* Queue.offer(outgoingMessagesQueue, message)
-        }
-      }).pipe(
-        Effect.withSpan('@livestore/common:leader-thread:devtools:sendToDevtools'),
-        Effect.interruptible,
-        Effect.ignoreLogged,
-      )
-
-    // broadcastCallbacks.add((message) => sendMessage(message))
-
-    const { connectedClientSessionPullQueues, syncProcessor } = yield* LeaderThreadCtx
-    const { localHead } = yield* syncProcessor.syncState
-
-    // TODO close queue when devtools disconnects
-    const pullQueue = yield* connectedClientSessionPullQueues.makeQueue(localHead)
-
-    yield* Stream.fromQueue(pullQueue).pipe(
-      Stream.tap((msg) =>
-        Effect.gen(function* () {
-          if (msg.payload._tag === 'upstream-advance') {
-            for (const mutationEventEncoded of msg.payload.newEvents) {
-              yield* sendMessage(
-                Devtools.MutationBroadcast.make({
-                  mutationEventEncoded,
-
-                  liveStoreVersion,
-                }),
-              )
-            }
-          } else {
-            yield* Effect.logWarning('TODO implement rebases in devtools')
-          }
-        }),
-      ),
-      Stream.runDrain,
-      Effect.forkScoped,
-    )
-
-    yield* devtoolsCoordinatorChannel.listen.pipe(
-      Stream.flatten(),
-      // Stream.tapLogWithLabel('@livestore/common:leader-thread:devtools:onPortMessage'),
-      Stream.tap((msg) =>
-        Effect.gen(function* () {
-          // yield* Effect.logDebug(`[@livestore/common:leader-thread:devtools] message from port: ${msg._tag}`, msg)
-          // if (msg._tag === 'LSD.MessagePortForStoreRes') {
-          //   yield* Deferred.succeed(storeMessagePortDeferred, msg.port)
-          // } else {
-          yield* PubSub.publish(incomingMessagesPubSub, msg)
-          // }
-        }),
-      ),
+    yield* listenToDevtools({
+      incomingMessages: Stream.fromQueue(extraIncomingMessagesQueue),
+      sendMessage: () => Effect.void,
+    }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
+
+    const { persistenceInfo, devtoolsWebChannel } = yield* options.makeBootContext
+
+    const sendMessage: SendMessageToDevtools = (message) =>
+      devtoolsWebChannel
+        .send(message)
+        .pipe(
+          Effect.withSpan('@livestore/common:leader-thread:devtools:sendToDevtools'),
+          Effect.interruptible,
+          Effect.ignoreLogged,
+        )
+
+    const syncState = yield* syncProcessor.syncState
+    const mergeCounter = syncProcessor.getMergeCounter()
+
+    yield* syncProcessor.pull({ cursor: { mergeCounter, eventId: syncState.localHead } }).pipe(
+      Stream.tap(({ payload }) => sendMessage(Devtools.Leader.SyncPull.make({ payload, liveStoreVersion }))),
       Stream.runDrain,
-      Effect.withSpan(`@livestore/common:leader-thread:devtools:onPortMessage`),
-      Effect.ignoreLogged,
       Effect.forkScoped,
     )
 
-    // yield* sendMessage(Devtools.AppHostReady.make({ appHostId, liveStoreVersion, isLeader }), { force: true })
-
-    // yield* sendMessage(Devtools.MessagePortForStoreReq.make({ appHostId, liveStoreVersion, requestId: nanoid() }), {
-    //   force: true,
-    // })
-
     yield* listenToDevtools({
-      incomingMessages,
+      incomingMessages: devtoolsWebChannel.listen.pipe(Stream.flatten(), Stream.orDie),
       sendMessage,
-      // isConnected,
-      // disconnect,
-      // storeId,
-      // appHostId,
-      // isLeader,
       persistenceInfo,
-      shutdownChannel,
-    })
+    }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
   }).pipe(Effect.withSpan('@livestore/common:leader-thread:devtools:boot'))
 
 const listenToDevtools = ({
   incomingMessages,
   sendMessage,
-  // isConnected,
-  // disconnect,
-  // appHostId,
-  // storeId,
-  // isLeader,
   persistenceInfo,
-  shutdownChannel,
 }: {
-  incomingMessages: Stream.Stream<Devtools.MessageToAppLeader>
+  incomingMessages: Stream.Stream<Devtools.Leader.MessageToApp>
   sendMessage: SendMessageToDevtools
-  // isConnected: SubscriptionRef.SubscriptionRef<boolean>
-  // disconnect: Effect.Effect<void>
-  // appHostId: string
-  // storeId: string
-  // isLeader: boolean
-  persistenceInfo: PersistenceInfoPair
-  shutdownChannel: ShutdownChannel
+  persistenceInfo?: PersistenceInfoPair
 }) =>
   Effect.gen(function* () {
-    const innerWorkerCtx = yield* LeaderThreadCtx
-    const { syncBackend, makeSyncDb, db, dbLog, shutdownStateSubRef, syncProcessor } = innerWorkerCtx
+    const {
+      syncBackend,
+      makeSqliteDb,
+      dbReadModel,
+      dbEventlog,
+      shutdownStateSubRef,
+      shutdownChannel,
+      syncProcessor,
+      clientId,
+      devtools,
+    } = yield* LeaderThreadCtx
+
+    type SubscriptionId = string
+    const subscriptionFiberMap = yield* FiberMap.make<SubscriptionId>()
 
     type RequestId = string
-    const subscriptionFiberMap = yield* FiberMap.make<RequestId>()
+    const handledRequestIds = new Set<RequestId>()
 
     yield* incomingMessages.pipe(
       Stream.tap((decodedEvent) =>
         Effect.gen(function* () {
-          // yield* Effect.logDebug('[@livestore/common:leader-thread:devtools] incomingMessage', decodedEvent)
-
-          if (decodedEvent._tag === 'LSD.Disconnect') {
-            // yield* SubscriptionRef.set(isConnected, false)
+          const { requestId } = decodedEvent
+          const reqPayload = { requestId, liveStoreVersion, clientId }
 
-            // yield* disconnect
+          // yield* Effect.logDebug(
+          //   `[@livestore/common:leader-thread:devtools] incomingMessage: ${decodedEvent._tag} (${requestId})`,
+          //   decodedEvent,
+          // )
 
-            // TODO is there a better place for this?
-            // yield* sendMessage(Devtools.AppHostReady.make({ appHostId, liveStoreVersion, isLeader }), {
-            //   force: true,
-            // })
+          if (decodedEvent._tag === 'LSD.Leader.Disconnect') {
+            return
+          }
 
+          // TODO we should try to move the duplicate message handling on the webmesh layer
+          // So far I could only observe this problem with webmesh proxy channels (e.g. for Expo)
+          // Proof: https://share.cleanshot.com/V9G87B0B
+          // Also see `store/devtools.ts` for same problem
+          if (handledRequestIds.has(requestId)) {
+            // yield* Effect.logWarning(`Duplicate message`, decodedEvent)
             return
           }
 
-          const { requestId } = decodedEvent
-          const reqPayload = { requestId, liveStoreVersion }
+          handledRequestIds.add(requestId)
 
           switch (decodedEvent._tag) {
-            case 'LSD.Ping': {
-              yield* sendMessage(Devtools.Pong.make({ ...reqPayload }))
+            case 'LSD.Leader.Ping': {
+              yield* sendMessage(Devtools.Leader.Pong.make({ ...reqPayload }))
               return
             }
             case 'LSD.Leader.SnapshotReq': {
-              const snapshot = db.export()
+              const snapshot = dbReadModel.export()
 
-              yield* sendMessage(Devtools.SnapshotRes.make({ snapshot, ...reqPayload }))
+              yield* sendMessage(Devtools.Leader.SnapshotRes.make({ snapshot, ...reqPayload }))
 
               return
             }
-            case 'LSD.Leader.LoadDatabaseFileReq': {
+            case 'LSD.Leader.LoadDatabaseFile.Request': {
               const { data } = decodedEvent
 
               let tableNames: Set<string>
 
               try {
-                const tmpSyncDb = yield* makeSyncDb({ _tag: 'in-memory' })
-                tmpSyncDb.import(data)
-                const tableNameResults = tmpSyncDb.select<{ name: string }>(
+                const tmpDb = yield* makeSqliteDb({ _tag: 'in-memory' })
+                tmpDb.import(data)
+                const tableNameResults = tmpDb.select<{ name: string }>(
                   `select name from sqlite_master where type = 'table'`,
                 )
 
                 tableNames = new Set(tableNameResults.map((_) => _.name))
 
-                tmpSyncDb.close()
-              } catch (e) {
-                yield* Effect.logError(`Error importing database file`, e)
-                yield* sendMessage(Devtools.LoadDatabaseFileRes.make({ ...reqPayload, status: 'unsupported-file' }))
+                tmpDb.close()
+              } catch (cause) {
+                yield* Effect.logError(`Error importing database file`, cause)
+                yield* sendMessage(
+                  Devtools.Leader.LoadDatabaseFile.Error.make({
+                    ...reqPayload,
+                    cause: { _tag: 'unexpected-error', cause },
+                  }),
+                )
 
                 return
               }
 
-              if (tableNames.has(MUTATION_LOG_META_TABLE)) {
-                yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
-
-                dbLog.import(data)
+              try {
+                if (tableNames.has(EVENTLOG_META_TABLE)) {
+                  // Is eventlog
+                  yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
 
-                db.destroy()
-              } else if (tableNames.has(SCHEMA_META_TABLE) && tableNames.has(SCHEMA_MUTATIONS_META_TABLE)) {
-                yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
+                  dbEventlog.import(data)
 
-                db.import(data)
+                  dbReadModel.destroy()
+                } else if (tableNames.has(SCHEMA_META_TABLE) && tableNames.has(SCHEMA_EVENT_DEFS_META_TABLE)) {
+                  // Is read model
+                  yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
 
-                dbLog.destroy()
-              } else {
-                yield* sendMessage(Devtools.LoadDatabaseFileRes.make({ ...reqPayload, status: 'unsupported-database' }))
-                return
-              }
+                  dbReadModel.import(data)
 
-              yield* sendMessage(Devtools.LoadDatabaseFileRes.make({ ...reqPayload, status: 'ok' }))
+                  dbEventlog.destroy()
+                } else {
+                  yield* sendMessage(
+                    Devtools.Leader.LoadDatabaseFile.Error.make({
+                      ...reqPayload,
+                      cause: { _tag: 'unsupported-database' },
+                    }),
+                  )
+                  return
+                }
 
-              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' }))
+                yield* sendMessage(Devtools.Leader.LoadDatabaseFile.Success.make({ ...reqPayload }))
+                yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' })) ?? Effect.void
 
-              return
+                return
+              } catch (cause) {
+                yield* Effect.logError(`Error importing database file`, cause)
+                yield* sendMessage(
+                  Devtools.Leader.LoadDatabaseFile.Error.make({
+                    ...reqPayload,
+                    cause: { _tag: 'unexpected-error', cause },
+                  }),
+                )
+                return
+              }
             }
-            case 'LSD.Leader.ResetAllDataReq': {
+            case 'LSD.Leader.ResetAllData.Request': {
               const { mode } = decodedEvent
 
               yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
 
-              db.destroy()
+              dbReadModel.destroy()
 
               if (mode === 'all-data') {
-                dbLog.destroy()
+                dbEventlog.destroy()
               }
 
-              yield* sendMessage(Devtools.ResetAllDataRes.make({ ...reqPayload }))
+              yield* sendMessage(Devtools.Leader.ResetAllData.Success.make({ ...reqPayload }))
 
-              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-reset' }))
+              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-reset' })) ?? Effect.void
 
               return
             }
             case 'LSD.Leader.DatabaseFileInfoReq': {
+              if (persistenceInfo === undefined) {
+                console.log('[@livestore/common:leader-thread:devtools] persistenceInfo is required for this request')
+                return
+              }
+
               const dbSizeQuery = `SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size();`
-              const dbFileSize = db.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
-              const mutationLogFileSize = dbLog.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
+              const dbFileSize = dbReadModel.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
+              const eventlogFileSize = dbEventlog.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
 
               yield* sendMessage(
-                Devtools.DatabaseFileInfoRes.make({
-                  db: { fileSize: dbFileSize, persistenceInfo: persistenceInfo.db },
-                  mutationLog: { fileSize: mutationLogFileSize, persistenceInfo: persistenceInfo.mutationLog },
+                Devtools.Leader.DatabaseFileInfoRes.make({
+                  readModel: { fileSize: dbFileSize, persistenceInfo: persistenceInfo.readModel },
+                  eventlog: { fileSize: eventlogFileSize, persistenceInfo: persistenceInfo.eventlog },
                   ...reqPayload,
                 }),
               )
 
               return
             }
-            case 'LSD.Leader.MutationLogReq': {
-              const mutationLog = dbLog.export()
+            case 'LSD.Leader.EventlogReq': {
+              const eventlog = dbEventlog.export()
 
-              yield* sendMessage(Devtools.MutationLogRes.make({ mutationLog, ...reqPayload }))
+              yield* sendMessage(Devtools.Leader.EventlogRes.make({ eventlog, ...reqPayload }))
 
               return
             }
-            case 'LSD.Leader.RunMutationReq': {
-              yield* syncProcessor.pushPartial(decodedEvent.mutationEventEncoded)
+            case 'LSD.Leader.CommitEventReq': {
+              yield* syncProcessor.pushPartial({
+                event: decodedEvent.eventEncoded,
+                clientId: `devtools-${clientId}`,
+                sessionId: `devtools-${clientId}`,
+              })
 
-              yield* sendMessage(Devtools.RunMutationRes.make({ ...reqPayload }))
+              yield* sendMessage(Devtools.Leader.CommitEventRes.make({ ...reqPayload }))
 
               return
             }
             case 'LSD.Leader.SyncHistorySubscribe': {
-              const { requestId } = decodedEvent
+              const { subscriptionId } = decodedEvent
 
               if (syncBackend !== undefined) {
                 // TODO consider piggybacking on the existing leader-thread sync-pulling
                 yield* syncBackend.pull(Option.none()).pipe(
                   Stream.map((_) => _.batch),
                   Stream.flattenIterables,
-                  Stream.tap(({ mutationEventEncoded, metadata }) =>
-                    sendMessage(Devtools.SyncHistoryRes.make({ mutationEventEncoded, metadata, ...reqPayload })),
+                  Stream.tap(({ eventEncoded, metadata }) =>
+                    sendMessage(
+                      Devtools.Leader.SyncHistoryRes.make({
+                        eventEncoded,
+                        metadata,
+                        subscriptionId,
+                        ...reqPayload,
+                        requestId: nanoid(10),
+                      }),
+                    ),
                   ),
                   Stream.runDrain,
-                  Effect.acquireRelease(() => Effect.log('syncHistorySubscribe done')),
                   Effect.interruptible,
                   Effect.tapCauseLogPretty,
-                  FiberMap.run(subscriptionFiberMap, requestId),
+                  FiberMap.run(subscriptionFiberMap, subscriptionId),
                 )
               }
 
@@ -306,37 +274,42 @@ const listenToDevtools = ({
               return
             }
             case 'LSD.Leader.SyncingInfoReq': {
-              const syncingInfo = Devtools.SyncingInfo.make({
+              const syncingInfo = Devtools.Leader.SyncingInfo.make({
                 enabled: syncBackend !== undefined,
-                metadata: {},
+                metadata: syncBackend?.metadata ?? {},
               })
 
-              yield* sendMessage(Devtools.SyncingInfoRes.make({ syncingInfo, ...reqPayload }))
+              yield* sendMessage(Devtools.Leader.SyncingInfoRes.make({ syncingInfo, ...reqPayload }))
 
               return
             }
             case 'LSD.Leader.NetworkStatusSubscribe': {
               if (syncBackend !== undefined) {
-                const { requestId } = decodedEvent
+                const { subscriptionId } = decodedEvent
 
                 // TODO investigate and fix bug. seems that when sending messages right after
                 // the devtools have connected get sometimes lost
                 // This is probably the same "flaky databrowser loading" bug as we're seeing in the playwright tests
                 yield* Effect.sleep(1000)
 
-                yield* syncBackend.isConnected.changes.pipe(
-                  Stream.tap((isConnected) =>
+                yield* Stream.zipLatest(
+                  syncBackend.isConnected.changes,
+                  devtools.enabled ? devtools.syncBackendLatchState.changes : Stream.make({ latchClosed: false }),
+                ).pipe(
+                  Stream.tap(([isConnected, { latchClosed }]) =>
                     sendMessage(
-                      Devtools.NetworkStatusRes.make({
-                        networkStatus: { isConnected, timestampMs: Date.now() },
+                      Devtools.Leader.NetworkStatusRes.make({
+                        networkStatus: { isConnected, timestampMs: Date.now(), latchClosed },
+                        subscriptionId,
                         ...reqPayload,
+                        requestId: nanoid(10),
                       }),
                     ),
                   ),
                   Stream.runDrain,
                   Effect.interruptible,
                   Effect.tapCauseLogPretty,
-                  FiberMap.run(subscriptionFiberMap, requestId),
+                  FiberMap.run(subscriptionFiberMap, subscriptionId),
                 )
               }
 
@@ -349,6 +322,56 @@ const listenToDevtools = ({
 
               return
             }
+            case 'LSD.Leader.SyncHeadSubscribe': {
+              const { subscriptionId } = decodedEvent
+
+              yield* syncProcessor.syncState.changes.pipe(
+                Stream.tap((syncState) =>
+                  sendMessage(
+                    Devtools.Leader.SyncHeadRes.make({
+                      local: syncState.localHead,
+                      upstream: syncState.upstreamHead,
+                      subscriptionId,
+                      ...reqPayload,
+                      requestId: nanoid(10),
+                    }),
+                  ),
+                ),
+                Stream.runDrain,
+                Effect.interruptible,
+                Effect.tapCauseLogPretty,
+                FiberMap.run(subscriptionFiberMap, subscriptionId),
+              )
+
+              return
+            }
+            case 'LSD.Leader.SyncHeadUnsubscribe': {
+              const { subscriptionId } = decodedEvent
+
+              yield* FiberMap.remove(subscriptionFiberMap, subscriptionId)
+
+              return
+            }
+            case 'LSD.Leader.SetSyncLatch.Request': {
+              const { closeLatch } = decodedEvent
+
+              if (devtools.enabled === false) return
+
+              if (closeLatch === true) {
+                yield* devtools.syncBackendLatch.close
+              } else {
+                yield* devtools.syncBackendLatch.open
+              }
+
+              yield* SubscriptionRef.set(devtools.syncBackendLatchState, { latchClosed: closeLatch })
+
+              yield* sendMessage(Devtools.Leader.SetSyncLatch.Success.make({ ...reqPayload }))
+
+              return
+            }
+            default: {
+              yield* Effect.logWarning(`TODO implement devtools message`, decodedEvent)
+            }
           }
         }).pipe(Effect.withSpan(`@livestore/common:leader-thread:onDevtoolsMessage:${decodedEvent._tag}`)),
       ),