@livestore/common 0.3.0-dev.9 → 0.3.0

package/src/leader-thread/leader-worker-devtools.ts

@@ -1,11 +1,12 @@
 import { Effect, FiberMap, Option, Stream, SubscriptionRef } from '@livestore/utils/effect'
+import { nanoid } from '@livestore/utils/nanoid'
 
 import { Devtools, IntentionalShutdownCause, liveStoreVersion, UnexpectedError } from '../index.js'
-import { MUTATION_LOG_META_TABLE, SCHEMA_META_TABLE, SCHEMA_MUTATIONS_META_TABLE } from '../schema/mod.js'
+import { SystemTables } from '../schema/mod.js'
 import type { DevtoolsOptions, PersistenceInfoPair } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
-type SendMessageToDevtools = (message: Devtools.MessageFromAppLeader) => Effect.Effect<void>
+type SendMessageToDevtools = (message: Devtools.Leader.MessageFromApp) => Effect.Effect<void>
 
 // TODO bind scope to the webchannel lifetime
 export const bootDevtools = (options: DevtoolsOptions) =>
@@ -14,51 +15,56 @@ export const bootDevtools = (options: DevtoolsOptions) =>
       return
     }
 
-    const { connectedClientSessionPullQueues, syncProcessor, extraIncomingMessagesQueue } = yield* LeaderThreadCtx
+    const { syncProcessor, extraIncomingMessagesQueue, clientId, storeId } = yield* LeaderThreadCtx
 
     yield* listenToDevtools({
       incomingMessages: Stream.fromQueue(extraIncomingMessagesQueue),
       sendMessage: () => Effect.void,
     }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
 
-    const { persistenceInfo, devtoolsWebChannel } = yield* options.makeContext
+    const { node, persistenceInfo, mode } = yield* options.boot
 
-    const sendMessage: SendMessageToDevtools = (message) =>
-      devtoolsWebChannel
-        .send(message)
-        .pipe(
-          Effect.withSpan('@livestore/common:leader-thread:devtools:sendToDevtools'),
-          Effect.interruptible,
-          Effect.ignoreLogged,
-        )
-
-    const { localHead } = yield* syncProcessor.syncState
-
-    // TODO close queue when devtools disconnects
-    const pullQueue = yield* connectedClientSessionPullQueues.makeQueue(localHead)
-
-    yield* Stream.fromQueue(pullQueue).pipe(
-      Stream.tap((msg) =>
+    yield* node.listenForChannel.pipe(
+      Stream.filter(
+        (res) =>
+          Devtools.isChannelName.devtoolsClientLeader(res.channelName, { storeId, clientId }) && res.mode === mode,
+      ),
+      Stream.tap(({ channelName, source }) =>
         Effect.gen(function* () {
-          if (msg.payload._tag === 'upstream-advance') {
-            for (const mutationEventEncoded of msg.payload.newEvents) {
-              // TODO refactor with push semantics
-              yield* sendMessage(Devtools.MutationBroadcast.make({ mutationEventEncoded, liveStoreVersion }))
-            }
-          } else {
-            yield* Effect.logWarning('TODO implement rebases in devtools')
-          }
-        }),
+          const channel = yield* node.makeChannel({
+            target: source,
+            channelName,
+            schema: { listen: Devtools.Leader.MessageToApp, send: Devtools.Leader.MessageFromApp },
+            mode,
+          })
+
+          const sendMessage: SendMessageToDevtools = (message) =>
+            channel
+              .send(message)
+              .pipe(
+                Effect.withSpan('@livestore/common:leader-thread:devtools:sendToDevtools'),
+                Effect.interruptible,
+                Effect.ignoreLogged,
+              )
+
+          const syncState = yield* syncProcessor.syncState
+          const mergeCounter = syncProcessor.getMergeCounter()
+
+          yield* syncProcessor.pull({ cursor: { mergeCounter, eventNum: syncState.localHead } }).pipe(
+            Stream.tap(({ payload }) => sendMessage(Devtools.Leader.SyncPull.make({ payload, liveStoreVersion }))),
+            Stream.runDrain,
+            Effect.forkScoped,
+          )
+
+          yield* listenToDevtools({
+            incomingMessages: channel.listen.pipe(Stream.flatten(), Stream.orDie),
+            sendMessage,
+            persistenceInfo,
+          })
+        }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped),
       ),
       Stream.runDrain,
-      Effect.forkScoped,
     )
-
-    yield* listenToDevtools({
-      incomingMessages: devtoolsWebChannel.listen.pipe(Stream.flatten(), Stream.orDie),
-      sendMessage,
-      persistenceInfo,
-    }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
   }).pipe(Effect.withSpan('@livestore/common:leader-thread:devtools:boot'))
 
 const listenToDevtools = ({
@@ -66,98 +72,149 @@ const listenToDevtools = ({
   sendMessage,
   persistenceInfo,
 }: {
-  incomingMessages: Stream.Stream<Devtools.MessageToAppLeader>
+  incomingMessages: Stream.Stream<Devtools.Leader.MessageToApp>
   sendMessage: SendMessageToDevtools
   persistenceInfo?: PersistenceInfoPair
 }) =>
   Effect.gen(function* () {
-    const { syncBackend, makeSyncDb, db, dbLog, shutdownStateSubRef, shutdownChannel, syncProcessor } =
-      yield* LeaderThreadCtx
+    const {
+      syncBackend,
+      makeSqliteDb,
+      dbState,
+      dbEventlog,
+      shutdownStateSubRef,
+      shutdownChannel,
+      syncProcessor,
+      clientId,
+      devtools,
+    } = yield* LeaderThreadCtx
+
+    type SubscriptionId = string
+    const subscriptionFiberMap = yield* FiberMap.make<SubscriptionId>()
 
     type RequestId = string
-    const subscriptionFiberMap = yield* FiberMap.make<RequestId>()
+    const handledRequestIds = new Set<RequestId>()
 
     yield* incomingMessages.pipe(
       Stream.tap((decodedEvent) =>
         Effect.gen(function* () {
-          // yield* Effect.logDebug('[@livestore/common:leader-thread:devtools] incomingMessage', decodedEvent)
+          const { requestId } = decodedEvent
+          const reqPayload = { requestId, liveStoreVersion, clientId }
+
+          // yield* Effect.logDebug(
+          //   `[@livestore/common:leader-thread:devtools] incomingMessage: ${decodedEvent._tag} (${requestId})`,
+          //   decodedEvent,
+          // )
 
-          if (decodedEvent._tag === 'LSD.Disconnect') {
+          if (decodedEvent._tag === 'LSD.Leader.Disconnect') {
             return
           }
 
-          const { requestId } = decodedEvent
-          const reqPayload = { requestId, liveStoreVersion }
+          // TODO we should try to move the duplicate message handling on the webmesh layer
+          // So far I could only observe this problem with webmesh proxy channels (e.g. for Expo)
+          // Proof: https://share.cleanshot.com/V9G87B0B
+          // Also see `store/devtools.ts` for same problem
+          if (handledRequestIds.has(requestId)) {
+            // yield* Effect.logWarning(`Duplicate message`, decodedEvent)
+            return
+          }
+
+          handledRequestIds.add(requestId)
 
           switch (decodedEvent._tag) {
-            case 'LSD.Ping': {
-              yield* sendMessage(Devtools.Pong.make({ ...reqPayload }))
+            case 'LSD.Leader.Ping': {
+              yield* sendMessage(Devtools.Leader.Pong.make({ ...reqPayload }))
               return
             }
             case 'LSD.Leader.SnapshotReq': {
-              const snapshot = db.export()
+              const snapshot = dbState.export()
 
-              yield* sendMessage(Devtools.SnapshotRes.make({ snapshot, ...reqPayload }))
+              yield* sendMessage(Devtools.Leader.SnapshotRes.make({ snapshot, ...reqPayload }))
 
               return
             }
-            case 'LSD.Leader.LoadDatabaseFileReq': {
+            case 'LSD.Leader.LoadDatabaseFile.Request': {
               const { data } = decodedEvent
 
               let tableNames: Set<string>
 
               try {
-                const tmpSyncDb = yield* makeSyncDb({ _tag: 'in-memory' })
-                tmpSyncDb.import(data)
-                const tableNameResults = tmpSyncDb.select<{ name: string }>(
+                const tmpDb = yield* makeSqliteDb({ _tag: 'in-memory' })
+                tmpDb.import(data)
+                const tableNameResults = tmpDb.select<{ name: string }>(
                   `select name from sqlite_master where type = 'table'`,
                 )
 
                 tableNames = new Set(tableNameResults.map((_) => _.name))
 
-                tmpSyncDb.close()
-              } catch (e) {
-                yield* Effect.logError(`Error importing database file`, e)
-                yield* sendMessage(Devtools.LoadDatabaseFileRes.make({ ...reqPayload, status: 'unsupported-file' }))
+                tmpDb.close()
+              } catch (cause) {
+                yield* Effect.logError(`Error importing database file`, cause)
+                yield* sendMessage(
+                  Devtools.Leader.LoadDatabaseFile.Error.make({
+                    ...reqPayload,
+                    cause: { _tag: 'unexpected-error', cause },
+                  }),
+                )
 
                 return
               }
 
-              if (tableNames.has(MUTATION_LOG_META_TABLE)) {
-                yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
-
-                dbLog.import(data)
-
-                db.destroy()
-              } else if (tableNames.has(SCHEMA_META_TABLE) && tableNames.has(SCHEMA_MUTATIONS_META_TABLE)) {
-                yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
-
-                db.import(data)
+              try {
+                if (tableNames.has(SystemTables.EVENTLOG_META_TABLE)) {
+                  // Is eventlog db
+                  yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
+
+                  dbEventlog.import(data)
+
+                  dbState.destroy()
+                } else if (
+                  tableNames.has(SystemTables.SCHEMA_META_TABLE) &&
+                  tableNames.has(SystemTables.SCHEMA_EVENT_DEFS_META_TABLE)
+                ) {
+                  // Is state db
+                  yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
+
+                  dbState.import(data)
+
+                  dbEventlog.destroy()
+                } else {
+                  yield* sendMessage(
+                    Devtools.Leader.LoadDatabaseFile.Error.make({
+                      ...reqPayload,
+                      cause: { _tag: 'unsupported-database' },
+                    }),
+                  )
+                  return
+                }
+
+                yield* sendMessage(Devtools.Leader.LoadDatabaseFile.Success.make({ ...reqPayload }))
+                yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' })) ?? Effect.void
 
-                dbLog.destroy()
-              } else {
-                yield* sendMessage(Devtools.LoadDatabaseFileRes.make({ ...reqPayload, status: 'unsupported-database' }))
+                return
+              } catch (cause) {
+                yield* Effect.logError(`Error importing database file`, cause)
+                yield* sendMessage(
+                  Devtools.Leader.LoadDatabaseFile.Error.make({
+                    ...reqPayload,
+                    cause: { _tag: 'unexpected-error', cause },
+                  }),
+                )
                 return
               }
-
-              yield* sendMessage(Devtools.LoadDatabaseFileRes.make({ ...reqPayload, status: 'ok' }))
-
-              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' })) ?? Effect.void
-
-              return
             }
-            case 'LSD.Leader.ResetAllDataReq': {
+            case 'LSD.Leader.ResetAllData.Request': {
               const { mode } = decodedEvent
 
               yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
 
-              db.destroy()
+              dbState.destroy()
 
               if (mode === 'all-data') {
-                dbLog.destroy()
+                dbEventlog.destroy()
               }
 
-              yield* sendMessage(Devtools.ResetAllDataRes.make({ ...reqPayload }))
+              yield* sendMessage(Devtools.Leader.ResetAllData.Success.make({ ...reqPayload }))
 
               yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-reset' })) ?? Effect.void
 
@@ -170,49 +227,60 @@ const listenToDevtools = ({
               }
 
               const dbSizeQuery = `SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size();`
-              const dbFileSize = db.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
-              const mutationLogFileSize = dbLog.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
+              const dbFileSize = dbState.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
+              const eventlogFileSize = dbEventlog.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
 
               yield* sendMessage(
-                Devtools.DatabaseFileInfoRes.make({
-                  db: { fileSize: dbFileSize, persistenceInfo: persistenceInfo.db },
-                  mutationLog: { fileSize: mutationLogFileSize, persistenceInfo: persistenceInfo.mutationLog },
+                Devtools.Leader.DatabaseFileInfoRes.make({
+                  state: { fileSize: dbFileSize, persistenceInfo: persistenceInfo.state },
+                  eventlog: { fileSize: eventlogFileSize, persistenceInfo: persistenceInfo.eventlog },
                   ...reqPayload,
                 }),
               )
 
               return
             }
-            case 'LSD.Leader.MutationLogReq': {
-              const mutationLog = dbLog.export()
+            case 'LSD.Leader.EventlogReq': {
+              const eventlog = dbEventlog.export()
 
-              yield* sendMessage(Devtools.MutationLogRes.make({ mutationLog, ...reqPayload }))
+              yield* sendMessage(Devtools.Leader.EventlogRes.make({ eventlog, ...reqPayload }))
 
               return
             }
-            case 'LSD.Leader.RunMutationReq': {
-              yield* syncProcessor.pushPartial(decodedEvent.mutationEventEncoded)
+            case 'LSD.Leader.CommitEventReq': {
+              yield* syncProcessor.pushPartial({
+                event: decodedEvent.eventEncoded,
+                clientId: `devtools-${clientId}`,
+                sessionId: `devtools-${clientId}`,
+              })
 
-              yield* sendMessage(Devtools.RunMutationRes.make({ ...reqPayload }))
+              yield* sendMessage(Devtools.Leader.CommitEventRes.make({ ...reqPayload }))
 
               return
             }
             case 'LSD.Leader.SyncHistorySubscribe': {
-              const { requestId } = decodedEvent
+              const { subscriptionId } = decodedEvent
 
               if (syncBackend !== undefined) {
                 // TODO consider piggybacking on the existing leader-thread sync-pulling
                 yield* syncBackend.pull(Option.none()).pipe(
                   Stream.map((_) => _.batch),
                   Stream.flattenIterables,
-                  Stream.tap(({ mutationEventEncoded, metadata }) =>
-                    sendMessage(Devtools.SyncHistoryRes.make({ mutationEventEncoded, metadata, ...reqPayload })),
+                  Stream.tap(({ eventEncoded, metadata }) =>
+                    sendMessage(
+                      Devtools.Leader.SyncHistoryRes.make({
+                        eventEncoded,
+                        metadata,
+                        subscriptionId,
+                        ...reqPayload,
+                        requestId: nanoid(10),
+                      }),
+                    ),
                   ),
                   Stream.runDrain,
-                  Effect.acquireRelease(() => Effect.log('syncHistorySubscribe done')),
                   Effect.interruptible,
                   Effect.tapCauseLogPretty,
-                  FiberMap.run(subscriptionFiberMap, requestId),
+                  FiberMap.run(subscriptionFiberMap, subscriptionId),
                 )
               }
 
@@ -227,37 +295,42 @@ const listenToDevtools = ({
               return
             }
             case 'LSD.Leader.SyncingInfoReq': {
-              const syncingInfo = Devtools.SyncingInfo.make({
+              const syncingInfo = Devtools.Leader.SyncingInfo.make({
                 enabled: syncBackend !== undefined,
-                metadata: {},
+                metadata: syncBackend?.metadata ?? {},
               })
 
-              yield* sendMessage(Devtools.SyncingInfoRes.make({ syncingInfo, ...reqPayload }))
+              yield* sendMessage(Devtools.Leader.SyncingInfoRes.make({ syncingInfo, ...reqPayload }))
 
               return
             }
             case 'LSD.Leader.NetworkStatusSubscribe': {
               if (syncBackend !== undefined) {
-                const { requestId } = decodedEvent
+                const { subscriptionId } = decodedEvent
 
                 // TODO investigate and fix bug. seems that when sending messages right after
                 // the devtools have connected get sometimes lost
                 // This is probably the same "flaky databrowser loading" bug as we're seeing in the playwright tests
                 yield* Effect.sleep(1000)
 
-                yield* syncBackend.isConnected.changes.pipe(
-                  Stream.tap((isConnected) =>
+                yield* Stream.zipLatest(
+                  syncBackend.isConnected.changes,
+                  devtools.enabled ? devtools.syncBackendLatchState.changes : Stream.make({ latchClosed: false }),
+                ).pipe(
+                  Stream.tap(([isConnected, { latchClosed }]) =>
                     sendMessage(
-                      Devtools.NetworkStatusRes.make({
-                        networkStatus: { isConnected, timestampMs: Date.now() },
+                      Devtools.Leader.NetworkStatusRes.make({
+                        networkStatus: { isConnected, timestampMs: Date.now(), latchClosed },
+                        subscriptionId,
                         ...reqPayload,
+                        requestId: nanoid(10),
                       }),
                     ),
                   ),
                   Stream.runDrain,
                   Effect.interruptible,
                   Effect.tapCauseLogPretty,
-                  FiberMap.run(subscriptionFiberMap, requestId),
+                  FiberMap.run(subscriptionFiberMap, subscriptionId),
                 )
               }
 
@@ -270,6 +343,56 @@ const listenToDevtools = ({
 
               return
             }
+            case 'LSD.Leader.SyncHeadSubscribe': {
+              const { subscriptionId } = decodedEvent
+
+              yield* syncProcessor.syncState.changes.pipe(
+                Stream.tap((syncState) =>
+                  sendMessage(
+                    Devtools.Leader.SyncHeadRes.make({
+                      local: syncState.localHead,
+                      upstream: syncState.upstreamHead,
+                      subscriptionId,
+                      ...reqPayload,
+                      requestId: nanoid(10),
+                    }),
+                  ),
+                ),
+                Stream.runDrain,
+                Effect.interruptible,
+                Effect.tapCauseLogPretty,
+                FiberMap.run(subscriptionFiberMap, subscriptionId),
+              )
+
+              return
+            }
+            case 'LSD.Leader.SyncHeadUnsubscribe': {
+              const { subscriptionId } = decodedEvent
+
+              yield* FiberMap.remove(subscriptionFiberMap, subscriptionId)
+
+              return
+            }
+            case 'LSD.Leader.SetSyncLatch.Request': {
+              const { closeLatch } = decodedEvent
+
+              if (devtools.enabled === false) return
+
+              if (closeLatch === true) {
+                yield* devtools.syncBackendLatch.close
+              } else {
+                yield* devtools.syncBackendLatch.open
+              }
+
+              yield* SubscriptionRef.set(devtools.syncBackendLatchState, { latchClosed: closeLatch })
+
+              yield* sendMessage(Devtools.Leader.SetSyncLatch.Success.make({ ...reqPayload }))
+
+              return
+            }
+            default: {
+              yield* Effect.logWarning(`TODO implement devtools message`, decodedEvent)
+            }
           }
         }).pipe(Effect.withSpan(`@livestore/common:leader-thread:onDevtoolsMessage:${decodedEvent._tag}`)),
       ),