@livestore/common 0.4.0-dev.9 → 0.4.0

package/src/leader-thread/leader-worker-devtools.ts

@@ -1,70 +1,106 @@
 import { Effect, FiberMap, Option, Stream, SubscriptionRef } from '@livestore/utils/effect'
 import { nanoid } from '@livestore/utils/nanoid'
 
-import { Devtools, IntentionalShutdownCause, liveStoreVersion, UnexpectedError } from '../index.ts'
+import {
+  Devtools,
+  devtoolsProtocolVersion,
+  IntentionalShutdownCause,
+  isDevtoolsProtocolVersionSupported,
+  liveStoreVersion,
+  resolveDevtoolsProtocolVersion,
+  UnknownError,
+} from '../index.ts'
 import { SystemTables } from '../schema/mod.ts'
 import type { DevtoolsOptions, PersistenceInfoPair } from './types.ts'
 import { LeaderThreadCtx } from './types.ts'
 
 type SendMessageToDevtools = (message: Devtools.Leader.MessageFromApp) => Effect.Effect<void>
 
-// TODO bind scope to the webchannel lifetime
-export const bootDevtools = (options: DevtoolsOptions) =>
-  Effect.gen(function* () {
-    if (options.enabled === false) {
-      return
-    }
-
-    const { syncProcessor, extraIncomingMessagesQueue, clientId, storeId } = yield* LeaderThreadCtx
-
-    yield* listenToDevtools({
-      incomingMessages: Stream.fromQueue(extraIncomingMessagesQueue),
-      sendMessage: () => Effect.void,
-    }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
-
-    const { node, persistenceInfo, mode } = yield* options.boot
-
-    yield* node.listenForChannel.pipe(
-      Stream.filter(
-        (res) =>
-          Devtools.isChannelName.devtoolsClientLeader(res.channelName, { storeId, clientId }) && res.mode === mode,
-      ),
-      Stream.tap(({ channelName, source }) =>
-        Effect.gen(function* () {
-          const channel = yield* node.makeChannel({
-            target: source,
-            channelName,
-            schema: { listen: Devtools.Leader.MessageToApp, send: Devtools.Leader.MessageFromApp },
-            mode,
-          })
-
-          const sendMessage: SendMessageToDevtools = (message) =>
-            channel
-              .send(message)
-              .pipe(
-                Effect.withSpan('@livestore/common:leader-thread:devtools:sendToDevtools'),
-                Effect.interruptible,
-                Effect.ignoreLogged,
-              )
-
-          const syncState = yield* syncProcessor.syncState
+/**
+ * Type guard for DevtoolsViteNotInstalledError.
+ * This error is defined in @livestore/adapter-node but we need to handle it here.
+ */
+const isDevtoolsViteNotInstalledError = (
+  error: unknown,
+): error is { _tag: 'DevtoolsViteNotInstalledError'; message: string } =>
+  typeof error === 'object' && error !== null && '_tag' in error && error._tag === 'DevtoolsViteNotInstalledError'
 
-          yield* syncProcessor.pull({ cursor: syncState.localHead }).pipe(
-            Stream.tap(({ payload }) => sendMessage(Devtools.Leader.SyncPull.make({ payload, liveStoreVersion }))),
-            Stream.runDrain,
-            Effect.forkScoped,
-          )
-
-          yield* listenToDevtools({
-            incomingMessages: channel.listen.pipe(Stream.flatten(), Stream.orDie),
-            sendMessage,
-            persistenceInfo,
-          })
-        }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped),
+// TODO bind scope to the webchannel lifetime
+export const bootDevtools = Effect.fn('@livestore/common:leader-thread:devtools:boot')(function* (
+  options: DevtoolsOptions,
+) {
+  if (options.enabled === false) {
+    return
+  }
+
+  const { syncProcessor, extraIncomingMessagesQueue, clientId, storeId } = yield* LeaderThreadCtx
+
+  yield* listenToDevtools({
+    incomingMessages: Stream.fromQueue(extraIncomingMessagesQueue),
+    sendMessage: () => Effect.void,
+  }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
+
+  const bootResult = yield* options.boot.pipe(
+    Effect.map(Option.some),
+    Effect.catchIf(isDevtoolsViteNotInstalledError, (error) =>
+      Effect.logWarning(`[@livestore/devtools] ${error.message} Devtools will be disabled.`).pipe(
+        Effect.as(Option.none()),
       ),
-      Stream.runDrain,
-    )
-  }).pipe(Effect.withSpan('@livestore/common:leader-thread:devtools:boot'))
+    ),
+    Effect.catchAllCause((cause) =>
+      Effect.logWarning(
+        `[@livestore/devtools] Failed to start devtools server. Devtools will be disabled.`,
+        cause,
+      ).pipe(Effect.as(Option.none())),
+    ),
+  )
+
+  if (Option.isNone(bootResult) === true) {
+    return
+  }
+
+  const { node, persistenceInfo, mode } = bootResult.value
+
+  yield* node.listenForChannel.pipe(
+    Stream.filter(
+      (res) => Devtools.isChannelName.devtoolsClientLeader(res.channelName, { storeId, clientId }) && res.mode === mode,
+    ),
+    Stream.tap(({ channelName, source }) =>
+      Effect.gen(function* () {
+        const channel = yield* node.makeChannel({
+          target: source,
+          channelName,
+          schema: { listen: Devtools.Leader.MessageToApp, send: Devtools.Leader.MessageFromApp },
+          mode,
+        })
+
+        const sendMessage: SendMessageToDevtools = (message) =>
+          channel
+            .send(message)
+            .pipe(
+              Effect.withSpan('@livestore/common:leader-thread:devtools:sendToDevtools'),
+              Effect.interruptible,
+              Effect.ignoreLogged,
+            )
+
+        const syncState = yield* syncProcessor.syncState
+
+        yield* syncProcessor.pull({ cursor: syncState.localHead }).pipe(
+          Stream.tap(({ payload }) => sendMessage(Devtools.Leader.SyncPull.make({ payload, liveStoreVersion }))),
+          Stream.runDrain,
+          Effect.forkScoped,
+        )
+
+        yield* listenToDevtools({
+          incomingMessages: channel.listen.pipe(Stream.flatten(), Stream.orDie),
+          sendMessage,
+          persistenceInfo,
+        })
+      }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped),
+    ),
+    Stream.runDrain,
+  )
+})
 
 const listenToDevtools = ({
   incomingMessages,
@@ -94,6 +130,22 @@ const listenToDevtools = ({
     type RequestId = string
     const handledRequestIds = new Set<RequestId>()
 
+    type LoadDatabaseKind = 'state' | 'eventlog'
+    const loadDatabaseBatchTracker = new Map<string, Set<LoadDatabaseKind>>()
+
+    const registerBatchProgress = (batchId: string, kind: LoadDatabaseKind) => {
+      const entry = loadDatabaseBatchTracker.get(batchId) ?? new Set<LoadDatabaseKind>()
+      entry.add(kind)
+      loadDatabaseBatchTracker.set(batchId, entry)
+      const finished = entry.has('state') && entry.has('eventlog')
+
+      if (finished === true) {
+        loadDatabaseBatchTracker.delete(batchId)
+      }
+
+      return finished
+    }
+
     yield* incomingMessages.pipe(
       Stream.tap((decodedEvent) =>
         Effect.gen(function* () {
@@ -113,7 +165,7 @@ const listenToDevtools = ({
           // So far I could only observe this problem with webmesh proxy channels (e.g. for Expo)
           // Proof: https://share.cleanshot.com/V9G87B0B
           // Also see `store/devtools.ts` for same problem
-          if (handledRequestIds.has(requestId)) {
+          if (handledRequestIds.has(requestId) === true) {
             // yield* Effect.logWarning(`Duplicate message`, decodedEvent)
             return
           }
@@ -122,7 +174,19 @@ const listenToDevtools = ({
 
           switch (decodedEvent._tag) {
             case 'LSD.Leader.Ping': {
-              yield* sendMessage(Devtools.Leader.Pong.make({ ...reqPayload }))
+              if (isDevtoolsProtocolVersionSupported(decodedEvent.devtoolsProtocolVersion) === false) {
+                yield* sendMessage(
+                  Devtools.Leader.VersionMismatch.make({
+                    ...reqPayload,
+                    appVersion: liveStoreVersion,
+                    receivedVersion: decodedEvent.liveStoreVersion,
+                    appDevtoolsProtocolVersion: devtoolsProtocolVersion,
+                    receivedDevtoolsProtocolVersion: resolveDevtoolsProtocolVersion(decodedEvent.devtoolsProtocolVersion),
+                  }),
+                )
+                return
+              }
+              yield* sendMessage(Devtools.Leader.Pong.make({ ...reqPayload, devtoolsProtocolVersion }))
               return
             }
             case 'LSD.Leader.SnapshotReq': {
@@ -133,74 +197,85 @@ const listenToDevtools = ({
               return
             }
             case 'LSD.Leader.LoadDatabaseFile.Request': {
-              const { data } = decodedEvent
+              const { data, batchId } = decodedEvent
 
-              let tableNames: Set<string>
-
-              try {
-                const tmpDb = yield* makeSqliteDb({ _tag: 'in-memory' })
-                tmpDb.import(data)
-                const tableNameResults = tmpDb.select<{ name: string }>(
-                  `select name from sqlite_master where type = 'table'`,
-                )
-
-                tableNames = new Set(tableNameResults.map((_) => _.name))
-
-                tmpDb.close()
-              } catch (cause) {
-                yield* Effect.logError(`Error importing database file`, cause)
-                yield* sendMessage(
-                  Devtools.Leader.LoadDatabaseFile.Error.make({
-                    ...reqPayload,
-                    cause: { _tag: 'unexpected-error', cause },
-                  }),
+              const handleLoadDb = Effect.gen(function* () {
+                const tableNames = yield* Effect.acquireRelease(makeSqliteDb({ _tag: 'in-memory' }), (db) =>
+                  Effect.sync(() => db.close()),
+                ).pipe(
+                  Effect.flatMap((db) =>
+                    Effect.try(() => {
+                      db.import(data)
+                      const rows = db.select<{ name: string }>(`select name from sqlite_master where type = 'table'`)
+                      return new Set(rows.map((r) => r.name))
+                    }),
+                  ),
                 )
 
-                return
-              }
+                let databaseKind: LoadDatabaseKind | undefined
 
-              try {
-                if (tableNames.has(SystemTables.EVENTLOG_META_TABLE)) {
-                  // Is eventlog db
+                if (tableNames.has(SystemTables.EVENTLOG_META_TABLE) === true) {
+                  databaseKind = 'eventlog'
                   yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
+                  yield* Effect.try(() => dbEventlog.import(data))
 
-                  dbEventlog.import(data)
-
-                  dbState.destroy()
+                  if (batchId === undefined) {
+                    yield* Effect.try(() => dbState.destroy())
+                  }
                 } else if (
-                  tableNames.has(SystemTables.SCHEMA_META_TABLE) &&
-                  tableNames.has(SystemTables.SCHEMA_EVENT_DEFS_META_TABLE)
+                  tableNames.has(SystemTables.SCHEMA_META_TABLE) === true &&
+                  tableNames.has(SystemTables.SCHEMA_EVENT_DEFS_META_TABLE) === true
                 ) {
-                  // Is state db
+                  databaseKind = 'state'
                   yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
+                  yield* Effect.try(() => dbState.import(data))
 
-                  dbState.import(data)
-
-                  dbEventlog.destroy()
+                  if (batchId === undefined) {
+                    yield* Effect.try(() => dbEventlog.destroy())
+                  }
                 } else {
-                  yield* sendMessage(
-                    Devtools.Leader.LoadDatabaseFile.Error.make({
-                      ...reqPayload,
-                      cause: { _tag: 'unsupported-database' },
-                    }),
-                  )
-                  return
+                  return yield* Effect.fail({ _tag: 'unsupported-database' } as const)
                 }
 
+                const resolvedDatabaseKind = databaseKind
+                if (resolvedDatabaseKind === undefined) {
+                  return yield* Effect.fail({ _tag: 'unsupported-database' } as const)
+                }
+
+                const shouldShutdown =
+                  batchId === undefined ? true : registerBatchProgress(batchId, resolvedDatabaseKind)
+
                 yield* sendMessage(Devtools.Leader.LoadDatabaseFile.Success.make({ ...reqPayload }))
-                yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' })) ?? Effect.void
 
-                return
-              } catch (cause) {
-                yield* Effect.logError(`Error importing database file`, cause)
-                yield* sendMessage(
-                  Devtools.Leader.LoadDatabaseFile.Error.make({
-                    ...reqPayload,
-                    cause: { _tag: 'unexpected-error', cause },
-                  }),
-                )
-                return
-              }
+                if (shouldShutdown === true) {
+                  yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' }))
+                }
+              })
+
+              yield* handleLoadDb.pipe(
+                Effect.catchTag('unsupported-database', () =>
+                  sendMessage(
+                    Devtools.Leader.LoadDatabaseFile.Error.make({
+                      ...reqPayload,
+                      cause: { _tag: 'unsupported-database' as const },
+                    }),
+                  ),
+                ),
+                Effect.catchAll((cause) =>
+                  Effect.logWarning('Error importing database file', cause).pipe(
+                    Effect.zipRight(
+                      sendMessage(
+                        Devtools.Leader.LoadDatabaseFile.Error.make({
+                          ...reqPayload,
+                          cause: { _tag: 'unknown-error' as const, cause },
+                        }),
+                      ),
+                    ),
+                  ),
+                ),
+              )
+
+              return
             }
             case 'LSD.Leader.ResetAllData.Request': {
               const { mode } = decodedEvent
@@ -215,7 +290,7 @@ const listenToDevtools = ({
 
               yield* sendMessage(Devtools.Leader.ResetAllData.Success.make({ ...reqPayload }))
 
-              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-reset' })) ?? Effect.void
+              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-reset' }))
 
               return
             }
@@ -286,10 +361,10 @@ const listenToDevtools = ({
               return
             }
             case 'LSD.Leader.SyncHistoryUnsubscribe': {
-              const { requestId } = decodedEvent
-              console.log('LSD.SyncHistoryUnsubscribe', requestId)
+              const unsubscribeRequestId = decodedEvent.requestId
+              console.log('LSD.SyncHistoryUnsubscribe', unsubscribeRequestId)
 
-              yield* FiberMap.remove(subscriptionFiberMap, requestId)
+              yield* FiberMap.remove(subscriptionFiberMap, unsubscribeRequestId)
 
               return
             }
@@ -314,12 +389,18 @@ const listenToDevtools = ({
 
                 yield* Stream.zipLatest(
                   syncBackend.isConnected.changes,
-                  devtools.enabled ? devtools.syncBackendLatchState.changes : Stream.make({ latchClosed: false }),
+                  devtools.enabled === true
+                    ? devtools.syncBackendLatchState.changes
+                    : Stream.make({ latchClosed: false }),
                 ).pipe(
                   Stream.tap(([isConnected, { latchClosed }]) =>
                     sendMessage(
                       Devtools.Leader.NetworkStatusRes.make({
-                        networkStatus: { isConnected, timestampMs: Date.now(), latchClosed },
+                        networkStatus: {
+                          isConnected,
+                          timestampMs: Date.now(),
+                          devtools: { latchClosed },
+                        },
                         subscriptionId,
                         ...reqPayload,
                         requestId: nanoid(10),
@@ -336,9 +417,9 @@ const listenToDevtools = ({
               return
             }
             case 'LSD.Leader.NetworkStatusUnsubscribe': {
-              const { requestId } = decodedEvent
+              const unsubscribeRequestId = decodedEvent.requestId
 
-              yield* FiberMap.remove(subscriptionFiberMap, requestId)
+              yield* FiberMap.remove(subscriptionFiberMap, unsubscribeRequestId)
 
               return
             }
@@ -395,7 +476,7 @@ const listenToDevtools = ({
           }
         }).pipe(Effect.withSpan(`@livestore/common:leader-thread:onDevtoolsMessage:${decodedEvent._tag}`)),
       ),
-      UnexpectedError.mapToUnexpectedErrorStream,
+      UnknownError.mapToUnknownErrorStream,
       Stream.runDrain,
     )
   })

package/src/leader-thread/make-leader-thread-layer.test.ts

@@ -0,0 +1,44 @@
+import { Vitest } from '@livestore/utils-dev/node-vitest'
+import { Effect, Stream, SubscriptionRef } from '@livestore/utils/effect'
+
+import { makeMockSyncBackend } from '../sync/mock-sync-backend.ts'
+import type { SyncBackend } from '../sync/sync.ts'
+import { makeNetworkStatusSubscribable } from './make-leader-thread-layer.ts'
+import type { DevtoolsContext } from './types.ts'
+
+Vitest.describe('makeNetworkStatusSubscribable', () => {
+  Vitest.scopedLive('tracks sync backend connectivity and devtools latch state', () =>
+    Effect.gen(function* () {
+      const mockBackend = yield* makeMockSyncBackend({ startConnected: false })
+      const syncBackend = yield* mockBackend.makeSyncBackend
+      const latchStateRef = yield* SubscriptionRef.make<{ latchClosed: boolean }>({ latchClosed: false })
+
+      const devtoolsContext: DevtoolsContext = {
+        enabled: true,
+        syncBackendLatch: yield* Effect.makeLatch(true),
+        syncBackendLatchState: latchStateRef,
+      }
+
+      const networkStatus = yield* makeNetworkStatusSubscribable({ syncBackend, devtoolsContext })
+
+      const initial = yield* networkStatus
+      Vitest.expect(initial.isConnected).toBe(false)
+      Vitest.expect(initial.devtools.latchClosed).toBe(false)
+
+      const waitFor = (predicate: (status: SyncBackend.NetworkStatus) => boolean) =>
+        networkStatus.changes.pipe(Stream.filter(predicate), Stream.runHead, Effect.flatten)
+
+      const onlineFiber = yield* waitFor((status) => status.isConnected).pipe(Effect.forkScoped)
+      yield* mockBackend.connect
+      const online = yield* onlineFiber
+      Vitest.expect(online.isConnected).toBe(true)
+      Vitest.expect(online.timestampMs).toBeGreaterThan(initial.timestampMs)
+
+      const latchedFiber = yield* waitFor((status) => status.devtools.latchClosed).pipe(Effect.forkScoped)
+      yield* SubscriptionRef.set(latchStateRef, { latchClosed: true })
+      const latched = yield* latchedFiber
+      Vitest.expect(latched.devtools.latchClosed).toBe(true)
+      Vitest.expect(latched.timestampMs).toBeGreaterThanOrEqual(online.timestampMs)
+    }),
+  )
+})