@livestore/adapter-web 0.3.0-dev.24 → 0.3.0-dev.25

package/src/web-worker/common/persisted-sqlite.ts

@@ -1,7 +1,7 @@
-import { liveStoreStorageFormatVersion } from '@livestore/common'
+import { liveStoreStorageFormatVersion, UnexpectedError } from '@livestore/common'
 import type { LiveStoreSchema } from '@livestore/common/schema'
 import { decodeSAHPoolFilename, HEADER_OFFSET_DATA } from '@livestore/sqlite-wasm/browser'
-import { Effect, Schema } from '@livestore/utils/effect'
+import { Effect, Schedule, Schema } from '@livestore/utils/effect'
 
 import * as OpfsUtils from '../../opfs-utils.js'
 import type * as WorkerSchema from './worker-schema.js'
@@ -87,7 +87,12 @@ export const resetPersistedDataFromClientSession = ({
   Effect.gen(function* () {
     const directory = sanitizeOpfsDir(storageOptions.directory, storeId)
     yield* opfsDeleteAbs(directory)
-  }).pipe(Effect.withSpan('@livestore/adapter-web:resetPersistedDataFromClientSession'))
+  }).pipe(
+    Effect.retry({
+      schedule: Schedule.exponentialBackoff10Sec,
+    }),
+    Effect.withSpan('@livestore/adapter-web:resetPersistedDataFromClientSession'),
+  )
 
 const opfsDeleteAbs = (absPath: string) =>
   Effect.promise(async () => {
@@ -114,7 +119,10 @@ const opfsDeleteAbs = (absPath: string) =>
         throw error
       }
     }
-  }).pipe(Effect.withSpan('@livestore/adapter-web:worker:opfsDeleteFile', { attributes: { absFilePath: absPath } }))
+  }).pipe(
+    UnexpectedError.mapToUnexpectedError,
+    Effect.withSpan('@livestore/adapter-web:worker:opfsDeleteFile', { attributes: { absFilePath: absPath } }),
+  )
 
 export const sanitizeOpfsDir = (directory: string | undefined, storeId: string) => {
   // Root dir should be `''` not `/`

package/src/web-worker/common/worker-schema.ts

@@ -8,7 +8,7 @@ import {
   UnexpectedError,
 } from '@livestore/common'
 import { EventId, MutationEvent } from '@livestore/common/schema'
-import * as WebMeshWorker from '@livestore/devtools-web-common/worker'
+import * as WebmeshWorker from '@livestore/devtools-web-common/worker'
 import { Schema, Transferable } from '@livestore/utils/effect'
 
 export const StorageTypeOpfs = Schema.Struct({
@@ -159,7 +159,7 @@ export namespace LeaderWorkerInner {
     NetworkStatusStream,
     Shutdown,
     ExtraDevtoolsMessage,
-    WebMeshWorker.Schema.CreateConnection,
+    WebmeshWorker.Schema.CreateConnection,
   )
   export type Request = typeof Request.Type
 }
@@ -202,6 +202,6 @@ export namespace SharedWorker {
     LeaderWorkerInner.Shutdown,
     LeaderWorkerInner.ExtraDevtoolsMessage,
 
-    WebMeshWorker.Schema.CreateConnection,
+    WebmeshWorker.Schema.CreateConnection,
   ) {}
 }

package/src/web-worker/leader-worker/make-leader-worker.ts

@@ -10,7 +10,7 @@ import {
 import type { LiveStoreSchema } from '@livestore/common/schema'
 import { MutationEvent } from '@livestore/common/schema'
 import { makeChannelForConnectedMeshNode } from '@livestore/devtools-web-common/web-channel'
-import * as WebMeshWorker from '@livestore/devtools-web-common/worker'
+import * as WebmeshWorker from '@livestore/devtools-web-common/worker'
 import { sqliteDbFactory } from '@livestore/sqlite-wasm/browser'
 import { loadSqlite3Wasm } from '@livestore/sqlite-wasm/load-wasm'
 import { isDevEnv, LS_DEV } from '@livestore/utils'
@@ -97,7 +97,7 @@ const makeWorkerRunnerOuter = (
           Effect.scoped,
           Effect.withSpan('@livestore/adapter-web:worker:wrapper:InitialMessage:innerFiber'),
           Effect.tapCauseLogPretty,
-          Effect.provide(WebMeshWorker.CacheService.layer({ nodeName: `leader-${storeId}-${clientId}` })),
+          Effect.provide(WebmeshWorker.CacheService.layer({ nodeName: `leader-${storeId}-${clientId}` })),
           Effect.forkScoped,
         )
 
@@ -242,7 +242,7 @@ const makeWorkerRunnerInner = ({ schema, sync: syncOptions }: WorkerOptions) =>
         UnexpectedError.mapToUnexpectedError,
         Effect.withSpan('@livestore/adapter-web:worker:ExtraDevtoolsMessage'),
       ),
-    'DevtoolsWebCommon.CreateConnection': WebMeshWorker.CreateConnection,
+    'DevtoolsWebCommon.CreateConnection': WebmeshWorker.CreateConnection,
   })
 
 const makeDevtoolsOptions = ({
@@ -253,12 +253,12 @@ const makeDevtoolsOptions = ({
   devtoolsEnabled: boolean
   dbReadModel: SqliteDb
   dbMutationLog: SqliteDb
-}): Effect.Effect<DevtoolsOptions, UnexpectedError, Scope.Scope | WebMeshWorker.CacheService> =>
+}): Effect.Effect<DevtoolsOptions, UnexpectedError, Scope.Scope | WebmeshWorker.CacheService> =>
   Effect.gen(function* () {
     if (devtoolsEnabled === false) {
       return { enabled: false }
     }
-    const { node } = yield* WebMeshWorker.CacheService
+    const { node } = yield* WebmeshWorker.CacheService
 
     return {
       enabled: true,

package/src/web-worker/shared-worker/make-shared-worker.ts

@@ -1,12 +1,11 @@
 import { UnexpectedError } from '@livestore/common'
 import { connectViaWorker } from '@livestore/devtools-web-common/web-channel'
-import * as WebMeshWorker from '@livestore/devtools-web-common/worker'
+import * as WebmeshWorker from '@livestore/devtools-web-common/worker'
 import { isDevEnv, isNotUndefined, LS_DEV } from '@livestore/utils'
 import {
   BrowserWorker,
   BrowserWorkerRunner,
   Deferred,
-  Duration,
   Effect,
   Exit,
   FetchHttpClient,
@@ -15,7 +14,6 @@ import {
   Logger,
   LogLevel,
   ParseResult,
-  Queue,
   Ref,
   Schema,
   Scope,
@@ -65,6 +63,7 @@ const makeWorkerRunner = Effect.gen(function* () {
       Effect.andThen((worker) => worker.executeEffect(req) as Effect.Effect<unknown, unknown, never>),
       // Effect.tap((_) => Effect.log(`forwardRequest: ${req._tag}`, _)),
       // Effect.tapError((cause) => Effect.logError(`forwardRequest err: ${req._tag}`, cause)),
+      Effect.interruptible,
       Effect.logWarnIfTakesLongerThan({
         label: `@livestore/adapter-web:shared-worker:forwardRequest:${req._tag}`,
         duration: 500,
@@ -80,57 +79,35 @@ const makeWorkerRunner = Effect.gen(function* () {
       Effect.tapCauseLogPretty,
     ) as any
 
-  // const forwardRequestStream = <TReq extends WorkerSchema.DedicatedWorkerInner.Request>(
-  //   req: TReq,
-  // ): TReq extends Serializable.WithResult<infer A, infer _I, infer _E, infer _EI, infer _R>
-  //   ? Stream.Stream<A, UnexpectedError, never>
-  //   : never =>
-  //   waitForWorker.pipe(
-  //     Effect.logBefore(`forwardRequestStream: ${req._tag}`),
-  //     Effect.andThen((worker) => worker.execute(req) as Stream.Stream<unknown, unknown, never>),
-  //     Effect.interruptible,
-  //     UnexpectedError.mapToUnexpectedError,
-  //     Effect.tapCauseLogPretty,
-  //     Stream.unwrap,
-  //     Stream.ensuring(Effect.logDebug(`shutting down stream for ${req._tag}`)),
-  //     UnexpectedError.mapToUnexpectedErrorStream,
-  //   ) as any
-
-  // TODO bring back the `forwardRequestStream` impl above. Needs debugging with Tim Smart
-  // It seems the in-progress streams are not being closed properly if the worker is closed (e.g. by closing the leader tab)
   const forwardRequestStream = <TReq extends WorkerSchema.LeaderWorkerInner.Request>(
     req: TReq,
   ): TReq extends Schema.WithResult<infer A, infer _I, infer _E, infer _EI, infer _R>
     ? Stream.Stream<A, UnexpectedError, never>
     : never =>
     Effect.gen(function* () {
+      yield* Effect.logDebug(`forwardRequestStream: ${req._tag}`)
       const { worker, scope } = yield* SubscriptionRef.waitUntil(leaderWorkerContextSubRef, isNotUndefined)
-      const queue = yield* Queue.unbounded()
+      const stream = worker.execute(req) as Stream.Stream<unknown, unknown, never>
 
-      yield* Scope.addFinalizer(scope, Queue.shutdown(queue))
+      // It seems the request stream is not automatically interrupted when the scope shuts down
+      // so we need to manually interrupt it when the scope shuts down
+      const shutdownDeferred = yield* Deferred.make<void>()
+      yield* Scope.addFinalizer(scope, Deferred.succeed(shutdownDeferred, undefined))
 
-      const workerStream = worker.execute(req) as Stream.Stream<unknown, unknown, never>
+      // Here we're creating an empty stream that will finish when the scope shuts down
+      const scopeShutdownStream = Effect.gen(function* () {
+        yield* shutdownDeferred
+        return Stream.empty
+      }).pipe(Stream.unwrap)
 
-      yield* workerStream.pipe(
-        Stream.tap((_) => Queue.offer(queue, _)),
-        Stream.runDrain,
-        Effect.interruptible,
-        Effect.forkIn(scope),
-      )
-
-      return Stream.fromQueue(queue)
+      return Stream.merge(stream, scopeShutdownStream, { haltStrategy: 'either' })
     }).pipe(
+      Effect.interruptible,
       UnexpectedError.mapToUnexpectedError,
       Effect.tapCauseLogPretty,
       Stream.unwrap,
-      Stream.mapError((cause) =>
-        Schema.is(UnexpectedError)(cause)
-          ? cause
-          : ParseResult.isParseError(cause) || Schema.is(WorkerError.WorkerError)(cause)
-            ? new UnexpectedError({ cause })
-            : cause,
-      ),
-      // Stream.ensuring(Effect.logDebug(`shutting down stream for ${req._tag}`)),
+      Stream.ensuring(Effect.logDebug(`shutting down stream for ${req._tag}`)),
+      UnexpectedError.mapToUnexpectedErrorStream,
     ) as any
 
   const resetCurrentWorkerCtx = Effect.gen(function* () {
@@ -142,14 +119,10 @@ const makeWorkerRunner = Effect.gen(function* () {
       yield* Effect.yieldNow()
 
       yield* Scope.close(prevWorker.scope, Exit.void).pipe(
-        // TODO there still seem to be scenarios where it takes longer than 1 second which is leading to problems
-        Effect.timeout(Duration.seconds(1)),
         Effect.logWarnIfTakesLongerThan({
           label: '@livestore/adapter-web:shared-worker:close-previous-worker',
           duration: 500,
         }),
-        // Effect.catchTag('TimeoutException', () => Scope.close(prevWorker.scope, Exit.fail('boom'))),
-        Effect.ignoreLogged,
       )
     }
   }).pipe(Effect.withSpan('@livestore/adapter-web:shared-worker:resetCurrentWorkerCtx'))
@@ -208,26 +181,6 @@ const makeWorkerRunner = Effect.gen(function* () {
 
         const scope = yield* Scope.make()
 
-        const workerDeferred = yield* Deferred.make<
-          Worker.SerializedWorkerPool<WorkerSchema.LeaderWorkerInner.Request>,
-          UnexpectedError
-        >()
-        // TODO we could also keep the pool instance around to re-use it by removing the previous worker and adding a new one
-        yield* Worker.makePoolSerialized<WorkerSchema.LeaderWorkerInner.Request>({
-          size: 1,
-          concurrency: 100,
-          initialMessage: () => initialMessagePayload.initialMessage,
-        }).pipe(
-          Effect.tap((worker) => Deferred.succeed(workerDeferred, worker)),
-          Effect.provide(BrowserWorker.layer(() => port)),
-          Effect.catchAllCause((cause) => new UnexpectedError({ cause })),
-          Effect.tapError((cause) => Deferred.fail(workerDeferred, cause)),
-          Effect.withSpan('@livestore/adapter-web:shared-worker:makeWorkerProxyFromPort'),
-          Effect.tapCauseLogPretty,
-          Scope.extend(scope),
-          Effect.forkIn(scope),
-        )
-
         yield* Effect.gen(function* () {
           const shutdownChannel = yield* makeShutdownChannel(initialMessagePayload.initialMessage.storeId)
 
@@ -235,22 +188,32 @@ const makeWorkerRunner = Effect.gen(function* () {
             Stream.flatten(),
             Stream.tap(() => reset),
             Stream.runDrain,
+            Effect.tapCauseLogPretty,
+            Effect.forkScoped,
           )
-        }).pipe(Effect.tapCauseLogPretty, Scope.extend(scope), Effect.forkIn(scope))
 
-        const worker = yield* workerDeferred
+          const workerLayer = yield* Layer.build(BrowserWorker.layer(() => port))
 
-        // Prepare the web mesh connection for leader worker to be able to connect to the devtools
-        const { node } = yield* WebMeshWorker.CacheService
-        const { storeId, clientId } = initialMessagePayload.initialMessage
+          const worker = yield* Worker.makePoolSerialized<WorkerSchema.LeaderWorkerInner.Request>({
+            size: 1,
+            concurrency: 100,
+            initialMessage: () => initialMessagePayload.initialMessage,
+          }).pipe(
+            Effect.provide(workerLayer),
+            Effect.withSpan('@livestore/adapter-web:shared-worker:makeWorkerProxyFromPort'),
+          )
 
-        yield* connectViaWorker({ node, worker, target: `leader-${storeId}-${clientId}` }).pipe(
-          Effect.tapCauseLogPretty,
-          Scope.extend(scope),
-          Effect.forkIn(scope),
-        )
+          // Prepare the web mesh connection for leader worker to be able to connect to the devtools
+          const { node } = yield* WebmeshWorker.CacheService
+          const { storeId, clientId } = initialMessagePayload.initialMessage
 
-        yield* SubscriptionRef.set(leaderWorkerContextSubRef, { worker, scope })
+          yield* connectViaWorker({ node, worker, target: `leader-${storeId}-${clientId}` }).pipe(
+            Effect.tapCauseLogPretty,
+            Effect.forkScoped,
+          )
+
+          yield* SubscriptionRef.set(leaderWorkerContextSubRef, { worker, scope })
+        }).pipe(Effect.tapCauseLogPretty, Scope.extend(scope), Effect.forkIn(scope))
       }).pipe(
         Effect.withSpan('@livestore/adapter-web:shared-worker:updateMessagePort'),
         UnexpectedError.mapToUnexpectedError,
@@ -272,9 +235,7 @@ const makeWorkerRunner = Effect.gen(function* () {
     ExtraDevtoolsMessage: forwardRequest,
 
     // Accept devtools connections (from leader and client sessions)
-    'DevtoolsWebCommon.CreateConnection': WebMeshWorker.CreateConnection,
-
-    // ...devtoolsWebBridge.handlers,
+    'DevtoolsWebCommon.CreateConnection': WebmeshWorker.CreateConnection,
   })
 }).pipe(Layer.unwrapScoped)
 
@@ -288,7 +249,7 @@ export const makeWorker = () => {
     Effect.annotateLogs({ thread: self.name }),
     Effect.provide(Logger.prettyWithThread(self.name)),
     Effect.provide(FetchHttpClient.layer),
-    Effect.provide(WebMeshWorker.CacheService.layer({ nodeName: 'shared-worker' })),
+    Effect.provide(WebmeshWorker.CacheService.layer({ nodeName: 'shared-worker' })),
     LS_DEV ? TaskTracing.withAsyncTaggingTracing((name) => (console as any).createTask(name)) : identity,
     // TODO remove type-cast (currently needed to silence a tsc bug)
     (_) => _ as any as Effect.Effect<void, any>,