@livestore/adapter-node 0.4.0 → 0.5.0-dev.0

package/src/make-leader-worker.ts

@@ -6,8 +6,6 @@ if (process.execArgv.includes('--inspect') === true) {
   inspector.waitForDebugger()
 }
 
-import type * as otel from '@opentelemetry/api'
-
 import type { SyncOptions } from '@livestore/common'
 import { LogConfig, UnknownError } from '@livestore/common'
 import type { StreamEventsOptions } from '@livestore/common/leader-thread'
@@ -16,8 +14,22 @@ import type { LiveStoreSchema } from '@livestore/common/schema'
 import { LiveStoreEvent } from '@livestore/common/schema'
 import { loadSqlite3Wasm } from '@livestore/sqlite-wasm/load-wasm'
 import { sqliteDbFactory } from '@livestore/sqlite-wasm/node'
-import { Effect, FetchHttpClient, Layer, OtelTracer, Schema, Stream, WorkerRunner } from '@livestore/utils/effect'
+import { isDevEnv } from '@livestore/utils'
+import {
+  Effect,
+  FetchHttpClient,
+  Layer,
+  OtelTracer,
+  Queue,
+  References,
+  RpcServer,
+  RpcWorker,
+  Schema,
+  Scope,
+  Stream,
+} from '@livestore/utils/effect'
 import { PlatformNode } from '@livestore/utils/node'
+import type * as otel from '@opentelemetry/api'
 
 import type { TestingOverrides } from './leader-thread-shared.ts'
 import { makeLeaderThread } from './leader-thread-shared.ts'
@@ -33,7 +45,7 @@ export type WorkerOptions = {
     serviceName?: string
   }
   testing?: TestingOverrides
-} & LogConfig.WithLoggerOptions
+} & LogConfig.LoggerOptions
 
 export const getWorkerArgs = () => Schema.decodeSync(WorkerSchema.WorkerArgv)(process.argv[2]!)
 
@@ -42,139 +54,163 @@ export const makeWorker = (options: WorkerOptions) => {
 }
 
 export const makeWorkerEffect = (options: WorkerOptions) => {
-  const TracingLive = options.otelOptions?.tracer !== undefined
-    ? Layer.unwrapEffect(Effect.map(OtelTracer.make, Layer.setTracer)).pipe(
-        Layer.provideMerge(Layer.succeed(OtelTracer.OtelTracer, options.otelOptions.tracer)),
-      )
-    : undefined
+  const TracingLive =
+    options.otelOptions?.tracer !== undefined
+      ? OtelTracer.layerWithoutOtelTracer.pipe(
+          Layer.provideMerge(Layer.succeed(OtelTracer.OtelTracer, options.otelOptions.tracer)),
+        )
+      : Layer.empty
 
   // Merge the runtime dependencies once so we can provide them together without chaining Effect.provide.
-  const runtimeLayer = Layer.mergeAll(
-    FetchHttpClient.layer,
-    PlatformNode.NodeFileSystem.layer,
-    TracingLive ?? Layer.empty,
-  )
+  const runtimeLayer = Layer.mergeAll(FetchHttpClient.layer, PlatformNode.NodeFileSystem.layer, TracingLive)
 
-  return WorkerRunner.layerSerialized(WorkerSchema.LeaderWorkerInnerRequest, {
-    InitialMessage: (args) =>
-      Effect.gen(function* () {
-        const sqlite3 = yield* Effect.promise(() => loadSqlite3Wasm()).pipe(
-          Effect.withSpan('@livestore/adapter-node:leader-thread:loadSqlite3Wasm'),
-        )
-        const makeSqliteDb = yield* sqliteDbFactory({ sqlite3 })
-        return yield* makeLeaderThread({
-          ...args,
-          syncOptions: options.sync,
-          schema: options.schema,
-          testing: options.testing,
-          makeSqliteDb,
-          syncPayloadEncoded: args.syncPayloadEncoded,
-          syncPayloadSchema: options.syncPayloadSchema,
-        })
-      }).pipe(Layer.unwrapScoped),
-    PushToLeader: ({ batch }) =>
-      Effect.andThen(LeaderThreadCtx, (_) =>
-        _.syncProcessor.push(
-          batch.map((item) => new LiveStoreEvent.Client.EncodedWithMeta(item)),
-          // We'll wait in order to keep back pressure on the client session
-          { waitForProcessing: true },
-        ),
-      ).pipe(Effect.uninterruptible, Effect.withSpan('@livestore/adapter-node:worker:PushToLeader')),
-    BootStatusStream: () =>
-      Effect.andThen(LeaderThreadCtx, (_) => Stream.fromQueue(_.bootStatusQueue)).pipe(Stream.unwrap),
-    PullStream: ({ cursor }) =>
-      Effect.gen(function* () {
-        const { syncProcessor } = yield* LeaderThreadCtx
-        return syncProcessor.pull({ cursor })
-      }).pipe(Stream.unwrapScoped),
-    StreamEvents: (options: WorkerSchema.LeaderWorkerInnerStreamEvents) =>
-      LeaderThreadCtx.pipe(
-        Effect.map(({ dbEventlog, syncProcessor }) => {
-          const { _tag: _ignored, ...payload } = options
-          const streamOptions = payload as StreamEventsOptions
-          return streamEventsWithSyncState({
-            dbEventlog,
-            syncState: syncProcessor.syncState,
-            options: streamOptions,
-          })
-        }),
-        Stream.unwrapScoped,
-        Stream.withSpan('@livestore/adapter-node:worker:StreamEvents'),
-      ),
-    Export: () =>
-      Effect.andThen(LeaderThreadCtx, (_) => _.dbState.export()).pipe(
-        Effect.withSpan('@livestore/adapter-node:worker:Export'),
-      ),
-    ExportEventlog: () =>
-      Effect.andThen(LeaderThreadCtx, (_) => _.dbEventlog.export()).pipe(
-        Effect.withSpan('@livestore/adapter-node:worker:ExportEventlog'),
-      ),
-    GetLeaderHead: Effect.fn('@livestore/adapter-node:worker:GetLeaderHead')(function* () {
-      const workerCtx = yield* LeaderThreadCtx
-      return Eventlog.getClientHeadFromDb(workerCtx.dbEventlog)
-    }),
-    GetLeaderSyncState: Effect.fn('@livestore/adapter-node:worker:GetLeaderSyncState')(function* () {
-      const workerCtx = yield* LeaderThreadCtx
-      return yield* workerCtx.syncProcessor.syncState
-    }),
-    SyncStateStream: () =>
-      Effect.gen(function* () {
-        const workerCtx = yield* LeaderThreadCtx
-        return workerCtx.syncProcessor.syncState.changes
-      }).pipe(Stream.unwrapScoped),
-    GetNetworkStatus: Effect.fn('@livestore/adapter-node:worker:GetNetworkStatus')(function* () {
-      const workerCtx = yield* LeaderThreadCtx
-      return yield* workerCtx.networkStatus
-    }),
-    NetworkStatusStream: () =>
-      Effect.gen(function* () {
-        const workerCtx = yield* LeaderThreadCtx
-        return workerCtx.networkStatus.changes
-      }).pipe(Stream.unwrapScoped),
-    GetRecreateSnapshot: Effect.fn('@livestore/adapter-node:worker:GetRecreateSnapshot')(function* () {
-      const workerCtx = yield* LeaderThreadCtx
-      // const result = yield* Deferred.await(workerCtx.initialSetupDeferred)
-      // NOTE we can only return the cached snapshot once as it's transferred (i.e. disposed), so we need to set it to undefined
-      // const cachedSnapshot =
-      //   result._tag === 'Recreate' ? yield* Ref.getAndSet(result.snapshotRef, undefined) : undefined
-      // return cachedSnapshot ?? workerCtx.db.export()
-      const snapshot = workerCtx.dbState.export()
-      return { snapshot, migrationsReport: workerCtx.initialState.migrationsReport }
-    }),
-    Shutdown: Effect.fn('@livestore/adapter-node:worker:Shutdown')(function* () {
-      // const { db, dbEventlog } = yield* LeaderThreadCtx
-      yield* Effect.logDebug('[@livestore/adapter-node:worker] Shutdown')
-
-      // if (devtools.enabled) {
-      //   yield* FiberSet.clear(devtools.connections)
-      // }
-      // db.close()
-      // dbEventlog.close()
-
-      // Buy some time for Otel to flush
-      // TODO find a cleaner way to do this
-      // yield* Effect.sleep(1000)
-    }),
-    ExtraDevtoolsMessage: ({ message }) =>
-      Effect.andThen(LeaderThreadCtx, (_) => _.extraIncomingMessagesQueue.offer(message)).pipe(
-        Effect.withSpan('@livestore/adapter-node:worker:ExtraDevtoolsMessage'),
-      ),
-  }).pipe(
-    Layer.provide(PlatformNode.NodeWorkerRunner.layer),
-    WorkerRunner.launch,
+  return RpcServer.make(WorkerSchema.LeaderWorkerInnerRpcs).pipe(
+    Effect.provide(makeWorkerRunnerInner(options)),
+    Effect.provide(RpcServer.layerProtocolWorkerRunner),
+    Effect.provide(PlatformNode.NodeWorkerRunner.layer),
     Effect.scoped,
     Effect.tapCauseLogPretty,
     Effect.annotateLogs({
       thread: options.otelOptions?.serviceName ?? 'livestore-node-leader-thread',
       processId: process.pid,
     }),
-    LogConfig.withLoggerConfig(
-      { logger: options.logger, logLevel: options.logLevel },
-      { threadName: options.otelOptions?.serviceName ?? 'livestore-node-leader-thread' },
+    Effect.provide(
+      Layer.mergeAll(
+        options.logger ?? Layer.empty,
+        Layer.succeed(References.MinimumLogLevel, options.logLevel ?? (isDevEnv() === true ? 'Debug' : 'Info')),
+      ),
     ),
-    // TODO bring back with Effect 4 once it's easier to work with replacing loggers.
-    // We basically only want to provide this logger if it's replacing the default logger, not if there's a custom logger already provided.
-    // Effect.provide(Logger.prettyWithThread(options.otelOptions?.serviceName ?? 'livestore-node-leader-thread')),
     Effect.provide(runtimeLayer),
   )
 }
+
+const makeWorkerRunnerInner = (options: WorkerOptions) =>
+  WorkerSchema.LeaderWorkerInnerRpcs.toLayer(
+    Effect.gen(function* () {
+      const leaderThreadScope = yield* Scope.make()
+      yield* Effect.addFinalizer((exit) => Scope.close(leaderThreadScope, exit))
+
+      const leaderThreadContextOnce = yield* Effect.cached(
+        Effect.gen(function* () {
+          const { storeId, clientId, storage, devtools, syncPayloadEncoded } = yield* RpcWorker.initialMessage(
+            WorkerSchema.LeaderWorkerInnerInitialMessage.payloadSchema,
+          )
+
+          const sqlite3 = yield* Effect.promise(() => loadSqlite3Wasm()).pipe(
+            Effect.withSpan('@livestore/adapter-node:leader-thread:loadSqlite3Wasm'),
+          )
+          const makeSqliteDb = yield* sqliteDbFactory({ sqlite3 })
+
+          const layer = yield* makeLeaderThread({
+            storeId,
+            clientId,
+            storage,
+            devtools,
+            syncOptions: options.sync,
+            schema: options.schema,
+            testing: options.testing,
+            makeSqliteDb,
+            syncPayloadEncoded,
+            syncPayloadSchema: options.syncPayloadSchema,
+          })
+
+          return yield* Layer.buildWithScope(layer, leaderThreadScope)
+        }).pipe(
+          Scope.provide(leaderThreadScope),
+          Effect.tapCauseLogPretty,
+          UnknownError.mapToUnknownError,
+          Effect.withSpan('@livestore/adapter-node:worker:InitialMessage'),
+          Effect.orDie,
+        ),
+      )
+
+      const provideLeaderThread = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
+        Effect.gen(function* () {
+          const leaderThreadContext = yield* leaderThreadContextOnce
+          return yield* effect.pipe(Effect.provide(leaderThreadContext))
+        })
+
+      return WorkerSchema.LeaderWorkerInnerRpcs.of({
+        GetRecreateSnapshot: () =>
+          Effect.gen(function* () {
+            const workerCtx = yield* LeaderThreadCtx
+            const snapshot = workerCtx.dbState.export()
+            return { snapshot, migrationsReport: workerCtx.initialState.migrationsReport }
+          }).pipe(provideLeaderThread, Effect.withSpan('@livestore/adapter-node:worker:GetRecreateSnapshot')),
+        PullStream: ({ cursor }) =>
+          Effect.gen(function* () {
+            const { syncProcessor } = yield* LeaderThreadCtx
+            return syncProcessor.pull({ cursor })
+          }).pipe(provideLeaderThread, Stream.unwrap),
+        PushToLeader: ({ batch }) =>
+          Effect.andThen(LeaderThreadCtx, ({ syncProcessor }) =>
+            syncProcessor.push(batch.map((event) => new LiveStoreEvent.Client.EncodedWithMeta(event))),
+          ).pipe(provideLeaderThread, Effect.uninterruptible, Effect.withSpan('@livestore/adapter-node:worker:PushToLeader')),
+        StreamEvents: (options) =>
+          LeaderThreadCtx.pipe(
+            Effect.map(({ dbEventlog, syncProcessor }) =>
+              streamEventsWithSyncState({
+                dbEventlog,
+                syncState: syncProcessor.syncState,
+                options: options as StreamEventsOptions,
+              }),
+            ),
+            provideLeaderThread,
+            Stream.unwrap,
+            Stream.withSpan('@livestore/adapter-node:worker:StreamEvents'),
+          ),
+        Export: () =>
+          LeaderThreadCtx.pipe(
+            Effect.flatMap((_) => Effect.sync(() => _.dbState.export())),
+            provideLeaderThread,
+            Effect.withSpan('@livestore/adapter-node:worker:Export'),
+          ),
+        ExportEventlog: () =>
+          LeaderThreadCtx.pipe(
+            Effect.flatMap((_) => Effect.sync(() => _.dbEventlog.export())),
+            provideLeaderThread,
+            Effect.withSpan('@livestore/adapter-node:worker:ExportEventlog'),
+          ),
+        BootStatusStream: () =>
+          LeaderThreadCtx.pipe(
+            Effect.map((_) => Stream.fromQueue(_.bootStatusQueue)),
+            provideLeaderThread,
+            Stream.unwrap,
+          ),
+        GetLeaderHead: () =>
+          Effect.gen(function* () {
+            const workerCtx = yield* LeaderThreadCtx
+            return Eventlog.getClientHeadFromDb(workerCtx.dbEventlog)
+          }).pipe(provideLeaderThread, Effect.withSpan('@livestore/adapter-node:worker:GetLeaderHead')),
+        GetLeaderSyncState: () =>
+          Effect.gen(function* () {
+            const workerCtx = yield* LeaderThreadCtx
+            return yield* workerCtx.syncProcessor.syncState
+          }).pipe(provideLeaderThread, Effect.withSpan('@livestore/adapter-node:worker:GetLeaderSyncState')),
+        SyncStateStream: () =>
+          Effect.gen(function* () {
+            const workerCtx = yield* LeaderThreadCtx
+            return workerCtx.syncProcessor.syncState.changes
+          }).pipe(provideLeaderThread, Stream.unwrap),
+        GetNetworkStatus: () =>
+          Effect.gen(function* () {
+            const workerCtx = yield* LeaderThreadCtx
+            return yield* workerCtx.networkStatus
+          }).pipe(provideLeaderThread, Effect.withSpan('@livestore/adapter-node:worker:GetNetworkStatus')),
+        NetworkStatusStream: () =>
+          Effect.gen(function* () {
+            const workerCtx = yield* LeaderThreadCtx
+            return workerCtx.networkStatus.changes
+          }).pipe(provideLeaderThread, Stream.unwrap),
+        Shutdown: Effect.fn('@livestore/adapter-node:worker:Shutdown')(function* () {
+          yield* Effect.logDebug('[@livestore/adapter-node:worker] Shutdown')
+        }),
+        ExtraDevtoolsMessage: ({ message }) =>
+          Effect.andThen(LeaderThreadCtx, (_) => Queue.offer(_.extraIncomingMessagesQueue, message)).pipe(
+            provideLeaderThread,
+            Effect.asVoid,
+            Effect.withSpan('@livestore/adapter-node:worker:ExtraDevtoolsMessage'),
+          ),
+      })
+    }),
+  )

package/src/rpc-worker.ts

@@ -0,0 +1,28 @@
+import { Effect, RpcClientError, Schema, Stream } from '@livestore/utils/effect'
+
+export type WithoutRpcClientError<E> = E extends { readonly _tag: infer Tag }
+  ? 'RpcClientError' extends Tag
+    ? never
+    : E
+  : E
+
+export const isRpcClientError = (error: unknown): error is RpcClientError.RpcClientError =>
+  Schema.is(RpcClientError.RpcClientError)(error)
+
+export const dieOnRpcClientError = <A, E, R>(
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, WithoutRpcClientError<E>, R> =>
+  (effect as Effect.Effect<A, RpcClientError.RpcClientError | WithoutRpcClientError<E>, R>).pipe(
+    Effect.catchIf(isRpcClientError, (error) => Effect.die(error)),
+  ) as Effect.Effect<A, WithoutRpcClientError<E>, R>
+
+export const dieOnRpcClientErrorStream = <A, E, R>(
+  stream: Stream.Stream<A, E, R>,
+): Stream.Stream<A, WithoutRpcClientError<E>, R> =>
+  (stream as Stream.Stream<A, RpcClientError.RpcClientError | WithoutRpcClientError<E>, R>).pipe(
+    Stream.catchIf(
+      isRpcClientError,
+      (error) => Stream.die(error),
+      (error) => Stream.fail(error),
+    ),
+  ) as Stream.Stream<A, WithoutRpcClientError<E>, R>

package/src/webchannel.ts

@@ -1,14 +1,13 @@
 import type { BroadcastChannel as NodeBroadcastChannel } from 'node:worker_threads'
 
-import type { Either, ParseResult } from '@livestore/utils/effect'
-import { Deferred, Effect, Exit, Schema, Scope, Stream, WebChannel } from '@livestore/utils/effect'
+import { Deferred, Effect, Exit, Queue, Schema, Scope, Stream, WebChannel } from '@livestore/utils/effect'
 
 export const makeBroadcastChannel = <Msg, MsgEncoded>({
   channelName,
   schema,
 }: {
   channelName: string
-  schema: Schema.Schema<Msg, MsgEncoded>
+  schema: Schema.Codec<Msg, MsgEncoded>
 }): Effect.Effect<WebChannel.WebChannel<Msg, Msg>, never, Scope.Scope> =>
   Effect.scopeWithCloseable((scope) =>
     Effect.gen(function* () {
@@ -21,39 +20,33 @@ export const makeBroadcastChannel = <Msg, MsgEncoded>({
       // from `node:worker_threads` is not yet stable in Deno
       const channel = new globalThis.BroadcastChannel(channelName) as any as NodeBroadcastChannel
 
-      yield* Effect.addFinalizer(() => Effect.try(() => channel.close()).pipe(Effect.ignoreLogged))
+      yield* Effect.addFinalizer(() =>
+        Effect.try(() => {
+          // NOTE not all BroadcastChannel implementations have the `unref` method
+          if (typeof channel.unref === 'function') {
+            channel.unref()
+          }
+          channel.close()
+        }).pipe(Effect.ignore),
+      )
 
       const send = (message: Msg) =>
         Effect.gen(function* () {
-          const messageEncoded = yield* Schema.encode(schema)(message)
+          const messageEncoded = yield* Schema.encodeEffect(schema)(message)
           channel.postMessage(messageEncoded)
         })
 
-      // TODO also listen to `messageerror` in parallel
-      // const listen = Stream.fromEventListener<MessageEvent>(channel, 'message').pipe(
-      //   Stream.map((_) => Schema.decodeEither(listenSchema)(_.data)),
-      // )
-
-      const listen = Stream.asyncPush<Either.Either<Msg, ParseResult.ParseError>>((emit) =>
-        Effect.acquireRelease(
-          Effect.gen(function* () {
-            channel.onmessage = (event: any) => {
-              return emit.single(Schema.decodeEither(schema)(event.data))
-            }
+      // In Effect v4, forking a consumer is not a readiness handshake. Install the message
+      // listener while constructing the channel and buffer into a scoped queue so early messages
+      // are not lost before `listen` is pulled.
+      const messageQueue = yield* Effect.acquireRelease(Queue.unbounded<unknown>(), Queue.shutdown)
+      channel.onmessage = (event: any) => {
+        Queue.offerUnsafe(messageQueue, event.data)
+      }
 
-            return channel
-          }),
-          (channel) =>
-            Effect.sync(() => {
-              // NOTE not all BroadcastChannel implementations have the `unref` method
-              if (typeof channel.unref === 'function') {
-                channel.unref()
-              }
-            }),
-        ),
-      )
+      const listen = Stream.fromQueue(messageQueue).pipe(Stream.map((data) => Schema.decodeUnknownResult(schema)(data)))
 
-      const closedDeferred = yield* Deferred.make<void>().pipe(Effect.acquireRelease(Deferred.done(Exit.void)))
+      const closedDeferred = yield* Effect.acquireRelease(Deferred.make<void>(), Deferred.done(Exit.void))
       const supportsTransferables = false
 
       return {