@livestore/adapter-node 0.4.0-dev.9 → 0.5.0-dev.0

package/src/make-leader-worker.ts

@@ -1,30 +1,32 @@
 import './thread-polyfill.ts'
-
 import inspector from 'node:inspector'
 
-if (process.execArgv.includes('--inspect')) {
+if (process.execArgv.includes('--inspect') === true) {
   inspector.open()
   inspector.waitForDebugger()
 }
 
 import type { SyncOptions } from '@livestore/common'
-import { UnexpectedError } from '@livestore/common'
-import { Eventlog, LeaderThreadCtx } from '@livestore/common/leader-thread'
+import { LogConfig, UnknownError } from '@livestore/common'
+import type { StreamEventsOptions } from '@livestore/common/leader-thread'
+import { Eventlog, LeaderThreadCtx, streamEventsWithSyncState } from '@livestore/common/leader-thread'
 import type { LiveStoreSchema } from '@livestore/common/schema'
 import { LiveStoreEvent } from '@livestore/common/schema'
 import { loadSqlite3Wasm } from '@livestore/sqlite-wasm/load-wasm'
 import { sqliteDbFactory } from '@livestore/sqlite-wasm/node'
+import { isDevEnv } from '@livestore/utils'
 import {
   Effect,
   FetchHttpClient,
-  identity,
   Layer,
-  Logger,
-  LogLevel,
   OtelTracer,
+  Queue,
+  References,
+  RpcServer,
+  RpcWorker,
   Schema,
+  Scope,
   Stream,
-  WorkerRunner,
 } from '@livestore/utils/effect'
 import { PlatformNode } from '@livestore/utils/node'
 import type * as otel from '@opentelemetry/api'
@@ -36,132 +38,179 @@ import * as WorkerSchema from './worker-schema.ts'
 export type WorkerOptions = {
   schema: LiveStoreSchema
   sync?: SyncOptions
+  syncPayloadSchema?: Schema.Schema<any>
   otelOptions?: {
     tracer?: otel.Tracer
     /** @default 'livestore-node-leader-thread' */
     serviceName?: string
   }
   testing?: TestingOverrides
-}
+} & LogConfig.LoggerOptions
 
 export const getWorkerArgs = () => Schema.decodeSync(WorkerSchema.WorkerArgv)(process.argv[2]!)
 
 export const makeWorker = (options: WorkerOptions) => {
-  makeWorkerEffect(options).pipe(
-    Effect.provide(Logger.prettyWithThread(options.otelOptions?.serviceName ?? 'livestore-node-leader-thread')),
-    PlatformNode.NodeRuntime.runMain,
-  )
+  makeWorkerEffect(options).pipe(PlatformNode.NodeRuntime.runMain)
 }
 
 export const makeWorkerEffect = (options: WorkerOptions) => {
-  const TracingLive = options.otelOptions?.tracer
-    ? Layer.unwrapEffect(Effect.map(OtelTracer.make, Layer.setTracer)).pipe(
-        Layer.provideMerge(Layer.succeed(OtelTracer.OtelTracer, options.otelOptions.tracer)),
-      )
-    : undefined
-
-  return WorkerRunner.layerSerialized(WorkerSchema.LeaderWorkerInnerRequest, {
-    InitialMessage: (args) =>
-      Effect.gen(function* () {
-        const sqlite3 = yield* Effect.promise(() => loadSqlite3Wasm()).pipe(
-          Effect.withSpan('@livestore/adapter-node:leader-thread:loadSqlite3Wasm'),
+  const TracingLive =
+    options.otelOptions?.tracer !== undefined
+      ? OtelTracer.layerWithoutOtelTracer.pipe(
+          Layer.provideMerge(Layer.succeed(OtelTracer.OtelTracer, options.otelOptions.tracer)),
         )
-        const makeSqliteDb = yield* sqliteDbFactory({ sqlite3 })
-        return yield* makeLeaderThread({
-          ...args,
-          syncOptions: options.sync,
-          schema: options.schema,
-          testing: options.testing,
-          makeSqliteDb,
-        })
-      }).pipe(Layer.unwrapScoped),
-    PushToLeader: ({ batch }) =>
-      Effect.andThen(LeaderThreadCtx, (_) =>
-        _.syncProcessor.push(
-          batch.map((item) => new LiveStoreEvent.EncodedWithMeta(item)),
-          // We'll wait in order to keep back pressure on the client session
-          { waitForProcessing: true },
-        ),
-      ).pipe(Effect.uninterruptible, Effect.withSpan('@livestore/adapter-node:worker:PushToLeader')),
-    BootStatusStream: () =>
-      Effect.andThen(LeaderThreadCtx, (_) => Stream.fromQueue(_.bootStatusQueue)).pipe(Stream.unwrap),
-    PullStream: ({ cursor }) =>
-      Effect.gen(function* () {
-        const { syncProcessor } = yield* LeaderThreadCtx
-        return syncProcessor.pull({ cursor })
-      }).pipe(Stream.unwrapScoped),
-    Export: () =>
-      Effect.andThen(LeaderThreadCtx, (_) => _.dbState.export()).pipe(
-        UnexpectedError.mapToUnexpectedError,
-        Effect.withSpan('@livestore/adapter-node:worker:Export'),
-      ),
-    ExportEventlog: () =>
-      Effect.andThen(LeaderThreadCtx, (_) => _.dbEventlog.export()).pipe(
-        UnexpectedError.mapToUnexpectedError,
-        Effect.withSpan('@livestore/adapter-node:worker:ExportEventlog'),
-      ),
-    GetLeaderHead: () =>
-      Effect.gen(function* () {
-        const workerCtx = yield* LeaderThreadCtx
-        return Eventlog.getClientHeadFromDb(workerCtx.dbEventlog)
-      }).pipe(UnexpectedError.mapToUnexpectedError, Effect.withSpan('@livestore/adapter-node:worker:GetLeaderHead')),
-    GetLeaderSyncState: () =>
-      Effect.gen(function* () {
-        const workerCtx = yield* LeaderThreadCtx
-        return yield* workerCtx.syncProcessor.syncState
-      }).pipe(
-        UnexpectedError.mapToUnexpectedError,
-        Effect.withSpan('@livestore/adapter-node:worker:GetLeaderSyncState'),
-      ),
-    GetRecreateSnapshot: () =>
-      Effect.gen(function* () {
-        const workerCtx = yield* LeaderThreadCtx
-        // const result = yield* Deferred.await(workerCtx.initialSetupDeferred)
-        // NOTE we can only return the cached snapshot once as it's transferred (i.e. disposed), so we need to set it to undefined
-        // const cachedSnapshot =
-        //   result._tag === 'Recreate' ? yield* Ref.getAndSet(result.snapshotRef, undefined) : undefined
-        // return cachedSnapshot ?? workerCtx.db.export()
-        const snapshot = workerCtx.dbState.export()
-        return { snapshot, migrationsReport: workerCtx.initialState.migrationsReport }
-      }).pipe(
-        UnexpectedError.mapToUnexpectedError,
-        Effect.withSpan('@livestore/adapter-node:worker:GetRecreateSnapshot'),
-      ),
-    Shutdown: () =>
-      Effect.gen(function* () {
-        // const { db, dbEventlog } = yield* LeaderThreadCtx
-        yield* Effect.logDebug('[@livestore/adapter-node:worker] Shutdown')
-
-        // if (devtools.enabled) {
-        //   yield* FiberSet.clear(devtools.connections)
-        // }
-        // db.close()
-        // dbEventlog.close()
-
-        // Buy some time for Otel to flush
-        // TODO find a cleaner way to do this
-        // yield* Effect.sleep(1000)
-      }).pipe(UnexpectedError.mapToUnexpectedError, Effect.withSpan('@livestore/adapter-node:worker:Shutdown')),
-    ExtraDevtoolsMessage: ({ message }) =>
-      Effect.andThen(LeaderThreadCtx, (_) => _.extraIncomingMessagesQueue.offer(message)).pipe(
-        UnexpectedError.mapToUnexpectedError,
-        Effect.withSpan('@livestore/adapter-node:worker:ExtraDevtoolsMessage'),
-      ),
-  }).pipe(
-    Layer.provide(PlatformNode.NodeWorkerRunner.layer),
-    WorkerRunner.launch,
+      : Layer.empty
+
+  // Merge the runtime dependencies once so we can provide them together without chaining Effect.provide.
+  const runtimeLayer = Layer.mergeAll(FetchHttpClient.layer, PlatformNode.NodeFileSystem.layer, TracingLive)
+
+  return RpcServer.make(WorkerSchema.LeaderWorkerInnerRpcs).pipe(
+    Effect.provide(makeWorkerRunnerInner(options)),
+    Effect.provide(RpcServer.layerProtocolWorkerRunner),
+    Effect.provide(PlatformNode.NodeWorkerRunner.layer),
     Effect.scoped,
     Effect.tapCauseLogPretty,
     Effect.annotateLogs({
       thread: options.otelOptions?.serviceName ?? 'livestore-node-leader-thread',
       processId: process.pid,
     }),
-    // TODO bring back with Effect 4 once it's easier to work with replacing loggers.
-    // We basically only want to provide this logger if it's replacing the default logger, not if there's a custom logger already provided.
-    // Effect.provide(Logger.prettyWithThread(options.otelOptions?.serviceName ?? 'livestore-node-leader-thread')),
-    Effect.provide(FetchHttpClient.layer),
-    Effect.provide(PlatformNode.NodeFileSystem.layer),
-    TracingLive ? Effect.provide(TracingLive) : identity,
-    Logger.withMinimumLogLevel(LogLevel.Debug),
+    Effect.provide(
+      Layer.mergeAll(
+        options.logger ?? Layer.empty,
+        Layer.succeed(References.MinimumLogLevel, options.logLevel ?? (isDevEnv() === true ? 'Debug' : 'Info')),
+      ),
+    ),
+    Effect.provide(runtimeLayer),
   )
 }
+
+const makeWorkerRunnerInner = (options: WorkerOptions) =>
+  WorkerSchema.LeaderWorkerInnerRpcs.toLayer(
+    Effect.gen(function* () {
+      const leaderThreadScope = yield* Scope.make()
+      yield* Effect.addFinalizer((exit) => Scope.close(leaderThreadScope, exit))
+
+      const leaderThreadContextOnce = yield* Effect.cached(
+        Effect.gen(function* () {
+          const { storeId, clientId, storage, devtools, syncPayloadEncoded } = yield* RpcWorker.initialMessage(
+            WorkerSchema.LeaderWorkerInnerInitialMessage.payloadSchema,
+          )
+
+          const sqlite3 = yield* Effect.promise(() => loadSqlite3Wasm()).pipe(
+            Effect.withSpan('@livestore/adapter-node:leader-thread:loadSqlite3Wasm'),
+          )
+          const makeSqliteDb = yield* sqliteDbFactory({ sqlite3 })
+
+          const layer = yield* makeLeaderThread({
+            storeId,
+            clientId,
+            storage,
+            devtools,
+            syncOptions: options.sync,
+            schema: options.schema,
+            testing: options.testing,
+            makeSqliteDb,
+            syncPayloadEncoded,
+            syncPayloadSchema: options.syncPayloadSchema,
+          })
+
+          return yield* Layer.buildWithScope(layer, leaderThreadScope)
+        }).pipe(
+          Scope.provide(leaderThreadScope),
+          Effect.tapCauseLogPretty,
+          UnknownError.mapToUnknownError,
+          Effect.withSpan('@livestore/adapter-node:worker:InitialMessage'),
+          Effect.orDie,
+        ),
+      )
+
+      const provideLeaderThread = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
+        Effect.gen(function* () {
+          const leaderThreadContext = yield* leaderThreadContextOnce
+          return yield* effect.pipe(Effect.provide(leaderThreadContext))
+        })
+
+      return WorkerSchema.LeaderWorkerInnerRpcs.of({
+        GetRecreateSnapshot: () =>
+          Effect.gen(function* () {
+            const workerCtx = yield* LeaderThreadCtx
+            const snapshot = workerCtx.dbState.export()
+            return { snapshot, migrationsReport: workerCtx.initialState.migrationsReport }
+          }).pipe(provideLeaderThread, Effect.withSpan('@livestore/adapter-node:worker:GetRecreateSnapshot')),
+        PullStream: ({ cursor }) =>
+          Effect.gen(function* () {
+            const { syncProcessor } = yield* LeaderThreadCtx
+            return syncProcessor.pull({ cursor })
+          }).pipe(provideLeaderThread, Stream.unwrap),
+        PushToLeader: ({ batch }) =>
+          Effect.andThen(LeaderThreadCtx, ({ syncProcessor }) =>
+            syncProcessor.push(batch.map((event) => new LiveStoreEvent.Client.EncodedWithMeta(event))),
+          ).pipe(provideLeaderThread, Effect.uninterruptible, Effect.withSpan('@livestore/adapter-node:worker:PushToLeader')),
+        StreamEvents: (options) =>
+          LeaderThreadCtx.pipe(
+            Effect.map(({ dbEventlog, syncProcessor }) =>
+              streamEventsWithSyncState({
+                dbEventlog,
+                syncState: syncProcessor.syncState,
+                options: options as StreamEventsOptions,
+              }),
+            ),
+            provideLeaderThread,
+            Stream.unwrap,
+            Stream.withSpan('@livestore/adapter-node:worker:StreamEvents'),
+          ),
+        Export: () =>
+          LeaderThreadCtx.pipe(
+            Effect.flatMap((_) => Effect.sync(() => _.dbState.export())),
+            provideLeaderThread,
+            Effect.withSpan('@livestore/adapter-node:worker:Export'),
+          ),
+        ExportEventlog: () =>
+          LeaderThreadCtx.pipe(
+            Effect.flatMap((_) => Effect.sync(() => _.dbEventlog.export())),
+            provideLeaderThread,
+            Effect.withSpan('@livestore/adapter-node:worker:ExportEventlog'),
+          ),
+        BootStatusStream: () =>
+          LeaderThreadCtx.pipe(
+            Effect.map((_) => Stream.fromQueue(_.bootStatusQueue)),
+            provideLeaderThread,
+            Stream.unwrap,
+          ),
+        GetLeaderHead: () =>
+          Effect.gen(function* () {
+            const workerCtx = yield* LeaderThreadCtx
+            return Eventlog.getClientHeadFromDb(workerCtx.dbEventlog)
+          }).pipe(provideLeaderThread, Effect.withSpan('@livestore/adapter-node:worker:GetLeaderHead')),
+        GetLeaderSyncState: () =>
+          Effect.gen(function* () {
+            const workerCtx = yield* LeaderThreadCtx
+            return yield* workerCtx.syncProcessor.syncState
+          }).pipe(provideLeaderThread, Effect.withSpan('@livestore/adapter-node:worker:GetLeaderSyncState')),
+        SyncStateStream: () =>
+          Effect.gen(function* () {
+            const workerCtx = yield* LeaderThreadCtx
+            return workerCtx.syncProcessor.syncState.changes
+          }).pipe(provideLeaderThread, Stream.unwrap),
+        GetNetworkStatus: () =>
+          Effect.gen(function* () {
+            const workerCtx = yield* LeaderThreadCtx
+            return yield* workerCtx.networkStatus
+          }).pipe(provideLeaderThread, Effect.withSpan('@livestore/adapter-node:worker:GetNetworkStatus')),
+        NetworkStatusStream: () =>
+          Effect.gen(function* () {
+            const workerCtx = yield* LeaderThreadCtx
+            return workerCtx.networkStatus.changes
+          }).pipe(provideLeaderThread, Stream.unwrap),
+        Shutdown: Effect.fn('@livestore/adapter-node:worker:Shutdown')(function* () {
+          yield* Effect.logDebug('[@livestore/adapter-node:worker] Shutdown')
+        }),
+        ExtraDevtoolsMessage: ({ message }) =>
+          Effect.andThen(LeaderThreadCtx, (_) => Queue.offer(_.extraIncomingMessagesQueue, message)).pipe(
+            provideLeaderThread,
+            Effect.asVoid,
+            Effect.withSpan('@livestore/adapter-node:worker:ExtraDevtoolsMessage'),
+          ),
+      })
+    }),
+  )

package/src/rpc-worker.ts

@@ -0,0 +1,28 @@
+import { Effect, RpcClientError, Schema, Stream } from '@livestore/utils/effect'
+
+export type WithoutRpcClientError<E> = E extends { readonly _tag: infer Tag }
+  ? 'RpcClientError' extends Tag
+    ? never
+    : E
+  : E
+
+export const isRpcClientError = (error: unknown): error is RpcClientError.RpcClientError =>
+  Schema.is(RpcClientError.RpcClientError)(error)
+
+export const dieOnRpcClientError = <A, E, R>(
+  effect: Effect.Effect<A, E, R>,
+): Effect.Effect<A, WithoutRpcClientError<E>, R> =>
+  (effect as Effect.Effect<A, RpcClientError.RpcClientError | WithoutRpcClientError<E>, R>).pipe(
+    Effect.catchIf(isRpcClientError, (error) => Effect.die(error)),
+  ) as Effect.Effect<A, WithoutRpcClientError<E>, R>
+
+export const dieOnRpcClientErrorStream = <A, E, R>(
+  stream: Stream.Stream<A, E, R>,
+): Stream.Stream<A, WithoutRpcClientError<E>, R> =>
+  (stream as Stream.Stream<A, RpcClientError.RpcClientError | WithoutRpcClientError<E>, R>).pipe(
+    Stream.catchIf(
+      isRpcClientError,
+      (error) => Stream.die(error),
+      (error) => Stream.fail(error),
+    ),
+  ) as Stream.Stream<A, WithoutRpcClientError<E>, R>

package/src/shutdown-channel.ts

@@ -1,5 +1,6 @@
 import { ShutdownChannel } from '@livestore/common/leader-thread'
 import type { Effect, Scope } from '@livestore/utils/effect'
+
 import { makeBroadcastChannel } from './webchannel.ts'
 
 export const makeShutdownChannel = (

package/src/webchannel.ts

@@ -1,14 +1,13 @@
 import type { BroadcastChannel as NodeBroadcastChannel } from 'node:worker_threads'
 
-import type { Either, ParseResult } from '@livestore/utils/effect'
-import { Deferred, Effect, Exit, Schema, Scope, Stream, WebChannel } from '@livestore/utils/effect'
+import { Deferred, Effect, Exit, Queue, Schema, Scope, Stream, WebChannel } from '@livestore/utils/effect'
 
 export const makeBroadcastChannel = <Msg, MsgEncoded>({
   channelName,
   schema,
 }: {
   channelName: string
-  schema: Schema.Schema<Msg, MsgEncoded>
+  schema: Schema.Codec<Msg, MsgEncoded>
 }): Effect.Effect<WebChannel.WebChannel<Msg, Msg>, never, Scope.Scope> =>
   Effect.scopeWithCloseable((scope) =>
     Effect.gen(function* () {
@@ -21,39 +20,33 @@ export const makeBroadcastChannel = <Msg, MsgEncoded>({
       // from `node:worker_threads` is not yet stable in Deno
       const channel = new globalThis.BroadcastChannel(channelName) as any as NodeBroadcastChannel
 
-      yield* Effect.addFinalizer(() => Effect.try(() => channel.close()).pipe(Effect.ignoreLogged))
+      yield* Effect.addFinalizer(() =>
+        Effect.try(() => {
+          // NOTE not all BroadcastChannel implementations have the `unref` method
+          if (typeof channel.unref === 'function') {
+            channel.unref()
+          }
+          channel.close()
+        }).pipe(Effect.ignore),
+      )
 
       const send = (message: Msg) =>
         Effect.gen(function* () {
-          const messageEncoded = yield* Schema.encode(schema)(message)
+          const messageEncoded = yield* Schema.encodeEffect(schema)(message)
           channel.postMessage(messageEncoded)
         })
 
-      // TODO also listen to `messageerror` in parallel
-      // const listen = Stream.fromEventListener<MessageEvent>(channel, 'message').pipe(
-      //   Stream.map((_) => Schema.decodeEither(listenSchema)(_.data)),
-      // )
-
-      const listen = Stream.asyncPush<Either.Either<Msg, ParseResult.ParseError>>((emit) =>
-        Effect.acquireRelease(
-          Effect.gen(function* () {
-            channel.onmessage = (event: any) => {
-              return emit.single(Schema.decodeEither(schema)(event.data))
-            }
+      // In Effect v4, forking a consumer is not a readiness handshake. Install the message
+      // listener while constructing the channel and buffer into a scoped queue so early messages
+      // are not lost before `listen` is pulled.
+      const messageQueue = yield* Effect.acquireRelease(Queue.unbounded<unknown>(), Queue.shutdown)
+      channel.onmessage = (event: any) => {
+        Queue.offerUnsafe(messageQueue, event.data)
+      }
 
-            return channel
-          }),
-          (channel) =>
-            Effect.sync(() => {
-              // NOTE not all BroadcastChannel implementations have the `unref` method
-              if (typeof channel.unref === 'function') {
-                channel.unref()
-              }
-            }),
-        ),
-      )
+      const listen = Stream.fromQueue(messageQueue).pipe(Stream.map((data) => Schema.decodeUnknownResult(schema)(data)))
 
-      const closedDeferred = yield* Deferred.make<void>().pipe(Effect.acquireRelease(Deferred.done(Exit.void)))
+      const closedDeferred = yield* Effect.acquireRelease(Deferred.make<void>(), Deferred.done(Exit.void))
       const supportsTransferables = false
 
       return {