@livestore/adapter-node 0.4.0-dev.2 → 0.4.0-dev.21

package/src/make-leader-worker.ts

@@ -8,24 +8,14 @@ if (process.execArgv.includes('--inspect')) {
 }
 
 import type { SyncOptions } from '@livestore/common'
-import { UnexpectedError } from '@livestore/common'
-import { Eventlog, LeaderThreadCtx } from '@livestore/common/leader-thread'
+import { LogConfig, UnknownError } from '@livestore/common'
+import type { StreamEventsOptions } from '@livestore/common/leader-thread'
+import { Eventlog, LeaderThreadCtx, streamEventsWithSyncState } from '@livestore/common/leader-thread'
 import type { LiveStoreSchema } from '@livestore/common/schema'
 import { LiveStoreEvent } from '@livestore/common/schema'
 import { loadSqlite3Wasm } from '@livestore/sqlite-wasm/load-wasm'
 import { sqliteDbFactory } from '@livestore/sqlite-wasm/node'
-import {
-  Effect,
-  FetchHttpClient,
-  identity,
-  Layer,
-  Logger,
-  LogLevel,
-  OtelTracer,
-  Schema,
-  Stream,
-  WorkerRunner,
-} from '@livestore/utils/effect'
+import { Effect, FetchHttpClient, Layer, OtelTracer, Schema, Stream, WorkerRunner } from '@livestore/utils/effect'
 import { PlatformNode } from '@livestore/utils/node'
 import type * as otel from '@opentelemetry/api'
 
@@ -36,21 +26,19 @@ import * as WorkerSchema from './worker-schema.ts'
 export type WorkerOptions = {
   schema: LiveStoreSchema
   sync?: SyncOptions
+  syncPayloadSchema?: Schema.Schema<any>
   otelOptions?: {
     tracer?: otel.Tracer
     /** @default 'livestore-node-leader-thread' */
     serviceName?: string
   }
   testing?: TestingOverrides
-}
+} & LogConfig.WithLoggerOptions
 
 export const getWorkerArgs = () => Schema.decodeSync(WorkerSchema.WorkerArgv)(process.argv[2]!)
 
 export const makeWorker = (options: WorkerOptions) => {
-  makeWorkerEffect(options).pipe(
-    Effect.provide(Logger.prettyWithThread(options.otelOptions?.serviceName ?? 'livestore-node-leader-thread')),
-    PlatformNode.NodeRuntime.runMain,
-  )
+  makeWorkerEffect(options).pipe(PlatformNode.NodeRuntime.runMain)
 }
 
 export const makeWorkerEffect = (options: WorkerOptions) => {
@@ -60,6 +48,13 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
       )
     : undefined
 
+  // Merge the runtime dependencies once so we can provide them together without chaining Effect.provide.
+  const runtimeLayer = Layer.mergeAll(
+    FetchHttpClient.layer,
+    PlatformNode.NodeFileSystem.layer,
+    TracingLive ?? Layer.empty,
+  )
+
   return WorkerRunner.layerSerialized(WorkerSchema.LeaderWorkerInnerRequest, {
     InitialMessage: (args) =>
       Effect.gen(function* () {
@@ -73,12 +68,14 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
           schema: options.schema,
           testing: options.testing,
           makeSqliteDb,
+          syncPayloadEncoded: args.syncPayloadEncoded,
+          syncPayloadSchema: options.syncPayloadSchema,
         })
       }).pipe(Layer.unwrapScoped),
     PushToLeader: ({ batch }) =>
       Effect.andThen(LeaderThreadCtx, (_) =>
         _.syncProcessor.push(
-          batch.map((item) => new LiveStoreEvent.EncodedWithMeta(item)),
+          batch.map((item) => new LiveStoreEvent.Client.EncodedWithMeta(item)),
           // We'll wait in order to keep back pressure on the client session
           { waitForProcessing: true },
         ),
@@ -90,29 +87,55 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
         const { syncProcessor } = yield* LeaderThreadCtx
         return syncProcessor.pull({ cursor })
       }).pipe(Stream.unwrapScoped),
+    StreamEvents: (options: WorkerSchema.LeaderWorkerInnerStreamEvents) =>
+      LeaderThreadCtx.pipe(
+        Effect.map(({ dbEventlog, syncProcessor }) => {
+          const { _tag: _ignored, ...payload } = options
+          const streamOptions = payload as StreamEventsOptions
+          return streamEventsWithSyncState({
+            dbEventlog,
+            syncState: syncProcessor.syncState,
+            options: streamOptions,
+          })
+        }),
+        Stream.unwrapScoped,
+        Stream.withSpan('@livestore/adapter-node:worker:StreamEvents'),
+      ),
     Export: () =>
       Effect.andThen(LeaderThreadCtx, (_) => _.dbState.export()).pipe(
-        UnexpectedError.mapToUnexpectedError,
+        UnknownError.mapToUnknownError,
         Effect.withSpan('@livestore/adapter-node:worker:Export'),
       ),
     ExportEventlog: () =>
       Effect.andThen(LeaderThreadCtx, (_) => _.dbEventlog.export()).pipe(
-        UnexpectedError.mapToUnexpectedError,
+        UnknownError.mapToUnknownError,
         Effect.withSpan('@livestore/adapter-node:worker:ExportEventlog'),
       ),
     GetLeaderHead: () =>
       Effect.gen(function* () {
         const workerCtx = yield* LeaderThreadCtx
         return Eventlog.getClientHeadFromDb(workerCtx.dbEventlog)
-      }).pipe(UnexpectedError.mapToUnexpectedError, Effect.withSpan('@livestore/adapter-node:worker:GetLeaderHead')),
+      }).pipe(UnknownError.mapToUnknownError, Effect.withSpan('@livestore/adapter-node:worker:GetLeaderHead')),
     GetLeaderSyncState: () =>
       Effect.gen(function* () {
         const workerCtx = yield* LeaderThreadCtx
         return yield* workerCtx.syncProcessor.syncState
-      }).pipe(
-        UnexpectedError.mapToUnexpectedError,
-        Effect.withSpan('@livestore/adapter-node:worker:GetLeaderSyncState'),
-      ),
+      }).pipe(UnknownError.mapToUnknownError, Effect.withSpan('@livestore/adapter-node:worker:GetLeaderSyncState')),
+    SyncStateStream: () =>
+      Effect.gen(function* () {
+        const workerCtx = yield* LeaderThreadCtx
+        return workerCtx.syncProcessor.syncState.changes
+      }).pipe(Stream.unwrapScoped),
+    GetNetworkStatus: () =>
+      Effect.gen(function* () {
+        const workerCtx = yield* LeaderThreadCtx
+        return yield* workerCtx.networkStatus
+      }).pipe(UnknownError.mapToUnknownError, Effect.withSpan('@livestore/adapter-node:worker:GetNetworkStatus')),
+    NetworkStatusStream: () =>
+      Effect.gen(function* () {
+        const workerCtx = yield* LeaderThreadCtx
+        return workerCtx.networkStatus.changes
+      }).pipe(Stream.unwrapScoped),
     GetRecreateSnapshot: () =>
       Effect.gen(function* () {
         const workerCtx = yield* LeaderThreadCtx
@@ -123,11 +146,9 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
         // return cachedSnapshot ?? workerCtx.db.export()
         const snapshot = workerCtx.dbState.export()
         return { snapshot, migrationsReport: workerCtx.initialState.migrationsReport }
-      }).pipe(
-        UnexpectedError.mapToUnexpectedError,
-        Effect.withSpan('@livestore/adapter-node:worker:GetRecreateSnapshot'),
-      ),
+      }).pipe(UnknownError.mapToUnknownError, Effect.withSpan('@livestore/adapter-node:worker:GetRecreateSnapshot')),
     Shutdown: () =>
+      // @effect-diagnostics-next-line unnecessaryEffectGen:off
       Effect.gen(function* () {
         // const { db, dbEventlog } = yield* LeaderThreadCtx
         yield* Effect.logDebug('[@livestore/adapter-node:worker] Shutdown')
@@ -141,10 +162,10 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
         // Buy some time for Otel to flush
         // TODO find a cleaner way to do this
         // yield* Effect.sleep(1000)
-      }).pipe(UnexpectedError.mapToUnexpectedError, Effect.withSpan('@livestore/adapter-node:worker:Shutdown')),
+      }).pipe(UnknownError.mapToUnknownError, Effect.withSpan('@livestore/adapter-node:worker:Shutdown')),
     ExtraDevtoolsMessage: ({ message }) =>
       Effect.andThen(LeaderThreadCtx, (_) => _.extraIncomingMessagesQueue.offer(message)).pipe(
-        UnexpectedError.mapToUnexpectedError,
+        UnknownError.mapToUnknownError,
         Effect.withSpan('@livestore/adapter-node:worker:ExtraDevtoolsMessage'),
       ),
   }).pipe(
@@ -156,12 +177,13 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
       thread: options.otelOptions?.serviceName ?? 'livestore-node-leader-thread',
       processId: process.pid,
     }),
+    LogConfig.withLoggerConfig(
+      { logger: options.logger, logLevel: options.logLevel },
+      { threadName: options.otelOptions?.serviceName ?? 'livestore-node-leader-thread' },
+    ),
     // TODO bring back with Effect 4 once it's easier to work with replacing loggers.
     // We basically only want to provide this logger if it's replacing the default logger, not if there's a custom logger already provided.
     // Effect.provide(Logger.prettyWithThread(options.otelOptions?.serviceName ?? 'livestore-node-leader-thread')),
-    Effect.provide(FetchHttpClient.layer),
-    Effect.provide(PlatformNode.NodeFileSystem.layer),
-    TracingLive ? Effect.provide(TracingLive) : identity,
-    Logger.withMinimumLogLevel(LogLevel.Debug),
+    Effect.provide(runtimeLayer),
   )
 }

package/src/worker-schema.ts

@@ -1,4 +1,13 @@
-import { BootStatus, Devtools, LeaderAheadError, MigrationsReport, SyncState, UnexpectedError } from '@livestore/common'
+import {
+  BootStatus,
+  Devtools,
+  LeaderAheadError,
+  MigrationsReport,
+  SyncBackend,
+  SyncState,
+  UnknownError,
+} from '@livestore/common'
+import { StreamEventsOptionsFields } from '@livestore/common/leader-thread'
 import { EventSequenceNumber, LiveStoreEvent } from '@livestore/common/schema'
 import { Schema, Transferable } from '@livestore/utils/effect'
 
@@ -7,6 +16,7 @@ export const WorkerArgv = Schema.parseJson(
     clientId: Schema.String,
     storeId: Schema.String,
     sessionId: Schema.String,
+    extraArgs: Schema.UndefinedOr(Schema.JsonValue),
   }),
 )
 
@@ -52,7 +62,7 @@ export class LeaderWorkerOuterInitialMessage extends Schema.TaggedRequest<Leader
   {
     payload: { port: Transferable.MessagePort },
     success: Schema.Void,
-    failure: UnexpectedError,
+    failure: UnknownError,
   },
 ) {}
 
@@ -65,7 +75,7 @@ export class LeaderWorkerInnerInitialMessage extends Schema.TaggedRequest<Leader
       storeId: Schema.String,
       clientId: Schema.String,
       storage: StorageType,
-      syncPayload: Schema.UndefinedOr(Schema.JsonValue),
+      syncPayloadEncoded: Schema.UndefinedOr(Schema.JsonValue),
       devtools: Schema.Union(
         Schema.Struct({
           enabled: Schema.Literal(true),
@@ -79,7 +89,7 @@ export class LeaderWorkerInnerInitialMessage extends Schema.TaggedRequest<Leader
       ),
     },
     success: Schema.Void,
-    failure: UnexpectedError,
+    failure: UnknownError,
   },
 ) {}
 
@@ -88,35 +98,44 @@ export class LeaderWorkerInnerBootStatusStream extends Schema.TaggedRequest<Lead
   {
     payload: {},
     success: BootStatus,
-    failure: UnexpectedError,
+    failure: UnknownError,
   },
 ) {}
 
 export class LeaderWorkerInnerPullStream extends Schema.TaggedRequest<LeaderWorkerInnerPullStream>()('PullStream', {
   payload: {
-    cursor: EventSequenceNumber.EventSequenceNumber,
+    cursor: Schema.typeSchema(EventSequenceNumber.Client.Composite),
   },
   success: Schema.Struct({
     payload: SyncState.PayloadUpstream,
   }),
-  failure: UnexpectedError,
+  failure: UnknownError,
 }) {}
 
+export class LeaderWorkerInnerStreamEvents extends Schema.TaggedRequest<LeaderWorkerInnerStreamEvents>()(
+  'StreamEvents',
+  {
+    payload: StreamEventsOptionsFields,
+    success: LiveStoreEvent.Client.Encoded,
+    failure: UnknownError,
+  },
+) {}
+
 export class LeaderWorkerInnerPushToLeader extends Schema.TaggedRequest<LeaderWorkerInnerPushToLeader>()(
   'PushToLeader',
   {
     payload: {
-      batch: Schema.Array(LiveStoreEvent.AnyEncoded),
+      batch: Schema.Array(Schema.typeSchema(LiveStoreEvent.Client.Encoded)),
     },
-    success: Schema.Void,
-    failure: Schema.Union(UnexpectedError, LeaderAheadError),
+    success: Schema.Void as Schema.Schema<void>,
+    failure: Schema.Union(UnknownError, LeaderAheadError),
   },
 ) {}
 
 export class LeaderWorkerInnerExport extends Schema.TaggedRequest<LeaderWorkerInnerExport>()('Export', {
   payload: {},
   success: Transferable.Uint8Array as Schema.Schema<Uint8Array<ArrayBuffer>>,
-  failure: UnexpectedError,
+  failure: UnknownError,
 }) {}
 
 export class LeaderWorkerInnerGetRecreateSnapshot extends Schema.TaggedRequest<LeaderWorkerInnerGetRecreateSnapshot>()(
@@ -127,7 +146,7 @@ export class LeaderWorkerInnerGetRecreateSnapshot extends Schema.TaggedRequest<L
       snapshot: Transferable.Uint8Array as Schema.Schema<Uint8Array<ArrayBuffer>>,
       migrationsReport: MigrationsReport,
     }),
-    failure: UnexpectedError,
+    failure: UnknownError,
   },
 ) {}
 
@@ -136,7 +155,7 @@ export class LeaderWorkerInnerExportEventlog extends Schema.TaggedRequest<Leader
   {
     payload: {},
     success: Transferable.Uint8Array as Schema.Schema<Uint8Array<ArrayBuffer>>,
-    failure: UnexpectedError,
+    failure: UnknownError,
   },
 ) {}
 
@@ -144,8 +163,8 @@ export class LeaderWorkerInnerGetLeaderHead extends Schema.TaggedRequest<LeaderW
   'GetLeaderHead',
   {
     payload: {},
-    success: EventSequenceNumber.EventSequenceNumber,
-    failure: UnexpectedError,
+    success: Schema.typeSchema(EventSequenceNumber.Client.Composite),
+    failure: UnknownError,
   },
 ) {}
 
@@ -154,14 +173,41 @@ export class LeaderWorkerInnerGetLeaderSyncState extends Schema.TaggedRequest<Le
   {
     payload: {},
     success: SyncState.SyncState,
-    failure: UnexpectedError,
+    failure: UnknownError,
+  },
+) {}
+
+export class LeaderWorkerInnerSyncStateStream extends Schema.TaggedRequest<LeaderWorkerInnerSyncStateStream>()(
+  'SyncStateStream',
+  {
+    payload: {},
+    success: SyncState.SyncState,
+    failure: UnknownError,
+  },
+) {}
+
+export class LeaderWorkerInnerGetNetworkStatus extends Schema.TaggedRequest<LeaderWorkerInnerGetNetworkStatus>()(
+  'GetNetworkStatus',
+  {
+    payload: {},
+    success: SyncBackend.NetworkStatus,
+    failure: UnknownError,
+  },
+) {}
+
+export class LeaderWorkerInnerNetworkStatusStream extends Schema.TaggedRequest<LeaderWorkerInnerNetworkStatusStream>()(
+  'NetworkStatusStream',
+  {
+    payload: {},
+    success: SyncBackend.NetworkStatus,
+    failure: UnknownError,
   },
 ) {}
 
 export class LeaderWorkerInnerShutdown extends Schema.TaggedRequest<LeaderWorkerInnerShutdown>()('Shutdown', {
   payload: {},
   success: Schema.Void,
-  failure: UnexpectedError,
+  failure: UnknownError,
 }) {}
 
 export class LeaderWorkerInnerExtraDevtoolsMessage extends Schema.TaggedRequest<LeaderWorkerInnerExtraDevtoolsMessage>()(
@@ -171,7 +217,7 @@ export class LeaderWorkerInnerExtraDevtoolsMessage extends Schema.TaggedRequest<
       message: Devtools.Leader.MessageToApp,
     },
     success: Schema.Void,
-    failure: UnexpectedError,
+    failure: UnknownError,
   },
 ) {}
 
@@ -179,12 +225,16 @@ export const LeaderWorkerInnerRequest = Schema.Union(
   LeaderWorkerInnerInitialMessage,
   LeaderWorkerInnerBootStatusStream,
   LeaderWorkerInnerPullStream,
+  LeaderWorkerInnerStreamEvents,
   LeaderWorkerInnerPushToLeader,
   LeaderWorkerInnerExport,
   LeaderWorkerInnerGetRecreateSnapshot,
   LeaderWorkerInnerExportEventlog,
   LeaderWorkerInnerGetLeaderHead,
   LeaderWorkerInnerGetLeaderSyncState,
+  LeaderWorkerInnerSyncStateStream,
+  LeaderWorkerInnerGetNetworkStatus,
+  LeaderWorkerInnerNetworkStatusStream,
   LeaderWorkerInnerShutdown,
   LeaderWorkerInnerExtraDevtoolsMessage,
 )