@livestore/adapter-node 0.4.0-dev.9 → 0.4.0

package/src/leader-thread-shared.ts

@@ -1,13 +1,13 @@
 import inspector from 'node:inspector'
 import path from 'node:path'
 
-if (process.execArgv.includes('--inspect')) {
+if (process.execArgv.includes('--inspect') === true) {
   inspector.open()
   inspector.waitForDebugger()
 }
 
 import type { ClientSessionLeaderThreadProxy, MakeSqliteDb, SqliteDb, SyncOptions } from '@livestore/common'
-import { Devtools, liveStoreStorageFormatVersion, migrateDb, UnexpectedError } from '@livestore/common'
+import { Devtools, liveStoreStorageFormatVersion, migrateDb, UnknownError } from '@livestore/common'
 import type { DevtoolsOptions, LeaderSqliteDb, LeaderThreadCtx } from '@livestore/common/leader-thread'
 import { configureConnection, makeLeaderThreadLayer } from '@livestore/common/leader-thread'
 import type { LiveStoreSchema } from '@livestore/common/schema'
@@ -30,7 +30,7 @@ export type TestingOverrides = {
       dbEventlog: SqliteDb
       dbState: SqliteDb
     },
-    UnexpectedError
+    UnknownError
   >
 }
 
@@ -42,7 +42,8 @@ export interface MakeLeaderThreadArgs {
   makeSqliteDb: MakeNodeSqliteDb
   devtools: WorkerSchema.LeaderWorkerInnerInitialMessage['devtools']
   schema: LiveStoreSchema
-  syncPayload: Schema.JsonValue | undefined
+  syncPayloadEncoded: Schema.JsonValue | undefined
+  syncPayloadSchema: Schema.Schema<any> | undefined
   testing: TestingOverrides | undefined
 }
 
@@ -54,21 +55,22 @@ export const makeLeaderThread = ({
   storage,
   devtools,
   schema,
-  syncPayload,
+  syncPayloadEncoded,
+  syncPayloadSchema,
   testing,
 }: MakeLeaderThreadArgs): Effect.Effect<
-  Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient | FileSystem.FileSystem>,
-  UnexpectedError,
+  Layer.Layer<LeaderThreadCtx, UnknownError, Scope.Scope | HttpClient.HttpClient | FileSystem.FileSystem>,
+  UnknownError,
   Scope.Scope
 > =>
   Effect.gen(function* () {
-    const runtime = yield* Effect.runtime<never>()
+    const runtime = yield* Effect.runtime()
 
     const schemaHashSuffix =
       schema.state.sqlite.migrations.strategy === 'manual' ? 'fixed' : schema.state.sqlite.hash.toString()
 
     const makeDb = (kind: 'state' | 'eventlog') => {
-      if (testing?.makeLeaderThread) {
+      if (testing?.makeLeaderThread !== undefined) {
         return testing
           .makeLeaderThread(makeSqliteDb)
           .pipe(Effect.map(({ dbEventlog, dbState }) => (kind === 'state' ? dbState : dbEventlog)))
@@ -113,11 +115,12 @@ export const makeLeaderThread = ({
       dbEventlog,
       devtoolsOptions,
       shutdownChannel,
-      syncPayload,
+      syncPayloadEncoded,
+      syncPayloadSchema,
     })
   }).pipe(
     Effect.tapCauseLogPretty,
-    UnexpectedError.mapToUnexpectedError,
+    UnknownError.mapToUnknownError,
     Effect.withSpan('@livestore/adapter-node:makeLeaderThread', {
       attributes: { storeId, clientId, storage, devtools, syncOptions },
     }),
@@ -137,7 +140,7 @@ const makeDevtoolsOptions = ({
   storeId: string
   clientId: string
   devtools: WorkerSchema.LeaderWorkerInnerInitialMessage['devtools']
-}): Effect.Effect<DevtoolsOptions, UnexpectedError, Scope.Scope> =>
+}): Effect.Effect<DevtoolsOptions, UnknownError, Scope.Scope> =>
   Effect.gen(function* () {
     if (devtools.enabled === false) {
       return {
@@ -161,6 +164,7 @@ const makeDevtoolsOptions = ({
               sessionId: 'static', // TODO make this dynamic
               schemaAlias: devtools.schemaAlias,
               isLeader: true,
+              origin: undefined,
             }),
             port: devtools.port,
             host: devtools.host,
@@ -180,7 +184,7 @@ const makeDevtoolsOptions = ({
           eventlog: dbEventlog.metadata.persistenceInfo,
         }
 
-        return { node, persistenceInfo, mode: 'proxy' }
+        return { node, persistenceInfo, mode: 'proxy' as const }
       }),
     }
   })

package/src/make-leader-worker.ts

@@ -1,33 +1,23 @@
 import './thread-polyfill.ts'
-
 import inspector from 'node:inspector'
 
-if (process.execArgv.includes('--inspect')) {
+if (process.execArgv.includes('--inspect') === true) {
   inspector.open()
   inspector.waitForDebugger()
 }
 
+import type * as otel from '@opentelemetry/api'
+
 import type { SyncOptions } from '@livestore/common'
-import { UnexpectedError } from '@livestore/common'
-import { Eventlog, LeaderThreadCtx } from '@livestore/common/leader-thread'
+import { LogConfig, UnknownError } from '@livestore/common'
+import type { StreamEventsOptions } from '@livestore/common/leader-thread'
+import { Eventlog, LeaderThreadCtx, streamEventsWithSyncState } from '@livestore/common/leader-thread'
 import type { LiveStoreSchema } from '@livestore/common/schema'
 import { LiveStoreEvent } from '@livestore/common/schema'
 import { loadSqlite3Wasm } from '@livestore/sqlite-wasm/load-wasm'
 import { sqliteDbFactory } from '@livestore/sqlite-wasm/node'
-import {
-  Effect,
-  FetchHttpClient,
-  identity,
-  Layer,
-  Logger,
-  LogLevel,
-  OtelTracer,
-  Schema,
-  Stream,
-  WorkerRunner,
-} from '@livestore/utils/effect'
+import { Effect, FetchHttpClient, Layer, OtelTracer, Schema, Stream, WorkerRunner } from '@livestore/utils/effect'
 import { PlatformNode } from '@livestore/utils/node'
-import type * as otel from '@opentelemetry/api'
 
 import type { TestingOverrides } from './leader-thread-shared.ts'
 import { makeLeaderThread } from './leader-thread-shared.ts'
@@ -36,30 +26,35 @@ import * as WorkerSchema from './worker-schema.ts'
 export type WorkerOptions = {
   schema: LiveStoreSchema
   sync?: SyncOptions
+  syncPayloadSchema?: Schema.Schema<any>
   otelOptions?: {
     tracer?: otel.Tracer
     /** @default 'livestore-node-leader-thread' */
     serviceName?: string
   }
   testing?: TestingOverrides
-}
+} & LogConfig.WithLoggerOptions
 
 export const getWorkerArgs = () => Schema.decodeSync(WorkerSchema.WorkerArgv)(process.argv[2]!)
 
 export const makeWorker = (options: WorkerOptions) => {
-  makeWorkerEffect(options).pipe(
-    Effect.provide(Logger.prettyWithThread(options.otelOptions?.serviceName ?? 'livestore-node-leader-thread')),
-    PlatformNode.NodeRuntime.runMain,
-  )
+  makeWorkerEffect(options).pipe(PlatformNode.NodeRuntime.runMain)
 }
 
 export const makeWorkerEffect = (options: WorkerOptions) => {
-  const TracingLive = options.otelOptions?.tracer
+  const TracingLive = options.otelOptions?.tracer !== undefined
     ? Layer.unwrapEffect(Effect.map(OtelTracer.make, Layer.setTracer)).pipe(
         Layer.provideMerge(Layer.succeed(OtelTracer.OtelTracer, options.otelOptions.tracer)),
       )
     : undefined
 
+  // Merge the runtime dependencies once so we can provide them together without chaining Effect.provide.
+  const runtimeLayer = Layer.mergeAll(
+    FetchHttpClient.layer,
+    PlatformNode.NodeFileSystem.layer,
+    TracingLive ?? Layer.empty,
+  )
+
   return WorkerRunner.layerSerialized(WorkerSchema.LeaderWorkerInnerRequest, {
     InitialMessage: (args) =>
       Effect.gen(function* () {
@@ -73,12 +68,14 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
           schema: options.schema,
           testing: options.testing,
           makeSqliteDb,
+          syncPayloadEncoded: args.syncPayloadEncoded,
+          syncPayloadSchema: options.syncPayloadSchema,
         })
       }).pipe(Layer.unwrapScoped),
     PushToLeader: ({ batch }) =>
       Effect.andThen(LeaderThreadCtx, (_) =>
         _.syncProcessor.push(
-          batch.map((item) => new LiveStoreEvent.EncodedWithMeta(item)),
+          batch.map((item) => new LiveStoreEvent.Client.EncodedWithMeta(item)),
           // We'll wait in order to keep back pressure on the client session
           { waitForProcessing: true },
         ),
@@ -90,61 +87,76 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
         const { syncProcessor } = yield* LeaderThreadCtx
         return syncProcessor.pull({ cursor })
       }).pipe(Stream.unwrapScoped),
+    StreamEvents: (options: WorkerSchema.LeaderWorkerInnerStreamEvents) =>
+      LeaderThreadCtx.pipe(
+        Effect.map(({ dbEventlog, syncProcessor }) => {
+          const { _tag: _ignored, ...payload } = options
+          const streamOptions = payload as StreamEventsOptions
+          return streamEventsWithSyncState({
+            dbEventlog,
+            syncState: syncProcessor.syncState,
+            options: streamOptions,
+          })
+        }),
+        Stream.unwrapScoped,
+        Stream.withSpan('@livestore/adapter-node:worker:StreamEvents'),
+      ),
     Export: () =>
       Effect.andThen(LeaderThreadCtx, (_) => _.dbState.export()).pipe(
-        UnexpectedError.mapToUnexpectedError,
         Effect.withSpan('@livestore/adapter-node:worker:Export'),
       ),
     ExportEventlog: () =>
       Effect.andThen(LeaderThreadCtx, (_) => _.dbEventlog.export()).pipe(
-        UnexpectedError.mapToUnexpectedError,
         Effect.withSpan('@livestore/adapter-node:worker:ExportEventlog'),
       ),
-    GetLeaderHead: () =>
-      Effect.gen(function* () {
-        const workerCtx = yield* LeaderThreadCtx
-        return Eventlog.getClientHeadFromDb(workerCtx.dbEventlog)
-      }).pipe(UnexpectedError.mapToUnexpectedError, Effect.withSpan('@livestore/adapter-node:worker:GetLeaderHead')),
-    GetLeaderSyncState: () =>
+    GetLeaderHead: Effect.fn('@livestore/adapter-node:worker:GetLeaderHead')(function* () {
+      const workerCtx = yield* LeaderThreadCtx
+      return Eventlog.getClientHeadFromDb(workerCtx.dbEventlog)
+    }),
+    GetLeaderSyncState: Effect.fn('@livestore/adapter-node:worker:GetLeaderSyncState')(function* () {
+      const workerCtx = yield* LeaderThreadCtx
+      return yield* workerCtx.syncProcessor.syncState
+    }),
+    SyncStateStream: () =>
       Effect.gen(function* () {
         const workerCtx = yield* LeaderThreadCtx
-        return yield* workerCtx.syncProcessor.syncState
-      }).pipe(
-        UnexpectedError.mapToUnexpectedError,
-        Effect.withSpan('@livestore/adapter-node:worker:GetLeaderSyncState'),
-      ),
-    GetRecreateSnapshot: () =>
+        return workerCtx.syncProcessor.syncState.changes
+      }).pipe(Stream.unwrapScoped),
+    GetNetworkStatus: Effect.fn('@livestore/adapter-node:worker:GetNetworkStatus')(function* () {
+      const workerCtx = yield* LeaderThreadCtx
+      return yield* workerCtx.networkStatus
+    }),
+    NetworkStatusStream: () =>
       Effect.gen(function* () {
         const workerCtx = yield* LeaderThreadCtx
-        // const result = yield* Deferred.await(workerCtx.initialSetupDeferred)
-        // NOTE we can only return the cached snapshot once as it's transferred (i.e. disposed), so we need to set it to undefined
-        // const cachedSnapshot =
-        //   result._tag === 'Recreate' ? yield* Ref.getAndSet(result.snapshotRef, undefined) : undefined
-        // return cachedSnapshot ?? workerCtx.db.export()
-        const snapshot = workerCtx.dbState.export()
-        return { snapshot, migrationsReport: workerCtx.initialState.migrationsReport }
-      }).pipe(
-        UnexpectedError.mapToUnexpectedError,
-        Effect.withSpan('@livestore/adapter-node:worker:GetRecreateSnapshot'),
-      ),
-    Shutdown: () =>
-      Effect.gen(function* () {
-        // const { db, dbEventlog } = yield* LeaderThreadCtx
-        yield* Effect.logDebug('[@livestore/adapter-node:worker] Shutdown')
+        return workerCtx.networkStatus.changes
+      }).pipe(Stream.unwrapScoped),
+    GetRecreateSnapshot: Effect.fn('@livestore/adapter-node:worker:GetRecreateSnapshot')(function* () {
+      const workerCtx = yield* LeaderThreadCtx
+      // const result = yield* Deferred.await(workerCtx.initialSetupDeferred)
+      // NOTE we can only return the cached snapshot once as it's transferred (i.e. disposed), so we need to set it to undefined
+      // const cachedSnapshot =
+      //   result._tag === 'Recreate' ? yield* Ref.getAndSet(result.snapshotRef, undefined) : undefined
+      // return cachedSnapshot ?? workerCtx.db.export()
+      const snapshot = workerCtx.dbState.export()
+      return { snapshot, migrationsReport: workerCtx.initialState.migrationsReport }
+    }),
+    Shutdown: Effect.fn('@livestore/adapter-node:worker:Shutdown')(function* () {
+      // const { db, dbEventlog } = yield* LeaderThreadCtx
+      yield* Effect.logDebug('[@livestore/adapter-node:worker] Shutdown')
 
-        // if (devtools.enabled) {
-        //   yield* FiberSet.clear(devtools.connections)
-        // }
-        // db.close()
-        // dbEventlog.close()
+      // if (devtools.enabled) {
+      //   yield* FiberSet.clear(devtools.connections)
+      // }
+      // db.close()
+      // dbEventlog.close()
 
-        // Buy some time for Otel to flush
-        // TODO find a cleaner way to do this
-        // yield* Effect.sleep(1000)
-      }).pipe(UnexpectedError.mapToUnexpectedError, Effect.withSpan('@livestore/adapter-node:worker:Shutdown')),
+      // Buy some time for Otel to flush
+      // TODO find a cleaner way to do this
+      // yield* Effect.sleep(1000)
+    }),
     ExtraDevtoolsMessage: ({ message }) =>
       Effect.andThen(LeaderThreadCtx, (_) => _.extraIncomingMessagesQueue.offer(message)).pipe(
-        UnexpectedError.mapToUnexpectedError,
         Effect.withSpan('@livestore/adapter-node:worker:ExtraDevtoolsMessage'),
       ),
   }).pipe(
@@ -156,12 +168,13 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
       thread: options.otelOptions?.serviceName ?? 'livestore-node-leader-thread',
       processId: process.pid,
     }),
+    LogConfig.withLoggerConfig(
+      { logger: options.logger, logLevel: options.logLevel },
+      { threadName: options.otelOptions?.serviceName ?? 'livestore-node-leader-thread' },
+    ),
     // TODO bring back with Effect 4 once it's easier to work with replacing loggers.
     // We basically only want to provide this logger if it's replacing the default logger, not if there's a custom logger already provided.
     // Effect.provide(Logger.prettyWithThread(options.otelOptions?.serviceName ?? 'livestore-node-leader-thread')),
-    Effect.provide(FetchHttpClient.layer),
-    Effect.provide(PlatformNode.NodeFileSystem.layer),
-    TracingLive ? Effect.provide(TracingLive) : identity,
-    Logger.withMinimumLogLevel(LogLevel.Debug),
+    Effect.provide(runtimeLayer),
   )
 }

package/src/shutdown-channel.ts

@@ -1,5 +1,6 @@
 import { ShutdownChannel } from '@livestore/common/leader-thread'
 import type { Effect, Scope } from '@livestore/utils/effect'
+
 import { makeBroadcastChannel } from './webchannel.ts'
 
 export const makeShutdownChannel = (

package/src/worker-schema.ts

@@ -1,4 +1,13 @@
-import { BootStatus, Devtools, LeaderAheadError, MigrationsReport, SyncState, UnexpectedError } from '@livestore/common'
+import {
+  BootStatus,
+  Devtools,
+  RejectedPushError,
+  MigrationsReport,
+  SyncBackend,
+  SyncState,
+  UnknownError,
+} from '@livestore/common'
+import { StreamEventsOptionsFields } from '@livestore/common/leader-thread'
 import { EventSequenceNumber, LiveStoreEvent } from '@livestore/common/schema'
 import { Schema, Transferable } from '@livestore/utils/effect'
 
@@ -53,7 +62,7 @@ export class LeaderWorkerOuterInitialMessage extends Schema.TaggedRequest<Leader
   {
     payload: { port: Transferable.MessagePort },
     success: Schema.Void,
-    failure: UnexpectedError,
+    failure: Schema.Never,
   },
 ) {}
 
@@ -66,7 +75,7 @@ export class LeaderWorkerInnerInitialMessage extends Schema.TaggedRequest<Leader
       storeId: Schema.String,
       clientId: Schema.String,
       storage: StorageType,
-      syncPayload: Schema.UndefinedOr(Schema.JsonValue),
+      syncPayloadEncoded: Schema.UndefinedOr(Schema.JsonValue),
       devtools: Schema.Union(
         Schema.Struct({
           enabled: Schema.Literal(true),
@@ -80,7 +89,7 @@ export class LeaderWorkerInnerInitialMessage extends Schema.TaggedRequest<Leader
       ),
     },
     success: Schema.Void,
-    failure: UnexpectedError,
+    failure: UnknownError,
   },
 ) {}
 
@@ -89,35 +98,44 @@ export class LeaderWorkerInnerBootStatusStream extends Schema.TaggedRequest<Lead
   {
     payload: {},
     success: BootStatus,
-    failure: UnexpectedError,
+    failure: Schema.Never,
   },
 ) {}
 
 export class LeaderWorkerInnerPullStream extends Schema.TaggedRequest<LeaderWorkerInnerPullStream>()('PullStream', {
   payload: {
-    cursor: EventSequenceNumber.EventSequenceNumber,
+    cursor: Schema.typeSchema(EventSequenceNumber.Client.Composite),
   },
   success: Schema.Struct({
     payload: SyncState.PayloadUpstream,
   }),
-  failure: UnexpectedError,
+  failure: Schema.Never,
 }) {}
 
+export class LeaderWorkerInnerStreamEvents extends Schema.TaggedRequest<LeaderWorkerInnerStreamEvents>()(
+  'StreamEvents',
+  {
+    payload: StreamEventsOptionsFields,
+    success: LiveStoreEvent.Client.Encoded,
+    failure: Schema.Never,
+  },
+) {}
+
 export class LeaderWorkerInnerPushToLeader extends Schema.TaggedRequest<LeaderWorkerInnerPushToLeader>()(
   'PushToLeader',
   {
     payload: {
-      batch: Schema.Array(LiveStoreEvent.AnyEncoded),
+      batch: Schema.Array(Schema.typeSchema(LiveStoreEvent.Client.Encoded)),
     },
-    success: Schema.Void,
-    failure: Schema.Union(UnexpectedError, LeaderAheadError),
+    success: Schema.Void as Schema.Schema<void>,
+    failure: RejectedPushError,
   },
 ) {}
 
 export class LeaderWorkerInnerExport extends Schema.TaggedRequest<LeaderWorkerInnerExport>()('Export', {
   payload: {},
   success: Transferable.Uint8Array as Schema.Schema<Uint8Array<ArrayBuffer>>,
-  failure: UnexpectedError,
+  failure: Schema.Never,
 }) {}
 
 export class LeaderWorkerInnerGetRecreateSnapshot extends Schema.TaggedRequest<LeaderWorkerInnerGetRecreateSnapshot>()(
@@ -128,7 +146,7 @@ export class LeaderWorkerInnerGetRecreateSnapshot extends Schema.TaggedRequest<L
       snapshot: Transferable.Uint8Array as Schema.Schema<Uint8Array<ArrayBuffer>>,
       migrationsReport: MigrationsReport,
     }),
-    failure: UnexpectedError,
+    failure: Schema.Never,
   },
 ) {}
 
@@ -137,7 +155,7 @@ export class LeaderWorkerInnerExportEventlog extends Schema.TaggedRequest<Leader
   {
     payload: {},
     success: Transferable.Uint8Array as Schema.Schema<Uint8Array<ArrayBuffer>>,
-    failure: UnexpectedError,
+    failure: Schema.Never,
   },
 ) {}
 
@@ -145,8 +163,8 @@ export class LeaderWorkerInnerGetLeaderHead extends Schema.TaggedRequest<LeaderW
   'GetLeaderHead',
   {
     payload: {},
-    success: EventSequenceNumber.EventSequenceNumber,
-    failure: UnexpectedError,
+    success: Schema.typeSchema(EventSequenceNumber.Client.Composite),
+    failure: Schema.Never,
   },
 ) {}
 
@@ -155,14 +173,41 @@ export class LeaderWorkerInnerGetLeaderSyncState extends Schema.TaggedRequest<Le
   {
     payload: {},
     success: SyncState.SyncState,
-    failure: UnexpectedError,
+    failure: Schema.Never,
+  },
+) {}
+
+export class LeaderWorkerInnerSyncStateStream extends Schema.TaggedRequest<LeaderWorkerInnerSyncStateStream>()(
+  'SyncStateStream',
+  {
+    payload: {},
+    success: SyncState.SyncState,
+    failure: Schema.Never,
+  },
+) {}
+
+export class LeaderWorkerInnerGetNetworkStatus extends Schema.TaggedRequest<LeaderWorkerInnerGetNetworkStatus>()(
+  'GetNetworkStatus',
+  {
+    payload: {},
+    success: SyncBackend.NetworkStatus,
+    failure: Schema.Never,
+  },
+) {}
+
+export class LeaderWorkerInnerNetworkStatusStream extends Schema.TaggedRequest<LeaderWorkerInnerNetworkStatusStream>()(
+  'NetworkStatusStream',
+  {
+    payload: {},
+    success: SyncBackend.NetworkStatus,
+    failure: Schema.Never,
   },
 ) {}
 
 export class LeaderWorkerInnerShutdown extends Schema.TaggedRequest<LeaderWorkerInnerShutdown>()('Shutdown', {
   payload: {},
   success: Schema.Void,
-  failure: UnexpectedError,
+  failure: Schema.Never,
 }) {}
 
 export class LeaderWorkerInnerExtraDevtoolsMessage extends Schema.TaggedRequest<LeaderWorkerInnerExtraDevtoolsMessage>()(
@@ -172,7 +217,7 @@ export class LeaderWorkerInnerExtraDevtoolsMessage extends Schema.TaggedRequest<
       message: Devtools.Leader.MessageToApp,
     },
     success: Schema.Void,
-    failure: UnexpectedError,
+    failure: Schema.Never,
   },
 ) {}
 
@@ -180,12 +225,16 @@ export const LeaderWorkerInnerRequest = Schema.Union(
   LeaderWorkerInnerInitialMessage,
   LeaderWorkerInnerBootStatusStream,
   LeaderWorkerInnerPullStream,
+  LeaderWorkerInnerStreamEvents,
   LeaderWorkerInnerPushToLeader,
   LeaderWorkerInnerExport,
   LeaderWorkerInnerGetRecreateSnapshot,
   LeaderWorkerInnerExportEventlog,
   LeaderWorkerInnerGetLeaderHead,
   LeaderWorkerInnerGetLeaderSyncState,
+  LeaderWorkerInnerSyncStateStream,
+  LeaderWorkerInnerGetNetworkStatus,
+  LeaderWorkerInnerNetworkStatusStream,
   LeaderWorkerInnerShutdown,
   LeaderWorkerInnerExtraDevtoolsMessage,
 )