@livestore/adapter-node 0.4.0-dev.1 → 0.4.0-dev.11

package/src/client-session/adapter.ts

@@ -1,10 +1,11 @@
 import { hostname } from 'node:os'
+import path from 'node:path'
 import * as WT from 'node:worker_threads'
 import {
   type Adapter,
   type BootStatus,
   ClientSessionLeaderThreadProxy,
-  type IntentionalShutdownCause,
+  IntentionalShutdownCause,
   type LockStatus,
   type MakeSqliteDb,
   makeClientSession,
@@ -17,17 +18,21 @@ import type { LiveStoreSchema } from '@livestore/common/schema'
 import { LiveStoreEvent } from '@livestore/common/schema'
 import { loadSqlite3Wasm } from '@livestore/sqlite-wasm/load-wasm'
 import { sqliteDbFactory } from '@livestore/sqlite-wasm/node'
+import { omitUndefineds } from '@livestore/utils'
 import {
   Cause,
   Effect,
   Exit,
   FetchHttpClient,
   Fiber,
+  FileSystem,
   Layer,
   ParseResult,
   Queue,
+  Schedule,
   Schema,
   Stream,
+  Subscribable,
   SubscriptionRef,
   Worker,
   WorkerError,
@@ -50,6 +55,13 @@ export interface NodeAdapterOptions {
    */
   sessionId?: string
 
+  /**
+   * Warning: This will reset both the app and eventlog database. This should only be used during development.
+   *
+   * @default false
+   */
+  resetPersistence?: boolean
+
   devtools?: {
     schemaPath: string | URL
     /**
@@ -89,13 +101,19 @@ export const makeAdapter = ({
  */
 export const makeWorkerAdapter = ({
   workerUrl,
+  workerExtraArgs,
   ...options
 }: NodeAdapterOptions & {
   /**
    * Example: `new URL('./livestore.worker.ts', import.meta.url)`
    */
   workerUrl: URL
-}): Adapter => makeAdapterImpl({ ...options, leaderThread: { _tag: 'multi-threaded', workerUrl } })
+  /**
+   * Extra arguments to pass to the worker which can be accessed in the worker
+   * via `getWorkerArgs()`
+   */
+  workerExtraArgs?: Schema.JsonValue
+}): Adapter => makeAdapterImpl({ ...options, leaderThread: { _tag: 'multi-threaded', workerUrl, workerExtraArgs } })
 
 const makeAdapterImpl = ({
   storage,
@@ -104,6 +122,7 @@ const makeAdapterImpl = ({
   // TODO make this dynamic and actually support multiple sessions
   sessionId = 'static',
   testing,
+  resetPersistence = false,
   leaderThread: leaderThreadInput,
 }: NodeAdapterOptions & {
   leaderThread:
@@ -114,6 +133,7 @@ const makeAdapterImpl = ({
     | {
         _tag: 'multi-threaded'
         workerUrl: URL
+        workerExtraArgs: Schema.JsonValue | undefined
       }
 }): Adapter =>
   ((adapterArgs) =>
@@ -135,6 +155,14 @@ const makeAdapterImpl = ({
 
       const shutdownChannel = yield* makeShutdownChannel(storeId)
 
+      if (resetPersistence === true) {
+        yield* shutdownChannel
+          .send(IntentionalShutdownCause.make({ reason: 'adapter-reset' }))
+          .pipe(UnexpectedError.mapToUnexpectedError)
+
+        yield* resetNodePersistence({ storage, storeId })
+      }
+
       yield* shutdownChannel.listen.pipe(
         Stream.flatten(),
         Stream.tap((cause) =>
@@ -173,11 +201,13 @@ const makeAdapterImpl = ({
               clientId,
               schema,
               makeSqliteDb,
-              syncOptions: leaderThreadInput.sync,
-              syncPayload,
               devtools: devtoolsOptions,
               storage,
-              testing,
+              ...omitUndefineds({
+                syncOptions: leaderThreadInput.sync,
+                syncPayload,
+                testing,
+              }),
             }).pipe(UnexpectedError.mapToUnexpectedError)
           : yield* makeWorkerLeaderThread({
               shutdown,
@@ -185,6 +215,7 @@ const makeAdapterImpl = ({
               clientId,
               sessionId,
               workerUrl: leaderThreadInput.workerUrl,
+              workerExtraArgs: leaderThreadInput.workerExtraArgs,
               storage,
               devtools: devtoolsOptions,
               bootStatusQueue,
@@ -219,10 +250,38 @@ const makeAdapterImpl = ({
       return clientSession
     }).pipe(
       Effect.withSpan('@livestore/adapter-node:adapter'),
-      Effect.provide(PlatformNode.NodeFileSystem.layer),
-      Effect.provide(FetchHttpClient.layer),
+      Effect.provide(Layer.mergeAll(PlatformNode.NodeFileSystem.layer, FetchHttpClient.layer)),
     )) satisfies Adapter
 
+const resetNodePersistence = ({
+  storage,
+  storeId,
+}: {
+  storage: WorkerSchema.StorageType
+  storeId: string
+}): Effect.Effect<void, UnexpectedError, FileSystem.FileSystem> => {
+  if (storage.type !== 'fs') {
+    return Effect.void
+  }
+
+  const directory = path.join(storage.baseDirectory ?? '', storeId)
+
+  return Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem
+
+    const directoryExists = yield* fs.exists(directory).pipe(UnexpectedError.mapToUnexpectedError)
+
+    if (directoryExists === false) {
+      return
+    }
+
+    yield* fs.remove(directory, { recursive: true }).pipe(UnexpectedError.mapToUnexpectedError)
+  }).pipe(
+    Effect.retry({ schedule: Schedule.exponentialBackoff10Sec }),
+    Effect.withSpan('@livestore/adapter-node:resetPersistence', { attributes: { directory } }),
+  )
+}
+
 const makeLocalLeaderThread = ({
   storeId,
   clientId,
@@ -257,12 +316,13 @@ const makeLocalLeaderThread = ({
         syncPayload,
         devtools,
         makeSqliteDb,
-        testing: testing?.overrides,
+        ...omitUndefineds({ testing: testing?.overrides }),
       }).pipe(Layer.unwrapScoped),
     )
 
     return yield* Effect.gen(function* () {
-      const { dbState, dbEventlog, syncProcessor, extraIncomingMessagesQueue, initialState } = yield* LeaderThreadCtx
+      const { dbState, dbEventlog, syncProcessor, extraIncomingMessagesQueue, initialState, networkStatus } =
+        yield* LeaderThreadCtx
 
       const initialLeaderHead = Eventlog.getClientHeadFromDb(dbEventlog)
 
@@ -279,10 +339,11 @@ const makeLocalLeaderThread = ({
           initialState: { leaderHead: initialLeaderHead, migrationsReport: initialState.migrationsReport },
           export: Effect.sync(() => dbState.export()),
           getEventlogData: Effect.sync(() => dbEventlog.export()),
-          getSyncState: syncProcessor.syncState,
+          syncState: syncProcessor.syncState,
           sendDevtoolsMessage: (message) => extraIncomingMessagesQueue.offer(message),
+          networkStatus,
         },
-        { overrides: testing?.overrides?.clientSession?.leaderThreadProxy },
+        { ...omitUndefineds({ overrides: testing?.overrides?.clientSession?.leaderThreadProxy }) },
       )
 
       const initialSnapshot = dbState.export()
@@ -297,6 +358,7 @@ const makeWorkerLeaderThread = ({
   clientId,
   sessionId,
   workerUrl,
+  workerExtraArgs,
   storage,
   devtools,
   bootStatusQueue,
@@ -308,6 +370,7 @@ const makeWorkerLeaderThread = ({
   clientId: string
   sessionId: string
   workerUrl: URL
+  workerExtraArgs: Schema.JsonValue | undefined
   storage: WorkerSchema.StorageType
   devtools: WorkerSchema.LeaderWorkerInnerInitialMessage['devtools']
   bootStatusQueue: Queue.Queue<BootStatus>
@@ -319,7 +382,7 @@ const makeWorkerLeaderThread = ({
   Effect.gen(function* () {
     const nodeWorker = new WT.Worker(workerUrl, {
       execArgv: process.env.DEBUG_WORKER ? ['--inspect --enable-source-maps'] : ['--enable-source-maps'],
-      argv: [Schema.encodeSync(WorkerSchema.WorkerArgv)({ storeId, clientId, sessionId })],
+      argv: [Schema.encodeSync(WorkerSchema.WorkerArgv)({ storeId, clientId, sessionId, extraArgs: workerExtraArgs })],
     })
     const nodeWorkerLayer = yield* Layer.build(PlatformNode.NodeWorker.layer(() => nodeWorker))
 
@@ -423,18 +486,25 @@ const makeWorkerLeaderThread = ({
           Effect.withSpan('@livestore/adapter-node:client-session:export'),
         ),
         getEventlogData: Effect.dieMessage('Not implemented'),
-        getSyncState: runInWorker(new WorkerSchema.LeaderWorkerInnerGetLeaderSyncState()).pipe(
-          UnexpectedError.mapToUnexpectedError,
-          Effect.withSpan('@livestore/adapter-node:client-session:getLeaderSyncState'),
-        ),
+        syncState: Subscribable.make({
+          get: runInWorker(new WorkerSchema.LeaderWorkerInnerGetLeaderSyncState()).pipe(
+            UnexpectedError.mapToUnexpectedError,
+            Effect.withSpan('@livestore/adapter-node:client-session:getLeaderSyncState'),
+          ),
+          changes: runInWorkerStream(new WorkerSchema.LeaderWorkerInnerSyncStateStream()).pipe(Stream.orDie),
+        }),
         sendDevtoolsMessage: (message) =>
           runInWorker(new WorkerSchema.LeaderWorkerInnerExtraDevtoolsMessage({ message })).pipe(
             UnexpectedError.mapToUnexpectedError,
             Effect.withSpan('@livestore/adapter-node:client-session:devtoolsMessageForLeader'),
           ),
+        networkStatus: Subscribable.make({
+          get: runInWorker(new WorkerSchema.LeaderWorkerInnerGetNetworkStatus()).pipe(Effect.orDie),
+          changes: runInWorkerStream(new WorkerSchema.LeaderWorkerInnerNetworkStatusStream()).pipe(Stream.orDie),
+        }),
       },
       {
-        overrides: testing?.overrides?.clientSession?.leaderThreadProxy,
+        ...omitUndefineds({ overrides: testing?.overrides?.clientSession?.leaderThreadProxy }),
       },
     )
 

package/src/make-leader-worker.ts

@@ -17,7 +17,6 @@ import { sqliteDbFactory } from '@livestore/sqlite-wasm/node'
 import {
   Effect,
   FetchHttpClient,
-  identity,
   Layer,
   Logger,
   LogLevel,
@@ -60,6 +59,13 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
       )
     : undefined
 
+  // Merge the runtime dependencies once so we can provide them together without chaining Effect.provide.
+  const runtimeLayer = Layer.mergeAll(
+    FetchHttpClient.layer,
+    PlatformNode.NodeFileSystem.layer,
+    TracingLive ?? Layer.empty,
+  )
+
   return WorkerRunner.layerSerialized(WorkerSchema.LeaderWorkerInnerRequest, {
     InitialMessage: (args) =>
       Effect.gen(function* () {
@@ -113,6 +119,21 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
         UnexpectedError.mapToUnexpectedError,
         Effect.withSpan('@livestore/adapter-node:worker:GetLeaderSyncState'),
       ),
+    SyncStateStream: () =>
+      Effect.gen(function* () {
+        const workerCtx = yield* LeaderThreadCtx
+        return workerCtx.syncProcessor.syncState.changes
+      }).pipe(Stream.unwrapScoped),
+    GetNetworkStatus: () =>
+      Effect.gen(function* () {
+        const workerCtx = yield* LeaderThreadCtx
+        return yield* workerCtx.networkStatus
+      }).pipe(UnexpectedError.mapToUnexpectedError, Effect.withSpan('@livestore/adapter-node:worker:GetNetworkStatus')),
+    NetworkStatusStream: () =>
+      Effect.gen(function* () {
+        const workerCtx = yield* LeaderThreadCtx
+        return workerCtx.networkStatus.changes
+      }).pipe(Stream.unwrapScoped),
     GetRecreateSnapshot: () =>
       Effect.gen(function* () {
         const workerCtx = yield* LeaderThreadCtx
@@ -159,9 +180,7 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
     // TODO bring back with Effect 4 once it's easier to work with replacing loggers.
     // We basically only want to provide this logger if it's replacing the default logger, not if there's a custom logger already provided.
     // Effect.provide(Logger.prettyWithThread(options.otelOptions?.serviceName ?? 'livestore-node-leader-thread')),
-    Effect.provide(FetchHttpClient.layer),
-    Effect.provide(PlatformNode.NodeFileSystem.layer),
-    TracingLive ? Effect.provide(TracingLive) : identity,
+    Effect.provide(runtimeLayer),
     Logger.withMinimumLogLevel(LogLevel.Debug),
   )
 }

package/src/worker-schema.ts

@@ -1,4 +1,12 @@
-import { BootStatus, Devtools, LeaderAheadError, MigrationsReport, SyncState, UnexpectedError } from '@livestore/common'
+import {
+  BootStatus,
+  Devtools,
+  LeaderAheadError,
+  MigrationsReport,
+  SyncBackend,
+  SyncState,
+  UnexpectedError,
+} from '@livestore/common'
 import { EventSequenceNumber, LiveStoreEvent } from '@livestore/common/schema'
 import { Schema, Transferable } from '@livestore/utils/effect'
 
@@ -7,6 +15,7 @@ export const WorkerArgv = Schema.parseJson(
     clientId: Schema.String,
     storeId: Schema.String,
     sessionId: Schema.String,
+    extraArgs: Schema.UndefinedOr(Schema.JsonValue),
   }),
 )
 
@@ -158,6 +167,33 @@ export class LeaderWorkerInnerGetLeaderSyncState extends Schema.TaggedRequest<Le
   },
 ) {}
 
+export class LeaderWorkerInnerSyncStateStream extends Schema.TaggedRequest<LeaderWorkerInnerSyncStateStream>()(
+  'SyncStateStream',
+  {
+    payload: {},
+    success: SyncState.SyncState,
+    failure: UnexpectedError,
+  },
+) {}
+
+export class LeaderWorkerInnerGetNetworkStatus extends Schema.TaggedRequest<LeaderWorkerInnerGetNetworkStatus>()(
+  'GetNetworkStatus',
+  {
+    payload: {},
+    success: SyncBackend.NetworkStatus,
+    failure: UnexpectedError,
+  },
+) {}
+
+export class LeaderWorkerInnerNetworkStatusStream extends Schema.TaggedRequest<LeaderWorkerInnerNetworkStatusStream>()(
+  'NetworkStatusStream',
+  {
+    payload: {},
+    success: SyncBackend.NetworkStatus,
+    failure: UnexpectedError,
+  },
+) {}
+
 export class LeaderWorkerInnerShutdown extends Schema.TaggedRequest<LeaderWorkerInnerShutdown>()('Shutdown', {
   payload: {},
   success: Schema.Void,
@@ -185,6 +221,9 @@ export const LeaderWorkerInnerRequest = Schema.Union(
   LeaderWorkerInnerExportEventlog,
   LeaderWorkerInnerGetLeaderHead,
   LeaderWorkerInnerGetLeaderSyncState,
+  LeaderWorkerInnerSyncStateStream,
+  LeaderWorkerInnerGetNetworkStatus,
+  LeaderWorkerInnerNetworkStatusStream,
   LeaderWorkerInnerShutdown,
   LeaderWorkerInnerExtraDevtoolsMessage,
 )