@livestore/adapter-node 0.3.0-dev.32 → 0.3.0-dev.34

package/src/make-leader-worker.ts

@@ -1,7 +1,6 @@
 import './thread-polyfill.js'
 
 import inspector from 'node:inspector'
-import path from 'node:path'
 
 if (process.execArgv.includes('--inspect')) {
   inspector.open()
@@ -9,15 +8,12 @@ if (process.execArgv.includes('--inspect')) {
 }
 
 import type { SyncOptions } from '@livestore/common'
-import { Devtools, liveStoreStorageFormatVersion, UnexpectedError } from '@livestore/common'
-import type { DevtoolsOptions, LeaderSqliteDb } from '@livestore/common/leader-thread'
-import { configureConnection, Eventlog, LeaderThreadCtx, makeLeaderThreadLayer } from '@livestore/common/leader-thread'
+import { UnexpectedError } from '@livestore/common'
+import { Eventlog, LeaderThreadCtx } from '@livestore/common/leader-thread'
 import type { LiveStoreSchema } from '@livestore/common/schema'
 import { LiveStoreEvent } from '@livestore/common/schema'
-import { makeNodeDevtoolsChannel } from '@livestore/devtools-node-common/web-channel'
 import { loadSqlite3Wasm } from '@livestore/sqlite-wasm/load-wasm'
 import { sqliteDbFactory } from '@livestore/sqlite-wasm/node'
-import type { FileSystem, HttpClient, Scope } from '@livestore/utils/effect'
 import {
   Effect,
   FetchHttpClient,
@@ -33,17 +29,19 @@ import {
 import { PlatformNode } from '@livestore/utils/node'
 import type * as otel from '@opentelemetry/api'
 
-import { startDevtoolsServer } from './devtools/devtools-server.js'
-import { makeShutdownChannel } from './shutdown-channel.js'
+import type { TestingOverrides } from './leader-thread-shared.js'
+import { makeLeaderThread } from './leader-thread-shared.js'
 import * as WorkerSchema from './worker-schema.js'
 
 export type WorkerOptions = {
+  schema: LiveStoreSchema
   sync?: SyncOptions
   otelOptions?: {
     tracer?: otel.Tracer
     /** @default 'livestore-node-leader-thread' */
     serviceName?: string
   }
+  testing?: TestingOverrides
 }
 
 export const getWorkerArgs = () => Schema.decodeSync(WorkerSchema.WorkerArgv)(process.argv[2]!)
@@ -60,7 +58,20 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
     : undefined
 
   return WorkerRunner.layerSerialized(WorkerSchema.LeaderWorkerInner.Request, {
-    InitialMessage: (args) => makeLeaderThread({ ...args, syncOptions: options.sync }),
+    InitialMessage: (args) =>
+      Effect.gen(function* () {
+        const sqlite3 = yield* Effect.promise(() => loadSqlite3Wasm()).pipe(
+          Effect.withSpan('@livestore/adapter-node:leader-thread:loadSqlite3Wasm'),
+        )
+        const makeSqliteDb = yield* sqliteDbFactory({ sqlite3 })
+        return yield* makeLeaderThread({
+          ...args,
+          syncOptions: options.sync,
+          schema: options.schema,
+          testing: options.testing,
+          makeSqliteDb,
+        })
+      }).pipe(Layer.unwrapScoped),
     PushToLeader: ({ batch }) =>
       Effect.andThen(LeaderThreadCtx, (_) =>
         _.syncProcessor.push(
@@ -77,7 +88,7 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
         return syncProcessor.pull({ cursor })
       }).pipe(Stream.unwrapScoped),
     Export: () =>
-      Effect.andThen(LeaderThreadCtx, (_) => _.dbReadModel.export()).pipe(
+      Effect.andThen(LeaderThreadCtx, (_) => _.dbState.export()).pipe(
         UnexpectedError.mapToUnexpectedError,
         Effect.withSpan('@livestore/adapter-node:worker:Export'),
       ),
@@ -107,7 +118,7 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
         // const cachedSnapshot =
         //   result._tag === 'Recreate' ? yield* Ref.getAndSet(result.snapshotRef, undefined) : undefined
         // return cachedSnapshot ?? workerCtx.db.export()
-        const snapshot = workerCtx.dbReadModel.export()
+        const snapshot = workerCtx.dbState.export()
         return { snapshot, migrationsReport: workerCtx.initialState.migrationsReport }
       }).pipe(
         UnexpectedError.mapToUnexpectedError,
@@ -149,130 +160,3 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
     Logger.withMinimumLogLevel(LogLevel.Debug),
   )
 }
-
-const makeLeaderThread = ({
-  storeId,
-  clientId,
-  syncOptions,
-  baseDirectory,
-  devtools,
-  schemaPath,
-  syncPayload,
-}: WorkerSchema.LeaderWorkerInner.InitialMessage & {
-  syncOptions: SyncOptions | undefined
-  schemaPath: string
-}): Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient | FileSystem.FileSystem> =>
-  Effect.gen(function* () {
-    const schema = yield* Effect.promise(() => import(schemaPath).then((m) => m.schema as LiveStoreSchema))
-
-    const sqlite3 = yield* Effect.promise(() => loadSqlite3Wasm()).pipe(
-      Effect.withSpan('@livestore/adapter-node:leader-thread:loadSqlite3Wasm'),
-    )
-    const makeSqliteDb = yield* sqliteDbFactory({ sqlite3 })
-    const runtime = yield* Effect.runtime<never>()
-
-    const schemaHashSuffix = schema.migrationOptions.strategy === 'manual' ? 'fixed' : schema.hash.toString()
-
-    const makeDb = (kind: 'app' | 'eventlog') =>
-      makeSqliteDb({
-        _tag: 'fs',
-        directory: path.join(baseDirectory ?? '', storeId),
-        fileName: kind === 'app' ? getAppDbFileName(schemaHashSuffix) : `eventlog@${liveStoreStorageFormatVersion}.db`,
-        // TODO enable WAL for nodejs
-        configureDb: (db) =>
-          configureConnection(db, { foreignKeys: true }).pipe(Effect.provide(runtime), Effect.runSync),
-      }).pipe(Effect.acquireRelease((db) => Effect.sync(() => db.close())))
-
-    // Might involve some async work, so we're running them concurrently
-    const [dbReadModel, dbEventlog] = yield* Effect.all([makeDb('app'), makeDb('eventlog')], { concurrency: 2 })
-
-    const devtoolsOptions = yield* makeDevtoolsOptions({
-      devtoolsEnabled: devtools.enabled,
-      devtoolsPort: devtools.port,
-      devtoolsHost: devtools.host,
-      dbReadModel,
-      dbEventlog,
-      storeId,
-      clientId,
-      schemaPath,
-    })
-
-    const shutdownChannel = yield* makeShutdownChannel(storeId)
-
-    return makeLeaderThreadLayer({
-      schema,
-      storeId,
-      clientId,
-      makeSqliteDb,
-      syncOptions,
-      dbReadModel,
-      dbEventlog,
-      devtoolsOptions,
-      shutdownChannel,
-      syncPayload,
-    })
-  }).pipe(
-    Effect.tapCauseLogPretty,
-    UnexpectedError.mapToUnexpectedError,
-    Effect.withSpan('@livestore/adapter-node:worker:InitialMessage'),
-    Layer.unwrapScoped,
-  )
-
-const getAppDbFileName = (suffix: string) => `app${suffix}@${liveStoreStorageFormatVersion}.db`
-
-const makeDevtoolsOptions = ({
-  devtoolsEnabled,
-  dbReadModel,
-  dbEventlog,
-  storeId,
-  clientId,
-  devtoolsPort,
-  devtoolsHost,
-  schemaPath,
-}: {
-  devtoolsEnabled: boolean
-  dbReadModel: LeaderSqliteDb
-  dbEventlog: LeaderSqliteDb
-  storeId: string
-  clientId: string
-  devtoolsPort: number
-  devtoolsHost: string
-  schemaPath: string
-}): Effect.Effect<DevtoolsOptions, UnexpectedError, Scope.Scope> =>
-  Effect.gen(function* () {
-    if (devtoolsEnabled === false) {
-      return {
-        enabled: false,
-      }
-    }
-
-    return {
-      enabled: true,
-      makeBootContext: Effect.gen(function* () {
-        // TODO instead of failing when the port is already in use, we should try to use that WS server instead of starting a new one
-        yield* startDevtoolsServer({
-          schemaPath,
-          storeId,
-          clientId,
-          sessionId: 'static', // TODO make this dynamic
-          port: devtoolsPort,
-          host: devtoolsHost,
-        }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
-
-        const devtoolsWebChannel = yield* makeNodeDevtoolsChannel({
-          nodeName: `leader-${storeId}-${clientId}`,
-          target: `devtools-${storeId}-${clientId}-static`,
-          url: `ws://localhost:${devtoolsPort}`,
-          schema: { listen: Devtools.Leader.MessageToApp, send: Devtools.Leader.MessageFromApp },
-        })
-
-        return {
-          devtoolsWebChannel,
-          persistenceInfo: {
-            readModel: dbReadModel.metadata.persistenceInfo,
-            eventlog: dbEventlog.metadata.persistenceInfo,
-          },
-        }
-      }).pipe(Effect.provide(FetchHttpClient.layer)),
-    }
-  })

package/src/webchannel.ts

@@ -1,4 +1,4 @@
-import { BroadcastChannel } from 'node:worker_threads'
+import { type BroadcastChannel as NodeBroadcastChannel } from 'node:worker_threads'
 
 import type { Either, ParseResult } from '@livestore/utils/effect'
 import { Deferred, Effect, Exit, Schema, Scope, Stream, WebChannel } from '@livestore/utils/effect'
@@ -12,7 +12,14 @@ export const makeBroadcastChannel = <Msg, MsgEncoded>({
 }): Effect.Effect<WebChannel.WebChannel<Msg, Msg>, never, Scope.Scope> =>
   Effect.scopeWithCloseable((scope) =>
     Effect.gen(function* () {
-      const channel = new BroadcastChannel(channelName)
+      if (globalThis.BroadcastChannel === undefined) {
+        yield* Effect.logWarning('BroadcastChannel is not supported in this environment. Using a NoopChannel instead.')
+        return (yield* WebChannel.noopChannel()) as any as WebChannel.WebChannel<Msg, Msg>
+      }
+
+      // NOTE we're using `globalThis.BroadcastChannel` here because `BroadcastChannel`
+      // from `node:worker_threads` is not yet stable in Deno
+      const channel = new globalThis.BroadcastChannel(channelName) as any as NodeBroadcastChannel
 
       yield* Effect.addFinalizer(() => Effect.try(() => channel.close()).pipe(Effect.ignoreLogged))
 
@@ -37,7 +44,13 @@ export const makeBroadcastChannel = <Msg, MsgEncoded>({
 
             return channel
           }),
-          (channel) => Effect.sync(() => channel.unref()),
+          (channel) =>
+            Effect.sync(() => {
+              // NOTE not all BroadcastChannel implementations have the `unref` method
+              if (typeof channel.unref === 'function') {
+                channel.unref()
+              }
+            }),
         ),
       )
 

package/src/worker-schema.ts

@@ -18,31 +18,25 @@ export const WorkerArgv = Schema.parseJson(
   }),
 )
 
-export const StorageTypeOpfs = Schema.Struct({
-  type: Schema.Literal('opfs'),
+export const StorageTypeInMemory = Schema.Struct({
+  type: Schema.Literal('in-memory'),
+})
+
+export type StorageTypeInMemory = typeof StorageTypeInMemory.Type
+
+export const StorageTypeFs = Schema.Struct({
+  type: Schema.Literal('fs'),
   /**
-   * Default is `livestore-${storeId}`
+   * Where to store the database files
    *
-   * When providing this option, make sure to include the `storeId` in the path to avoid
-   * conflicts with other LiveStore apps.
+   * @default Current working directory
    */
-  directory: Schema.optional(Schema.String),
+  baseDirectory: Schema.String,
 })
 
-export type StorageTypeOpfs = typeof StorageTypeOpfs.Type
+export type StorageTypeFs = typeof StorageTypeFs.Type
 
-// export const StorageTypeIndexeddb = Schema.Struct({
-//   type: Schema.Literal('indexeddb'),
-//   /** @default "livestore" */
-//   databaseName: Schema.optionalWith(Schema.String, { default: () => 'livestore' }),
-//   /** @default "livestore-" */
-//   storeNamePrefix: Schema.optionalWith(Schema.String, { default: () => 'livestore-' }),
-// })
-
-export const StorageType = Schema.Union(
-  StorageTypeOpfs,
-  // StorageTypeIndexeddb
-)
+export const StorageType = Schema.Union(StorageTypeInMemory, StorageTypeFs)
 export type StorageType = typeof StorageType.Type
 export type StorageTypeEncoded = typeof StorageType.Encoded
 
@@ -71,14 +65,17 @@ export namespace LeaderWorkerInner {
     payload: {
       storeId: Schema.String,
       clientId: Schema.String,
-      baseDirectory: Schema.optional(Schema.String),
-      schemaPath: Schema.String,
+      storage: StorageType,
       syncPayload: Schema.UndefinedOr(Schema.JsonValue),
-      devtools: Schema.Struct({
-        port: Schema.Number,
-        host: Schema.String,
-        enabled: Schema.Boolean,
-      }),
+      devtools: Schema.Union(
+        Schema.Struct({
+          enabled: Schema.Literal(true),
+          schemaPath: Schema.String,
+          port: Schema.Number,
+          host: Schema.String,
+        }),
+        Schema.Struct({ enabled: Schema.Literal(false) }),
+      ),
     },
     success: Schema.Void,
     failure: UnexpectedError,

package/rollup.config.mjs

@@ -1,24 +0,0 @@
-import commonjs from '@rollup/plugin-commonjs'
-import { nodeResolve } from '@rollup/plugin-node-resolve'
-import terser from '@rollup/plugin-terser'
-
-export default {
-  input: 'dist/leader-thread.js',
-  output: {
-    file: 'dist/leader-thread.bundle.js',
-    // dir: 'dist/leader-thread-bundle',
-    format: 'esm',
-    // inlineDynamicImports: true,
-  },
-  external: ['@livestore/sqlite-wasm', '@opentelemetry/otlp-exporter-base'],
-  plugins: [
-    nodeResolve({
-      // esnext is needed for @opentelemetry/* packages
-      mainFields: ['esnext', 'module', 'main'],
-    }),
-    commonjs(),
-    terser(),
-  ],
-  // Needed for @opentelemetry/* packages
-  // inlineDynamicImports: true,
-}

package/src/client-session/in-memory-adapter.ts

@@ -1,170 +0,0 @@
-import type {
-  Adapter,
-  ClientSession,
-  ClientSessionLeaderThreadProxy,
-  LockStatus,
-  MakeSqliteDb,
-  SqliteDb,
-  SyncOptions,
-} from '@livestore/common'
-import { UnexpectedError } from '@livestore/common'
-import { Eventlog, LeaderThreadCtx, makeLeaderThreadLayer } from '@livestore/common/leader-thread'
-import type { LiveStoreSchema } from '@livestore/common/schema'
-import { LiveStoreEvent } from '@livestore/common/schema'
-import { sqliteDbFactory } from '@livestore/sqlite-wasm/browser'
-import { loadSqlite3Wasm } from '@livestore/sqlite-wasm/load-wasm'
-import type { Schema } from '@livestore/utils/effect'
-import { Cause, Effect, FetchHttpClient, Layer, Stream, SubscriptionRef } from '@livestore/utils/effect'
-import { nanoid } from '@livestore/utils/nanoid'
-
-import { makeShutdownChannel } from '../shutdown-channel.js'
-
-// TODO unify in-memory adapter with other in-memory adapter implementations
-
-export interface InMemoryAdapterOptions {
-  sync?: SyncOptions
-  /**
-   * @default 'in-memory'
-   */
-  clientId?: string
-  /**
-   * @default nanoid(6)
-   */
-  sessionId?: string
-
-  /** Only used internally for testing */
-  testing?: {
-    overrides?: TestingOverrides
-  }
-}
-
-export type TestingOverrides = {
-  clientSession?: {
-    leaderThreadProxy?: Partial<ClientSessionLeaderThreadProxy>
-  }
-  makeLeaderThread?: {
-    dbEventlog?: (makeSqliteDb: MakeSqliteDb) => Effect.Effect<SqliteDb, UnexpectedError>
-  }
-}
-
-/** NOTE: This adapter is currently only used for testing */
-export const makeInMemoryAdapter =
-  ({ sync: syncOptions, clientId = 'in-memory', sessionId = nanoid(6), testing }: InMemoryAdapterOptions): Adapter =>
-  ({
-    schema,
-    storeId,
-    shutdown,
-    syncPayload,
-    // devtoolsEnabled, bootStatusQueue, shutdown, connectDevtoolsToStore
-  }) =>
-    Effect.gen(function* () {
-      const sqlite3 = yield* Effect.promise(() => loadSqlite3Wasm())
-
-      const makeSqliteDb = sqliteDbFactory({ sqlite3 })
-      const sqliteDb = yield* makeSqliteDb({ _tag: 'in-memory' })
-
-      const lockStatus = yield* SubscriptionRef.make<LockStatus>('has-lock')
-
-      const shutdownChannel = yield* makeShutdownChannel(storeId)
-
-      yield* shutdownChannel.listen.pipe(
-        Stream.flatten(),
-        Stream.tap((error) => Effect.sync(() => shutdown(Cause.fail(error)))),
-        Stream.runDrain,
-        Effect.interruptible,
-        Effect.tapCauseLogPretty,
-        Effect.forkScoped,
-      )
-
-      const { leaderThread, initialSnapshot } = yield* makeLeaderThread({
-        storeId,
-        clientId,
-        schema,
-        makeSqliteDb,
-        syncOptions,
-        syncPayload,
-        testing,
-      })
-
-      sqliteDb.import(initialSnapshot)
-
-      const clientSession = {
-        sqliteDb,
-        devtools: { enabled: false },
-        clientId,
-        sessionId,
-        lockStatus,
-        leaderThread,
-        shutdown,
-      } satisfies ClientSession
-
-      return clientSession
-    }).pipe(UnexpectedError.mapToUnexpectedError)
-
-const makeLeaderThread = ({
-  storeId,
-  clientId,
-  schema,
-  makeSqliteDb,
-  syncOptions,
-  syncPayload,
-  testing,
-}: {
-  storeId: string
-  clientId: string
-  schema: LiveStoreSchema
-  makeSqliteDb: MakeSqliteDb
-  syncOptions: SyncOptions | undefined
-  syncPayload: Schema.JsonValue | undefined
-  testing?: {
-    overrides?: TestingOverrides
-  }
-}) =>
-  Effect.gen(function* () {
-    const layer = yield* Layer.memoize(
-      makeLeaderThreadLayer({
-        clientId,
-        dbReadModel: yield* makeSqliteDb({ _tag: 'in-memory' }),
-        dbEventlog: testing?.overrides?.makeLeaderThread?.dbEventlog
-          ? yield* testing.overrides.makeLeaderThread.dbEventlog(makeSqliteDb)
-          : yield* makeSqliteDb({ _tag: 'in-memory' }),
-        devtoolsOptions: { enabled: false },
-        makeSqliteDb,
-        schema,
-        // NOTE we're creating a separate channel here since you can't listen to your own channel messages
-        shutdownChannel: yield* makeShutdownChannel(storeId),
-        storeId,
-        syncOptions,
-        syncPayload,
-      }).pipe(Layer.provideMerge(FetchHttpClient.layer)),
-    )
-
-    return yield* Effect.gen(function* () {
-      const { dbReadModel, dbEventlog, syncProcessor, extraIncomingMessagesQueue, initialState } =
-        yield* LeaderThreadCtx
-
-      const initialLeaderHead = Eventlog.getClientHeadFromDb(dbEventlog)
-
-      const leaderThread = {
-        events: {
-          pull:
-            testing?.overrides?.clientSession?.leaderThreadProxy?.events?.pull ??
-            (({ cursor }) => syncProcessor.pull({ cursor })),
-          push: (batch) =>
-            syncProcessor.push(
-              batch.map((item) => new LiveStoreEvent.EncodedWithMeta(item)),
-              { waitForProcessing: true },
-            ),
-        },
-        initialState: { leaderHead: initialLeaderHead, migrationsReport: initialState.migrationsReport },
-        export: Effect.sync(() => dbReadModel.export()),
-        getEventlogData: Effect.sync(() => dbEventlog.export()),
-        getSyncState: syncProcessor.syncState,
-        sendDevtoolsMessage: (message) => extraIncomingMessagesQueue.offer(message),
-      } satisfies ClientSessionLeaderThreadProxy
-
-      const initialSnapshot = dbReadModel.export()
-
-      return { leaderThread, initialSnapshot }
-    }).pipe(Effect.provide(layer))
-  })

package/tsconfig.json

@@ -1,17 +0,0 @@
-{
-  "extends": "../../../tsconfig.base.json",
-  "compilerOptions": {
-    "outDir": "./dist",
-    "rootDir": "./src",
-    "resolveJsonModule": true,
-    "tsBuildInfoFile": "./dist/.tsbuildinfo"
-  },
-  "include": ["./src"],
-  "references": [
-    { "path": "../common" },
-    { "path": "../utils" },
-    { "path": "../devtools-node-common" },
-    { "path": "../webmesh" },
-    { "path": "../sqlite-wasm" }
-  ]
-}