@livestore/adapter-node 0.0.0-snapshot-5a6440f111a5a7c2441a649866587d5d51e22820.3 → 0.0.0-snapshot-412a36a7e6c9b0e9e237b553fd0522aed285228f

package/src/make-leader-worker.ts

@@ -1,7 +1,6 @@
 import './thread-polyfill.js'
 
 import inspector from 'node:inspector'
-import path from 'node:path'
 
 if (process.execArgv.includes('--inspect')) {
   inspector.open()
@@ -9,15 +8,12 @@ if (process.execArgv.includes('--inspect')) {
 }
 
 import type { SyncOptions } from '@livestore/common'
-import { Devtools, liveStoreStorageFormatVersion, UnexpectedError } from '@livestore/common'
-import type { DevtoolsOptions, LeaderSqliteDb } from '@livestore/common/leader-thread'
-import { configureConnection, Eventlog, LeaderThreadCtx, makeLeaderThreadLayer } from '@livestore/common/leader-thread'
+import { UnexpectedError } from '@livestore/common'
+import { Eventlog, LeaderThreadCtx } from '@livestore/common/leader-thread'
 import type { LiveStoreSchema } from '@livestore/common/schema'
 import { LiveStoreEvent } from '@livestore/common/schema'
-import { makeNodeDevtoolsChannel } from '@livestore/devtools-node-common/web-channel'
 import { loadSqlite3Wasm } from '@livestore/sqlite-wasm/load-wasm'
 import { sqliteDbFactory } from '@livestore/sqlite-wasm/node'
-import type { FileSystem, HttpClient, Scope } from '@livestore/utils/effect'
 import {
   Effect,
   FetchHttpClient,
@@ -33,17 +29,19 @@ import {
 import { PlatformNode } from '@livestore/utils/node'
 import type * as otel from '@opentelemetry/api'
 
-import { startDevtoolsServer } from './devtools/devtools-server.js'
-import { makeShutdownChannel } from './shutdown-channel.js'
+import type { TestingOverrides } from './leader-thread-shared.js'
+import { makeLeaderThread } from './leader-thread-shared.js'
 import * as WorkerSchema from './worker-schema.js'
 
 export type WorkerOptions = {
+  schema: LiveStoreSchema
   sync?: SyncOptions
   otelOptions?: {
     tracer?: otel.Tracer
     /** @default 'livestore-node-leader-thread' */
     serviceName?: string
   }
+  testing?: TestingOverrides
 }
 
 export const getWorkerArgs = () => Schema.decodeSync(WorkerSchema.WorkerArgv)(process.argv[2]!)
@@ -60,7 +58,20 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
     : undefined
 
   return WorkerRunner.layerSerialized(WorkerSchema.LeaderWorkerInner.Request, {
-    InitialMessage: (args) => makeLeaderThread({ ...args, syncOptions: options.sync }),
+    InitialMessage: (args) =>
+      Effect.gen(function* () {
+        const sqlite3 = yield* Effect.promise(() => loadSqlite3Wasm()).pipe(
+          Effect.withSpan('@livestore/adapter-node:leader-thread:loadSqlite3Wasm'),
+        )
+        const makeSqliteDb = yield* sqliteDbFactory({ sqlite3 })
+        return yield* makeLeaderThread({
+          ...args,
+          syncOptions: options.sync,
+          schema: options.schema,
+          testing: options.testing,
+          makeSqliteDb,
+        })
+      }).pipe(Layer.unwrapScoped),
     PushToLeader: ({ batch }) =>
       Effect.andThen(LeaderThreadCtx, (_) =>
         _.syncProcessor.push(
@@ -149,132 +160,3 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
     Logger.withMinimumLogLevel(LogLevel.Debug),
   )
 }
-
-const makeLeaderThread = ({
-  storeId,
-  clientId,
-  syncOptions,
-  baseDirectory,
-  devtools,
-  schemaPath,
-  syncPayload,
-}: WorkerSchema.LeaderWorkerInner.InitialMessage & {
-  syncOptions: SyncOptions | undefined
-  schemaPath: string
-}): Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient | FileSystem.FileSystem> =>
-  Effect.gen(function* () {
-    const schema = yield* Effect.promise(() => import(schemaPath).then((m) => m.schema as LiveStoreSchema))
-
-    const sqlite3 = yield* Effect.promise(() => loadSqlite3Wasm()).pipe(
-      Effect.withSpan('@livestore/adapter-node:leader-thread:loadSqlite3Wasm'),
-    )
-    const makeSqliteDb = yield* sqliteDbFactory({ sqlite3 })
-    const runtime = yield* Effect.runtime<never>()
-
-    const schemaHashSuffix =
-      schema.state.sqlite.migrations.strategy === 'manual' ? 'fixed' : schema.state.sqlite.hash.toString()
-
-    const makeDb = (kind: 'state' | 'eventlog') =>
-      makeSqliteDb({
-        _tag: 'fs',
-        directory: path.join(baseDirectory ?? '', storeId),
-        fileName:
-          kind === 'state' ? getStateDbFileName(schemaHashSuffix) : `eventlog@${liveStoreStorageFormatVersion}.db`,
-        // TODO enable WAL for nodejs
-        configureDb: (db) =>
-          configureConnection(db, { foreignKeys: true }).pipe(Effect.provide(runtime), Effect.runSync),
-      }).pipe(Effect.acquireRelease((db) => Effect.sync(() => db.close())))
-
-    // Might involve some async work, so we're running them concurrently
-    const [dbState, dbEventlog] = yield* Effect.all([makeDb('state'), makeDb('eventlog')], { concurrency: 2 })
-
-    const devtoolsOptions = yield* makeDevtoolsOptions({
-      devtoolsEnabled: devtools.enabled,
-      devtoolsPort: devtools.port,
-      devtoolsHost: devtools.host,
-      dbState,
-      dbEventlog,
-      storeId,
-      clientId,
-      schemaPath,
-    })
-
-    const shutdownChannel = yield* makeShutdownChannel(storeId)
-
-    return makeLeaderThreadLayer({
-      schema,
-      storeId,
-      clientId,
-      makeSqliteDb,
-      syncOptions,
-      dbState,
-      dbEventlog,
-      devtoolsOptions,
-      shutdownChannel,
-      syncPayload,
-    })
-  }).pipe(
-    Effect.tapCauseLogPretty,
-    UnexpectedError.mapToUnexpectedError,
-    Effect.withSpan('@livestore/adapter-node:worker:InitialMessage'),
-    Layer.unwrapScoped,
-  )
-
-const getStateDbFileName = (suffix: string) => `state${suffix}@${liveStoreStorageFormatVersion}.db`
-
-const makeDevtoolsOptions = ({
-  devtoolsEnabled,
-  dbState,
-  dbEventlog,
-  storeId,
-  clientId,
-  devtoolsPort,
-  devtoolsHost,
-  schemaPath,
-}: {
-  devtoolsEnabled: boolean
-  dbState: LeaderSqliteDb
-  dbEventlog: LeaderSqliteDb
-  storeId: string
-  clientId: string
-  devtoolsPort: number
-  devtoolsHost: string
-  schemaPath: string
-}): Effect.Effect<DevtoolsOptions, UnexpectedError, Scope.Scope> =>
-  Effect.gen(function* () {
-    if (devtoolsEnabled === false) {
-      return {
-        enabled: false,
-      }
-    }
-
-    return {
-      enabled: true,
-      makeBootContext: Effect.gen(function* () {
-        // TODO instead of failing when the port is already in use, we should try to use that WS server instead of starting a new one
-        yield* startDevtoolsServer({
-          schemaPath,
-          storeId,
-          clientId,
-          sessionId: 'static', // TODO make this dynamic
-          port: devtoolsPort,
-          host: devtoolsHost,
-        }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
-
-        const devtoolsWebChannel = yield* makeNodeDevtoolsChannel({
-          nodeName: `leader-${storeId}-${clientId}`,
-          target: `devtools-${storeId}-${clientId}-static`,
-          url: `ws://localhost:${devtoolsPort}`,
-          schema: { listen: Devtools.Leader.MessageToApp, send: Devtools.Leader.MessageFromApp },
-        })
-
-        return {
-          devtoolsWebChannel,
-          persistenceInfo: {
-            state: dbState.metadata.persistenceInfo,
-            eventlog: dbEventlog.metadata.persistenceInfo,
-          },
-        }
-      }).pipe(Effect.provide(FetchHttpClient.layer)),
-    }
-  })

package/src/worker-schema.ts

@@ -18,31 +18,25 @@ export const WorkerArgv = Schema.parseJson(
   }),
 )
 
-export const StorageTypeOpfs = Schema.Struct({
-  type: Schema.Literal('opfs'),
+export const StorageTypeInMemory = Schema.Struct({
+  type: Schema.Literal('in-memory'),
+})
+
+export type StorageTypeInMemory = typeof StorageTypeInMemory.Type
+
+export const StorageTypeFs = Schema.Struct({
+  type: Schema.Literal('fs'),
   /**
-   * Default is `livestore-${storeId}`
+   * Where to store the database files
    *
-   * When providing this option, make sure to include the `storeId` in the path to avoid
-   * conflicts with other LiveStore apps.
+   * @default Current working directory
    */
-  directory: Schema.optional(Schema.String),
+  baseDirectory: Schema.String,
 })
 
-export type StorageTypeOpfs = typeof StorageTypeOpfs.Type
+export type StorageTypeFs = typeof StorageTypeFs.Type
 
-// export const StorageTypeIndexeddb = Schema.Struct({
-//   type: Schema.Literal('indexeddb'),
-//   /** @default "livestore" */
-//   databaseName: Schema.optionalWith(Schema.String, { default: () => 'livestore' }),
-//   /** @default "livestore-" */
-//   storeNamePrefix: Schema.optionalWith(Schema.String, { default: () => 'livestore-' }),
-// })
-
-export const StorageType = Schema.Union(
-  StorageTypeOpfs,
-  // StorageTypeIndexeddb
-)
+export const StorageType = Schema.Union(StorageTypeInMemory, StorageTypeFs)
 export type StorageType = typeof StorageType.Type
 export type StorageTypeEncoded = typeof StorageType.Encoded
 
@@ -71,14 +65,17 @@ export namespace LeaderWorkerInner {
     payload: {
       storeId: Schema.String,
       clientId: Schema.String,
-      baseDirectory: Schema.optional(Schema.String),
-      schemaPath: Schema.String,
+      storage: StorageType,
       syncPayload: Schema.UndefinedOr(Schema.JsonValue),
-      devtools: Schema.Struct({
-        port: Schema.Number,
-        host: Schema.String,
-        enabled: Schema.Boolean,
-      }),
+      devtools: Schema.Union(
+        Schema.Struct({
+          enabled: Schema.Literal(true),
+          schemaPath: Schema.String,
+          port: Schema.Number,
+          host: Schema.String,
+        }),
+        Schema.Struct({ enabled: Schema.Literal(false) }),
+      ),
     },
     success: Schema.Void,
     failure: UnexpectedError,

package/src/client-session/in-memory-adapter.ts

@@ -1,169 +0,0 @@
-import type {
-  Adapter,
-  ClientSession,
-  ClientSessionLeaderThreadProxy,
-  LockStatus,
-  MakeSqliteDb,
-  SqliteDb,
-  SyncOptions,
-} from '@livestore/common'
-import { UnexpectedError } from '@livestore/common'
-import { Eventlog, LeaderThreadCtx, makeLeaderThreadLayer } from '@livestore/common/leader-thread'
-import type { LiveStoreSchema } from '@livestore/common/schema'
-import { LiveStoreEvent } from '@livestore/common/schema'
-import { sqliteDbFactory } from '@livestore/sqlite-wasm/browser'
-import { loadSqlite3Wasm } from '@livestore/sqlite-wasm/load-wasm'
-import type { Schema } from '@livestore/utils/effect'
-import { Cause, Effect, FetchHttpClient, Layer, Stream, SubscriptionRef } from '@livestore/utils/effect'
-import { nanoid } from '@livestore/utils/nanoid'
-
-import { makeShutdownChannel } from '../shutdown-channel.js'
-
-// TODO unify in-memory adapter with other in-memory adapter implementations
-
-export interface InMemoryAdapterOptions {
-  sync?: SyncOptions
-  /**
-   * @default 'in-memory'
-   */
-  clientId?: string
-  /**
-   * @default nanoid(6)
-   */
-  sessionId?: string
-
-  /** Only used internally for testing */
-  testing?: {
-    overrides?: TestingOverrides
-  }
-}
-
-export type TestingOverrides = {
-  clientSession?: {
-    leaderThreadProxy?: Partial<ClientSessionLeaderThreadProxy>
-  }
-  makeLeaderThread?: {
-    dbEventlog?: (makeSqliteDb: MakeSqliteDb) => Effect.Effect<SqliteDb, UnexpectedError>
-  }
-}
-
-/** NOTE: This adapter is currently only used for testing */
-export const makeInMemoryAdapter =
-  ({ sync: syncOptions, clientId = 'in-memory', sessionId = nanoid(6), testing }: InMemoryAdapterOptions): Adapter =>
-  ({
-    schema,
-    storeId,
-    shutdown,
-    syncPayload,
-    // devtoolsEnabled, bootStatusQueue, shutdown, connectDevtoolsToStore
-  }) =>
-    Effect.gen(function* () {
-      const sqlite3 = yield* Effect.promise(() => loadSqlite3Wasm())
-
-      const makeSqliteDb = sqliteDbFactory({ sqlite3 })
-      const sqliteDb = yield* makeSqliteDb({ _tag: 'in-memory' })
-
-      const lockStatus = yield* SubscriptionRef.make<LockStatus>('has-lock')
-
-      const shutdownChannel = yield* makeShutdownChannel(storeId)
-
-      yield* shutdownChannel.listen.pipe(
-        Stream.flatten(),
-        Stream.tap((error) => Effect.sync(() => shutdown(Cause.fail(error)))),
-        Stream.runDrain,
-        Effect.interruptible,
-        Effect.tapCauseLogPretty,
-        Effect.forkScoped,
-      )
-
-      const { leaderThread, initialSnapshot } = yield* makeLeaderThread({
-        storeId,
-        clientId,
-        schema,
-        makeSqliteDb,
-        syncOptions,
-        syncPayload,
-        testing,
-      })
-
-      sqliteDb.import(initialSnapshot)
-
-      const clientSession = {
-        sqliteDb,
-        devtools: { enabled: false },
-        clientId,
-        sessionId,
-        lockStatus,
-        leaderThread,
-        shutdown,
-      } satisfies ClientSession
-
-      return clientSession
-    }).pipe(UnexpectedError.mapToUnexpectedError)
-
-const makeLeaderThread = ({
-  storeId,
-  clientId,
-  schema,
-  makeSqliteDb,
-  syncOptions,
-  syncPayload,
-  testing,
-}: {
-  storeId: string
-  clientId: string
-  schema: LiveStoreSchema
-  makeSqliteDb: MakeSqliteDb
-  syncOptions: SyncOptions | undefined
-  syncPayload: Schema.JsonValue | undefined
-  testing?: {
-    overrides?: TestingOverrides
-  }
-}) =>
-  Effect.gen(function* () {
-    const layer = yield* Layer.memoize(
-      makeLeaderThreadLayer({
-        clientId,
-        dbState: yield* makeSqliteDb({ _tag: 'in-memory' }),
-        dbEventlog: testing?.overrides?.makeLeaderThread?.dbEventlog
-          ? yield* testing.overrides.makeLeaderThread.dbEventlog(makeSqliteDb)
-          : yield* makeSqliteDb({ _tag: 'in-memory' }),
-        devtoolsOptions: { enabled: false },
-        makeSqliteDb,
-        schema,
-        // NOTE we're creating a separate channel here since you can't listen to your own channel messages
-        shutdownChannel: yield* makeShutdownChannel(storeId),
-        storeId,
-        syncOptions,
-        syncPayload,
-      }).pipe(Layer.provideMerge(FetchHttpClient.layer)),
-    )
-
-    return yield* Effect.gen(function* () {
-      const { dbState, dbEventlog, syncProcessor, extraIncomingMessagesQueue, initialState } = yield* LeaderThreadCtx
-
-      const initialLeaderHead = Eventlog.getClientHeadFromDb(dbEventlog)
-
-      const leaderThread = {
-        events: {
-          pull:
-            testing?.overrides?.clientSession?.leaderThreadProxy?.events?.pull ??
-            (({ cursor }) => syncProcessor.pull({ cursor })),
-          push: (batch) =>
-            syncProcessor.push(
-              batch.map((item) => new LiveStoreEvent.EncodedWithMeta(item)),
-              { waitForProcessing: true },
-            ),
-        },
-        initialState: { leaderHead: initialLeaderHead, migrationsReport: initialState.migrationsReport },
-        export: Effect.sync(() => dbState.export()),
-        getEventlogData: Effect.sync(() => dbEventlog.export()),
-        getSyncState: syncProcessor.syncState,
-        sendDevtoolsMessage: (message) => extraIncomingMessagesQueue.offer(message),
-      } satisfies ClientSessionLeaderThreadProxy
-
-      const initialSnapshot = dbState.export()
-
-      return { leaderThread, initialSnapshot }
-    }).pipe(Effect.provide(layer))
-  })