@livestore/adapter-node 0.0.0-snapshot-a953343ad2d7468c6573bcb5e26f0eab4302078f

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@livestore/adapter-node",
+  "version": "0.0.0-snapshot-a953343ad2d7468c6573bcb5e26f0eab4302078f",
+  "type": "module",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./devtools": {
+      "types": "./dist/devtools/mod.d.ts",
+      "default": "./dist/devtools/mod.js"
+    },
+    "./worker": {
+      "types": "./dist/make-leader-worker.d.ts",
+      "default": "./dist/make-leader-worker.js"
+    }
+  },
+  "types": "./dist/index.d.ts",
+  "typesVersions": {
+    "*": {
+      "./devtools": [
+        "./dist/devtools/mod.d.ts"
+      ]
+    }
+  },
+  "dependencies": {
+    "@livestore/devtools-vite": "0.3.0-dev.15",
+    "@opentelemetry/api": "1.9.0",
+    "@opentelemetry/otlp-exporter-base": "0.57.2",
+    "vite": "6.1.0",
+    "ws": "8.18.0",
+    "@livestore/adapter-web": "0.0.0-snapshot-a953343ad2d7468c6573bcb5e26f0eab4302078f",
+    "@livestore/common": "0.0.0-snapshot-a953343ad2d7468c6573bcb5e26f0eab4302078f",
+    "@livestore/devtools-node-common": "0.0.0-snapshot-a953343ad2d7468c6573bcb5e26f0eab4302078f",
+    "@livestore/sqlite-wasm": "0.0.0-snapshot-a953343ad2d7468c6573bcb5e26f0eab4302078f",
+    "@livestore/utils": "0.0.0-snapshot-a953343ad2d7468c6573bcb5e26f0eab4302078f",
+    "@livestore/webmesh": "0.0.0-snapshot-a953343ad2d7468c6573bcb5e26f0eab4302078f"
+  },
+  "devDependencies": {
+    "@rollup/plugin-commonjs": "^28.0.1",
+    "@rollup/plugin-node-resolve": "^16.0.0",
+    "@rollup/plugin-terser": "^0.4.4",
+    "@types/ws": "^8.5.12",
+    "rollup": "^4.27.4"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "test": "echo No tests yet"
+  }
+}

package/rollup.config.mjs

@@ -0,0 +1,24 @@
+import commonjs from '@rollup/plugin-commonjs'
+import { nodeResolve } from '@rollup/plugin-node-resolve'
+import terser from '@rollup/plugin-terser'
+
+export default {
+  input: 'dist/leader-thread.js',
+  output: {
+    file: 'dist/leader-thread.bundle.js',
+    // dir: 'dist/leader-thread-bundle',
+    format: 'esm',
+    // inlineDynamicImports: true,
+  },
+  external: ['@livestore/sqlite-wasm', '@opentelemetry/otlp-exporter-base'],
+  plugins: [
+    nodeResolve({
+      // esnext is needed for @opentelemetry/* packages
+      mainFields: ['esnext', 'module', 'main'],
+    }),
+    commonjs(),
+    terser(),
+  ],
+  // Needed for @opentelemetry/* packages
+  // inlineDynamicImports: true,
+}

package/src/client-session/index.ts

@@ -0,0 +1,295 @@
+import { hostname } from 'node:os'
+import * as WT from 'node:worker_threads'
+
+import type {
+  Adapter,
+  ClientSession,
+  ClientSessionLeaderThreadProxy,
+  IntentionalShutdownCause,
+  LockStatus,
+  NetworkStatus,
+} from '@livestore/common'
+import { Devtools, UnexpectedError } from '@livestore/common'
+import type { MutationEvent } from '@livestore/common/schema'
+import { makeNodeDevtoolsChannel } from '@livestore/devtools-node-common/web-channel'
+import { loadSqlite3Wasm } from '@livestore/sqlite-wasm/load-wasm'
+import { sqliteDbFactory } from '@livestore/sqlite-wasm/node'
+import type { Cause } from '@livestore/utils/effect'
+import {
+  BucketQueue,
+  Effect,
+  Fiber,
+  ParseResult,
+  Schema,
+  Stream,
+  SubscriptionRef,
+  Worker,
+  WorkerError,
+} from '@livestore/utils/effect'
+import { PlatformNode } from '@livestore/utils/node'
+
+import * as WorkerSchema from '../worker-schema.js'
+
+export interface NodeAdapterOptions {
+  /**
+   * Example: `new URL('./livestore.worker.js', import.meta.url)`
+   */
+  workerUrl: URL
+  /** Needed for the worker and the devtools */
+  schemaPath: string
+  /** Where to store the database files */
+  baseDirectory?: string
+  /** The default is the hostname of the current machine */
+  clientId?: string
+  devtools?: {
+    /**
+     * Where to run the devtools server (via Vite)
+     *
+     * @default 4242
+     */
+    port: number
+  }
+}
+
+export const makeNodeAdapter = ({
+  workerUrl,
+  schemaPath,
+  baseDirectory,
+  devtools: devtoolsOptions = { port: 4242 },
+  clientId = hostname(),
+}: NodeAdapterOptions): Adapter =>
+  (({ storeId, devtoolsEnabled, shutdown, connectDevtoolsToStore }) =>
+    Effect.gen(function* () {
+      // TODO make this dynamic and actually support multiple sessions
+      const sessionId = 'static'
+
+      const sqlite3 = yield* Effect.promise(() => loadSqlite3Wasm())
+      const makeSqliteDb = yield* sqliteDbFactory({ sqlite3 })
+
+      // TODO consider bringing back happy-path initialisation boost
+      // const fileData = yield* fs.readFile(dbFilePath).pipe(Effect.either)
+      // if (fileData._tag === 'Right') {
+      //   syncInMemoryDb.import(fileData.right)
+      // } else {
+      //   yield* Effect.logWarning('Failed to load database file', fileData.left)
+      // }
+
+      const syncInMemoryDb = yield* makeSqliteDb({ _tag: 'in-memory' }).pipe(Effect.orDie)
+
+      // TODO actually implement this multi-session support
+      const lockStatus = yield* SubscriptionRef.make<LockStatus>('has-lock')
+
+      const { leaderThread, initialSnapshot } = yield* makeLeaderThread({
+        shutdown,
+        storeId,
+        clientId,
+        sessionId,
+        workerUrl,
+        baseDirectory,
+        devtoolsEnabled,
+        devtoolsOptions,
+        schemaPath,
+      })
+
+      syncInMemoryDb.import(initialSnapshot)
+
+      if (devtoolsEnabled) {
+        yield* Effect.gen(function* () {
+          const storeDevtoolsChannel = yield* makeNodeDevtoolsChannel({
+            nodeName: `client-session-${storeId}-${clientId}-${sessionId}`,
+            target: `devtools`,
+            url: `ws://localhost:${devtoolsOptions.port}`,
+            schema: {
+              listen: Devtools.ClientSession.MessageToApp,
+              send: Devtools.ClientSession.MessageFromApp,
+            },
+          })
+
+          yield* connectDevtoolsToStore(storeDevtoolsChannel)
+        }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
+      }
+
+      const devtools: ClientSession['devtools'] = devtoolsEnabled
+        ? { enabled: true, pullLatch: yield* Effect.makeLatch(true), pushLatch: yield* Effect.makeLatch(true) }
+        : { enabled: false }
+
+      const clientSession = {
+        sqliteDb: syncInMemoryDb,
+        leaderThread,
+        devtools,
+        lockStatus,
+        clientId,
+        sessionId,
+        shutdown,
+      } satisfies ClientSession
+
+      return clientSession
+    }).pipe(
+      Effect.withSpan('@livestore/adapter-node:adapter'),
+      Effect.parallelFinalizers,
+      Effect.provide(PlatformNode.NodeFileSystem.layer),
+    )) satisfies Adapter
+
+const makeLeaderThread = ({
+  shutdown,
+  storeId,
+  clientId,
+  sessionId,
+  workerUrl,
+  baseDirectory,
+  devtoolsEnabled,
+  devtoolsOptions,
+  schemaPath,
+}: {
+  shutdown: (cause: Cause.Cause<UnexpectedError | IntentionalShutdownCause>) => Effect.Effect<void>
+  storeId: string
+  clientId: string
+  sessionId: string
+  workerUrl: URL
+  baseDirectory: string | undefined
+  devtoolsEnabled: boolean
+  devtoolsOptions: { port: number }
+  schemaPath: string
+}) =>
+  Effect.gen(function* () {
+    const nodeWorker = new WT.Worker(workerUrl, {
+      execArgv: process.env.DEBUG_WORKER ? ['--inspect --enable-source-maps'] : ['--enable-source-maps'],
+      argv: [Schema.encodeSync(WorkerSchema.WorkerArgv)({ storeId, clientId, sessionId })],
+    })
+
+    const leaderThreadFiber = yield* Worker.makePoolSerialized<typeof WorkerSchema.LeaderWorkerInner.Request.Type>({
+      size: 1,
+      concurrency: 100,
+      initialMessage: () =>
+        new WorkerSchema.LeaderWorkerInner.InitialMessage({
+          storeId,
+          clientId,
+          baseDirectory,
+          devtools: { enabled: devtoolsEnabled, port: devtoolsOptions.port },
+          schemaPath,
+        }),
+    }).pipe(
+      Effect.provide(PlatformNode.NodeWorker.layer(() => nodeWorker)),
+      UnexpectedError.mapToUnexpectedError,
+      Effect.tapErrorCause(shutdown),
+      Effect.withSpan('@livestore/adapter-node:adapter:setupLeaderThread'),
+      Effect.tapCauseLogPretty,
+      Effect.forkScoped,
+    )
+
+    yield* Effect.addFinalizer(() =>
+      Effect.gen(function* () {
+        // We first try to gracefully shutdown the leader worker and then forcefully terminate it
+        yield* Effect.raceFirst(
+          runInWorker(new WorkerSchema.LeaderWorkerInner.Shutdown()).pipe(Effect.andThen(() => nodeWorker.terminate())),
+
+          Effect.sync(() => {
+            console.warn('[@livestore/adapter-node:adapter] Worker did not gracefully shutdown in time, terminating it')
+            nodeWorker.terminate()
+          }).pipe(Effect.delay(1000)),
+        ).pipe(Effect.exit) // The disconnect is to prevent the interrupt to bubble out
+      }).pipe(Effect.withSpan('@livestore/adapter-node:adapter:shutdown'), Effect.tapCauseLogPretty, Effect.orDie),
+    )
+
+    const runInWorker = <TReq extends typeof WorkerSchema.LeaderWorkerInner.Request.Type>(
+      req: TReq,
+    ): TReq extends Schema.WithResult<infer A, infer _I, infer _E, infer _EI, infer R>
+      ? Effect.Effect<A, UnexpectedError, R>
+      : never =>
+      Fiber.join(leaderThreadFiber).pipe(
+        Effect.flatMap((worker) => worker.executeEffect(req) as any),
+        Effect.logWarnIfTakesLongerThan({
+          label: `@livestore/adapter-node:client-session:runInWorker:${req._tag}`,
+          duration: 2000,
+        }),
+        Effect.withSpan(`@livestore/adapter-node:client-session:runInWorker:${req._tag}`),
+        Effect.mapError((cause) =>
+          Schema.is(UnexpectedError)(cause)
+            ? cause
+            : ParseResult.isParseError(cause) || Schema.is(WorkerError.WorkerError)(cause)
+              ? new UnexpectedError({ cause })
+              : cause,
+        ),
+        Effect.catchAllDefect((cause) => new UnexpectedError({ cause })),
+      ) as any
+
+    const runInWorkerStream = <TReq extends typeof WorkerSchema.LeaderWorkerInner.Request.Type>(
+      req: TReq,
+    ): TReq extends Schema.WithResult<infer A, infer _I, infer _E, infer _EI, infer R>
+      ? Stream.Stream<A, UnexpectedError, R>
+      : never =>
+      Effect.gen(function* () {
+        const sharedWorker = yield* Fiber.join(leaderThreadFiber)
+        return sharedWorker.execute(req as any).pipe(
+          Stream.mapError((cause) =>
+            Schema.is(UnexpectedError)(cause)
+              ? cause
+              : ParseResult.isParseError(cause) || Schema.is(WorkerError.WorkerError)(cause)
+                ? new UnexpectedError({ cause })
+                : cause,
+          ),
+          Stream.withSpan(`@livestore/adapter-node:client-session:runInWorkerStream:${req._tag}`),
+        )
+      }).pipe(Stream.unwrap) as any
+
+    const initialLeaderHead = yield* runInWorker(new WorkerSchema.LeaderWorkerInner.GetLeaderHead())
+
+    const networkStatus = yield* SubscriptionRef.make<NetworkStatus>({
+      isConnected: true,
+      timestampMs: Date.now(),
+      latchClosed: false,
+    })
+
+    const pushQueue = yield* BucketQueue.make<MutationEvent.AnyEncoded>()
+
+    yield* Effect.gen(function* () {
+      const batch = yield* BucketQueue.takeBetween(pushQueue, 1, 100)
+      yield* runInWorker(new WorkerSchema.LeaderWorkerInner.PushToLeader({ batch })).pipe(
+        Effect.withSpan('@livestore/adapter-node:client-session:pushToLeader', {
+          attributes: { batchSize: batch.length },
+        }),
+        // We can ignore the error here because the ClientSessionSyncProcessor will retry after rebasing
+        Effect.ignoreLogged,
+      )
+    }).pipe(Effect.forever, Effect.interruptible, Effect.tapCauseLogPretty, Effect.forkScoped)
+
+    const bootResult = yield* runInWorker(new WorkerSchema.LeaderWorkerInner.GetRecreateSnapshot()).pipe(
+      Effect.timeout(10_000),
+      UnexpectedError.mapToUnexpectedError,
+      Effect.withSpan('@livestore/adapter-node:client-session:export'),
+    )
+
+    const leaderThread = {
+      networkStatus,
+      mutations: {
+        pull: runInWorkerStream(new WorkerSchema.LeaderWorkerInner.PullStream({ cursor: initialLeaderHead })).pipe(
+          Stream.orDie,
+        ),
+        // NOTE instead of sending the worker message right away, we're batching the events in order to
+        // - maintain a consistent order of events
+        // - improve efficiency by reducing the number of messages
+        push: (batch) => BucketQueue.offerAll(pushQueue, batch),
+      },
+      initialState: {
+        leaderHead: initialLeaderHead,
+        migrationsReport: bootResult.migrationsReport,
+      },
+      export: runInWorker(new WorkerSchema.LeaderWorkerInner.Export()).pipe(
+        Effect.timeout(10_000),
+        UnexpectedError.mapToUnexpectedError,
+        Effect.withSpan('@livestore/adapter-node:client-session:export'),
+      ),
+      getMutationLogData: Effect.dieMessage('Not implemented'),
+      getSyncState: runInWorker(new WorkerSchema.LeaderWorkerInner.GetLeaderSyncState()).pipe(
+        UnexpectedError.mapToUnexpectedError,
+        Effect.withSpan('@livestore/adapter-node:client-session:getLeaderSyncState'),
+      ),
+      sendDevtoolsMessage: (message) =>
+        runInWorker(new WorkerSchema.LeaderWorkerInner.ExtraDevtoolsMessage({ message })).pipe(
+          UnexpectedError.mapToUnexpectedError,
+          Effect.withSpan('@livestore/adapter-node:client-session:devtoolsMessageForLeader'),
+        ),
+    } satisfies ClientSessionLeaderThreadProxy
+
+    return { leaderThread, initialSnapshot: bootResult.snapshot }
+  })

package/src/devtools/devtools-server.ts

@@ -0,0 +1,88 @@
+import http from 'node:http'
+import path from 'node:path'
+
+import { UnexpectedError } from '@livestore/common'
+import type { Scope } from '@livestore/utils/effect'
+import { Effect } from '@livestore/utils/effect'
+import { makeWebSocketServer } from '@livestore/webmesh/websocket-server'
+
+import { makeViteServer } from './vite-dev-server.js'
+
+/**
+ * Starts a devtools HTTP/WS server which serves ...
+ * - the Devtools UI via Vite
+ * - the Devtools Protocol via WebSocket Webmesh
+ */
+export const startDevtoolsServer = ({
+  schemaPath,
+  storeId,
+  clientId,
+  sessionId,
+  port,
+}: {
+  schemaPath: string
+  storeId: string
+  clientId: string
+  sessionId: string
+  port: number
+}): Effect.Effect<void, UnexpectedError, Scope.Scope> =>
+  Effect.gen(function* () {
+    const httpServer = http.createServer()
+    const webSocketServer = yield* makeWebSocketServer({ relayNodeName: 'ws' })
+
+    yield* Effect.addFinalizer(() => Effect.sync(() => httpServer.close()))
+
+    // Handle upgrade manually
+    httpServer.on('upgrade', (request, socket, head) => {
+      webSocketServer.handleUpgrade(request, socket, head, (ws) => {
+        webSocketServer.emit('connection', ws, request)
+      })
+    })
+
+    const startServer = (port: number) =>
+      Effect.async<void, UnexpectedError>((cb) => {
+        httpServer.on('error', (err: any) => {
+          cb(UnexpectedError.make({ cause: err }))
+        })
+
+        httpServer.listen(port, () => {
+          cb(Effect.succeed(undefined))
+        })
+      })
+
+    yield* startServer(port)
+
+    yield* Effect.logDebug(
+      `[@livestore/adapter-node:devtools] LiveStore devtools are available at http://localhost:${port}/livestore-devtools`,
+    )
+
+    const viteServer = yield* makeViteServer({
+      mode: { _tag: 'node', storeId, clientId, sessionId, url: `ws://localhost:${port}` },
+      schemaPath: path.resolve(process.cwd(), schemaPath),
+      viteConfig: (viteConfig) => {
+        viteConfig.server ??= {}
+        viteConfig.server.fs ??= {}
+
+        // TODO move this into the example code
+        // Point to Overtone monorepo root
+        viteConfig.server.fs.allow ??= []
+        viteConfig.server.fs.allow.push(process.env.WORKSPACE_ROOT + '/../..')
+
+        viteConfig.optimizeDeps ??= {}
+        viteConfig.optimizeDeps.force = true
+
+        return viteConfig
+      },
+    })
+
+    yield* Effect.addFinalizer(() => Effect.promise(() => viteServer.close()))
+
+    httpServer.on('request', (req, res) => {
+      if (req.url === '/' || req.url === '') {
+        res.writeHead(302, { Location: '/livestore-devtools' })
+        res.end()
+      } else if (req.url?.startsWith('/livestore-devtools')) {
+        return viteServer.middlewares(req, res as any)
+      }
+    })
+  }).pipe(Effect.withSpan('@livestore/adapter-node:devtools:startDevtoolsServer'))

package/src/devtools/mod.ts

@@ -0,0 +1,2 @@
+export { makeViteServer } from './vite-dev-server.js'
+export type { Options } from './types.js'

package/src/devtools/types.d.ts

@@ -0,0 +1,33 @@
+import type * as http from 'node:http'
+
+import type * as Vite from 'vite'
+
+export type Middleware = (req: http.IncomingMessage, res: http.ServerResponse, next: () => void) => void
+
+export type Options = {
+  viteConfig?: (config: Vite.UserConfig) => Vite.UserConfig
+  /**
+   * Path to the file exporting the LiveStore schema as `export const schema = ...`
+   * File path must be relative to the project root and will be imported via Vite.
+   *
+   * Example: `./src/schema.ts`
+   */
+  schemaPath: string
+  // TODO consolidate with `Mode` in `@livestore/devtools-react/devtools-api.ts`
+  /**
+   * The mode of the devtools server.
+   *
+   * @default 'node'
+   */
+  mode:
+    | {
+        _tag: 'node'
+        storeId: string
+        clientId: string
+        sessionId: string
+        url: string
+      }
+    | {
+        _tag: 'expo'
+      }
+}

package/src/devtools/vite-dev-server.ts

@@ -0,0 +1,122 @@
+import * as http from 'node:http'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import { UnexpectedError } from '@livestore/common'
+import { Effect } from '@livestore/utils/effect'
+import * as Vite from 'vite'
+
+import type { Options } from './types.js'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+export const makeViteServer = (options: Options): Effect.Effect<Vite.ViteDevServer, UnexpectedError> =>
+  Effect.gen(function* () {
+    const hmrPort = yield* getFreePort
+
+    const cwd = process.cwd()
+
+    const defaultViteConfig = Vite.defineConfig({
+      server: {
+        middlewareMode: true,
+        hmr: {
+          port: hmrPort,
+        },
+        fs: {
+          // Adds `node_modules` so we can import `@livestore/wa-sqlite` for WASM to work
+          allow: [path.resolve(__dirname, '..', '..')],
+        },
+      },
+      resolve: {
+        alias: {
+          '@schema': path.resolve(cwd, options.schemaPath),
+        },
+      },
+      appType: 'spa',
+      optimizeDeps: {
+        // TODO remove once fixed https://github.com/vitejs/vite/issues/8427
+        exclude: ['@livestore/wa-sqlite'],
+      },
+      root: __dirname,
+      base: '/livestore-devtools/',
+      plugins: [virtualHtmlPlugin(options.mode)],
+      clearScreen: false,
+      logLevel: 'silent',
+    })
+
+    const viteConfig = options.viteConfig?.(defaultViteConfig) ?? defaultViteConfig
+
+    const viteServer = yield* Effect.promise(() => Vite.createServer(viteConfig)).pipe(
+      UnexpectedError.mapToUnexpectedError,
+    )
+
+    return viteServer
+  }).pipe(Effect.withSpan('@livestore/adapter-node:devtools:makeViteServer'))
+
+// TODO unify this with `@livestore/devtools-vite/plugin.ts`
+const virtualHtmlPlugin = (mode: Options['mode']): Vite.Plugin => ({
+  name: 'virtual-html',
+  configureServer: (server) => {
+    return () => {
+      server.middlewares.use(async (req, res, next) => {
+        if (req.url === '/' || req.url === '' || req.url === '/index.html') {
+          const html = `
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="UTF-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1.0" />
+<meta name="livestore-devtools" content="true" />
+<title>LiveStore Devtools</title>
+</head>
+<body>
+<div id="root"></div>
+<script type="module">
+import '@livestore/devtools-react/index.css'
+import { mountDevtools } from '@livestore/devtools-react'
+import sharedWorker from '@livestore/adapter-web/shared-worker?sharedworker'
+import { schema } from '@schema'
+
+mountDevtools({
+  schema,
+  rootEl: document.getElementById('root'),
+  sharedWorker,
+  mode: ${JSON.stringify(mode)},
+  license: ${JSON.stringify(process.env.LSD_LICENSE)},
+})
+</script>
+</body>
+</html>
+          `
+          const transformedHtml = await server.transformIndexHtml(req.url, html)
+          res.statusCode = 200
+          res.setHeader('Content-Type', 'text/html')
+          res.end(transformedHtml)
+        } else {
+          next()
+        }
+      })
+    }
+  },
+})
+
+export const getFreePort = Effect.async<number, UnexpectedError>((cb) => {
+  const server = http.createServer()
+
+  // Listen on port 0 to get an available port
+  server.listen(0, () => {
+    const address = server.address()
+
+    if (address && typeof address === 'object') {
+      const port = address.port
+      server.close(() => cb(Effect.succeed(port)))
+    } else {
+      server.close(() => cb(UnexpectedError.make({ cause: 'Failed to get a free port' })))
+    }
+  })
+
+  // Error handling in case the server encounters an error
+  server.on('error', (err) => {
+    server.close(() => cb(UnexpectedError.make({ cause: err })))
+  })
+})