@livestore/adapter-node 0.0.0-snapshot-2b8a9de3ec1a701aca891ebc2c98eb328274ae9e → 0.0.0-snapshot-2c861249e50661661613204300b1fc0d902c2e46

package/src/client-session/in-memory-adapter.ts

@@ -4,12 +4,13 @@ import type {
   ClientSessionLeaderThreadProxy,
   LockStatus,
   MakeSqliteDb,
+  SqliteDb,
   SyncOptions,
 } from '@livestore/common'
 import { UnexpectedError } from '@livestore/common'
-import { getClientHeadFromDb, LeaderThreadCtx, makeLeaderThreadLayer } from '@livestore/common/leader-thread'
+import { Eventlog, LeaderThreadCtx, makeLeaderThreadLayer } from '@livestore/common/leader-thread'
 import type { LiveStoreSchema } from '@livestore/common/schema'
-import { MutationEvent } from '@livestore/common/schema'
+import { LiveStoreEvent } from '@livestore/common/schema'
 import { sqliteDbFactory } from '@livestore/sqlite-wasm/browser'
 import { loadSqlite3Wasm } from '@livestore/sqlite-wasm/load-wasm'
 import type { Schema } from '@livestore/utils/effect'
@@ -33,9 +34,16 @@ export interface InMemoryAdapterOptions {
 
   /** Only used internally for testing */
   testing?: {
-    overrides?: {
-      leaderThread?: Partial<ClientSessionLeaderThreadProxy>
-    }
+    overrides?: TestingOverrides
+  }
+}
+
+export type TestingOverrides = {
+  clientSession?: {
+    leaderThreadProxy?: Partial<ClientSessionLeaderThreadProxy>
+  }
+  makeLeaderThread?: {
+    dbEventlog?: (makeSqliteDb: MakeSqliteDb) => Effect.Effect<SqliteDb, UnexpectedError>
   }
 }
 
@@ -109,9 +117,7 @@ const makeLeaderThread = ({
   syncOptions: SyncOptions | undefined
   syncPayload: Schema.JsonValue | undefined
   testing?: {
-    overrides?: {
-      leaderThread?: Partial<ClientSessionLeaderThreadProxy>
-    }
+    overrides?: TestingOverrides
   }
 }) =>
   Effect.gen(function* () {
@@ -119,7 +125,9 @@ const makeLeaderThread = ({
       makeLeaderThreadLayer({
         clientId,
         dbReadModel: yield* makeSqliteDb({ _tag: 'in-memory' }),
-        dbMutationLog: yield* makeSqliteDb({ _tag: 'in-memory' }),
+        dbEventlog: testing?.overrides?.makeLeaderThread?.dbEventlog
+          ? yield* testing.overrides.makeLeaderThread.dbEventlog(makeSqliteDb)
+          : yield* makeSqliteDb({ _tag: 'in-memory' }),
         devtoolsOptions: { enabled: false },
         makeSqliteDb,
         schema,
@@ -132,41 +140,30 @@ const makeLeaderThread = ({
     )
 
     return yield* Effect.gen(function* () {
-      const {
-        dbReadModel: db,
-        dbMutationLog,
-        syncProcessor,
-        connectedClientSessionPullQueues,
-        extraIncomingMessagesQueue,
-        initialState,
-      } = yield* LeaderThreadCtx
-
-      const initialLeaderHead = getClientHeadFromDb(dbMutationLog)
-      const pullQueue = yield* connectedClientSessionPullQueues.makeQueue(initialLeaderHead)
+      const { dbReadModel, dbEventlog, syncProcessor, extraIncomingMessagesQueue, initialState } =
+        yield* LeaderThreadCtx
+
+      const initialLeaderHead = Eventlog.getClientHeadFromDb(dbEventlog)
 
       const leaderThread = {
-        mutations: {
-          pull: testing?.overrides?.leaderThread?.mutations?.pull ?? (() => Stream.fromQueue(pullQueue)),
+        events: {
+          pull:
+            testing?.overrides?.clientSession?.leaderThreadProxy?.events?.pull ??
+            (({ cursor }) => syncProcessor.pull({ cursor })),
           push: (batch) =>
-            syncProcessor
-              .push(
-                batch.map((item) => new MutationEvent.EncodedWithMeta(item)),
-                { waitForProcessing: true },
-              )
-              .pipe(Effect.provide(layer), Effect.scoped),
+            syncProcessor.push(
+              batch.map((item) => new LiveStoreEvent.EncodedWithMeta(item)),
+              { waitForProcessing: true },
+            ),
         },
         initialState: { leaderHead: initialLeaderHead, migrationsReport: initialState.migrationsReport },
-        export: Effect.sync(() => db.export()),
-        getMutationLogData: Effect.sync(() => dbMutationLog.export()),
-        // TODO
-        networkStatus: SubscriptionRef.make({ isConnected: false, timestampMs: Date.now(), latchClosed: false }).pipe(
-          Effect.runSync,
-        ),
+        export: Effect.sync(() => dbReadModel.export()),
+        getEventlogData: Effect.sync(() => dbEventlog.export()),
         getSyncState: syncProcessor.syncState,
         sendDevtoolsMessage: (message) => extraIncomingMessagesQueue.offer(message),
       } satisfies ClientSessionLeaderThreadProxy
 
-      const initialSnapshot = db.export()
+      const initialSnapshot = dbReadModel.export()
 
       return { leaderThread, initialSnapshot }
     }).pipe(Effect.provide(layer))

package/src/client-session/persisted-adapter.ts

@@ -8,7 +8,6 @@ import type {
   ClientSessionLeaderThreadProxy,
   IntentionalShutdownCause,
   LockStatus,
-  NetworkStatus,
 } from '@livestore/common'
 import { Devtools, UnexpectedError } from '@livestore/common'
 import * as DevtoolsNode from '@livestore/devtools-node-common/web-channel'
@@ -289,12 +288,6 @@ const makeLeaderThread = ({
 
     const initialLeaderHead = yield* runInWorker(new WorkerSchema.LeaderWorkerInner.GetLeaderHead())
 
-    const networkStatus = yield* SubscriptionRef.make<NetworkStatus>({
-      isConnected: true,
-      timestampMs: Date.now(),
-      latchClosed: false,
-    })
-
     const bootResult = yield* runInWorker(new WorkerSchema.LeaderWorkerInner.GetRecreateSnapshot()).pipe(
       Effect.timeout(10_000),
       UnexpectedError.mapToUnexpectedError,
@@ -302,8 +295,7 @@ const makeLeaderThread = ({
     )
 
     const leaderThread = {
-      networkStatus,
-      mutations: {
+      events: {
         pull: ({ cursor }) =>
           runInWorkerStream(new WorkerSchema.LeaderWorkerInner.PullStream({ cursor })).pipe(Stream.orDie),
         push: (batch) =>
@@ -322,7 +314,7 @@ const makeLeaderThread = ({
         UnexpectedError.mapToUnexpectedError,
         Effect.withSpan('@livestore/adapter-node:client-session:export'),
       ),
-      getMutationLogData: Effect.dieMessage('Not implemented'),
+      getEventlogData: Effect.dieMessage('Not implemented'),
       getSyncState: runInWorker(new WorkerSchema.LeaderWorkerInner.GetLeaderSyncState()).pipe(
         UnexpectedError.mapToUnexpectedError,
         Effect.withSpan('@livestore/adapter-node:client-session:getLeaderSyncState'),

package/src/make-leader-worker.ts

@@ -8,17 +8,12 @@ if (process.execArgv.includes('--inspect')) {
   inspector.waitForDebugger()
 }
 
-import type { NetworkStatus, SyncOptions } from '@livestore/common'
+import type { SyncOptions } from '@livestore/common'
 import { Devtools, liveStoreStorageFormatVersion, UnexpectedError } from '@livestore/common'
 import type { DevtoolsOptions, LeaderSqliteDb } from '@livestore/common/leader-thread'
-import {
-  configureConnection,
-  getClientHeadFromDb,
-  LeaderThreadCtx,
-  makeLeaderThreadLayer,
-} from '@livestore/common/leader-thread'
+import { configureConnection, Eventlog, LeaderThreadCtx, makeLeaderThreadLayer } from '@livestore/common/leader-thread'
 import type { LiveStoreSchema } from '@livestore/common/schema'
-import { MutationEvent } from '@livestore/common/schema'
+import { LiveStoreEvent } from '@livestore/common/schema'
 import { makeNodeDevtoolsChannel } from '@livestore/devtools-node-common/web-channel'
 import { loadSqlite3Wasm } from '@livestore/sqlite-wasm/load-wasm'
 import { sqliteDbFactory } from '@livestore/sqlite-wasm/node'
@@ -69,7 +64,7 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
     PushToLeader: ({ batch }) =>
       Effect.andThen(LeaderThreadCtx, (_) =>
         _.syncProcessor.push(
-          batch.map((item) => new MutationEvent.EncodedWithMeta(item)),
+          batch.map((item) => new LiveStoreEvent.EncodedWithMeta(item)),
           // We'll wait in order to keep back pressure on the client session
           { waitForProcessing: true },
         ),
@@ -78,37 +73,24 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
       Effect.andThen(LeaderThreadCtx, (_) => Stream.fromQueue(_.bootStatusQueue)).pipe(Stream.unwrap),
     PullStream: ({ cursor }) =>
       Effect.gen(function* () {
-        const { connectedClientSessionPullQueues } = yield* LeaderThreadCtx
-        const pullQueue = yield* connectedClientSessionPullQueues.makeQueue(cursor)
-        return Stream.fromQueue(pullQueue)
+        const { syncProcessor } = yield* LeaderThreadCtx
+        return syncProcessor.pull({ cursor })
       }).pipe(Stream.unwrapScoped),
     Export: () =>
       Effect.andThen(LeaderThreadCtx, (_) => _.dbReadModel.export()).pipe(
         UnexpectedError.mapToUnexpectedError,
         Effect.withSpan('@livestore/adapter-node:worker:Export'),
       ),
-    ExportMutationlog: () =>
-      Effect.andThen(LeaderThreadCtx, (_) => _.dbMutationLog.export()).pipe(
+    ExportEventlog: () =>
+      Effect.andThen(LeaderThreadCtx, (_) => _.dbEventlog.export()).pipe(
         UnexpectedError.mapToUnexpectedError,
-        Effect.withSpan('@livestore/adapter-node:worker:ExportMutationlog'),
+        Effect.withSpan('@livestore/adapter-node:worker:ExportEventlog'),
       ),
     GetLeaderHead: () =>
       Effect.gen(function* () {
         const workerCtx = yield* LeaderThreadCtx
-        return getClientHeadFromDb(workerCtx.dbMutationLog)
+        return Eventlog.getClientHeadFromDb(workerCtx.dbEventlog)
       }).pipe(UnexpectedError.mapToUnexpectedError, Effect.withSpan('@livestore/adapter-node:worker:GetLeaderHead')),
-    NetworkStatusStream: () =>
-      Effect.gen(function* (_) {
-        const ctx = yield* LeaderThreadCtx
-
-        if (ctx.syncBackend === undefined) {
-          return Stream.make<[NetworkStatus]>({ isConnected: false, timestampMs: Date.now(), latchClosed: false })
-        }
-
-        return ctx.syncBackend.isConnected.changes.pipe(
-          Stream.map((isConnected) => ({ isConnected, timestampMs: Date.now(), latchClosed: false })),
-        )
-      }).pipe(Stream.unwrap),
     GetLeaderSyncState: () =>
       Effect.gen(function* () {
         const workerCtx = yield* LeaderThreadCtx
@@ -133,14 +115,14 @@ export const makeWorkerEffect = (options: WorkerOptions) => {
       ),
     Shutdown: () =>
       Effect.gen(function* () {
-        // const { db, dbMutationLog } = yield* LeaderThreadCtx
+        // const { db, dbEventlog } = yield* LeaderThreadCtx
         yield* Effect.logDebug('[@livestore/adapter-node:worker] Shutdown')
 
         // if (devtools.enabled) {
         //   yield* FiberSet.clear(devtools.connections)
         // }
         // db.close()
-        // dbMutationLog.close()
+        // dbEventlog.close()
 
         // Buy some time for Otel to flush
         // TODO find a cleaner way to do this
@@ -191,26 +173,25 @@ const makeLeaderThread = ({
 
     const schemaHashSuffix = schema.migrationOptions.strategy === 'manual' ? 'fixed' : schema.hash.toString()
 
-    const makeDb = (kind: 'app' | 'mutationlog') =>
+    const makeDb = (kind: 'app' | 'eventlog') =>
       makeSqliteDb({
         _tag: 'fs',
         directory: path.join(baseDirectory ?? '', storeId),
-        fileName:
-          kind === 'app' ? getAppDbFileName(schemaHashSuffix) : `mutationlog@${liveStoreStorageFormatVersion}.db`,
+        fileName: kind === 'app' ? getAppDbFileName(schemaHashSuffix) : `eventlog@${liveStoreStorageFormatVersion}.db`,
         // TODO enable WAL for nodejs
         configureDb: (db) =>
           configureConnection(db, { foreignKeys: true }).pipe(Effect.provide(runtime), Effect.runSync),
       }).pipe(Effect.acquireRelease((db) => Effect.sync(() => db.close())))
 
     // Might involve some async work, so we're running them concurrently
-    const [dbReadModel, dbMutationLog] = yield* Effect.all([makeDb('app'), makeDb('mutationlog')], { concurrency: 2 })
+    const [dbReadModel, dbEventlog] = yield* Effect.all([makeDb('app'), makeDb('eventlog')], { concurrency: 2 })
 
     const devtoolsOptions = yield* makeDevtoolsOptions({
       devtoolsEnabled: devtools.enabled,
       devtoolsPort: devtools.port,
       devtoolsHost: devtools.host,
       dbReadModel,
-      dbMutationLog,
+      dbEventlog,
       storeId,
       clientId,
       schemaPath,
@@ -225,7 +206,7 @@ const makeLeaderThread = ({
       makeSqliteDb,
       syncOptions,
       dbReadModel,
-      dbMutationLog,
+      dbEventlog,
       devtoolsOptions,
       shutdownChannel,
       syncPayload,
@@ -242,7 +223,7 @@ const getAppDbFileName = (suffix: string) => `app${suffix}@${liveStoreStorageFor
 const makeDevtoolsOptions = ({
   devtoolsEnabled,
   dbReadModel,
-  dbMutationLog,
+  dbEventlog,
   storeId,
   clientId,
   devtoolsPort,
@@ -251,7 +232,7 @@ const makeDevtoolsOptions = ({
 }: {
   devtoolsEnabled: boolean
   dbReadModel: LeaderSqliteDb
-  dbMutationLog: LeaderSqliteDb
+  dbEventlog: LeaderSqliteDb
   storeId: string
   clientId: string
   devtoolsPort: number
@@ -289,7 +270,7 @@ const makeDevtoolsOptions = ({
           devtoolsWebChannel,
           persistenceInfo: {
             readModel: dbReadModel.metadata.persistenceInfo,
-            mutationLog: dbMutationLog.metadata.persistenceInfo,
+            eventlog: dbEventlog.metadata.persistenceInfo,
           },
         }
       }).pipe(Effect.provide(FetchHttpClient.layer)),

package/src/worker-schema.ts

@@ -1,5 +1,13 @@
-import { BootStatus, Devtools, LeaderAheadError, MigrationsReport, SyncState, UnexpectedError } from '@livestore/common'
-import { EventId, MutationEvent } from '@livestore/common/schema'
+import {
+  BootStatus,
+  Devtools,
+  LeaderAheadError,
+  LeaderPullCursor,
+  MigrationsReport,
+  SyncState,
+  UnexpectedError,
+} from '@livestore/common'
+import { EventId, LiveStoreEvent } from '@livestore/common/schema'
 import { Schema, Transferable } from '@livestore/utils/effect'
 
 export const WorkerArgv = Schema.parseJson(
@@ -84,18 +92,18 @@ export namespace LeaderWorkerInner {
 
   export class PullStream extends Schema.TaggedRequest<PullStream>()('PullStream', {
     payload: {
-      cursor: EventId.EventId,
+      cursor: LeaderPullCursor,
     },
     success: Schema.Struct({
       payload: SyncState.PayloadUpstream,
-      remaining: Schema.Number,
+      mergeCounter: Schema.Number,
     }),
     failure: UnexpectedError,
   }) {}
 
   export class PushToLeader extends Schema.TaggedRequest<PushToLeader>()('PushToLeader', {
     payload: {
-      batch: Schema.Array(MutationEvent.AnyEncoded),
+      batch: Schema.Array(LiveStoreEvent.AnyEncoded),
     },
     success: Schema.Void,
     failure: Schema.Union(UnexpectedError, LeaderAheadError),
@@ -116,7 +124,7 @@ export namespace LeaderWorkerInner {
     failure: UnexpectedError,
   }) {}
 
-  export class ExportMutationlog extends Schema.TaggedRequest<ExportMutationlog>()('ExportMutationlog', {
+  export class ExportEventlog extends Schema.TaggedRequest<ExportEventlog>()('ExportEventlog', {
     payload: {},
     success: Transferable.Uint8Array,
     failure: UnexpectedError,
@@ -134,15 +142,6 @@ export namespace LeaderWorkerInner {
     failure: UnexpectedError,
   }) {}
 
-  export class NetworkStatusStream extends Schema.TaggedRequest<NetworkStatusStream>()('NetworkStatusStream', {
-    payload: {},
-    success: Schema.Struct({
-      isConnected: Schema.Boolean,
-      timestampMs: Schema.Number,
-    }),
-    failure: UnexpectedError,
-  }) {}
-
   export class Shutdown extends Schema.TaggedRequest<Shutdown>()('Shutdown', {
     payload: {},
     success: Schema.Void,
@@ -164,10 +163,9 @@ export namespace LeaderWorkerInner {
     PushToLeader,
     Export,
     GetRecreateSnapshot,
-    ExportMutationlog,
+    ExportEventlog,
     GetLeaderHead,
     GetLeaderSyncState,
-    NetworkStatusStream,
     Shutdown,
     ExtraDevtoolsMessage,
   )