@livestore/common 0.0.0-snapshot-2b8a9de3ec1a701aca891ebc2c98eb328274ae9e → 0.0.0-snapshot-2c861249e50661661613204300b1fc0d902c2e46

package/src/leader-thread/make-leader-thread-layer.ts

@@ -5,14 +5,13 @@ import type { BootStatus, MakeSqliteDb, MigrationsReport, SqliteError } from '..
 import { UnexpectedError } from '../adapter-types.js'
 import type * as Devtools from '../devtools/mod.js'
 import type { LiveStoreSchema } from '../schema/mod.js'
-import { EventId, MutationEvent, mutationLogMetaTable, SYNC_STATUS_TABLE, syncStatusTable } from '../schema/mod.js'
-import { migrateTable } from '../schema-management/migrations.js'
+import { LiveStoreEvent } from '../schema/mod.js'
 import type { InvalidPullError, IsOfflineError, SyncOptions } from '../sync/sync.js'
 import { sql } from '../util.js'
-import { execSql } from './connection.js'
+import { makeApplyEvent } from './apply-event.js'
+import * as Eventlog from './eventlog.js'
 import { bootDevtools } from './leader-worker-devtools.js'
 import { makeLeaderSyncProcessor } from './LeaderSyncProcessor.js'
-import { makePullQueueSet } from './pull-queue-set.js'
 import { recreateDb } from './recreate-db.js'
 import type { ShutdownChannel } from './shutdown-channel.js'
 import type {
@@ -24,6 +23,30 @@ import type {
 } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
+export interface MakeLeaderThreadLayerParams {
+  storeId: string
+  syncPayload: Schema.JsonValue | undefined
+  clientId: string
+  schema: LiveStoreSchema
+  makeSqliteDb: MakeSqliteDb
+  syncOptions: SyncOptions | undefined
+  dbReadModel: LeaderSqliteDb
+  dbEventlog: LeaderSqliteDb
+  devtoolsOptions: DevtoolsOptions
+  shutdownChannel: ShutdownChannel
+  params?: {
+    localPushBatchSize?: number
+    backendPushBatchSize?: number
+  }
+  testing?: {
+    syncProcessor?: {
+      delays?: {
+        localPushProcessing?: Effect.Effect<void>
+      }
+    }
+  }
+}
+
 export const makeLeaderThreadLayer = ({
   schema,
   storeId,
@@ -32,27 +55,21 @@ export const makeLeaderThreadLayer = ({
   makeSqliteDb,
   syncOptions,
   dbReadModel,
-  dbMutationLog,
+  dbEventlog,
   devtoolsOptions,
   shutdownChannel,
-}: {
-  storeId: string
-  syncPayload: Schema.JsonValue | undefined
-  clientId: string
-  schema: LiveStoreSchema
-  makeSqliteDb: MakeSqliteDb
-  syncOptions: SyncOptions | undefined
-  dbReadModel: LeaderSqliteDb
-  dbMutationLog: LeaderSqliteDb
-  devtoolsOptions: DevtoolsOptions
-  shutdownChannel: ShutdownChannel
-}): Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient> =>
+  params,
+  testing,
+}: MakeLeaderThreadLayerParams): Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient> =>
   Effect.gen(function* () {
     const bootStatusQueue = yield* Queue.unbounded<BootStatus>().pipe(Effect.acquireRelease(Queue.shutdown))
 
     // TODO do more validation here than just checking the count of tables
     // Either happens on initial boot or if schema changes
-    const dbMissing =
+    const dbEventlogMissing =
+      dbEventlog.select<{ count: number }>(sql`select count(*) as count from sqlite_master`)[0]!.count === 0
+
+    const dbReadModelMissing =
       dbReadModel.select<{ count: number }>(sql`select count(*) as count from sqlite_master`)[0]!.count === 0
 
     const syncBackend =
@@ -60,6 +77,11 @@ export const makeLeaderThreadLayer = ({
         ? undefined
         : yield* syncOptions.backend({ storeId, clientId, payload: syncPayload })
 
+    if (syncBackend !== undefined) {
+      // We're already connecting to the sync backend concurrently
+      yield* syncBackend.connect.pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
+    }
+
     const initialBlockingSyncContext = yield* makeInitialBlockingSyncContext({
       initialSyncOptions: syncOptions?.initialSyncOptions ?? { _tag: 'Skip' },
       bootStatusQueue,
@@ -67,11 +89,19 @@ export const makeLeaderThreadLayer = ({
 
     const syncProcessor = yield* makeLeaderSyncProcessor({
       schema,
-      dbMissing,
-      dbMutationLog,
+      dbEventlogMissing,
+      dbEventlog,
+      dbReadModel,
+      dbReadModelMissing,
       initialBlockingSyncContext,
-      clientId,
       onError: syncOptions?.onSyncError ?? 'ignore',
+      params: {
+        localPushBatchSize: params?.localPushBatchSize,
+        backendPushBatchSize: params?.backendPushBatchSize,
+      },
+      testing: {
+        delays: testing?.syncProcessor?.delays,
+      },
     })
 
     const extraIncomingMessagesQueue = yield* Queue.unbounded<Devtools.Leader.MessageToApp>().pipe(
@@ -86,20 +116,22 @@ export const makeLeaderThreadLayer = ({
         }
       : { enabled: false as const }
 
+    const applyEvent = yield* makeApplyEvent({ schema, dbReadModel, dbEventlog })
+
     const ctx = {
       schema,
       bootStatusQueue,
       storeId,
       clientId,
       dbReadModel,
-      dbMutationLog,
+      dbEventlog,
       makeSqliteDb,
-      mutationEventSchema: MutationEvent.makeMutationEventSchema(schema),
+      eventSchema: LiveStoreEvent.makeEventDefSchema(schema),
       shutdownStateSubRef: yield* SubscriptionRef.make<ShutdownState>('running'),
       shutdownChannel,
       syncBackend,
       syncProcessor,
-      connectedClientSessionPullQueues: yield* makePullQueueSet,
+      applyEvent,
       extraIncomingMessagesQueue,
       devtools: devtoolsContext,
       // State will be set during `bootLeaderThread`
@@ -112,7 +144,7 @@ export const makeLeaderThreadLayer = ({
     const layer = Layer.succeed(LeaderThreadCtx, ctx)
 
     ctx.initialState = yield* bootLeaderThread({
-      dbMissing,
+      dbReadModelMissing,
       initialBlockingSyncContext,
       devtoolsOptions,
     }).pipe(Effect.provide(layer))
@@ -122,6 +154,7 @@ export const makeLeaderThreadLayer = ({
     Effect.withSpan('@livestore/common:leader-thread:boot'),
     Effect.withSpanScoped('@livestore/common:leader-thread'),
     UnexpectedError.mapToUnexpectedError,
+    Effect.tapCauseLogPretty,
     Layer.unwrapScoped,
   )
 
@@ -135,7 +168,7 @@ const makeInitialBlockingSyncContext = ({
   Effect.gen(function* () {
     const ctx = {
       isDone: false,
-      processedMutations: 0,
+      processedEvents: 0,
       total: -1,
     }
 
@@ -158,10 +191,10 @@ const makeInitialBlockingSyncContext = ({
             ctx.total = remaining + processed
           }
 
-          ctx.processedMutations += processed
+          ctx.processedEvents += processed
           yield* Queue.offer(bootStatusQueue, {
             stage: 'syncing',
-            progress: { done: ctx.processedMutations, total: ctx.total },
+            progress: { done: ctx.processedEvents, total: ctx.total },
           })
 
           if (remaining === 0 && blockingDeferred !== undefined) {
@@ -177,11 +210,11 @@ const makeInitialBlockingSyncContext = ({
  * It also starts various background processes (e.g. syncing)
  */
 const bootLeaderThread = ({
-  dbMissing,
+  dbReadModelMissing,
   initialBlockingSyncContext,
   devtoolsOptions,
 }: {
-  dbMissing: boolean
+  dbReadModelMissing: boolean
   initialBlockingSyncContext: InitialBlockingSyncContext
   devtoolsOptions: DevtoolsOptions
 }): Effect.Effect<
@@ -190,46 +223,20 @@ const bootLeaderThread = ({
   LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
 > =>
   Effect.gen(function* () {
-    const { dbMutationLog, bootStatusQueue, syncProcessor } = yield* LeaderThreadCtx
-
-    yield* migrateTable({
-      db: dbMutationLog,
-      behaviour: 'create-if-not-exists',
-      tableAst: mutationLogMetaTable.sqliteDef.ast,
-      skipMetaTable: true,
-    })
-
-    yield* migrateTable({
-      db: dbMutationLog,
-      behaviour: 'create-if-not-exists',
-      tableAst: syncStatusTable.sqliteDef.ast,
-      skipMetaTable: true,
-    })
-
-    // Create sync status row if it doesn't exist
-    yield* execSql(
-      dbMutationLog,
-      sql`INSERT INTO ${SYNC_STATUS_TABLE} (head)
-          SELECT ${EventId.ROOT.global}
-          WHERE NOT EXISTS (SELECT 1 FROM ${SYNC_STATUS_TABLE})`,
-      {},
-    )
-
-    const dbReady = yield* Deferred.make<void>()
+    const { dbEventlog, bootStatusQueue, syncProcessor } = yield* LeaderThreadCtx
 
-    // We're already starting pulling from the sync backend concurrently but wait until the db is ready before
-    // processing any incoming mutations
-    const { initialLeaderHead } = yield* syncProcessor.boot({ dbReady })
+    yield* Eventlog.initEventlogDb(dbEventlog)
 
     let migrationsReport: MigrationsReport
-    if (dbMissing) {
+    if (dbReadModelMissing) {
       const recreateResult = yield* recreateDb
       migrationsReport = recreateResult.migrationsReport
     } else {
       migrationsReport = { migrations: [] }
     }
 
-    yield* Deferred.succeed(dbReady, void 0)
+    // NOTE the sync processor depends on the dbs being initialized properly
+    const { initialLeaderHead } = yield* syncProcessor.boot
 
     if (initialBlockingSyncContext.blockingDeferred !== undefined) {
       // Provides a syncing status right away before the first pull response comes in

package/src/leader-thread/mod.ts

@@ -3,4 +3,4 @@ export * from './types.js'
 export * as ShutdownChannel from './shutdown-channel.js'
 export * from './leader-worker-devtools.js'
 export * from './make-leader-thread-layer.js'
-export * from './mutationlog.js'
+export * as Eventlog from './eventlog.js'

package/src/leader-thread/recreate-db.ts

@@ -3,7 +3,7 @@ import type { HttpClient } from '@livestore/utils/effect'
 import { Effect, Queue } from '@livestore/utils/effect'
 
 import type { InvalidPullError, IsOfflineError, MigrationHooks, MigrationsReport, SqliteError } from '../index.js'
-import { initializeSingletonTables, migrateDb, rehydrateFromMutationLog, UnexpectedError } from '../index.js'
+import { migrateDb, rehydrateFromEventlog, UnexpectedError } from '../index.js'
 import { configureConnection } from './connection.js'
 import { LeaderThreadCtx } from './types.js'
 
@@ -12,7 +12,7 @@ export const recreateDb: Effect.Effect<
   UnexpectedError | SqliteError | IsOfflineError | InvalidPullError,
   LeaderThreadCtx | HttpClient.HttpClient
 > = Effect.gen(function* () {
-  const { dbReadModel, dbMutationLog, schema, bootStatusQueue } = yield* LeaderThreadCtx
+  const { dbReadModel, dbEventlog, schema, bootStatusQueue, applyEvent } = yield* LeaderThreadCtx
 
   const migrationOptions = schema.migrationOptions
   let migrationsReport: MigrationsReport
@@ -41,25 +41,24 @@ export const recreateDb: Effect.Effect<
           Queue.offer(bootStatusQueue, { stage: 'migrating', progress: { done, total } }),
       })
 
-      initializeSingletonTables(schema, tmpDb)
-
       yield* Effect.tryAll(() => hooks?.pre?.(tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
 
       return { migrationsReport, tmpDb }
     })
 
   switch (migrationOptions.strategy) {
-    case 'from-mutation-log': {
+    case 'from-eventlog': {
       const hooks = migrationOptions.hooks
       const initResult = yield* initDb(hooks)
 
       migrationsReport = initResult.migrationsReport
 
-      yield* rehydrateFromMutationLog({
-        db: initResult.tmpDb,
-        logDb: dbMutationLog,
+      yield* rehydrateFromEventlog({
+        // db: initResult.tmpDb,
+        dbEventlog,
         schema,
         migrationOptions,
+        applyEvent,
         onProgress: ({ done, total }) =>
           Queue.offer(bootStatusQueue, { stage: 'rehydrating', progress: { done, total } }),
       })

package/src/leader-thread/types.ts

@@ -5,12 +5,14 @@ import type {
   Option,
   Queue,
   Scope,
+  Stream,
   Subscribable,
   SubscriptionRef,
   WebChannel,
 } from '@livestore/utils/effect'
 import { Context, Schema } from '@livestore/utils/effect'
 
+import type { LeaderPullCursor, SqliteError } from '../adapter-types.js'
 import type {
   BootStatus,
   Devtools,
@@ -22,7 +24,7 @@ import type {
   SyncBackend,
   UnexpectedError,
 } from '../index.js'
-import type { EventId, LiveStoreSchema, MutationEvent } from '../schema/mod.js'
+import type { EventId, LiveStoreEvent, LiveStoreSchema } from '../schema/mod.js'
 import type * as SyncState from '../sync/syncstate.js'
 import type { ShutdownChannel } from './shutdown-channel.js'
 
@@ -50,7 +52,7 @@ export type InitialSyncInfo = Option.Option<{
 //   | { _tag: 'Reuse'; syncInfo: InitialSyncInfo }
 
 export type LeaderSqliteDb = SqliteDb<{ dbPointer: number; persistenceInfo: PersistenceInfo }>
-export type PersistenceInfoPair = { readModel: PersistenceInfo; mutationLog: PersistenceInfo }
+export type PersistenceInfoPair = { readModel: PersistenceInfo; eventlog: PersistenceInfo }
 
 export type DevtoolsOptions =
   | {
@@ -88,16 +90,16 @@ export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
     clientId: string
     makeSqliteDb: MakeSqliteDb
     dbReadModel: LeaderSqliteDb
-    dbMutationLog: LeaderSqliteDb
+    dbEventlog: LeaderSqliteDb
     bootStatusQueue: Queue.Queue<BootStatus>
     // TODO we should find a more elegant way to handle cases which need this ref for their implementation
     shutdownStateSubRef: SubscriptionRef.SubscriptionRef<ShutdownState>
     shutdownChannel: ShutdownChannel
-    mutationEventSchema: MutationEvent.ForMutationDefRecord<any>
+    eventSchema: LiveStoreEvent.ForEventDefRecord<any>
     devtools: DevtoolsContext
     syncBackend: SyncBackend | undefined
     syncProcessor: LeaderSyncProcessor
-    connectedClientSessionPullQueues: PullQueueSet
+    applyEvent: ApplyEvent
     initialState: {
       leaderHead: EventId.EventId
       migrationsReport: MigrationsReport
@@ -111,27 +113,41 @@ export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
   }
 >() {}
 
+export type ApplyEvent = (
+  eventEncoded: LiveStoreEvent.EncodedWithMeta,
+  options?: {
+    /** Needed for rehydrateFromEventlog */
+    skipEventlog?: boolean
+  },
+) => Effect.Effect<
+  { sessionChangeset: { _tag: 'sessionChangeset'; data: Uint8Array; debug: any } | { _tag: 'no-op' } },
+  SqliteError | UnexpectedError
+>
+
 export type InitialBlockingSyncContext = {
   blockingDeferred: Deferred.Deferred<void> | undefined
   update: (_: { remaining: number; processed: number }) => Effect.Effect<void>
 }
 
-export type PullQueueItem = {
-  payload: SyncState.PayloadUpstream
-  remaining: number
-}
-
 export interface LeaderSyncProcessor {
+  /** Used by client sessions to subscribe to upstream sync state changes */
+  pull: (args: {
+    cursor: LeaderPullCursor
+  }) => Stream.Stream<{ payload: typeof SyncState.PayloadUpstream.Type; mergeCounter: number }, UnexpectedError>
+  /** The `pullQueue` API can be used instead of `pull` when more convenient */
+  pullQueue: (args: {
+    cursor: LeaderPullCursor
+  }) => Effect.Effect<
+    Queue.Queue<{ payload: typeof SyncState.PayloadUpstream.Type; mergeCounter: number }>,
+    UnexpectedError,
+    Scope.Scope
+  >
+
+  /** Used by client sessions to push events to the leader thread */
   push: (
     /** `batch` needs to follow the same rules as `batch` in `SyncBackend.push` */
-    batch: ReadonlyArray<MutationEvent.EncodedWithMeta>,
+    batch: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>,
     options?: {
-      /**
-       * This generation number is used to automatically reject subsequent pushes
-       * of a previously rejected push from a client session. This might occur in
-       * certain concurrent scenarios.
-       */
-      // generation: number
       /**
        * If true, the effect will only finish when the local push has been processed (i.e. succeeded or was rejected).
        * @default false
@@ -140,24 +156,18 @@ export interface LeaderSyncProcessor {
     },
   ) => Effect.Effect<void, LeaderAheadError>
 
+  /** Currently only used by devtools which don't provide their own event numbers */
   pushPartial: (args: {
-    mutationEvent: MutationEvent.PartialAnyEncoded
+    event: LiveStoreEvent.PartialAnyEncoded
     clientId: string
     sessionId: string
-  }) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
-  boot: (args: {
-    dbReady: Deferred.Deferred<void>
-  }) => Effect.Effect<
+  }) => Effect.Effect<void, UnexpectedError>
+
+  boot: Effect.Effect<
     { initialLeaderHead: EventId.EventId },
     UnexpectedError,
     LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
   >
   syncState: Subscribable.Subscribable<SyncState.SyncState>
-}
-
-export interface PullQueueSet {
-  makeQueue: (
-    since: EventId.EventId,
-  ) => Effect.Effect<Queue.Queue<PullQueueItem>, UnexpectedError, Scope.Scope | LeaderThreadCtx>
-  offer: (item: PullQueueItem) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
+  getMergeCounter: () => number
 }

package/src/materializer-helper.ts

@@ -0,0 +1,110 @@
+import { isReadonlyArray } from '@livestore/utils'
+import { Schema } from '@livestore/utils/effect'
+
+import { SessionIdSymbol } from './adapter-types.js'
+import { isQueryBuilder } from './query-builder/api.js'
+import type { EventDef, Materializer, MaterializerResult } from './schema/EventDef.js'
+import type * as LiveStoreEvent from './schema/LiveStoreEvent.js'
+import type { BindValues } from './sql-queries/sql-queries.js'
+import type { PreparedBindValues } from './util.js'
+import { prepareBindValues } from './util.js'
+
+export const getExecArgsFromEvent = ({
+  eventDef: { eventDef, materializer },
+  event,
+}: {
+  eventDef: {
+    eventDef: EventDef.AnyWithoutFn
+    materializer: Materializer
+  }
+  /** Both encoded and decoded events are supported to reduce the number of times we need to decode/encode */
+  event:
+    | {
+        decoded: LiveStoreEvent.AnyDecoded | LiveStoreEvent.PartialAnyDecoded
+        encoded: undefined
+      }
+    | {
+        decoded: undefined
+        encoded: LiveStoreEvent.AnyEncoded | LiveStoreEvent.PartialAnyEncoded
+      }
+}): ReadonlyArray<{
+  statementSql: string
+  bindValues: PreparedBindValues
+  writeTables: ReadonlySet<string> | undefined
+}> => {
+  const eventArgsDecoded = event.decoded?.args ?? Schema.decodeUnknownSync(eventDef.schema)(event.encoded!.args)
+
+  const res = materializer(eventArgsDecoded, {
+    clientOnly: eventDef.options.clientOnly,
+    // TODO properly implement this
+    currentFacts: new Map(),
+  })
+
+  const statementRes = mapMaterializerResult(res)
+
+  return statementRes.map((statementRes) => {
+    const statementSql = statementRes.sql
+
+    const eventArgsEncoded = event.encoded?.args ?? Schema.encodeUnknownSync(eventDef.schema)(event.decoded!.args)
+    const bindValues = typeof statementRes === 'string' ? eventArgsEncoded : statementRes.bindValues
+
+    const writeTables = typeof statementRes === 'string' ? undefined : statementRes.writeTables
+
+    return { statementSql, bindValues: prepareBindValues(bindValues ?? {}, statementSql), writeTables }
+  })
+}
+
+const mapMaterializerResult = (
+  materializerResult: MaterializerResult | ReadonlyArray<MaterializerResult>,
+): ReadonlyArray<{
+  sql: string
+  bindValues: BindValues
+  writeTables: ReadonlySet<string> | undefined
+}> => {
+  if (isReadonlyArray(materializerResult)) {
+    return materializerResult.flatMap(mapMaterializerResult)
+  }
+  if (isQueryBuilder(materializerResult)) {
+    const { query, bindValues } = materializerResult.asSql()
+    return [{ sql: query, bindValues: bindValues as BindValues, writeTables: undefined }]
+  } else if (typeof materializerResult === 'string') {
+    return [{ sql: materializerResult, bindValues: {} as BindValues, writeTables: undefined }]
+  } else {
+    return [
+      {
+        sql: materializerResult.sql,
+        bindValues: materializerResult.bindValues,
+        writeTables: materializerResult.writeTables,
+      },
+    ]
+  }
+}
+
+// NOTE we should explore whether there is a more elegant solution
+// e.g. by leveraging the schema to replace the sessionIdSymbol
+export const replaceSessionIdSymbol = (
+  bindValues: Record<string, unknown> | ReadonlyArray<unknown>,
+  sessionId: string,
+) => {
+  deepReplaceValue(bindValues, SessionIdSymbol, sessionId)
+}
+
+const deepReplaceValue = <S, R>(input: any, searchValue: S, replaceValue: R): void => {
+  if (Array.isArray(input)) {
+    for (const i in input) {
+      if (input[i] === searchValue) {
+        input[i] = replaceValue
+      } else {
+        deepReplaceValue(input[i], searchValue, replaceValue)
+      }
+    }
+  } else if (typeof input === 'object' && input !== null) {
+    for (const key in input) {
+      if (input[key] === searchValue) {
+        input[key] = replaceValue
+      } else {
+        deepReplaceValue(input[key], searchValue, replaceValue)
+      }
+    }
+  }
+}