@livestore/common 0.3.0-dev.27 → 0.3.0-dev.29

package/src/leader-thread/make-leader-thread-layer.ts

@@ -5,13 +5,13 @@ import type { BootStatus, MakeSqliteDb, MigrationsReport, SqliteError } from '..
 import { UnexpectedError } from '../adapter-types.js'
 import type * as Devtools from '../devtools/mod.js'
 import type { LiveStoreSchema } from '../schema/mod.js'
-import { MutationEvent } from '../schema/mod.js'
+import { LiveStoreEvent } from '../schema/mod.js'
 import type { InvalidPullError, IsOfflineError, SyncOptions } from '../sync/sync.js'
 import { sql } from '../util.js'
-import { makeApplyMutation } from './apply-mutation.js'
+import { makeApplyEvent } from './apply-event.js'
+import * as Eventlog from './eventlog.js'
 import { bootDevtools } from './leader-worker-devtools.js'
 import { makeLeaderSyncProcessor } from './LeaderSyncProcessor.js'
-import * as Mutationlog from './mutationlog.js'
 import { recreateDb } from './recreate-db.js'
 import type { ShutdownChannel } from './shutdown-channel.js'
 import type {
@@ -23,6 +23,30 @@ import type {
 } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
+export interface MakeLeaderThreadLayerParams {
+  storeId: string
+  syncPayload: Schema.JsonValue | undefined
+  clientId: string
+  schema: LiveStoreSchema
+  makeSqliteDb: MakeSqliteDb
+  syncOptions: SyncOptions | undefined
+  dbReadModel: LeaderSqliteDb
+  dbEventlog: LeaderSqliteDb
+  devtoolsOptions: DevtoolsOptions
+  shutdownChannel: ShutdownChannel
+  params?: {
+    localPushBatchSize?: number
+    backendPushBatchSize?: number
+  }
+  testing?: {
+    syncProcessor?: {
+      delays?: {
+        localPushProcessing?: Effect.Effect<void>
+      }
+    }
+  }
+}
+
 export const makeLeaderThreadLayer = ({
   schema,
   storeId,
@@ -31,28 +55,19 @@ export const makeLeaderThreadLayer = ({
   makeSqliteDb,
   syncOptions,
   dbReadModel,
-  dbMutationLog,
+  dbEventlog,
   devtoolsOptions,
   shutdownChannel,
-}: {
-  storeId: string
-  syncPayload: Schema.JsonValue | undefined
-  clientId: string
-  schema: LiveStoreSchema
-  makeSqliteDb: MakeSqliteDb
-  syncOptions: SyncOptions | undefined
-  dbReadModel: LeaderSqliteDb
-  dbMutationLog: LeaderSqliteDb
-  devtoolsOptions: DevtoolsOptions
-  shutdownChannel: ShutdownChannel
-}): Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient> =>
+  params,
+  testing,
+}: MakeLeaderThreadLayerParams): Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient> =>
   Effect.gen(function* () {
     const bootStatusQueue = yield* Queue.unbounded<BootStatus>().pipe(Effect.acquireRelease(Queue.shutdown))
 
     // TODO do more validation here than just checking the count of tables
     // Either happens on initial boot or if schema changes
-    const dbMutationLogMissing =
-      dbMutationLog.select<{ count: number }>(sql`select count(*) as count from sqlite_master`)[0]!.count === 0
+    const dbEventlogMissing =
+      dbEventlog.select<{ count: number }>(sql`select count(*) as count from sqlite_master`)[0]!.count === 0
 
     const dbReadModelMissing =
       dbReadModel.select<{ count: number }>(sql`select count(*) as count from sqlite_master`)[0]!.count === 0
@@ -74,12 +89,19 @@ export const makeLeaderThreadLayer = ({
 
     const syncProcessor = yield* makeLeaderSyncProcessor({
       schema,
-      dbMutationLogMissing,
-      dbMutationLog,
+      dbEventlogMissing,
+      dbEventlog,
       dbReadModel,
       dbReadModelMissing,
       initialBlockingSyncContext,
       onError: syncOptions?.onSyncError ?? 'ignore',
+      params: {
+        localPushBatchSize: params?.localPushBatchSize,
+        backendPushBatchSize: params?.backendPushBatchSize,
+      },
+      testing: {
+        delays: testing?.syncProcessor?.delays,
+      },
     })
 
     const extraIncomingMessagesQueue = yield* Queue.unbounded<Devtools.Leader.MessageToApp>().pipe(
@@ -94,7 +116,7 @@ export const makeLeaderThreadLayer = ({
         }
       : { enabled: false as const }
 
-    const applyMutation = yield* makeApplyMutation({ schema, dbReadModel, dbMutationLog })
+    const applyEvent = yield* makeApplyEvent({ schema, dbReadModel, dbEventlog })
 
     const ctx = {
       schema,
@@ -102,14 +124,14 @@ export const makeLeaderThreadLayer = ({
       storeId,
       clientId,
       dbReadModel,
-      dbMutationLog,
+      dbEventlog,
       makeSqliteDb,
-      mutationEventSchema: MutationEvent.makeMutationEventSchema(schema),
+      eventSchema: LiveStoreEvent.makeEventDefSchema(schema),
       shutdownStateSubRef: yield* SubscriptionRef.make<ShutdownState>('running'),
       shutdownChannel,
       syncBackend,
       syncProcessor,
-      applyMutation,
+      applyEvent,
       extraIncomingMessagesQueue,
       devtools: devtoolsContext,
       // State will be set during `bootLeaderThread`
@@ -146,7 +168,7 @@ const makeInitialBlockingSyncContext = ({
   Effect.gen(function* () {
     const ctx = {
       isDone: false,
-      processedMutations: 0,
+      processedEvents: 0,
       total: -1,
     }
 
@@ -169,10 +191,10 @@ const makeInitialBlockingSyncContext = ({
             ctx.total = remaining + processed
           }
 
-          ctx.processedMutations += processed
+          ctx.processedEvents += processed
           yield* Queue.offer(bootStatusQueue, {
             stage: 'syncing',
-            progress: { done: ctx.processedMutations, total: ctx.total },
+            progress: { done: ctx.processedEvents, total: ctx.total },
           })
 
           if (remaining === 0 && blockingDeferred !== undefined) {
@@ -201,9 +223,9 @@ const bootLeaderThread = ({
   LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
 > =>
   Effect.gen(function* () {
-    const { dbMutationLog, bootStatusQueue, syncProcessor } = yield* LeaderThreadCtx
+    const { dbEventlog, bootStatusQueue, syncProcessor } = yield* LeaderThreadCtx
 
-    yield* Mutationlog.initMutationLogDb(dbMutationLog)
+    yield* Eventlog.initEventlogDb(dbEventlog)
 
     let migrationsReport: MigrationsReport
     if (dbReadModelMissing) {

package/src/leader-thread/mod.ts

@@ -3,4 +3,4 @@ export * from './types.js'
 export * as ShutdownChannel from './shutdown-channel.js'
 export * from './leader-worker-devtools.js'
 export * from './make-leader-thread-layer.js'
-export * as Mutationlog from './mutationlog.js'
+export * as Eventlog from './eventlog.js'

package/src/leader-thread/recreate-db.ts

@@ -3,7 +3,7 @@ import type { HttpClient } from '@livestore/utils/effect'
 import { Effect, Queue } from '@livestore/utils/effect'
 
 import type { InvalidPullError, IsOfflineError, MigrationHooks, MigrationsReport, SqliteError } from '../index.js'
-import { initializeSingletonTables, migrateDb, rehydrateFromMutationLog, UnexpectedError } from '../index.js'
+import { migrateDb, rehydrateFromEventlog, UnexpectedError } from '../index.js'
 import { configureConnection } from './connection.js'
 import { LeaderThreadCtx } from './types.js'
 
@@ -12,7 +12,7 @@ export const recreateDb: Effect.Effect<
   UnexpectedError | SqliteError | IsOfflineError | InvalidPullError,
   LeaderThreadCtx | HttpClient.HttpClient
 > = Effect.gen(function* () {
-  const { dbReadModel, dbMutationLog, schema, bootStatusQueue, applyMutation } = yield* LeaderThreadCtx
+  const { dbReadModel, dbEventlog, schema, bootStatusQueue, applyEvent } = yield* LeaderThreadCtx
 
   const migrationOptions = schema.migrationOptions
   let migrationsReport: MigrationsReport
@@ -41,26 +41,23 @@ export const recreateDb: Effect.Effect<
           Queue.offer(bootStatusQueue, { stage: 'migrating', progress: { done, total } }),
       })
 
-      initializeSingletonTables(schema, tmpDb)
-
       yield* Effect.tryAll(() => hooks?.pre?.(tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
 
       return { migrationsReport, tmpDb }
     })
 
   switch (migrationOptions.strategy) {
-    case 'from-mutation-log': {
+    case 'from-eventlog': {
       const hooks = migrationOptions.hooks
       const initResult = yield* initDb(hooks)
 
       migrationsReport = initResult.migrationsReport
 
-      yield* rehydrateFromMutationLog({
+      yield* rehydrateFromEventlog({
         // db: initResult.tmpDb,
-        dbMutationLog,
+        dbEventlog,
         schema,
-        migrationOptions,
-        applyMutation,
+        applyEvent,
         onProgress: ({ done, total }) =>
           Queue.offer(bootStatusQueue, { stage: 'rehydrating', progress: { done, total } }),
       })

package/src/leader-thread/types.ts

@@ -24,7 +24,7 @@ import type {
   SyncBackend,
   UnexpectedError,
 } from '../index.js'
-import type { EventId, LiveStoreSchema, MutationEvent } from '../schema/mod.js'
+import type { EventId, LiveStoreEvent, LiveStoreSchema } from '../schema/mod.js'
 import type * as SyncState from '../sync/syncstate.js'
 import type { ShutdownChannel } from './shutdown-channel.js'
 
@@ -52,7 +52,7 @@ export type InitialSyncInfo = Option.Option<{
 //   | { _tag: 'Reuse'; syncInfo: InitialSyncInfo }
 
 export type LeaderSqliteDb = SqliteDb<{ dbPointer: number; persistenceInfo: PersistenceInfo }>
-export type PersistenceInfoPair = { readModel: PersistenceInfo; mutationLog: PersistenceInfo }
+export type PersistenceInfoPair = { readModel: PersistenceInfo; eventlog: PersistenceInfo }
 
 export type DevtoolsOptions =
   | {
@@ -90,16 +90,16 @@ export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
     clientId: string
     makeSqliteDb: MakeSqliteDb
     dbReadModel: LeaderSqliteDb
-    dbMutationLog: LeaderSqliteDb
+    dbEventlog: LeaderSqliteDb
     bootStatusQueue: Queue.Queue<BootStatus>
     // TODO we should find a more elegant way to handle cases which need this ref for their implementation
     shutdownStateSubRef: SubscriptionRef.SubscriptionRef<ShutdownState>
     shutdownChannel: ShutdownChannel
-    mutationEventSchema: MutationEvent.ForMutationDefRecord<any>
+    eventSchema: LiveStoreEvent.ForEventDefRecord<any>
     devtools: DevtoolsContext
     syncBackend: SyncBackend | undefined
     syncProcessor: LeaderSyncProcessor
-    applyMutation: ApplyMutation
+    applyEvent: ApplyEvent
     initialState: {
       leaderHead: EventId.EventId
       migrationsReport: MigrationsReport
@@ -113,11 +113,11 @@ export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
   }
 >() {}
 
-export type ApplyMutation = (
-  mutationEventEncoded: MutationEvent.EncodedWithMeta,
+export type ApplyEvent = (
+  eventEncoded: LiveStoreEvent.EncodedWithMeta,
   options?: {
-    /** Needed for rehydrateFromMutationLog */
-    skipMutationLog?: boolean
+    /** Needed for rehydrateFromEventlog */
+    skipEventlog?: boolean
   },
 ) => Effect.Effect<
   { sessionChangeset: { _tag: 'sessionChangeset'; data: Uint8Array; debug: any } | { _tag: 'no-op' } },
@@ -143,10 +143,10 @@ export interface LeaderSyncProcessor {
     Scope.Scope
   >
 
-  /** Used by client sessions to push mutations to the leader thread */
+  /** Used by client sessions to push events to the leader thread */
   push: (
     /** `batch` needs to follow the same rules as `batch` in `SyncBackend.push` */
-    batch: ReadonlyArray<MutationEvent.EncodedWithMeta>,
+    batch: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>,
     options?: {
       /**
        * If true, the effect will only finish when the local push has been processed (i.e. succeeded or was rejected).
@@ -158,7 +158,7 @@ export interface LeaderSyncProcessor {
 
   /** Currently only used by devtools which don't provide their own event numbers */
   pushPartial: (args: {
-    mutationEvent: MutationEvent.PartialAnyEncoded
+    event: LiveStoreEvent.PartialAnyEncoded
     clientId: string
     sessionId: string
   }) => Effect.Effect<void, UnexpectedError>

package/src/materializer-helper.ts

@@ -0,0 +1,110 @@
+import { isReadonlyArray } from '@livestore/utils'
+import { Schema } from '@livestore/utils/effect'
+
+import { SessionIdSymbol } from './adapter-types.js'
+import { isQueryBuilder } from './query-builder/api.js'
+import type { EventDef, Materializer, MaterializerResult } from './schema/EventDef.js'
+import type * as LiveStoreEvent from './schema/LiveStoreEvent.js'
+import type { BindValues } from './sql-queries/sql-queries.js'
+import type { PreparedBindValues } from './util.js'
+import { prepareBindValues } from './util.js'
+
+export const getExecArgsFromEvent = ({
+  eventDef: { eventDef, materializer },
+  event,
+}: {
+  eventDef: {
+    eventDef: EventDef.AnyWithoutFn
+    materializer: Materializer
+  }
+  /** Both encoded and decoded events are supported to reduce the number of times we need to decode/encode */
+  event:
+    | {
+        decoded: LiveStoreEvent.AnyDecoded | LiveStoreEvent.PartialAnyDecoded
+        encoded: undefined
+      }
+    | {
+        decoded: undefined
+        encoded: LiveStoreEvent.AnyEncoded | LiveStoreEvent.PartialAnyEncoded
+      }
+}): ReadonlyArray<{
+  statementSql: string
+  bindValues: PreparedBindValues
+  writeTables: ReadonlySet<string> | undefined
+}> => {
+  const eventArgsDecoded = event.decoded?.args ?? Schema.decodeUnknownSync(eventDef.schema)(event.encoded!.args)
+
+  const res = materializer(eventArgsDecoded, {
+    clientOnly: eventDef.options.clientOnly,
+    // TODO properly implement this
+    currentFacts: new Map(),
+  })
+
+  const statementRes = mapMaterializerResult(res)
+
+  return statementRes.map((statementRes) => {
+    const statementSql = statementRes.sql
+
+    const eventArgsEncoded = event.encoded?.args ?? Schema.encodeUnknownSync(eventDef.schema)(event.decoded!.args)
+    const bindValues = typeof statementRes === 'string' ? eventArgsEncoded : statementRes.bindValues
+
+    const writeTables = typeof statementRes === 'string' ? undefined : statementRes.writeTables
+
+    return { statementSql, bindValues: prepareBindValues(bindValues ?? {}, statementSql), writeTables }
+  })
+}
+
+const mapMaterializerResult = (
+  materializerResult: MaterializerResult | ReadonlyArray<MaterializerResult>,
+): ReadonlyArray<{
+  sql: string
+  bindValues: BindValues
+  writeTables: ReadonlySet<string> | undefined
+}> => {
+  if (isReadonlyArray(materializerResult)) {
+    return materializerResult.flatMap(mapMaterializerResult)
+  }
+  if (isQueryBuilder(materializerResult)) {
+    const { query, bindValues } = materializerResult.asSql()
+    return [{ sql: query, bindValues: bindValues as BindValues, writeTables: undefined }]
+  } else if (typeof materializerResult === 'string') {
+    return [{ sql: materializerResult, bindValues: {} as BindValues, writeTables: undefined }]
+  } else {
+    return [
+      {
+        sql: materializerResult.sql,
+        bindValues: materializerResult.bindValues,
+        writeTables: materializerResult.writeTables,
+      },
+    ]
+  }
+}
+
+// NOTE we should explore whether there is a more elegant solution
+// e.g. by leveraging the schema to replace the sessionIdSymbol
+export const replaceSessionIdSymbol = (
+  bindValues: Record<string, unknown> | ReadonlyArray<unknown>,
+  sessionId: string,
+) => {
+  deepReplaceValue(bindValues, SessionIdSymbol, sessionId)
+}
+
+const deepReplaceValue = <S, R>(input: any, searchValue: S, replaceValue: R): void => {
+  if (Array.isArray(input)) {
+    for (const i in input) {
+      if (input[i] === searchValue) {
+        input[i] = replaceValue
+      } else {
+        deepReplaceValue(input[i], searchValue, replaceValue)
+      }
+    }
+  } else if (typeof input === 'object' && input !== null) {
+    for (const key in input) {
+      if (input[key] === searchValue) {
+        input[key] = replaceValue
+      } else {
+        deepReplaceValue(input[key], searchValue, replaceValue)
+      }
+    }
+  }
+}