@livestore/common 0.0.0-snapshot-2b8a9de3ec1a701aca891ebc2c98eb328274ae9e → 0.0.0-snapshot-2c861249e50661661613204300b1fc0d902c2e46

package/src/leader-thread/apply-event.ts

@@ -0,0 +1,197 @@
+import { LS_DEV, memoizeByRef, shouldNeverHappen } from '@livestore/utils'
+import { Effect, ReadonlyArray, Schema } from '@livestore/utils/effect'
+
+import type { SqliteDb } from '../adapter-types.js'
+import { getExecArgsFromEvent } from '../materializer-helper.js'
+import type { LiveStoreEvent, LiveStoreSchema, SessionChangesetMetaRow } from '../schema/mod.js'
+import {
+  EventId,
+  EVENTLOG_META_TABLE,
+  getEventDef,
+  SESSION_CHANGESET_META_TABLE,
+  sessionChangesetMetaTable,
+} from '../schema/mod.js'
+import { insertRow } from '../sql-queries/index.js'
+import { sql } from '../util.js'
+import { execSql, execSqlPrepared } from './connection.js'
+import * as Eventlog from './eventlog.js'
+import type { ApplyEvent } from './types.js'
+
+export const makeApplyEvent = ({
+  schema,
+  dbReadModel: db,
+  dbEventlog,
+}: {
+  schema: LiveStoreSchema
+  dbReadModel: SqliteDb
+  dbEventlog: SqliteDb
+}): Effect.Effect<ApplyEvent, never> =>
+  Effect.gen(function* () {
+    const shouldExcludeEventFromLog = makeShouldExcludeEventFromLog(schema)
+
+    const eventDefSchemaHashMap = new Map(
+      // TODO Running `Schema.hash` can be a bottleneck for larger schemas. There is an opportunity to run this
+      // at build time and lookup the pre-computed hash at runtime.
+      // Also see https://github.com/Effect-TS/effect/issues/2719
+      [...schema.eventsDefsMap.entries()].map(([k, v]) => [k, Schema.hash(v.schema)] as const),
+    )
+
+    return (eventEncoded, options) =>
+      Effect.gen(function* () {
+        const skipEventlog = options?.skipEventlog ?? false
+
+        const eventName = eventEncoded.name
+        const eventDef = getEventDef(schema, eventName)
+
+        const execArgsArr = getExecArgsFromEvent({
+          eventDef,
+          event: { decoded: undefined, encoded: eventEncoded },
+        })
+
+        // NOTE we might want to bring this back if we want to debug no-op events
+        // const makeExecuteOptions = (statementSql: string, bindValues: any) => ({
+        //   onRowsChanged: (rowsChanged: number) => {
+        //     if (rowsChanged === 0) {
+        //       console.warn(`Event "${eventDef.name}" did not affect any rows:`, statementSql, bindValues)
+        //     }
+        //   },
+        // })
+
+        // console.group('[@livestore/common:leader-thread:applyEvent]', { eventName })
+
+        const session = db.session()
+
+        for (const { statementSql, bindValues } of execArgsArr) {
+          // console.debug(eventName, statementSql, bindValues)
+          // TODO use cached prepared statements instead of exec
+          yield* execSqlPrepared(db, statementSql, bindValues)
+        }
+
+        const changeset = session.changeset()
+        session.finish()
+
+        // TODO use prepared statements
+        yield* execSql(
+          db,
+          ...insertRow({
+            tableName: SESSION_CHANGESET_META_TABLE,
+            columns: sessionChangesetMetaTable.sqliteDef.columns,
+            values: {
+              idGlobal: eventEncoded.id.global,
+              idClient: eventEncoded.id.client,
+              // NOTE the changeset will be empty (i.e. null) for no-op events
+              changeset: changeset ?? null,
+              debug: LS_DEV ? execArgsArr : null,
+            },
+          }),
+        )
+
+        // console.groupEnd()
+
+        // write to eventlog
+        const excludeFromEventlog = shouldExcludeEventFromLog(eventName, eventEncoded)
+        if (skipEventlog === false && excludeFromEventlog === false) {
+          const eventName = eventEncoded.name
+          const eventDefSchemaHash =
+            eventDefSchemaHashMap.get(eventName) ?? shouldNeverHappen(`Unknown event definition: ${eventName}`)
+
+          yield* Eventlog.insertIntoEventlog(
+            eventEncoded,
+            dbEventlog,
+            eventDefSchemaHash,
+            eventEncoded.clientId,
+            eventEncoded.sessionId,
+          )
+        } else {
+          //   console.debug('[@livestore/common:leader-thread] skipping eventlog write', mutation, statementSql, bindValues)
+        }
+
+        return {
+          sessionChangeset: changeset
+            ? {
+                _tag: 'sessionChangeset' as const,
+                data: changeset,
+                debug: LS_DEV ? execArgsArr : null,
+              }
+            : { _tag: 'no-op' as const },
+        }
+      }).pipe(
+        Effect.withSpan(`@livestore/common:leader-thread:applyEvent`, {
+          attributes: {
+            eventName: eventEncoded.name,
+            mutationId: eventEncoded.id,
+            'span.label': `${EventId.toString(eventEncoded.id)} ${eventEncoded.name}`,
+          },
+        }),
+        // Effect.logDuration('@livestore/common:leader-thread:applyEvent'),
+      )
+  })
+
+export const rollback = ({
+  db,
+  dbEventlog,
+  eventIdsToRollback,
+}: {
+  db: SqliteDb
+  dbEventlog: SqliteDb
+  eventIdsToRollback: EventId.EventId[]
+}) =>
+  Effect.gen(function* () {
+    const rollbackEvents = db
+      .select<SessionChangesetMetaRow>(
+        sql`SELECT * FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdsToRollback.map((id) => `(${id.global}, ${id.client})`).join(', ')})`,
+      )
+      .map((_) => ({ id: { global: _.idGlobal, client: _.idClient }, changeset: _.changeset, debug: _.debug }))
+      .toSorted((a, b) => EventId.compare(a.id, b.id))
+
+    // Apply changesets in reverse order
+    for (let i = rollbackEvents.length - 1; i >= 0; i--) {
+      const { changeset } = rollbackEvents[i]!
+      if (changeset !== null) {
+        db.makeChangeset(changeset).invert().apply()
+      }
+    }
+
+    const eventIdPairChunks = ReadonlyArray.chunksOf(100)(
+      eventIdsToRollback.map((id) => `(${id.global}, ${id.client})`),
+    )
+
+    // Delete the changeset rows
+    for (const eventIdPairChunk of eventIdPairChunks) {
+      db.execute(
+        sql`DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdPairChunk.join(', ')})`,
+      )
+    }
+
+    // Delete the eventlog rows
+    for (const eventIdPairChunk of eventIdPairChunks) {
+      dbEventlog.execute(
+        sql`DELETE FROM ${EVENTLOG_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdPairChunk.join(', ')})`,
+      )
+    }
+  }).pipe(
+    Effect.withSpan('@livestore/common:LeaderSyncProcessor:rollback', {
+      attributes: { count: eventIdsToRollback.length },
+    }),
+  )
+
+// TODO let's consider removing this "should exclude" mechanism in favour of log compaction etc
+const makeShouldExcludeEventFromLog = memoizeByRef((schema: LiveStoreSchema) => {
+  const migrationOptions = schema.migrationOptions
+  const eventlogExclude =
+    migrationOptions.strategy === 'from-eventlog'
+      ? (migrationOptions.excludeEvents ?? new Set(['livestore.RawSql']))
+      : new Set(['livestore.RawSql'])
+
+  return (eventName: string, eventEncoded: LiveStoreEvent.AnyEncoded): boolean => {
+    if (eventlogExclude.has(eventName)) return true
+
+    const eventDef = getEventDef(schema, eventName)
+    const execArgsArr = getExecArgsFromEvent({
+      eventDef,
+      event: { decoded: undefined, encoded: eventEncoded },
+    })
+
+    return execArgsArr.some((_) => _.statementSql.includes('__livestore'))
+  }
+})

package/src/leader-thread/eventlog.ts

@@ -0,0 +1,199 @@
+import { LS_DEV, shouldNeverHappen } from '@livestore/utils'
+import { Effect, Option, Schema } from '@livestore/utils/effect'
+
+import type { SqliteDb } from '../adapter-types.js'
+import * as EventId from '../schema/EventId.js'
+import * as LiveStoreEvent from '../schema/LiveStoreEvent.js'
+import {
+  EVENTLOG_META_TABLE,
+  eventlogMetaTable,
+  sessionChangesetMetaTable,
+  SYNC_STATUS_TABLE,
+  syncStatusTable,
+} from '../schema/system-tables.js'
+import { migrateTable } from '../schema-management/migrations.js'
+import { insertRow, updateRows } from '../sql-queries/sql-queries.js'
+import type { PreparedBindValues } from '../util.js'
+import { prepareBindValues, sql } from '../util.js'
+import { execSql } from './connection.js'
+import type { InitialSyncInfo } from './types.js'
+import { LeaderThreadCtx } from './types.js'
+
+export const initEventlogDb = (dbEventlog: SqliteDb) =>
+  Effect.gen(function* () {
+    yield* migrateTable({
+      db: dbEventlog,
+      behaviour: 'create-if-not-exists',
+      tableAst: eventlogMetaTable.sqliteDef.ast,
+      skipMetaTable: true,
+    })
+
+    yield* migrateTable({
+      db: dbEventlog,
+      behaviour: 'create-if-not-exists',
+      tableAst: syncStatusTable.sqliteDef.ast,
+      skipMetaTable: true,
+    })
+
+    // Create sync status row if it doesn't exist
+    yield* execSql(
+      dbEventlog,
+      sql`INSERT INTO ${SYNC_STATUS_TABLE} (head)
+          SELECT ${EventId.ROOT.global}
+          WHERE NOT EXISTS (SELECT 1 FROM ${SYNC_STATUS_TABLE})`,
+      {},
+    )
+  })
+
+/** Exclusive of the "since event" */
+export const getEventsSince = (
+  since: EventId.EventId,
+): Effect.Effect<ReadonlyArray<LiveStoreEvent.EncodedWithMeta>, never, LeaderThreadCtx> =>
+  Effect.gen(function* () {
+    const { dbEventlog, dbReadModel } = yield* LeaderThreadCtx
+
+    const query = eventlogMetaTable.where('idGlobal', '>=', since.global).asSql()
+    const pendingEventsRaw = dbEventlog.select(query.query, prepareBindValues(query.bindValues, query.query))
+    const pendingEvents = Schema.decodeUnknownSync(eventlogMetaTable.rowSchema.pipe(Schema.Array))(pendingEventsRaw)
+
+    const sessionChangesetRows = sessionChangesetMetaTable.where('idGlobal', '>=', since.global).asSql()
+    const sessionChangesetRowsRaw = dbReadModel.select(
+      sessionChangesetRows.query,
+      prepareBindValues(sessionChangesetRows.bindValues, sessionChangesetRows.query),
+    )
+    const sessionChangesetRowsDecoded = Schema.decodeUnknownSync(
+      sessionChangesetMetaTable.rowSchema.pipe(Schema.Array),
+    )(sessionChangesetRowsRaw)
+
+    return pendingEvents
+      .map((eventlogEvent) => {
+        const sessionChangeset = sessionChangesetRowsDecoded.find(
+          (readModelEvent) =>
+            readModelEvent.idGlobal === eventlogEvent.idGlobal && readModelEvent.idClient === eventlogEvent.idClient,
+        )
+        return LiveStoreEvent.EncodedWithMeta.make({
+          name: eventlogEvent.name,
+          args: eventlogEvent.argsJson,
+          id: { global: eventlogEvent.idGlobal, client: eventlogEvent.idClient },
+          parentId: { global: eventlogEvent.parentIdGlobal, client: eventlogEvent.parentIdClient },
+          clientId: eventlogEvent.clientId,
+          sessionId: eventlogEvent.sessionId,
+          meta: {
+            sessionChangeset:
+              sessionChangeset && sessionChangeset.changeset !== null
+                ? {
+                    _tag: 'sessionChangeset' as const,
+                    data: sessionChangeset.changeset,
+                    debug: sessionChangeset.debug,
+                  }
+                : { _tag: 'unset' as const },
+            syncMetadata: eventlogEvent.syncMetadataJson,
+          },
+        })
+      })
+      .filter((_) => EventId.compare(_.id, since) > 0)
+      .sort((a, b) => EventId.compare(a.id, b.id))
+  })
+
+export const getClientHeadFromDb = (dbEventlog: SqliteDb): EventId.EventId => {
+  const res = dbEventlog.select<{ idGlobal: EventId.GlobalEventId; idClient: EventId.ClientEventId }>(
+    sql`select idGlobal, idClient from ${EVENTLOG_META_TABLE} order by idGlobal DESC, idClient DESC limit 1`,
+  )[0]
+
+  return res ? { global: res.idGlobal, client: res.idClient } : EventId.ROOT
+}
+
+export const getBackendHeadFromDb = (dbEventlog: SqliteDb): EventId.GlobalEventId =>
+  dbEventlog.select<{ head: EventId.GlobalEventId }>(sql`select head from ${SYNC_STATUS_TABLE}`)[0]?.head ??
+  EventId.ROOT.global
+
+// TODO use prepared statements
+export const updateBackendHead = (dbEventlog: SqliteDb, head: EventId.EventId) =>
+  dbEventlog.execute(sql`UPDATE ${SYNC_STATUS_TABLE} SET head = ${head.global}`)
+
+export const insertIntoEventlog = (
+  eventEncoded: LiveStoreEvent.EncodedWithMeta,
+  dbEventlog: SqliteDb,
+  eventDefSchemaHash: number,
+  clientId: string,
+  sessionId: string,
+) =>
+  Effect.gen(function* () {
+    // Check history consistency during LS_DEV
+    if (LS_DEV && eventEncoded.parentId.global !== EventId.ROOT.global) {
+      const parentEventExists =
+        dbEventlog.select<{ count: number }>(
+          `SELECT COUNT(*) as count FROM ${EVENTLOG_META_TABLE} WHERE idGlobal = ? AND idClient = ?`,
+          [eventEncoded.parentId.global, eventEncoded.parentId.client] as any as PreparedBindValues,
+        )[0]!.count === 1
+
+      if (parentEventExists === false) {
+        shouldNeverHappen(
+          `Parent mutation ${eventEncoded.parentId.global},${eventEncoded.parentId.client} does not exist`,
+        )
+      }
+    }
+
+    // TODO use prepared statements
+    yield* execSql(
+      dbEventlog,
+      ...insertRow({
+        tableName: EVENTLOG_META_TABLE,
+        columns: eventlogMetaTable.sqliteDef.columns,
+        values: {
+          idGlobal: eventEncoded.id.global,
+          idClient: eventEncoded.id.client,
+          parentIdGlobal: eventEncoded.parentId.global,
+          parentIdClient: eventEncoded.parentId.client,
+          name: eventEncoded.name,
+          argsJson: eventEncoded.args ?? {},
+          clientId,
+          sessionId,
+          schemaHash: eventDefSchemaHash,
+          syncMetadataJson: eventEncoded.meta.syncMetadata,
+        },
+      }),
+    )
+  })
+
+export const updateSyncMetadata = (items: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>) =>
+  Effect.gen(function* () {
+    const { dbEventlog } = yield* LeaderThreadCtx
+
+    // TODO try to do this in a single query
+    for (let i = 0; i < items.length; i++) {
+      const event = items[i]!
+
+      yield* execSql(
+        dbEventlog,
+        ...updateRows({
+          tableName: EVENTLOG_META_TABLE,
+          columns: eventlogMetaTable.sqliteDef.columns,
+          where: { idGlobal: event.id.global, idClient: event.id.client },
+          updateValues: { syncMetadataJson: event.meta.syncMetadata },
+        }),
+      )
+    }
+  })
+
+export const getSyncBackendCursorInfo = (remoteHead: EventId.GlobalEventId) =>
+  Effect.gen(function* () {
+    const { dbEventlog } = yield* LeaderThreadCtx
+
+    if (remoteHead === EventId.ROOT.global) return Option.none()
+
+    const EventlogQuerySchema = Schema.Struct({
+      syncMetadataJson: Schema.parseJson(Schema.Option(Schema.JsonValue)),
+    }).pipe(Schema.pluck('syncMetadataJson'), Schema.Array, Schema.head)
+
+    const syncMetadataOption = yield* Effect.sync(() =>
+      dbEventlog.select<{ syncMetadataJson: string }>(
+        sql`SELECT syncMetadataJson FROM ${EVENTLOG_META_TABLE} WHERE idGlobal = ${remoteHead} ORDER BY idClient ASC LIMIT 1`,
+      ),
+    ).pipe(Effect.andThen(Schema.decode(EventlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie)
+
+    return Option.some({
+      cursor: { global: remoteHead, client: EventId.clientDefault },
+      metadata: syncMetadataOption,
+    }) satisfies InitialSyncInfo
+  }).pipe(Effect.withSpan('@livestore/common:eventlog:getSyncBackendCursorInfo', { attributes: { remoteHead } }))

package/src/leader-thread/leader-worker-devtools.ts

@@ -2,7 +2,7 @@ import { Effect, FiberMap, Option, Stream, SubscriptionRef } from '@livestore/ut
 import { nanoid } from '@livestore/utils/nanoid'
 
 import { Devtools, IntentionalShutdownCause, liveStoreVersion, UnexpectedError } from '../index.js'
-import { MUTATION_LOG_META_TABLE, SCHEMA_META_TABLE, SCHEMA_MUTATIONS_META_TABLE } from '../schema/mod.js'
+import { EVENTLOG_META_TABLE, SCHEMA_EVENT_DEFS_META_TABLE, SCHEMA_META_TABLE } from '../schema/mod.js'
 import type { DevtoolsOptions, PersistenceInfoPair } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
@@ -15,7 +15,7 @@ export const bootDevtools = (options: DevtoolsOptions) =>
       return
     }
 
-    const { connectedClientSessionPullQueues, syncProcessor, extraIncomingMessagesQueue } = yield* LeaderThreadCtx
+    const { syncProcessor, extraIncomingMessagesQueue } = yield* LeaderThreadCtx
 
     yield* listenToDevtools({
       incomingMessages: Stream.fromQueue(extraIncomingMessagesQueue),
@@ -33,13 +33,11 @@ export const bootDevtools = (options: DevtoolsOptions) =>
           Effect.ignoreLogged,
         )
 
-    const { localHead } = yield* syncProcessor.syncState
+    const syncState = yield* syncProcessor.syncState
+    const mergeCounter = syncProcessor.getMergeCounter()
 
-    // TODO close queue when devtools disconnects
-    const pullQueue = yield* connectedClientSessionPullQueues.makeQueue(localHead)
-
-    yield* Stream.fromQueue(pullQueue).pipe(
-      Stream.tap((msg) => sendMessage(Devtools.Leader.SyncPull.make({ payload: msg.payload, liveStoreVersion }))),
+    yield* syncProcessor.pull({ cursor: { mergeCounter, eventId: syncState.localHead } }).pipe(
+      Stream.tap(({ payload }) => sendMessage(Devtools.Leader.SyncPull.make({ payload, liveStoreVersion }))),
       Stream.runDrain,
       Effect.forkScoped,
     )
@@ -65,7 +63,7 @@ const listenToDevtools = ({
       syncBackend,
       makeSqliteDb,
       dbReadModel,
-      dbMutationLog,
+      dbEventlog,
       shutdownStateSubRef,
       shutdownChannel,
       syncProcessor,
@@ -145,20 +143,20 @@ const listenToDevtools = ({
               }
 
               try {
-                if (tableNames.has(MUTATION_LOG_META_TABLE)) {
-                  // Is mutation log
+                if (tableNames.has(EVENTLOG_META_TABLE)) {
+                  // Is eventlog
                   yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
 
-                  dbMutationLog.import(data)
+                  dbEventlog.import(data)
 
                   dbReadModel.destroy()
-                } else if (tableNames.has(SCHEMA_META_TABLE) && tableNames.has(SCHEMA_MUTATIONS_META_TABLE)) {
+                } else if (tableNames.has(SCHEMA_META_TABLE) && tableNames.has(SCHEMA_EVENT_DEFS_META_TABLE)) {
                   // Is read model
                   yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
 
                   dbReadModel.import(data)
 
-                  dbMutationLog.destroy()
+                  dbEventlog.destroy()
                 } else {
                   yield* sendMessage(
                     Devtools.Leader.LoadDatabaseFile.Error.make({
@@ -192,7 +190,7 @@ const listenToDevtools = ({
               dbReadModel.destroy()
 
               if (mode === 'all-data') {
-                dbMutationLog.destroy()
+                dbEventlog.destroy()
               }
 
               yield* sendMessage(Devtools.Leader.ResetAllData.Success.make({ ...reqPayload }))
@@ -209,33 +207,33 @@ const listenToDevtools = ({
 
               const dbSizeQuery = `SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size();`
               const dbFileSize = dbReadModel.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
-              const mutationLogFileSize = dbMutationLog.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
+              const eventlogFileSize = dbEventlog.select<{ size: number }>(dbSizeQuery, undefined)[0]!.size
 
               yield* sendMessage(
                 Devtools.Leader.DatabaseFileInfoRes.make({
                   readModel: { fileSize: dbFileSize, persistenceInfo: persistenceInfo.readModel },
-                  mutationLog: { fileSize: mutationLogFileSize, persistenceInfo: persistenceInfo.mutationLog },
+                  eventlog: { fileSize: eventlogFileSize, persistenceInfo: persistenceInfo.eventlog },
                   ...reqPayload,
                 }),
               )
 
               return
             }
-            case 'LSD.Leader.MutationLogReq': {
-              const mutationLog = dbMutationLog.export()
+            case 'LSD.Leader.EventlogReq': {
+              const eventlog = dbEventlog.export()
 
-              yield* sendMessage(Devtools.Leader.MutationLogRes.make({ mutationLog, ...reqPayload }))
+              yield* sendMessage(Devtools.Leader.EventlogRes.make({ eventlog, ...reqPayload }))
 
               return
             }
-            case 'LSD.Leader.RunMutationReq': {
+            case 'LSD.Leader.CommitEventReq': {
               yield* syncProcessor.pushPartial({
-                mutationEvent: decodedEvent.mutationEventEncoded,
+                event: decodedEvent.eventEncoded,
                 clientId: `devtools-${clientId}`,
                 sessionId: `devtools-${clientId}`,
               })
 
-              yield* sendMessage(Devtools.Leader.RunMutationRes.make({ ...reqPayload }))
+              yield* sendMessage(Devtools.Leader.CommitEventRes.make({ ...reqPayload }))
 
               return
             }
@@ -247,10 +245,10 @@ const listenToDevtools = ({
                 yield* syncBackend.pull(Option.none()).pipe(
                   Stream.map((_) => _.batch),
                   Stream.flattenIterables,
-                  Stream.tap(({ mutationEventEncoded, metadata }) =>
+                  Stream.tap(({ eventEncoded, metadata }) =>
                     sendMessage(
                       Devtools.Leader.SyncHistoryRes.make({
-                        mutationEventEncoded,
+                        eventEncoded,
                         metadata,
                         subscriptionId,
                         ...reqPayload,