@livestore/common 0.3.0-dev.26 → 0.3.0-dev.28

package/src/leader-thread/apply-mutation.ts

@@ -1,46 +1,43 @@
 import { LS_DEV, memoizeByRef, shouldNeverHappen } from '@livestore/utils'
-import type { Scope } from '@livestore/utils/effect'
-import { Effect, Option, Schema } from '@livestore/utils/effect'
+import { Effect, ReadonlyArray, Schema } from '@livestore/utils/effect'
 
-import type { PreparedBindValues, SqliteDb, SqliteError, UnexpectedError } from '../index.js'
+import type { SqliteDb } from '../adapter-types.js'
 import { getExecArgsFromMutation } from '../mutation.js'
+import type { LiveStoreSchema, MutationEvent, SessionChangesetMetaRow } from '../schema/mod.js'
 import {
   EventId,
   getMutationDef,
-  type LiveStoreSchema,
   MUTATION_LOG_META_TABLE,
-  type MutationEvent,
-  mutationLogMetaTable,
   SESSION_CHANGESET_META_TABLE,
   sessionChangesetMetaTable,
 } from '../schema/mod.js'
 import { insertRow } from '../sql-queries/index.js'
+import { sql } from '../util.js'
 import { execSql, execSqlPrepared } from './connection.js'
-import { LeaderThreadCtx } from './types.js'
-
-export type ApplyMutation = (
-  mutationEventEncoded: MutationEvent.AnyEncoded,
-  options?: {
-    /** Needed for rehydrateFromMutationLog */
-    skipMutationLog?: boolean
-  },
-) => Effect.Effect<void, SqliteError | UnexpectedError>
-
-export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope | LeaderThreadCtx> = Effect.gen(
-  function* () {
-    const leaderThreadCtx = yield* LeaderThreadCtx
-    const shouldExcludeMutationFromLog = makeShouldExcludeMutationFromLog(leaderThreadCtx.schema)
+import * as Mutationlog from './mutationlog.js'
+import type { ApplyMutation } from './types.js'
+
+export const makeApplyMutation = ({
+  schema,
+  dbReadModel: db,
+  dbMutationLog,
+}: {
+  schema: LiveStoreSchema
+  dbReadModel: SqliteDb
+  dbMutationLog: SqliteDb
+}): Effect.Effect<ApplyMutation, never> =>
+  Effect.gen(function* () {
+    const shouldExcludeMutationFromLog = makeShouldExcludeMutationFromLog(schema)
 
     const mutationDefSchemaHashMap = new Map(
       // TODO Running `Schema.hash` can be a bottleneck for larger schemas. There is an opportunity to run this
       // at build time and lookup the pre-computed hash at runtime.
       // Also see https://github.com/Effect-TS/effect/issues/2719
-      [...leaderThreadCtx.schema.mutations.map.entries()].map(([k, v]) => [k, Schema.hash(v.schema)] as const),
+      [...schema.mutations.map.entries()].map(([k, v]) => [k, Schema.hash(v.schema)] as const),
     )
 
     return (mutationEventEncoded, options) =>
       Effect.gen(function* () {
-        const { schema, dbReadModel: db, dbMutationLog } = leaderThreadCtx
         const skipMutationLog = options?.skipMutationLog ?? false
 
         const mutationName = mutationEventEncoded.mutation
@@ -84,7 +81,7 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
               idClient: mutationEventEncoded.id.client,
               // NOTE the changeset will be empty (i.e. null) for no-op mutations
               changeset: changeset ?? null,
-              debug: execArgsArr,
+              debug: LS_DEV ? execArgsArr : null,
             },
           }),
         )
@@ -94,76 +91,89 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
         // write to mutation_log
         const excludeFromMutationLog = shouldExcludeMutationFromLog(mutationName, mutationEventEncoded)
         if (skipMutationLog === false && excludeFromMutationLog === false) {
-          yield* insertIntoMutationLog(
+          const mutationName = mutationEventEncoded.mutation
+          const mutationDefSchemaHash =
+            mutationDefSchemaHashMap.get(mutationName) ?? shouldNeverHappen(`Unknown mutation: ${mutationName}`)
+
+          yield* Mutationlog.insertIntoMutationLog(
             mutationEventEncoded,
             dbMutationLog,
-            mutationDefSchemaHashMap,
+            mutationDefSchemaHash,
             mutationEventEncoded.clientId,
             mutationEventEncoded.sessionId,
           )
         } else {
           //   console.debug('[@livestore/common:leader-thread] skipping mutation log write', mutation, statementSql, bindValues)
         }
+
+        return {
+          sessionChangeset: changeset
+            ? {
+                _tag: 'sessionChangeset' as const,
+                data: changeset,
+                debug: LS_DEV ? execArgsArr : null,
+              }
+            : { _tag: 'no-op' as const },
+        }
       }).pipe(
         Effect.withSpan(`@livestore/common:leader-thread:applyMutation`, {
           attributes: {
             mutationName: mutationEventEncoded.mutation,
             mutationId: mutationEventEncoded.id,
-            'span.label': `(${mutationEventEncoded.id.global},${mutationEventEncoded.id.client}) ${mutationEventEncoded.mutation}`,
+            'span.label': `${EventId.toString(mutationEventEncoded.id)} ${mutationEventEncoded.mutation}`,
           },
         }),
         // Effect.logDuration('@livestore/common:leader-thread:applyMutation'),
       )
-  },
-)
-
-const insertIntoMutationLog = (
-  mutationEventEncoded: MutationEvent.AnyEncoded,
-  dbMutationLog: SqliteDb,
-  mutationDefSchemaHashMap: Map<string, number>,
-  clientId: string,
-  sessionId: string,
-) =>
+  })
+
+export const rollback = ({
+  db,
+  dbMutationLog,
+  eventIdsToRollback,
+}: {
+  db: SqliteDb
+  dbMutationLog: SqliteDb
+  eventIdsToRollback: EventId.EventId[]
+}) =>
   Effect.gen(function* () {
-    const mutationName = mutationEventEncoded.mutation
-    const mutationDefSchemaHash =
-      mutationDefSchemaHashMap.get(mutationName) ?? shouldNeverHappen(`Unknown mutation: ${mutationName}`)
-
-    if (LS_DEV && mutationEventEncoded.parentId.global !== EventId.ROOT.global) {
-      const parentMutationExists =
-        dbMutationLog.select<{ count: number }>(
-          `SELECT COUNT(*) as count FROM ${MUTATION_LOG_META_TABLE} WHERE idGlobal = ? AND idClient = ?`,
-          [mutationEventEncoded.parentId.global, mutationEventEncoded.parentId.client] as any as PreparedBindValues,
-        )[0]!.count === 1
-
-      if (parentMutationExists === false) {
-        shouldNeverHappen(
-          `Parent mutation ${mutationEventEncoded.parentId.global},${mutationEventEncoded.parentId.client} does not exist`,
-        )
+    const rollbackEvents = db
+      .select<SessionChangesetMetaRow>(
+        sql`SELECT * FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdsToRollback.map((id) => `(${id.global}, ${id.client})`).join(', ')})`,
+      )
+      .map((_) => ({ id: { global: _.idGlobal, client: _.idClient }, changeset: _.changeset, debug: _.debug }))
+      .toSorted((a, b) => EventId.compare(a.id, b.id))
+
+    // Apply changesets in reverse order
+    for (let i = rollbackEvents.length - 1; i >= 0; i--) {
+      const { changeset } = rollbackEvents[i]!
+      if (changeset !== null) {
+        db.makeChangeset(changeset).invert().apply()
       }
     }
 
-    // TODO use prepared statements
-    yield* execSql(
-      dbMutationLog,
-      ...insertRow({
-        tableName: MUTATION_LOG_META_TABLE,
-        columns: mutationLogMetaTable.sqliteDef.columns,
-        values: {
-          idGlobal: mutationEventEncoded.id.global,
-          idClient: mutationEventEncoded.id.client,
-          parentIdGlobal: mutationEventEncoded.parentId.global,
-          parentIdClient: mutationEventEncoded.parentId.client,
-          mutation: mutationEventEncoded.mutation,
-          argsJson: mutationEventEncoded.args ?? {},
-          clientId,
-          sessionId,
-          schemaHash: mutationDefSchemaHash,
-          syncMetadataJson: Option.none(),
-        },
-      }),
+    const eventIdPairChunks = ReadonlyArray.chunksOf(100)(
+      eventIdsToRollback.map((id) => `(${id.global}, ${id.client})`),
     )
-  })
+
+    // Delete the changeset rows
+    for (const eventIdPairChunk of eventIdPairChunks) {
+      db.execute(
+        sql`DELETE FROM ${SESSION_CHANGESET_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdPairChunk.join(', ')})`,
+      )
+    }
+
+    // Delete the mutation log rows
+    for (const eventIdPairChunk of eventIdPairChunks) {
+      dbMutationLog.execute(
+        sql`DELETE FROM ${MUTATION_LOG_META_TABLE} WHERE (idGlobal, idClient) IN (${eventIdPairChunk.join(', ')})`,
+      )
+    }
+  }).pipe(
+    Effect.withSpan('@livestore/common:LeaderSyncProcessor:rollback', {
+      attributes: { count: eventIdsToRollback.length },
+    }),
+  )
 
 // TODO let's consider removing this "should exclude" mechanism in favour of log compaction etc
 const makeShouldExcludeMutationFromLog = memoizeByRef((schema: LiveStoreSchema) => {

package/src/leader-thread/leader-worker-devtools.ts

@@ -15,7 +15,7 @@ export const bootDevtools = (options: DevtoolsOptions) =>
       return
     }
 
-    const { connectedClientSessionPullQueues, syncProcessor, extraIncomingMessagesQueue } = yield* LeaderThreadCtx
+    const { syncProcessor, extraIncomingMessagesQueue } = yield* LeaderThreadCtx
 
     yield* listenToDevtools({
       incomingMessages: Stream.fromQueue(extraIncomingMessagesQueue),
@@ -33,13 +33,11 @@ export const bootDevtools = (options: DevtoolsOptions) =>
           Effect.ignoreLogged,
         )
 
-    const { localHead } = yield* syncProcessor.syncState
+    const syncState = yield* syncProcessor.syncState
+    const mergeCounter = syncProcessor.getMergeCounter()
 
-    // TODO close queue when devtools disconnects
-    const pullQueue = yield* connectedClientSessionPullQueues.makeQueue(localHead)
-
-    yield* Stream.fromQueue(pullQueue).pipe(
-      Stream.tap((msg) => sendMessage(Devtools.Leader.SyncPull.make({ payload: msg.payload, liveStoreVersion }))),
+    yield* syncProcessor.pull({ cursor: { mergeCounter, eventId: syncState.localHead } }).pipe(
+      Stream.tap(({ payload }) => sendMessage(Devtools.Leader.SyncPull.make({ payload, liveStoreVersion }))),
       Stream.runDrain,
       Effect.forkScoped,
     )

package/src/leader-thread/make-leader-thread-layer.ts

@@ -5,14 +5,13 @@ import type { BootStatus, MakeSqliteDb, MigrationsReport, SqliteError } from '..
 import { UnexpectedError } from '../adapter-types.js'
 import type * as Devtools from '../devtools/mod.js'
 import type { LiveStoreSchema } from '../schema/mod.js'
-import { EventId, MutationEvent, mutationLogMetaTable, SYNC_STATUS_TABLE, syncStatusTable } from '../schema/mod.js'
-import { migrateTable } from '../schema-management/migrations.js'
+import { MutationEvent } from '../schema/mod.js'
 import type { InvalidPullError, IsOfflineError, SyncOptions } from '../sync/sync.js'
 import { sql } from '../util.js'
-import { execSql } from './connection.js'
+import { makeApplyMutation } from './apply-mutation.js'
 import { bootDevtools } from './leader-worker-devtools.js'
 import { makeLeaderSyncProcessor } from './LeaderSyncProcessor.js'
-import { makePullQueueSet } from './pull-queue-set.js'
+import * as Mutationlog from './mutationlog.js'
 import { recreateDb } from './recreate-db.js'
 import type { ShutdownChannel } from './shutdown-channel.js'
 import type {
@@ -24,6 +23,30 @@ import type {
 } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
+export interface MakeLeaderThreadLayerParams {
+  storeId: string
+  syncPayload: Schema.JsonValue | undefined
+  clientId: string
+  schema: LiveStoreSchema
+  makeSqliteDb: MakeSqliteDb
+  syncOptions: SyncOptions | undefined
+  dbReadModel: LeaderSqliteDb
+  dbMutationLog: LeaderSqliteDb
+  devtoolsOptions: DevtoolsOptions
+  shutdownChannel: ShutdownChannel
+  params?: {
+    localPushBatchSize?: number
+    backendPushBatchSize?: number
+  }
+  testing?: {
+    syncProcessor?: {
+      delays?: {
+        localPushProcessing?: Effect.Effect<void>
+      }
+    }
+  }
+}
+
 export const makeLeaderThreadLayer = ({
   schema,
   storeId,
@@ -35,24 +58,18 @@ export const makeLeaderThreadLayer = ({
   dbMutationLog,
   devtoolsOptions,
   shutdownChannel,
-}: {
-  storeId: string
-  syncPayload: Schema.JsonValue | undefined
-  clientId: string
-  schema: LiveStoreSchema
-  makeSqliteDb: MakeSqliteDb
-  syncOptions: SyncOptions | undefined
-  dbReadModel: LeaderSqliteDb
-  dbMutationLog: LeaderSqliteDb
-  devtoolsOptions: DevtoolsOptions
-  shutdownChannel: ShutdownChannel
-}): Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient> =>
+  params,
+  testing,
+}: MakeLeaderThreadLayerParams): Layer.Layer<LeaderThreadCtx, UnexpectedError, Scope.Scope | HttpClient.HttpClient> =>
   Effect.gen(function* () {
     const bootStatusQueue = yield* Queue.unbounded<BootStatus>().pipe(Effect.acquireRelease(Queue.shutdown))
 
     // TODO do more validation here than just checking the count of tables
     // Either happens on initial boot or if schema changes
-    const dbMissing =
+    const dbMutationLogMissing =
+      dbMutationLog.select<{ count: number }>(sql`select count(*) as count from sqlite_master`)[0]!.count === 0
+
+    const dbReadModelMissing =
       dbReadModel.select<{ count: number }>(sql`select count(*) as count from sqlite_master`)[0]!.count === 0
 
     const syncBackend =
@@ -60,6 +77,11 @@ export const makeLeaderThreadLayer = ({
         ? undefined
         : yield* syncOptions.backend({ storeId, clientId, payload: syncPayload })
 
+    if (syncBackend !== undefined) {
+      // We're already connecting to the sync backend concurrently
+      yield* syncBackend.connect.pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
+    }
+
     const initialBlockingSyncContext = yield* makeInitialBlockingSyncContext({
       initialSyncOptions: syncOptions?.initialSyncOptions ?? { _tag: 'Skip' },
       bootStatusQueue,
@@ -67,11 +89,19 @@ export const makeLeaderThreadLayer = ({
 
     const syncProcessor = yield* makeLeaderSyncProcessor({
       schema,
-      dbMissing,
+      dbMutationLogMissing,
       dbMutationLog,
+      dbReadModel,
+      dbReadModelMissing,
       initialBlockingSyncContext,
-      clientId,
       onError: syncOptions?.onSyncError ?? 'ignore',
+      params: {
+        localPushBatchSize: params?.localPushBatchSize,
+        backendPushBatchSize: params?.backendPushBatchSize,
+      },
+      testing: {
+        delays: testing?.syncProcessor?.delays,
+      },
     })
 
     const extraIncomingMessagesQueue = yield* Queue.unbounded<Devtools.Leader.MessageToApp>().pipe(
@@ -86,6 +116,8 @@ export const makeLeaderThreadLayer = ({
         }
       : { enabled: false as const }
 
+    const applyMutation = yield* makeApplyMutation({ schema, dbReadModel, dbMutationLog })
+
     const ctx = {
       schema,
       bootStatusQueue,
@@ -99,7 +131,7 @@ export const makeLeaderThreadLayer = ({
       shutdownChannel,
       syncBackend,
       syncProcessor,
-      connectedClientSessionPullQueues: yield* makePullQueueSet,
+      applyMutation,
       extraIncomingMessagesQueue,
       devtools: devtoolsContext,
       // State will be set during `bootLeaderThread`
@@ -112,7 +144,7 @@ export const makeLeaderThreadLayer = ({
     const layer = Layer.succeed(LeaderThreadCtx, ctx)
 
     ctx.initialState = yield* bootLeaderThread({
-      dbMissing,
+      dbReadModelMissing,
       initialBlockingSyncContext,
       devtoolsOptions,
     }).pipe(Effect.provide(layer))
@@ -122,6 +154,7 @@ export const makeLeaderThreadLayer = ({
     Effect.withSpan('@livestore/common:leader-thread:boot'),
     Effect.withSpanScoped('@livestore/common:leader-thread'),
     UnexpectedError.mapToUnexpectedError,
+    Effect.tapCauseLogPretty,
     Layer.unwrapScoped,
   )
 
@@ -177,11 +210,11 @@ const makeInitialBlockingSyncContext = ({
  * It also starts various background processes (e.g. syncing)
  */
 const bootLeaderThread = ({
-  dbMissing,
+  dbReadModelMissing,
   initialBlockingSyncContext,
   devtoolsOptions,
 }: {
-  dbMissing: boolean
+  dbReadModelMissing: boolean
   initialBlockingSyncContext: InitialBlockingSyncContext
   devtoolsOptions: DevtoolsOptions
 }): Effect.Effect<
@@ -192,44 +225,18 @@ const bootLeaderThread = ({
   Effect.gen(function* () {
     const { dbMutationLog, bootStatusQueue, syncProcessor } = yield* LeaderThreadCtx
 
-    yield* migrateTable({
-      db: dbMutationLog,
-      behaviour: 'create-if-not-exists',
-      tableAst: mutationLogMetaTable.sqliteDef.ast,
-      skipMetaTable: true,
-    })
-
-    yield* migrateTable({
-      db: dbMutationLog,
-      behaviour: 'create-if-not-exists',
-      tableAst: syncStatusTable.sqliteDef.ast,
-      skipMetaTable: true,
-    })
-
-    // Create sync status row if it doesn't exist
-    yield* execSql(
-      dbMutationLog,
-      sql`INSERT INTO ${SYNC_STATUS_TABLE} (head)
-          SELECT ${EventId.ROOT.global}
-          WHERE NOT EXISTS (SELECT 1 FROM ${SYNC_STATUS_TABLE})`,
-      {},
-    )
-
-    const dbReady = yield* Deferred.make<void>()
-
-    // We're already starting pulling from the sync backend concurrently but wait until the db is ready before
-    // processing any incoming mutations
-    const { initialLeaderHead } = yield* syncProcessor.boot({ dbReady })
+    yield* Mutationlog.initMutationLogDb(dbMutationLog)
 
     let migrationsReport: MigrationsReport
-    if (dbMissing) {
+    if (dbReadModelMissing) {
       const recreateResult = yield* recreateDb
       migrationsReport = recreateResult.migrationsReport
     } else {
       migrationsReport = { migrations: [] }
     }
 
-    yield* Deferred.succeed(dbReady, void 0)
+    // NOTE the sync processor depends on the dbs being initialized properly
+    const { initialLeaderHead } = yield* syncProcessor.boot
 
     if (initialBlockingSyncContext.blockingDeferred !== undefined) {
       // Provides a syncing status right away before the first pull response comes in

package/src/leader-thread/mod.ts

@@ -3,4 +3,4 @@ export * from './types.js'
 export * as ShutdownChannel from './shutdown-channel.js'
 export * from './leader-worker-devtools.js'
 export * from './make-leader-thread-layer.js'
-export * from './mutationlog.js'
+export * as Mutationlog from './mutationlog.js'

package/src/leader-thread/mutationlog.ts

@@ -1,18 +1,56 @@
-import { Effect, Schema } from '@livestore/utils/effect'
+import { LS_DEV, shouldNeverHappen } from '@livestore/utils'
+import { Effect, Option, Schema } from '@livestore/utils/effect'
 
 import type { SqliteDb } from '../adapter-types.js'
 import * as EventId from '../schema/EventId.js'
-import type * as MutationEvent from '../schema/MutationEvent.js'
-import { MUTATION_LOG_META_TABLE, mutationLogMetaTable, SYNC_STATUS_TABLE } from '../schema/system-tables.js'
+import * as MutationEvent from '../schema/MutationEvent.js'
+import {
+  MUTATION_LOG_META_TABLE,
+  mutationLogMetaTable,
+  sessionChangesetMetaTable,
+  SYNC_STATUS_TABLE,
+  syncStatusTable,
+} from '../schema/system-tables.js'
+import { migrateTable } from '../schema-management/migrations.js'
+import { insertRow, updateRows } from '../sql-queries/sql-queries.js'
+import type { PreparedBindValues } from '../util.js'
 import { prepareBindValues, sql } from '../util.js'
+import { execSql } from './connection.js'
+import type { InitialSyncInfo } from './types.js'
 import { LeaderThreadCtx } from './types.js'
 
+export const initMutationLogDb = (dbMutationLog: SqliteDb) =>
+  Effect.gen(function* () {
+    yield* migrateTable({
+      db: dbMutationLog,
+      behaviour: 'create-if-not-exists',
+      tableAst: mutationLogMetaTable.sqliteDef.ast,
+      skipMetaTable: true,
+    })
+
+    yield* migrateTable({
+      db: dbMutationLog,
+      behaviour: 'create-if-not-exists',
+      tableAst: syncStatusTable.sqliteDef.ast,
+      skipMetaTable: true,
+    })
+
+    // Create sync status row if it doesn't exist
+    yield* execSql(
+      dbMutationLog,
+      sql`INSERT INTO ${SYNC_STATUS_TABLE} (head)
+          SELECT ${EventId.ROOT.global}
+          WHERE NOT EXISTS (SELECT 1 FROM ${SYNC_STATUS_TABLE})`,
+      {},
+    )
+  })
+
 /** Exclusive of the "since event" */
 export const getMutationEventsSince = (
   since: EventId.EventId,
-): Effect.Effect<ReadonlyArray<MutationEvent.AnyEncoded>, never, LeaderThreadCtx> =>
+): Effect.Effect<ReadonlyArray<MutationEvent.EncodedWithMeta>, never, LeaderThreadCtx> =>
   Effect.gen(function* () {
-    const { dbMutationLog } = yield* LeaderThreadCtx
+    const { dbMutationLog, dbReadModel } = yield* LeaderThreadCtx
 
     const query = mutationLogMetaTable.query.where('idGlobal', '>=', since.global).asSql()
     const pendingMutationEventsRaw = dbMutationLog.select(query.query, prepareBindValues(query.bindValues, query.query))
@@ -20,16 +58,44 @@ export const getMutationEventsSince = (
       pendingMutationEventsRaw,
     )
 
+    const sessionChangesetRows = sessionChangesetMetaTable.query.where('idGlobal', '>=', since.global).asSql()
+    const sessionChangesetRowsRaw = dbReadModel.select(
+      sessionChangesetRows.query,
+      prepareBindValues(sessionChangesetRows.bindValues, sessionChangesetRows.query),
+    )
+    const sessionChangesetRowsDecoded = Schema.decodeUnknownSync(sessionChangesetMetaTable.schema.pipe(Schema.Array))(
+      sessionChangesetRowsRaw,
+    )
+
     return pendingMutationEvents
-      .map((_) => ({
-        mutation: _.mutation,
-        args: _.argsJson,
-        id: { global: _.idGlobal, client: _.idClient },
-        parentId: { global: _.parentIdGlobal, client: _.parentIdClient },
-        clientId: _.clientId,
-        sessionId: _.sessionId,
-      }))
+      .map((mutationLogEvent) => {
+        const sessionChangeset = sessionChangesetRowsDecoded.find(
+          (readModelEvent) =>
+            readModelEvent.idGlobal === mutationLogEvent.idGlobal &&
+            readModelEvent.idClient === mutationLogEvent.idClient,
+        )
+        return MutationEvent.EncodedWithMeta.make({
+          mutation: mutationLogEvent.mutation,
+          args: mutationLogEvent.argsJson,
+          id: { global: mutationLogEvent.idGlobal, client: mutationLogEvent.idClient },
+          parentId: { global: mutationLogEvent.parentIdGlobal, client: mutationLogEvent.parentIdClient },
+          clientId: mutationLogEvent.clientId,
+          sessionId: mutationLogEvent.sessionId,
+          meta: {
+            sessionChangeset:
+              sessionChangeset && sessionChangeset.changeset !== null
+                ? {
+                    _tag: 'sessionChangeset' as const,
+                    data: sessionChangeset.changeset,
+                    debug: sessionChangeset.debug,
+                  }
+                : { _tag: 'unset' as const },
+            syncMetadata: mutationLogEvent.syncMetadataJson,
+          },
+        })
+      })
       .filter((_) => EventId.compare(_.id, since) > 0)
+      .sort((a, b) => EventId.compare(a.id, b.id))
   })
 
 export const getClientHeadFromDb = (dbMutationLog: SqliteDb): EventId.EventId => {
@@ -47,3 +113,90 @@ export const getBackendHeadFromDb = (dbMutationLog: SqliteDb): EventId.GlobalEve
 // TODO use prepared statements
 export const updateBackendHead = (dbMutationLog: SqliteDb, head: EventId.EventId) =>
   dbMutationLog.execute(sql`UPDATE ${SYNC_STATUS_TABLE} SET head = ${head.global}`)
+
+export const insertIntoMutationLog = (
+  mutationEventEncoded: MutationEvent.EncodedWithMeta,
+  dbMutationLog: SqliteDb,
+  mutationDefSchemaHash: number,
+  clientId: string,
+  sessionId: string,
+) =>
+  Effect.gen(function* () {
+    // Check history consistency during LS_DEV
+    if (LS_DEV && mutationEventEncoded.parentId.global !== EventId.ROOT.global) {
+      const parentMutationExists =
+        dbMutationLog.select<{ count: number }>(
+          `SELECT COUNT(*) as count FROM ${MUTATION_LOG_META_TABLE} WHERE idGlobal = ? AND idClient = ?`,
+          [mutationEventEncoded.parentId.global, mutationEventEncoded.parentId.client] as any as PreparedBindValues,
+        )[0]!.count === 1
+
+      if (parentMutationExists === false) {
+        shouldNeverHappen(
+          `Parent mutation ${mutationEventEncoded.parentId.global},${mutationEventEncoded.parentId.client} does not exist`,
+        )
+      }
+    }
+
+    // TODO use prepared statements
+    yield* execSql(
+      dbMutationLog,
+      ...insertRow({
+        tableName: MUTATION_LOG_META_TABLE,
+        columns: mutationLogMetaTable.sqliteDef.columns,
+        values: {
+          idGlobal: mutationEventEncoded.id.global,
+          idClient: mutationEventEncoded.id.client,
+          parentIdGlobal: mutationEventEncoded.parentId.global,
+          parentIdClient: mutationEventEncoded.parentId.client,
+          mutation: mutationEventEncoded.mutation,
+          argsJson: mutationEventEncoded.args ?? {},
+          clientId,
+          sessionId,
+          schemaHash: mutationDefSchemaHash,
+          syncMetadataJson: mutationEventEncoded.meta.syncMetadata,
+        },
+      }),
+    )
+  })
+
+export const updateSyncMetadata = (items: ReadonlyArray<MutationEvent.EncodedWithMeta>) =>
+  Effect.gen(function* () {
+    const { dbMutationLog } = yield* LeaderThreadCtx
+
+    // TODO try to do this in a single query
+    for (let i = 0; i < items.length; i++) {
+      const mutationEvent = items[i]!
+
+      yield* execSql(
+        dbMutationLog,
+        ...updateRows({
+          tableName: MUTATION_LOG_META_TABLE,
+          columns: mutationLogMetaTable.sqliteDef.columns,
+          where: { idGlobal: mutationEvent.id.global, idClient: mutationEvent.id.client },
+          updateValues: { syncMetadataJson: mutationEvent.meta.syncMetadata },
+        }),
+      )
+    }
+  })
+
+export const getSyncBackendCursorInfo = (remoteHead: EventId.GlobalEventId) =>
+  Effect.gen(function* () {
+    const { dbMutationLog } = yield* LeaderThreadCtx
+
+    if (remoteHead === EventId.ROOT.global) return Option.none()
+
+    const MutationlogQuerySchema = Schema.Struct({
+      syncMetadataJson: Schema.parseJson(Schema.Option(Schema.JsonValue)),
+    }).pipe(Schema.pluck('syncMetadataJson'), Schema.Array, Schema.head)
+
+    const syncMetadataOption = yield* Effect.sync(() =>
+      dbMutationLog.select<{ syncMetadataJson: string }>(
+        sql`SELECT syncMetadataJson FROM ${MUTATION_LOG_META_TABLE} WHERE idGlobal = ${remoteHead} ORDER BY idClient ASC LIMIT 1`,
+      ),
+    ).pipe(Effect.andThen(Schema.decode(MutationlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie)
+
+    return Option.some({
+      cursor: { global: remoteHead, client: EventId.clientDefault },
+      metadata: syncMetadataOption,
+    }) satisfies InitialSyncInfo
+  }).pipe(Effect.withSpan('@livestore/common:mutationlog:getSyncBackendCursorInfo', { attributes: { remoteHead } }))