@livestore/common 0.0.0-snapshot-2b8a9de3ec1a701aca891ebc2c98eb328274ae9e → 0.0.0-snapshot-2ef046b02334f52613d31dbe06af53487685edc0

package/src/leader-thread/types.ts

@@ -5,12 +5,14 @@ import type {
   Option,
   Queue,
   Scope,
+  Stream,
   Subscribable,
   SubscriptionRef,
   WebChannel,
 } from '@livestore/utils/effect'
 import { Context, Schema } from '@livestore/utils/effect'
 
+import type { SqliteError } from '../adapter-types.js'
 import type {
   BootStatus,
   Devtools,
@@ -97,7 +99,7 @@ export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
     devtools: DevtoolsContext
     syncBackend: SyncBackend | undefined
     syncProcessor: LeaderSyncProcessor
-    connectedClientSessionPullQueues: PullQueueSet
+    applyMutation: ApplyMutation
     initialState: {
       leaderHead: EventId.EventId
       migrationsReport: MigrationsReport
@@ -111,27 +113,42 @@ export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
   }
 >() {}
 
+export type ApplyMutation = (
+  mutationEventEncoded: MutationEvent.EncodedWithMeta,
+  options?: {
+    /** Needed for rehydrateFromMutationLog */
+    skipMutationLog?: boolean
+  },
+) => Effect.Effect<
+  { sessionChangeset: { _tag: 'sessionChangeset'; data: Uint8Array; debug: any } | { _tag: 'no-op' } },
+  SqliteError | UnexpectedError
+>
+
 export type InitialBlockingSyncContext = {
   blockingDeferred: Deferred.Deferred<void> | undefined
   update: (_: { remaining: number; processed: number }) => Effect.Effect<void>
 }
 
-export type PullQueueItem = {
-  payload: SyncState.PayloadUpstream
-  remaining: number
-}
-
 export interface LeaderSyncProcessor {
+  /** Used by client sessions to subscribe to upstream sync state changes */
+  pull: (args: {
+    /** Leader merge counter */
+    cursor: number
+  }) => Stream.Stream<{ payload: typeof SyncState.PayloadUpstream.Type; mergeCounter: number }, UnexpectedError>
+  /** The `pullQueue` API can be used instead of `pull` when more convenient */
+  pullQueue: (args: {
+    cursor: number
+  }) => Effect.Effect<
+    Queue.Queue<{ payload: typeof SyncState.PayloadUpstream.Type; mergeCounter: number }>,
+    UnexpectedError,
+    Scope.Scope
+  >
+
+  /** Used by client sessions to push mutations to the leader thread */
   push: (
     /** `batch` needs to follow the same rules as `batch` in `SyncBackend.push` */
     batch: ReadonlyArray<MutationEvent.EncodedWithMeta>,
     options?: {
-      /**
-       * This generation number is used to automatically reject subsequent pushes
-       * of a previously rejected push from a client session. This might occur in
-       * certain concurrent scenarios.
-       */
-      // generation: number
       /**
        * If true, the effect will only finish when the local push has been processed (i.e. succeeded or was rejected).
        * @default false
@@ -140,24 +157,18 @@ export interface LeaderSyncProcessor {
     },
   ) => Effect.Effect<void, LeaderAheadError>
 
+  /** Currently only used by devtools which don't provide their own event numbers */
   pushPartial: (args: {
     mutationEvent: MutationEvent.PartialAnyEncoded
     clientId: string
     sessionId: string
-  }) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
-  boot: (args: {
-    dbReady: Deferred.Deferred<void>
-  }) => Effect.Effect<
+  }) => Effect.Effect<void, UnexpectedError>
+
+  boot: Effect.Effect<
     { initialLeaderHead: EventId.EventId },
     UnexpectedError,
     LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
   >
   syncState: Subscribable.Subscribable<SyncState.SyncState>
-}
-
-export interface PullQueueSet {
-  makeQueue: (
-    since: EventId.EventId,
-  ) => Effect.Effect<Queue.Queue<PullQueueItem>, UnexpectedError, Scope.Scope | LeaderThreadCtx>
-  offer: (item: PullQueueItem) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
+  getMergeCounter: () => number
 }

package/src/rehydrate-from-mutationlog.ts

@@ -2,35 +2,35 @@ import { memoizeByRef } from '@livestore/utils'
 import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
 
 import { type MigrationOptionsFromMutationLog, type SqliteDb, UnexpectedError } from './adapter-types.js'
-import { makeApplyMutation } from './leader-thread/apply-mutation.js'
-import type { LiveStoreSchema, MutationDef, MutationEvent, MutationLogMetaRow } from './schema/mod.js'
-import { EventId, getMutationDef, MUTATION_LOG_META_TABLE } from './schema/mod.js'
+import type { ApplyMutation } from './leader-thread/mod.js'
+import type { LiveStoreSchema, MutationDef, MutationLogMetaRow } from './schema/mod.js'
+import { EventId, getMutationDef, MUTATION_LOG_META_TABLE, MutationEvent } from './schema/mod.js'
 import type { PreparedBindValues } from './util.js'
 import { sql } from './util.js'
 
 export const rehydrateFromMutationLog = ({
-  logDb,
+  dbMutationLog,
   // TODO re-use this db when bringing back the boot in-memory db implementation
   // db,
   schema,
   migrationOptions,
   onProgress,
+  applyMutation,
 }: {
-  logDb: SqliteDb
-  db: SqliteDb
+  dbMutationLog: SqliteDb
+  // db: SqliteDb
   schema: LiveStoreSchema
   migrationOptions: MigrationOptionsFromMutationLog
   onProgress: (_: { done: number; total: number }) => Effect.Effect<void>
+  applyMutation: ApplyMutation
 }) =>
   Effect.gen(function* () {
-    const mutationsCount = logDb.select<{ count: number }>(
+    const mutationsCount = dbMutationLog.select<{ count: number }>(
       `SELECT COUNT(*) AS count FROM ${MUTATION_LOG_META_TABLE}`,
     )[0]!.count
 
     const hashMutation = memoizeByRef((mutation: MutationDef.Any) => Schema.hash(mutation.schema))
 
-    const applyMutation = yield* makeApplyMutation
-
     const processMutation = (row: MutationLogMetaRow) =>
       Effect.gen(function* () {
         const mutationDef = getMutationDef(schema, row.mutation)
@@ -59,21 +59,21 @@ This likely means the schema has changed in an incompatible way.
           ),
         )
 
-        const mutationEventEncoded = {
+        const mutationEventEncoded = MutationEvent.EncodedWithMeta.make({
           id: { global: row.idGlobal, client: row.idClient },
           parentId: { global: row.parentIdGlobal, client: row.parentIdClient },
           mutation: row.mutation,
           args,
           clientId: row.clientId,
           sessionId: row.sessionId,
-        } satisfies MutationEvent.AnyEncoded
+        })
 
         yield* applyMutation(mutationEventEncoded, { skipMutationLog: true })
       }).pipe(Effect.withSpan(`@livestore/common:rehydrateFromMutationLog:processMutation`))
 
     const CHUNK_SIZE = 100
 
-    const stmt = logDb.prepare(sql`\
+    const stmt = dbMutationLog.prepare(sql`\
 SELECT * FROM ${MUTATION_LOG_META_TABLE} 
 WHERE idGlobal > $idGlobal OR (idGlobal = $idGlobal AND idClient > $idClient)
 ORDER BY idGlobal ASC, idClient ASC

package/src/schema/EventId.ts

@@ -18,8 +18,15 @@ export const clientDefault = 0 as any as ClientEventId
  */
 export type EventId = { global: GlobalEventId; client: ClientEventId }
 
+// export const EventSequenceNumber = Schema.Struct({})
+// export const EventNumber = Schema.Struct({})
+
+/**
+ * NOTE: Client mutation events with a non-0 client id, won't be synced to the sync backend.
+ */
 export const EventId = Schema.Struct({
   global: GlobalEventId,
+  /** Only increments for clientOnly mutations */
   client: ClientEventId,
 }).annotations({ title: 'LiveStore.EventId' })
 
@@ -36,7 +43,7 @@ export const compare = (a: EventId, b: EventId) => {
 /**
  * Convert an event id to a string representation.
  */
-export const toString = (id: EventId) => `(${id.global},${id.client})`
+export const toString = (id: EventId) => (id.client === 0 ? `e${id.global}` : `e${id.global}+${id.client}`)
 
 /**
  * Convert a string representation of an event id to an event id.

package/src/schema/MutationEvent.ts

@@ -1,5 +1,5 @@
 import { memoizeByRef } from '@livestore/utils'
-import { Schema } from '@livestore/utils/effect'
+import { Option, Schema } from '@livestore/utils/effect'
 
 import * as EventId from './EventId.js'
 import type { MutationDef, MutationDefRecord } from './mutations.js'
@@ -155,16 +155,29 @@ export const makeMutationEventSchemaMemo = memoizeByRef(makeMutationEventSchema)
 export class EncodedWithMeta extends Schema.Class<EncodedWithMeta>('MutationEvent.EncodedWithMeta')({
   mutation: Schema.String,
   args: Schema.Any,
+  // TODO rename to `.num` / `.parentNum`
   id: EventId.EventId,
   parentId: EventId.EventId,
   clientId: Schema.String,
   sessionId: Schema.String,
   // TODO get rid of `meta` again by cleaning up the usage implementations
-  meta: Schema.optionalWith(
-    Schema.Any as Schema.Schema<{
-      sessionChangeset?: Uint8Array
-    }>,
-    { default: () => ({}) },
+  meta: Schema.Struct({
+    sessionChangeset: Schema.Union(
+      Schema.TaggedStruct('sessionChangeset', {
+        data: Schema.Uint8Array,
+        debug: Schema.Any.pipe(Schema.optional),
+      }),
+      Schema.TaggedStruct('no-op', {}),
+      Schema.TaggedStruct('unset', {}),
+    ),
+    syncMetadata: Schema.Option(Schema.JsonValue),
+  }).pipe(
+    Schema.mutable,
+    Schema.optional,
+    Schema.withDefaults({
+      constructor: () => ({ sessionChangeset: { _tag: 'unset' as const }, syncMetadata: Option.none() }),
+      decoding: () => ({ sessionChangeset: { _tag: 'unset' as const }, syncMetadata: Option.none() }),
+    }),
   ),
 }) {
   toJSON = (): any => {
@@ -172,7 +185,7 @@ export class EncodedWithMeta extends Schema.Class<EncodedWithMeta>('MutationEven
     // - More readable way to print the id + parentId
     // - not including `meta`, `clientId`, `sessionId`
     return {
-      id: `(${this.id.global},${this.id.client}) → (${this.parentId.global},${this.parentId.client})`,
+      id: `${EventId.toString(this.id)} → ${EventId.toString(this.parentId)}`,
       mutation: this.mutation,
       args: this.args,
     }
@@ -180,19 +193,20 @@ export class EncodedWithMeta extends Schema.Class<EncodedWithMeta>('MutationEven
 
   /**
    * Example: (global event)
-   * For event id (2,0) → (1,0) which should be rebased on event id (3,1) → (3,0)
-   * the resulting event id will be (4,0) → (3,0)
+   * For event id e2 → e1 which should be rebased on event id e3 → e2
+   * the resulting event id will be e4 → e3
    *
    * Example: (client event)
-   * For event id (2,1) → (2,0) which should be rebased on event id (3,0) → (2,0)
-   * the resulting event id will be (3,1) → (3,0)
+   * For event id e2+1 → e2 which should be rebased on event id e3 → e2
+   * the resulting event id will be e3+1 → e3
    *
-   * Syntax: (2,1) → (2,0)
-   *          ^ ^     ^ ^
-   *          | |     | +- client parent id
-   *          | |     +--- global parent id
+   * Syntax: e2+2 → e2+1
+   *          ^ ^    ^ ^
+   *          | |    | +- client parent id
+   *          | |    +--- global parent id
    *          | +-- client id
    *          +---- global id
+   * Client id is ommitted for global events
    */
   rebase = (parentId: EventId.EventId, isClient: boolean) =>
     new EncodedWithMeta({
@@ -200,11 +214,12 @@ export class EncodedWithMeta extends Schema.Class<EncodedWithMeta>('MutationEven
       ...EventId.nextPair(parentId, isClient),
     })
 
-  static fromGlobal = (mutationEvent: AnyEncodedGlobal) =>
+  static fromGlobal = (mutationEvent: AnyEncodedGlobal, syncMetadata: Option.Option<Schema.JsonValue>) =>
     new EncodedWithMeta({
       ...mutationEvent,
       id: { global: mutationEvent.id, client: EventId.clientDefault },
       parentId: { global: mutationEvent.parentId, client: EventId.clientDefault },
+      meta: { sessionChangeset: { _tag: 'unset' as const }, syncMetadata },
     })
 
   toGlobal = (): AnyEncodedGlobal => ({
@@ -214,6 +229,7 @@ export class EncodedWithMeta extends Schema.Class<EncodedWithMeta>('MutationEven
   })
 }
 
+/** NOTE `meta` is not considered for equality */
 export const isEqualEncoded = (a: AnyEncoded, b: AnyEncoded) =>
   a.id.global === b.id.global &&
   a.id.client === b.id.client &&

package/src/schema/schema.ts

@@ -112,7 +112,7 @@ export const makeSchema = <TInputSchema extends InputSchema>(
     _MutationDefMapType: Symbol.for('livestore.MutationDefMapType') as any,
     tables,
     mutations,
-    migrationOptions: inputSchema.migrations ?? { strategy: 'hard-reset' },
+    migrationOptions: inputSchema.migrations ?? { strategy: 'from-mutation-log' },
     hash,
   } satisfies LiveStoreSchema
 }

package/src/schema/system-tables.ts

@@ -60,7 +60,25 @@ export const sessionChangesetMetaTable = table(
 
 export type SessionChangesetMetaRow = FromTable.RowDecoded<typeof sessionChangesetMetaTable>
 
-export const systemTables = [schemaMetaTable, schemaMutationsMetaTable, sessionChangesetMetaTable]
+export const LEADER_MERGE_COUNTER_TABLE = '__livestore_leader_merge_counter'
+
+export const leaderMergeCounterTable = table(
+  LEADER_MERGE_COUNTER_TABLE,
+  {
+    id: SqliteDsl.integer({ primaryKey: true, schema: Schema.Literal(0) }),
+    mergeCounter: SqliteDsl.integer({ primaryKey: true }),
+  },
+  { disableAutomaticIdColumn: true },
+)
+
+export type LeaderMergeCounterRow = FromTable.RowDecoded<typeof leaderMergeCounterTable>
+
+export const systemTables = [
+  schemaMetaTable,
+  schemaMutationsMetaTable,
+  sessionChangesetMetaTable,
+  leaderMergeCounterTable,
+]
 
 /// Mutation log DB
 
@@ -72,6 +90,7 @@ export const MUTATION_LOG_META_TABLE = 'mutation_log'
 export const mutationLogMetaTable = table(
   MUTATION_LOG_META_TABLE,
   {
+    // TODO Adjust modeling so a global event never needs a client id component
     idGlobal: SqliteDsl.integer({ primaryKey: true, schema: EventId.GlobalEventId }),
     idClient: SqliteDsl.integer({ primaryKey: true, schema: EventId.ClientEventId }),
     parentIdGlobal: SqliteDsl.integer({ schema: EventId.GlobalEventId }),

package/src/sync/ClientSessionSyncProcessor.ts

@@ -5,8 +5,9 @@ import * as otel from '@opentelemetry/api'
 
 import type { ClientSession, UnexpectedError } from '../adapter-types.js'
 import * as EventId from '../schema/EventId.js'
-import { getMutationDef, type LiveStoreSchema } from '../schema/mod.js'
+import { getMutationDef, LEADER_MERGE_COUNTER_TABLE, type LiveStoreSchema } from '../schema/mod.js'
 import * as MutationEvent from '../schema/MutationEvent.js'
+import { sql } from '../util.js'
 import * as SyncState from './syncstate.js'
 
 /**
@@ -39,7 +40,7 @@ export const makeClientSessionSyncProcessor = ({
     options: { otelContext: otel.Context; withChangeset: boolean },
   ) => {
     writeTables: Set<string>
-    sessionChangeset: Uint8Array | undefined
+    sessionChangeset: { _tag: 'sessionChangeset'; data: Uint8Array; debug: any } | { _tag: 'no-op' } | { _tag: 'unset' }
   }
   rollback: (changeset: Uint8Array) => void
   refreshTables: (tables: Set<string>) => void
@@ -56,12 +57,12 @@ export const makeClientSessionSyncProcessor = ({
   const mutationEventSchema = MutationEvent.makeMutationEventSchemaMemo(schema)
 
   const syncStateRef = {
+    // The initial state is identical to the leader's initial state
     current: new SyncState.SyncState({
       localHead: clientSession.leaderThread.initialState.leaderHead,
       upstreamHead: clientSession.leaderThread.initialState.leaderHead,
+      // Given we're starting with the leader's snapshot, we don't have any pending mutations intially
       pending: [],
-      // TODO init rollbackTail from leader to be ready for backend rebasing
-      rollbackTail: [],
     }),
   }
 
@@ -76,13 +77,14 @@ export const makeClientSessionSyncProcessor = ({
     // TODO validate batch
 
     let baseEventId = syncStateRef.current.localHead
-    const encodedMutationEvents = batch.map((mutationEvent) => {
-      const mutationDef = getMutationDef(schema, mutationEvent.mutation)
+    const encodedMutationEvents = batch.map(({ mutation, args }) => {
+      const mutationDef = getMutationDef(schema, mutation)
       const nextIdPair = EventId.nextPair(baseEventId, mutationDef.options.clientOnly)
       baseEventId = nextIdPair.id
       return new MutationEvent.EncodedWithMeta(
         Schema.encodeUnknownSync(mutationEventSchema)({
-          ...mutationEvent,
+          mutation,
+          args,
           ...nextIdPair,
           clientId: clientSession.clientId,
           sessionId: clientSession.sessionId,
@@ -168,13 +170,17 @@ export const makeClientSessionSyncProcessor = ({
 
     yield* FiberHandle.run(leaderPushingFiberHandle, backgroundLeaderPushing)
 
+    const getMergeCounter = () =>
+      clientSession.sqliteDb.select<{ mergeCounter: number }>(
+        sql`SELECT mergeCounter FROM ${LEADER_MERGE_COUNTER_TABLE} WHERE id = 0`,
+      )[0]?.mergeCounter ?? 0
+
     // NOTE We need to lazily call `.pull` as we want the cursor to be updated
-    yield* Stream.suspend(() =>
-      clientSession.leaderThread.mutations.pull({ cursor: syncStateRef.current.localHead }),
-    ).pipe(
-      Stream.tap(({ payload, remaining }) =>
+    yield* Stream.suspend(() => clientSession.leaderThread.mutations.pull({ cursor: getMergeCounter() })).pipe(
+      Stream.tap(({ payload, mergeCounter: leaderMergeCounter }) =>
         Effect.gen(function* () {
-          // console.log('pulled payload from leader', { payload, remaining })
+          // yield* Effect.logDebug('ClientSessionSyncProcessor:pull', payload)
+
           if (clientSession.devtools.enabled) {
             yield* clientSession.devtools.pullLatch.await
           }
@@ -196,13 +202,13 @@ export const makeClientSessionSyncProcessor = ({
           syncStateUpdateQueue.offer(mergeResult.newSyncState).pipe(Effect.runSync)
 
           if (mergeResult._tag === 'rebase') {
-            span.addEvent('pull:rebase', {
+            span.addEvent('merge:pull:rebase', {
               payloadTag: payload._tag,
               payload: TRACE_VERBOSE ? JSON.stringify(payload) : undefined,
               newEventsCount: mergeResult.newEvents.length,
-              rollbackCount: mergeResult.eventsToRollback.length,
+              rollbackCount: mergeResult.rollbackEvents.length,
               res: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
-              remaining,
+              leaderMergeCounter,
             })
 
             debugInfo.rebaseCount++
@@ -216,28 +222,29 @@ export const makeClientSessionSyncProcessor = ({
 
             if (LS_DEV) {
               Effect.logDebug(
-                'pull:rebase: rollback',
-                mergeResult.eventsToRollback.length,
-                ...mergeResult.eventsToRollback.slice(0, 10).map((_) => _.toJSON()),
+                'merge:pull:rebase: rollback',
+                mergeResult.rollbackEvents.length,
+                ...mergeResult.rollbackEvents.slice(0, 10).map((_) => _.toJSON()),
+                { leaderMergeCounter },
               ).pipe(Effect.provide(runtime), Effect.runSync)
             }
 
-            for (let i = mergeResult.eventsToRollback.length - 1; i >= 0; i--) {
-              const event = mergeResult.eventsToRollback[i]!
-              if (event.meta.sessionChangeset) {
-                rollback(event.meta.sessionChangeset)
-                event.meta.sessionChangeset = undefined
+            for (let i = mergeResult.rollbackEvents.length - 1; i >= 0; i--) {
+              const event = mergeResult.rollbackEvents[i]!
+              if (event.meta.sessionChangeset._tag !== 'no-op' && event.meta.sessionChangeset._tag !== 'unset') {
+                rollback(event.meta.sessionChangeset.data)
+                event.meta.sessionChangeset = { _tag: 'unset' }
               }
             }
 
             yield* BucketQueue.offerAll(leaderPushQueue, mergeResult.newSyncState.pending)
           } else {
-            span.addEvent('pull:advance', {
+            span.addEvent('merge:pull:advance', {
               payloadTag: payload._tag,
               payload: TRACE_VERBOSE ? JSON.stringify(payload) : undefined,
               newEventsCount: mergeResult.newEvents.length,
               res: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
-              remaining,
+              leaderMergeCounter,
             })
 
             debugInfo.advanceCount++
@@ -247,6 +254,7 @@ export const makeClientSessionSyncProcessor = ({
 
           const writeTables = new Set<string>()
           for (const mutationEvent of mergeResult.newEvents) {
+            // TODO apply changeset if available (will require tracking of write tables as well)
             const decodedMutationEvent = Schema.decodeSync(mutationEventSchema)(mutationEvent)
             const res = applyMutation(decodedMutationEvent, { otelContext, withChangeset: true })
             for (const table of res.writeTables) {

package/src/sync/sync.ts

@@ -36,6 +36,10 @@ export type SyncBackendConstructor<TSyncMetadata = Schema.JsonValue> = (
 ) => Effect.Effect<SyncBackend<TSyncMetadata>, UnexpectedError, Scope.Scope | HttpClient.HttpClient>
 
 export type SyncBackend<TSyncMetadata = Schema.JsonValue> = {
+  /**
+   * Can be implemented to prepare a connection to the sync backend to speed up the first pull/push.
+   */
+  connect: Effect.Effect<void, IsOfflineError | UnexpectedError, HttpClient.HttpClient | Scope.Scope>
   pull: (
     args: Option.Option<{
       cursor: EventId.EventId
@@ -60,17 +64,10 @@ export type SyncBackend<TSyncMetadata = Schema.JsonValue> = {
      * - event ids must be in ascending order
      * */
     batch: ReadonlyArray<MutationEvent.AnyEncodedGlobal>,
-  ) => Effect.Effect<
-    {
-      /** Indexes are relative to `batch` */
-      metadata: ReadonlyArray<Option.Option<TSyncMetadata>>
-    },
-    IsOfflineError | InvalidPushError,
-    HttpClient.HttpClient
-  >
+  ) => Effect.Effect<void, IsOfflineError | InvalidPushError, HttpClient.HttpClient>
   isConnected: SubscriptionRef.SubscriptionRef<boolean>
   /**
-   * Metadata describing the sync backend.
+   * Metadata describing the sync backend. (Currently only used by devtools.)
    */
   metadata: { name: string; description: string } & Record<string, Schema.JsonValue>
 }