@livestore/common 0.3.0-dev.24 → 0.3.0-dev.26

package/src/sync/syncstate.ts

@@ -1,4 +1,4 @@
-import { casesHandled } from '@livestore/utils'
+import { casesHandled, shouldNeverHappen } from '@livestore/utils'
 import { Match, ReadonlyArray, Schema } from '@livestore/utils/effect'
 
 import { UnexpectedError } from '../adapter-types.js'
@@ -24,6 +24,9 @@ import * as MutationEvent from '../schema/MutationEvent.js'
  *   - Can be confirmed or rejected by the upstream.
  *   - Subject to rebase if rejected.
  * - **Rollback Tail**: Events that are kept around temporarily for potential rollback until confirmed by upstream.
+ *   - Currently only needed for ClientSessionSyncProcessor.
+ *   - Note: Confirmation of an event is stronger than acknowledgment of an event and can only be done by the
+ *     absolute authority in the sync hierarchy (i.e. the sync backend in our case).
  *
  * Payloads:
  * - `PayloadUpstreamRebase`: Upstream has performed a rebase, so downstream must roll back to the specified event
@@ -37,12 +40,19 @@ import * as MutationEvent from '../schema/MutationEvent.js'
  * 2. **Head Ordering**: Upstream Head ≤ Local Head.
  * 3. **ID Sequence**: Must follow the pattern (1,0)→(1,1)→(1,2)→(2,0).
  *
- * The `updateSyncState` function processes updates to the sync state based on incoming payloads,
+ * A few further notes to help form an intuition:
+ * - The goal is to keep the pending events as small as possible (i.e. to have synced with the next upstream node)
+ * - There are 2 cases for rebasing:
+ *   - The conflicting event only conflicts with the pending events -> only (some of) the pending events need to be rolled back
+ *   - The conflicting event conflicts even with the rollback tail (additionally to the pending events) -> events from both need to be rolled back
+ *
+ * The `merge` function processes updates to the sync state based on incoming payloads,
  * handling cases such as upstream rebase, advance, local push, and rollback tail trimming.
  */
 export class SyncState extends Schema.Class<SyncState>('SyncState')({
   pending: Schema.Array(MutationEvent.EncodedWithMeta),
   rollbackTail: Schema.Array(MutationEvent.EncodedWithMeta),
+  /** What this node expects the next upstream node to have as its own local head */
   upstreamHead: EventId.EventId,
   localHead: EventId.EventId,
 }) {
@@ -108,7 +118,7 @@ export class UpdateContext extends Schema.Class<UpdateContext>('UpdateContext')(
   }
 }
 
-export class UpdateResultAdvance extends Schema.Class<UpdateResultAdvance>('UpdateResultAdvance')({
+export class MergeResultAdvance extends Schema.Class<MergeResultAdvance>('MergeResultAdvance')({
   _tag: Schema.Literal('advance'),
   newSyncState: SyncState,
   /** Events which weren't pending before the update */
@@ -125,7 +135,7 @@ export class UpdateResultAdvance extends Schema.Class<UpdateResultAdvance>('Upda
   }
 }
 
-export class UpdateResultRebase extends Schema.Class<UpdateResultRebase>('UpdateResultRebase')({
+export class MergeResultRebase extends Schema.Class<MergeResultRebase>('MergeResultRebase')({
   _tag: Schema.Literal('rebase'),
   newSyncState: SyncState,
   /** Events which weren't pending before the update */
@@ -144,7 +154,7 @@ export class UpdateResultRebase extends Schema.Class<UpdateResultRebase>('Update
   }
 }
 
-export class UpdateResultReject extends Schema.Class<UpdateResultReject>('UpdateResultReject')({
+export class MergeResultReject extends Schema.Class<MergeResultReject>('MergeResultReject')({
   _tag: Schema.Literal('reject'),
   /** The minimum id that the new events must have */
   expectedMinimumId: EventId.EventId,
@@ -159,40 +169,40 @@ export class UpdateResultReject extends Schema.Class<UpdateResultReject>('Update
   }
 }
 
-export class UpdateResultUnexpectedError extends Schema.Class<UpdateResultUnexpectedError>(
-  'UpdateResultUnexpectedError',
-)({
+export class MergeResultUnexpectedError extends Schema.Class<MergeResultUnexpectedError>('MergeResultUnexpectedError')({
   _tag: Schema.Literal('unexpected-error'),
   cause: UnexpectedError,
 }) {}
 
-export class UpdateResult extends Schema.Union(
-  UpdateResultAdvance,
-  UpdateResultRebase,
-  UpdateResultReject,
-  UpdateResultUnexpectedError,
+export class MergeResult extends Schema.Union(
+  MergeResultAdvance,
+  MergeResultRebase,
+  MergeResultReject,
+  MergeResultUnexpectedError,
 ) {}
 
-const unexpectedError = (cause: unknown): UpdateResultUnexpectedError =>
-  UpdateResultUnexpectedError.make({
+const unexpectedError = (cause: unknown): MergeResultUnexpectedError =>
+  MergeResultUnexpectedError.make({
     _tag: 'unexpected-error',
     cause: new UnexpectedError({ cause }),
   })
 
-export const updateSyncState = ({
+export const merge = ({
   syncState,
   payload,
-  isLocalEvent,
+  isClientEvent,
   isEqualEvent,
-  ignoreLocalEvents = false,
+  ignoreClientEvents = false,
 }: {
   syncState: SyncState
   payload: typeof Payload.Type
-  isLocalEvent: (event: MutationEvent.EncodedWithMeta) => boolean
+  isClientEvent: (event: MutationEvent.EncodedWithMeta) => boolean
   isEqualEvent: (a: MutationEvent.EncodedWithMeta, b: MutationEvent.EncodedWithMeta) => boolean
-  /** This is used in the leader which should ignore local events when receiving an upstream-advance payload */
-  ignoreLocalEvents?: boolean
-}): typeof UpdateResult.Type => {
+  /** This is used in the leader which should ignore client events when receiving an upstream-advance payload */
+  ignoreClientEvents?: boolean
+}): typeof MergeResult.Type => {
+  validateSyncState(syncState)
+
   const trimRollbackTail = (
     rollbackTail: ReadonlyArray<MutationEvent.EncodedWithMeta>,
   ): ReadonlyArray<MutationEvent.EncodedWithMeta> => {
@@ -226,10 +236,10 @@ export const updateSyncState = ({
       const rebasedPending = rebaseEvents({
         events: syncState.pending,
         baseEventId: newUpstreamHead,
-        isLocalEvent,
+        isClientEvent,
       })
 
-      return UpdateResultRebase.make({
+      return MergeResultRebase.make({
         _tag: 'rebase',
         newSyncState: new SyncState({
           pending: rebasedPending,
@@ -243,9 +253,10 @@ export const updateSyncState = ({
       })
     }
 
+    // #region upstream-advance
     case 'upstream-advance': {
       if (payload.newEvents.length === 0) {
-        return UpdateResultAdvance.make({
+        return MergeResultAdvance.make({
           _tag: 'advance',
           newSyncState: new SyncState({
             pending: syncState.pending,
@@ -268,9 +279,23 @@ export const updateSyncState = ({
       }
 
       // Validate that incoming events are larger than upstream head
-      if (EventId.isGreaterThan(syncState.upstreamHead, payload.newEvents[0]!.id)) {
+      if (
+        EventId.isGreaterThan(syncState.upstreamHead, payload.newEvents[0]!.id) ||
+        EventId.isEqual(syncState.upstreamHead, payload.newEvents[0]!.id)
+      ) {
         return unexpectedError(
-          `Incoming events must be greater than upstream head. Expected greater than: [${syncState.upstreamHead.global},${syncState.upstreamHead.client}]. Received: [${payload.newEvents.map((e) => `(${e.id.global},${e.id.client})`).join(', ')}]`,
+          `Incoming events must be greater than upstream head. Expected greater than: (${syncState.upstreamHead.global},${syncState.upstreamHead.client}). Received: [${payload.newEvents.map((e) => `(${e.id.global},${e.id.client})`).join(', ')}]`,
+        )
+      }
+
+      // Validate that the parent id of the first incoming event is known
+      const knownEventGlobalIds = [...syncState.rollbackTail, ...syncState.pending].map((e) => e.id.global)
+      knownEventGlobalIds.push(syncState.upstreamHead.global)
+      const firstNewEvent = payload.newEvents[0]!
+      const hasUnknownParentId = knownEventGlobalIds.includes(firstNewEvent.parentId.global) === false
+      if (hasUnknownParentId) {
+        return unexpectedError(
+          `Incoming events must have a known parent id. Received: [${payload.newEvents.map((e) => `(${e.id.global},${e.id.client})`).join(', ')}]`,
         )
       }
 
@@ -280,10 +305,11 @@ export const updateSyncState = ({
         existingEvents: syncState.pending,
         incomingEvents: payload.newEvents,
         isEqualEvent,
-        isLocalEvent,
-        ignoreLocalEvents,
+        isClientEvent,
+        ignoreClientEvents,
       })
 
+      // No divergent pending events, thus we can just advance (some of) the pending events
       if (divergentPendingIndex === -1) {
         const pendingEventIds = new Set(syncState.pending.map((e) => `${e.id.global},${e.id.client}`))
         const newEvents = payload.newEvents.filter((e) => !pendingEventIds.has(`${e.id.global},${e.id.client}`))
@@ -292,17 +318,17 @@ export const updateSyncState = ({
         // we need to split the pending events into two groups:
         // - pendingMatching: The pending events up to point where they match the incoming events
         // - pendingRemaining: The pending events after the point where they match the incoming events
-        // The `localIndexOffset` is used to account for the local events that are being ignored
-        let localIndexOffset = 0
+        // The `clientIndexOffset` is used to account for the client events that are being ignored
+        let clientIndexOffset = 0
         const [pendingMatching, pendingRemaining] = ReadonlyArray.splitWhere(
           syncState.pending,
           (pendingEvent, index) => {
-            if (ignoreLocalEvents && isLocalEvent(pendingEvent)) {
-              localIndexOffset++
+            if (ignoreClientEvents && isClientEvent(pendingEvent)) {
+              clientIndexOffset++
               return false
             }
 
-            const newEvent = payload.newEvents.at(index - localIndexOffset)
+            const newEvent = payload.newEvents.at(index - clientIndexOffset)
             if (!newEvent) {
               return true
             }
@@ -320,7 +346,7 @@ export const updateSyncState = ({
           return true
         })
 
-        return UpdateResultAdvance.make({
+        return MergeResultAdvance.make({
           _tag: 'advance',
           newSyncState: new SyncState({
             pending: pendingRemaining,
@@ -336,18 +362,18 @@ export const updateSyncState = ({
         const rebasedPending = rebaseEvents({
           events: divergentPending,
           baseEventId: newUpstreamHead,
-          isLocalEvent,
+          isClientEvent,
         })
 
         const divergentNewEventsIndex = findDivergencePoint({
           existingEvents: payload.newEvents,
           incomingEvents: syncState.pending,
           isEqualEvent,
-          isLocalEvent,
-          ignoreLocalEvents,
+          isClientEvent,
+          ignoreClientEvents,
         })
 
-        return UpdateResultRebase.make({
+        return MergeResultRebase.make({
           _tag: 'rebase',
           newSyncState: new SyncState({
             pending: rebasedPending,
@@ -361,10 +387,11 @@ export const updateSyncState = ({
         })
       }
     }
+    // #endregion
 
     case 'local-push': {
       if (payload.newEvents.length === 0) {
-        return UpdateResultAdvance.make({
+        return MergeResultAdvance.make({
           _tag: 'advance',
           newSyncState: syncState,
           newEvents: [],
@@ -377,13 +404,13 @@ export const updateSyncState = ({
 
       if (invalidEventId) {
         const expectedMinimumId = EventId.nextPair(syncState.localHead, true).id
-        return UpdateResultReject.make({
+        return MergeResultReject.make({
           _tag: 'reject',
           expectedMinimumId,
           updateContext,
         })
       } else {
-        return UpdateResultAdvance.make({
+        return MergeResultAdvance.make({
           _tag: 'advance',
           newSyncState: new SyncState({
             pending: [...syncState.pending, ...payload.newEvents],
@@ -407,32 +434,32 @@ export const updateSyncState = ({
  * Gets the index relative to `existingEvents` where the divergence point is
  * by comparing each event in `existingEvents` to the corresponding event in `incomingEvents`
  */
-const findDivergencePoint = ({
+export const findDivergencePoint = ({
   existingEvents,
   incomingEvents,
   isEqualEvent,
-  isLocalEvent,
-  ignoreLocalEvents,
+  isClientEvent,
+  ignoreClientEvents,
 }: {
   existingEvents: ReadonlyArray<MutationEvent.EncodedWithMeta>
   incomingEvents: ReadonlyArray<MutationEvent.EncodedWithMeta>
   isEqualEvent: (a: MutationEvent.EncodedWithMeta, b: MutationEvent.EncodedWithMeta) => boolean
-  isLocalEvent: (event: MutationEvent.EncodedWithMeta) => boolean
-  ignoreLocalEvents: boolean
+  isClientEvent: (event: MutationEvent.EncodedWithMeta) => boolean
+  ignoreClientEvents: boolean
 }): number => {
-  if (ignoreLocalEvents) {
-    const filteredExistingEvents = existingEvents.filter((event) => !isLocalEvent(event))
-    const divergencePointWithoutLocalEvents = findDivergencePoint({
+  if (ignoreClientEvents) {
+    const filteredExistingEvents = existingEvents.filter((event) => !isClientEvent(event))
+    const divergencePointWithoutClientEvents = findDivergencePoint({
       existingEvents: filteredExistingEvents,
       incomingEvents,
       isEqualEvent,
-      isLocalEvent,
-      ignoreLocalEvents: false,
+      isClientEvent,
+      ignoreClientEvents: false,
     })
 
-    if (divergencePointWithoutLocalEvents === -1) return -1
+    if (divergencePointWithoutClientEvents === -1) return -1
 
-    const divergencePointEventId = existingEvents[divergencePointWithoutLocalEvents]!.id
+    const divergencePointEventId = existingEvents[divergencePointWithoutClientEvents]!.id
     // Now find the divergence point in the original array
     return existingEvents.findIndex((event) => EventId.isEqual(event.id, divergencePointEventId))
   }
@@ -447,15 +474,15 @@ const findDivergencePoint = ({
 const rebaseEvents = ({
   events,
   baseEventId,
-  isLocalEvent,
+  isClientEvent,
 }: {
   events: ReadonlyArray<MutationEvent.EncodedWithMeta>
   baseEventId: EventId.EventId
-  isLocalEvent: (event: MutationEvent.EncodedWithMeta) => boolean
+  isClientEvent: (event: MutationEvent.EncodedWithMeta) => boolean
 }): ReadonlyArray<MutationEvent.EncodedWithMeta> => {
   let prevEventId = baseEventId
   return events.map((event) => {
-    const isLocal = isLocalEvent(event)
+    const isLocal = isClientEvent(event)
     const newEvent = event.rebase(prevEventId, isLocal)
     prevEventId = newEvent.id
     return newEvent
@@ -469,4 +496,57 @@ const rebaseEvents = ({
  * it could make sense to "flatten" update results into a single update result which the client session
  * can process more efficiently which avoids push-threshing
  */
-const _flattenUpdateResults = (_updateResults: ReadonlyArray<UpdateResult>) => {}
+const _flattenMergeResults = (_updateResults: ReadonlyArray<MergeResult>) => {}
+
+const validateSyncState = (syncState: SyncState) => {
+  // Validate that the rollback tail and pending events together form a continuous chain of events / linked list via the parentId
+  const chain = [...syncState.rollbackTail, ...syncState.pending]
+  for (let i = 0; i < chain.length; i++) {
+    const event = chain[i]!
+    const nextEvent = chain[i + 1]
+    if (nextEvent === undefined) break // Reached end of chain
+
+    if (EventId.isGreaterThan(event.id, nextEvent.id)) {
+      shouldNeverHappen('Events must be sorted in ascending order by eventId', chain, {
+        event,
+        nextEvent,
+      })
+    }
+
+    // If the global id has increased, then the client id must be 0
+    const globalIdHasIncreased = nextEvent.id.global > event.id.global
+    if (globalIdHasIncreased) {
+      if (nextEvent.id.client !== 0) {
+        shouldNeverHappen(
+          `New global events must point to clientId 0 in the parentId. Received: (${nextEvent.id.global},${nextEvent.id.client})`,
+          chain,
+          {
+            event,
+            nextEvent,
+          },
+        )
+      }
+    } else {
+      // Otherwise, the parentId must be the same as the previous event's id
+      if (EventId.isEqual(nextEvent.parentId, event.id) === false) {
+        shouldNeverHappen('Events must be linked in a continuous chain via the parentId', chain, {
+          event,
+          nextEvent,
+        })
+      }
+    }
+  }
+
+  // TODO double check this
+  // const globalRollbackTail = syncState.rollbackTail.filter((event) => event.id.client === 0)
+  // // The parent of the first global rollback tail event ("oldest event") must be the upstream head (if there is a rollback tail)
+  // if (globalRollbackTail.length > 0) {
+  //   const firstRollbackTailEvent = globalRollbackTail[0]!
+  //   if (EventId.isEqual(firstRollbackTailEvent.parentId, syncState.upstreamHead) === false) {
+  //     shouldNeverHappen('The parent of the first rollback tail event must be the upstream head', chain, {
+  //       event: firstRollbackTailEvent,
+  //       upstreamHead: syncState.upstreamHead,
+  //     })
+  //   }
+  // }
+}

package/src/version.ts

@@ -2,7 +2,7 @@
 // import packageJson from '../package.json' with { type: 'json' }
 // export const liveStoreVersion = packageJson.version
 
-export const liveStoreVersion = '0.3.0-dev.24' as const
+export const liveStoreVersion = '0.3.0-dev.26' as const
 
 /**
  * This version number is incremented whenever the internal storage format changes in a breaking way.