@livestore/common 0.3.0-dev.2 → 0.3.0-dev.21

package/src/sync/syncstate.ts

@@ -1,6 +1,7 @@
-import { shouldNeverHappen } from '@livestore/utils'
-import { ReadonlyArray, Schema } from '@livestore/utils/effect'
+import { casesHandled } from '@livestore/utils'
+import { Match, ReadonlyArray, Schema } from '@livestore/utils/effect'
 
+import { UnexpectedError } from '../adapter-types.js'
 import * as EventId from '../schema/EventId.js'
 import * as MutationEvent from '../schema/MutationEvent.js'
 
@@ -39,19 +40,21 @@ import * as MutationEvent from '../schema/MutationEvent.js'
  * The `updateSyncState` function processes updates to the sync state based on incoming payloads,
  * handling cases such as upstream rebase, advance, local push, and rollback tail trimming.
  */
-export interface SyncState {
-  pending: ReadonlyArray<MutationEvent.EncodedWithMeta>
-  rollbackTail: ReadonlyArray<MutationEvent.EncodedWithMeta>
-  upstreamHead: EventId.EventId
-  localHead: EventId.EventId
-}
-
-export const SyncState = Schema.Struct({
+export class SyncState extends Schema.Class<SyncState>('SyncState')({
   pending: Schema.Array(MutationEvent.EncodedWithMeta),
   rollbackTail: Schema.Array(MutationEvent.EncodedWithMeta),
   upstreamHead: EventId.EventId,
   localHead: EventId.EventId,
-}).annotations({ title: 'SyncState' })
+}) {
+  toJSON = (): any => {
+    return {
+      pending: this.pending.map((e) => e.toJSON()),
+      rollbackTail: this.rollbackTail.map((e) => e.toJSON()),
+      upstreamHead: `(${this.upstreamHead.global},${this.upstreamHead.client})`,
+      localHead: `(${this.localHead.global},${this.localHead.client})`,
+    }
+  }
+}
 
 export class PayloadUpstreamRebase extends Schema.TaggedStruct('upstream-rebase', {
   /** Rollback until this event in the rollback tail (inclusive). Starting from the end of the rollback tail. */
@@ -77,31 +80,104 @@ export const PayloadUpstream = Schema.Union(PayloadUpstreamRebase, PayloadUpstre
 
 export type PayloadUpstream = typeof PayloadUpstream.Type
 
-export type UpdateResultAdvance = {
-  _tag: 'advance'
-  newSyncState: SyncState
-  previousSyncState: SyncState
+/** Only used for debugging purposes */
+export class UpdateContext extends Schema.Class<UpdateContext>('UpdateContext')({
+  payload: Payload,
+  syncState: SyncState,
+}) {
+  toJSON = (): any => {
+    const payload = Match.value(this.payload).pipe(
+      Match.tag('local-push', () => ({
+        _tag: 'local-push',
+        newEvents: this.payload.newEvents.map((e) => e.toJSON()),
+      })),
+      Match.tag('upstream-advance', () => ({
+        _tag: 'upstream-advance',
+        newEvents: this.payload.newEvents.map((e) => e.toJSON()),
+      })),
+      Match.tag('upstream-rebase', () => ({
+        _tag: 'upstream-rebase',
+        newEvents: this.payload.newEvents.map((e) => e.toJSON()),
+      })),
+      Match.exhaustive,
+    )
+    return {
+      payload,
+      syncState: this.syncState.toJSON(),
+    }
+  }
+}
+
+export class UpdateResultAdvance extends Schema.Class<UpdateResultAdvance>('UpdateResultAdvance')({
+  _tag: Schema.Literal('advance'),
+  newSyncState: SyncState,
   /** Events which weren't pending before the update */
-  newEvents: ReadonlyArray<MutationEvent.EncodedWithMeta>
+  newEvents: Schema.Array(MutationEvent.EncodedWithMeta),
+  updateContext: UpdateContext,
+}) {
+  toJSON = (): any => {
+    return {
+      _tag: this._tag,
+      newSyncState: this.newSyncState.toJSON(),
+      newEvents: this.newEvents.map((e) => e.toJSON()),
+      updateContext: this.updateContext.toJSON(),
+    }
+  }
 }
 
-export type UpdateResultRebase = {
-  _tag: 'rebase'
-  newSyncState: SyncState
-  previousSyncState: SyncState
+export class UpdateResultRebase extends Schema.Class<UpdateResultRebase>('UpdateResultRebase')({
+  _tag: Schema.Literal('rebase'),
+  newSyncState: SyncState,
   /** Events which weren't pending before the update */
-  newEvents: ReadonlyArray<MutationEvent.EncodedWithMeta>
-  eventsToRollback: ReadonlyArray<MutationEvent.EncodedWithMeta>
+  newEvents: Schema.Array(MutationEvent.EncodedWithMeta),
+  eventsToRollback: Schema.Array(MutationEvent.EncodedWithMeta),
+  updateContext: UpdateContext,
+}) {
+  toJSON = (): any => {
+    return {
+      _tag: this._tag,
+      newSyncState: this.newSyncState.toJSON(),
+      newEvents: this.newEvents.map((e) => e.toJSON()),
+      eventsToRollback: this.eventsToRollback.map((e) => e.toJSON()),
+      updateContext: this.updateContext.toJSON(),
+    }
+  }
 }
 
-export type UpdateResultReject = {
-  _tag: 'reject'
-  previousSyncState: SyncState
+export class UpdateResultReject extends Schema.Class<UpdateResultReject>('UpdateResultReject')({
+  _tag: Schema.Literal('reject'),
   /** The minimum id that the new events must have */
-  expectedMinimumId: EventId.EventId
+  expectedMinimumId: EventId.EventId,
+  updateContext: UpdateContext,
+}) {
+  toJSON = (): any => {
+    return {
+      _tag: this._tag,
+      expectedMinimumId: `(${this.expectedMinimumId.global},${this.expectedMinimumId.client})`,
+      updateContext: this.updateContext.toJSON(),
+    }
+  }
 }
 
-export type UpdateResult = UpdateResultAdvance | UpdateResultRebase | UpdateResultReject
+export class UpdateResultUnexpectedError extends Schema.Class<UpdateResultUnexpectedError>(
+  'UpdateResultUnexpectedError',
+)({
+  _tag: Schema.Literal('unexpected-error'),
+  cause: UnexpectedError,
+}) {}
+
+export class UpdateResult extends Schema.Union(
+  UpdateResultAdvance,
+  UpdateResultRebase,
+  UpdateResultReject,
+  UpdateResultUnexpectedError,
+) {}
+
+const unexpectedError = (cause: unknown): UpdateResultUnexpectedError =>
+  UpdateResultUnexpectedError.make({
+    _tag: 'unexpected-error',
+    cause: new UnexpectedError({ cause }),
+  })
 
 export const updateSyncState = ({
   syncState,
@@ -116,7 +192,7 @@ export const updateSyncState = ({
   isEqualEvent: (a: MutationEvent.EncodedWithMeta, b: MutationEvent.EncodedWithMeta) => boolean
   /** This is used in the leader which should ignore local events when receiving an upstream-advance payload */
   ignoreLocalEvents?: boolean
-}): UpdateResult => {
+}): typeof UpdateResult.Type => {
   const trimRollbackTail = (
     rollbackTail: ReadonlyArray<MutationEvent.EncodedWithMeta>,
   ): ReadonlyArray<MutationEvent.EncodedWithMeta> => {
@@ -127,6 +203,8 @@ export const updateSyncState = ({
     return rollbackTail.slice(index + 1)
   }
 
+  const updateContext = UpdateContext.make({ payload, syncState })
+
   switch (payload._tag) {
     case 'upstream-rebase': {
       // Find the index of the rollback event in the rollback tail
@@ -134,8 +212,8 @@ export const updateSyncState = ({
         EventId.isEqual(event.id, payload.rollbackUntil),
       )
       if (rollbackIndex === -1) {
-        return shouldNeverHappen(
-          `Rollback event not found in rollback tail. Rollback until: [${payload.rollbackUntil.global},${payload.rollbackUntil.local}]. Rollback tail: [${syncState.rollbackTail.map((e) => e.toString()).join(', ')}]`,
+        return unexpectedError(
+          `Rollback event not found in rollback tail. Rollback until: [${payload.rollbackUntil.global},${payload.rollbackUntil.client}]. Rollback tail: [${syncState.rollbackTail.map((e) => e.toString()).join(', ')}]`,
         )
       }
 
@@ -151,42 +229,51 @@ export const updateSyncState = ({
         isLocalEvent,
       })
 
-      return {
+      return UpdateResultRebase.make({
         _tag: 'rebase',
-        newSyncState: {
+        newSyncState: new SyncState({
           pending: rebasedPending,
           rollbackTail: trimRollbackTail([...syncState.rollbackTail.slice(0, rollbackIndex), ...payload.newEvents]),
           upstreamHead: newUpstreamHead,
           localHead: rebasedPending.at(-1)?.id ?? newUpstreamHead,
-        },
-        previousSyncState: syncState,
-        newEvents: payload.newEvents,
+        }),
+        newEvents: [...payload.newEvents, ...rebasedPending],
         eventsToRollback,
-      }
+        updateContext,
+      })
     }
 
     case 'upstream-advance': {
       if (payload.newEvents.length === 0) {
-        return {
+        return UpdateResultAdvance.make({
           _tag: 'advance',
-          newSyncState: {
+          newSyncState: new SyncState({
             pending: syncState.pending,
             rollbackTail: trimRollbackTail(syncState.rollbackTail),
             upstreamHead: syncState.upstreamHead,
             localHead: syncState.localHead,
-          },
-          previousSyncState: syncState,
+          }),
           newEvents: [],
-        }
+          updateContext,
+        })
       }
 
       // Validate that newEvents are sorted in ascending order by eventId
       for (let i = 1; i < payload.newEvents.length; i++) {
         if (EventId.isGreaterThan(payload.newEvents[i - 1]!.id, payload.newEvents[i]!.id)) {
-          return shouldNeverHappen('Events must be sorted in ascending order by eventId')
+          return unexpectedError(
+            `Events must be sorted in ascending order by eventId. Received: [${payload.newEvents.map((e) => `(${e.id.global},${e.id.client})`).join(', ')}]`,
+          )
         }
       }
 
+      // Validate that incoming events are larger than upstream head
+      if (EventId.isGreaterThan(syncState.upstreamHead, payload.newEvents[0]!.id)) {
+        return unexpectedError(
+          `Incoming events must be greater than upstream head. Expected greater than: [${syncState.upstreamHead.global},${syncState.upstreamHead.client}]. Received: [${payload.newEvents.map((e) => `(${e.id.global},${e.id.client})`).join(', ')}]`,
+        )
+      }
+
       const newUpstreamHead = payload.newEvents.at(-1)!.id
 
       const divergentPendingIndex = findDivergencePoint({
@@ -198,8 +285,8 @@ export const updateSyncState = ({
       })
 
       if (divergentPendingIndex === -1) {
-        const pendingEventIds = new Set(syncState.pending.map((e) => `${e.id.global},${e.id.local}`))
-        const newEvents = payload.newEvents.filter((e) => !pendingEventIds.has(`${e.id.global},${e.id.local}`))
+        const pendingEventIds = new Set(syncState.pending.map((e) => `${e.id.global},${e.id.client}`))
+        const newEvents = payload.newEvents.filter((e) => !pendingEventIds.has(`${e.id.global},${e.id.client}`))
 
         // In the case where the incoming events are a subset of the pending events,
         // we need to split the pending events into two groups:
@@ -225,7 +312,7 @@ export const updateSyncState = ({
 
         const seenEventIds = new Set<string>()
         const pendingAndNewEvents = [...pendingMatching, ...payload.newEvents].filter((event) => {
-          const eventIdStr = `${event.id.global},${event.id.local}`
+          const eventIdStr = `${event.id.global},${event.id.client}`
           if (seenEventIds.has(eventIdStr)) {
             return false
           }
@@ -233,17 +320,17 @@ export const updateSyncState = ({
           return true
         })
 
-        return {
+        return UpdateResultAdvance.make({
           _tag: 'advance',
-          newSyncState: {
+          newSyncState: new SyncState({
             pending: pendingRemaining,
             rollbackTail: trimRollbackTail([...syncState.rollbackTail, ...pendingAndNewEvents]),
             upstreamHead: newUpstreamHead,
             localHead: pendingRemaining.at(-1)?.id ?? newUpstreamHead,
-          },
-          previousSyncState: syncState,
+          }),
           newEvents,
-        }
+          updateContext,
+        })
       } else {
         const divergentPending = syncState.pending.slice(divergentPendingIndex)
         const rebasedPending = rebaseEvents({
@@ -260,24 +347,29 @@ export const updateSyncState = ({
           ignoreLocalEvents,
         })
 
-        return {
+        return UpdateResultRebase.make({
           _tag: 'rebase',
-          newSyncState: {
+          newSyncState: new SyncState({
             pending: rebasedPending,
             rollbackTail: trimRollbackTail([...syncState.rollbackTail, ...payload.newEvents]),
             upstreamHead: newUpstreamHead,
             localHead: rebasedPending.at(-1)!.id,
-          },
-          previousSyncState: syncState,
+          }),
           newEvents: [...payload.newEvents.slice(divergentNewEventsIndex), ...rebasedPending],
           eventsToRollback: [...syncState.rollbackTail, ...divergentPending],
-        }
+          updateContext,
+        })
       }
     }
 
     case 'local-push': {
       if (payload.newEvents.length === 0) {
-        return { _tag: 'advance', newSyncState: syncState, previousSyncState: syncState, newEvents: [] }
+        return UpdateResultAdvance.make({
+          _tag: 'advance',
+          newSyncState: syncState,
+          newEvents: [],
+          updateContext,
+        })
       }
 
       const newEventsFirst = payload.newEvents.at(0)!
@@ -285,43 +377,29 @@ export const updateSyncState = ({
 
       if (invalidEventId) {
         const expectedMinimumId = EventId.nextPair(syncState.localHead, true).id
-        return { _tag: 'reject', previousSyncState: syncState, expectedMinimumId }
+        return UpdateResultReject.make({
+          _tag: 'reject',
+          expectedMinimumId,
+          updateContext,
+        })
       } else {
-        return {
+        return UpdateResultAdvance.make({
           _tag: 'advance',
-          newSyncState: {
+          newSyncState: new SyncState({
             pending: [...syncState.pending, ...payload.newEvents],
             rollbackTail: syncState.rollbackTail,
             upstreamHead: syncState.upstreamHead,
             localHead: payload.newEvents.at(-1)!.id,
-          },
-          previousSyncState: syncState,
+          }),
           newEvents: payload.newEvents,
-        }
+          updateContext,
+        })
       }
     }
 
-    // case 'upstream-trim-rollback-tail': {
-    //   // Find the index of the new rollback start in the rollback tail
-    //   const startIndex = syncState.rollbackTail.findIndex((event) => eventIdsEqual(event.id, payload.trimRollbackUntil))
-    //   if (startIndex === -1) {
-    //     return shouldNeverHappen('New rollback start event not found in rollback tail')
-    //   }
-
-    //   // Keep only the events from the start index onwards
-    //   const newRollbackTail = syncState.rollbackTail.slice(startIndex)
-
-    //   return {
-    //     _tag: 'advance',
-    //     syncState: {
-    //       pending: syncState.pending,
-    //       rollbackTail: newRollbackTail,
-    //       upstreamHead: syncState.upstreamHead,
-    //       localHead: syncState.localHead,
-    //     },
-    //     newEvents: [],
-    //   }
-    // }
+    default: {
+      casesHandled(payload)
+    }
   }
 }
 
@@ -383,3 +461,12 @@ const rebaseEvents = ({
     return newEvent
   })
 }
+
+/**
+ * TODO: Implement this
+ *
+ * In certain scenarios e.g. when the client session has a queue of upstream update results,
+ * it could make sense to "flatten" update results into a single update result which the client session
+ * can process more efficiently which avoids push-threshing
+ */
+const _flattenUpdateResults = (_updateResults: ReadonlyArray<UpdateResult>) => {}

package/src/sync/validate-push-payload.ts

@@ -1,17 +1,20 @@
 import { Effect } from '@livestore/utils/effect'
 
-import type { MutationEvent } from '../schema/mod.js'
+import type { EventId, MutationEvent } from '../schema/mod.js'
 import { InvalidPushError } from './sync.js'
 
 // TODO proper batch validation
-export const validatePushPayload = (batch: ReadonlyArray<MutationEvent.AnyEncoded>, currentEventId: number) =>
+export const validatePushPayload = (
+  batch: ReadonlyArray<MutationEvent.AnyEncodedGlobal>,
+  currentEventId: EventId.GlobalEventId,
+) =>
   Effect.gen(function* () {
-    if (batch[0]!.id.global <= currentEventId) {
+    if (batch[0]!.id <= currentEventId) {
       return yield* InvalidPushError.make({
         reason: {
           _tag: 'ServerAhead',
           minimumExpectedId: currentEventId + 1,
-          providedId: batch[0]!.id.global,
+          providedId: batch[0]!.id,
         },
       })
     }

package/src/version.ts

@@ -2,7 +2,7 @@
 // import packageJson from '../package.json' with { type: 'json' }
 // export const liveStoreVersion = packageJson.version
 
-export const liveStoreVersion = '0.3.0-dev.0' as const
+export const liveStoreVersion = '0.3.0-dev.21' as const
 
 /**
  * This version number is incremented whenever the internal storage format changes in a breaking way.

package/tsconfig.json

@@ -7,5 +7,5 @@
     "tsBuildInfoFile": "./dist/.tsbuildinfo"
   },
   "include": ["./src"],
-  "references": [{ "path": "../db-schema" }, { "path": "../utils" }]
+  "references": [{ "path": "../utils" }]
 }

package/dist/devtools/devtools-bridge.d.ts

@@ -1,12 +0,0 @@
-import type { Effect, PubSub } from '@livestore/utils/effect';
-import type * as Devtools from './devtools-messages.js';
-export type PrepareDevtoolsBridge = {
-    /** Messages coming from the app host (usually responses to requests) */
-    responsePubSub: PubSub.PubSub<Devtools.MessageFromAppLeader | Devtools.MessageFromAppClientSession>;
-    sendToAppHost: (msg: Devtools.MessageToAppLeader | Devtools.MessageToAppClientSession) => Effect.Effect<void>;
-    appHostId: string;
-    copyToClipboard: (text: string) => Effect.Effect<void>;
-    sendEscapeKey?: Effect.Effect<void>;
-    isLeader: boolean;
-};
-//# sourceMappingURL=devtools-bridge.d.ts.map

package/dist/devtools/devtools-bridge.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"devtools-bridge.d.ts","sourceRoot":"","sources":["../../src/devtools/devtools-bridge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAE7D,OAAO,KAAK,KAAK,QAAQ,MAAM,wBAAwB,CAAA;AAEvD,MAAM,MAAM,qBAAqB,GAAG;IAClC,wEAAwE;IACxE,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,GAAG,QAAQ,CAAC,2BAA2B,CAAC,CAAA;IACnG,aAAa,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,kBAAkB,GAAG,QAAQ,CAAC,yBAAyB,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IAC7G,SAAS,EAAE,MAAM,CAAA;IACjB,eAAe,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACtD,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACnC,QAAQ,EAAE,OAAO,CAAA;CAClB,CAAA"}

package/dist/devtools/devtools-bridge.js

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=devtools-bridge.js.map

package/dist/devtools/devtools-bridge.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"devtools-bridge.js","sourceRoot":"","sources":["../../src/devtools/devtools-bridge.ts"],"names":[],"mappings":""}

package/dist/leader-thread/leader-sync-processor.d.ts

@@ -1,47 +0,0 @@
-import type { Scope } from '@livestore/utils/effect';
-import { Effect } from '@livestore/utils/effect';
-import type { SynchronousDatabase } from '../adapter-types.js';
-import { UnexpectedError } from '../adapter-types.js';
-import type { LiveStoreSchema } from '../schema/mod.js';
-import type { InitialBlockingSyncContext, SyncProcessor } from './types.js';
-/**
- * The general idea of the sync processor is to "follow the sync state"
- * and apply/rollback mutations as needed to the read model and mutation log.
- * The leader sync processor is also responsible for
- * - broadcasting mutations to client sessions via the pull queues.
- * - pushing mutations to the sync backend
- *
- * In the leader sync processor, pulling always has precedence over pushing.
- *
- * External events:
- * - Mutation pushed from client session
- * - Mutation pushed from devtools (via pushPartial)
- * - Mutation pulled from sync backend
- *
- * The machine can be in the following states:
- * - in-sync: fully synced with remote, now idling
- * - applying-syncstate-advance (with pointer to current progress in case of rebase interrupt)
- *
- * Transitions:
- * - in-sync -> applying-syncstate-advance
- * - applying-syncstate-advance -> in-sync
- * - applying-syncstate-advance -> applying-syncstate-advance (need to interrupt previous operation)
- *
- * Queuing vs interrupting behaviour:
- * - Operations caused by pull can never be interrupted
- * - Incoming pull can interrupt current push
- * - Incoming pull needs to wait to previous pull to finish
- * - Incoming push needs to wait to previous push to finish
- *
- * Backend pushing:
- * - continously push to backend
- * - only interrupted and restarted on rebase
- */
-export declare const makeLeaderSyncProcessor: ({ schema, dbMissing, dbLog, initialBlockingSyncContext, }: {
-    schema: LiveStoreSchema;
-    /** Only used to know whether we can safely query dbLog during setup execution */
-    dbMissing: boolean;
-    dbLog: SynchronousDatabase;
-    initialBlockingSyncContext: InitialBlockingSyncContext;
-}) => Effect.Effect<SyncProcessor, UnexpectedError, Scope.Scope>;
-//# sourceMappingURL=leader-sync-processor.d.ts.map

package/dist/leader-thread/leader-sync-processor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"leader-sync-processor.d.ts","sourceRoot":"","sources":["../../src/leader-thread/leader-sync-processor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAc,KAAK,EAAE,MAAM,yBAAyB,CAAA;AAChE,OAAO,EAGL,MAAM,EAUP,MAAM,yBAAyB,CAAA;AAGhC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AACrD,OAAO,KAAK,EAAE,eAAe,EAA2B,MAAM,kBAAkB,CAAA;AAehF,OAAO,KAAK,EAAE,0BAA0B,EAAmB,aAAa,EAAE,MAAM,YAAY,CAAA;AAuB5F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,eAAO,MAAM,uBAAuB,8DAKjC;IACD,MAAM,EAAE,eAAe,CAAA;IACvB,iFAAiF;IACjF,SAAS,EAAE,OAAO,CAAA;IAClB,KAAK,EAAE,mBAAmB,CAAA;IAC1B,0BAA0B,EAAE,0BAA0B,CAAA;CACvD,KAAG,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,eAAe,EAAE,KAAK,CAAC,KAAK,CAoOzD,CAAA"}