@livestore/common 0.0.0-snapshot-f5ece014485b5ab43795c218f7ba7cfa32c81707 → 0.0.0-snapshot-057a9e3a18ca69a310d4eb8cf35a34e94fa1841e

package/src/sync/ClientSessionSyncProcessor.ts

@@ -1,14 +1,23 @@
 /// <reference lib="dom" />
 import { LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
-import { Option, type Runtime, type Scope } from '@livestore/utils/effect'
-import { BucketQueue, Effect, FiberHandle, Queue, Schema, Stream, Subscribable } from '@livestore/utils/effect'
+import {
+  BucketQueue,
+  Effect,
+  FiberHandle,
+  Option,
+  Queue,
+  type Runtime,
+  Schema,
+  type Scope,
+  Stream,
+  Subscribable,
+} from '@livestore/utils/effect'
 import * as otel from '@opentelemetry/api'
 
-import type { ClientSession, UnexpectedError } from '../adapter-types.js'
+import { type ClientSession, SyncError, type UnexpectedError } from '../adapter-types.js'
 import * as EventSequenceNumber from '../schema/EventSequenceNumber.js'
 import * as LiveStoreEvent from '../schema/LiveStoreEvent.js'
-import { getEventDef, type LiveStoreSchema, SystemTables } from '../schema/mod.js'
-import { sql } from '../util.js'
+import { getEventDef, type LiveStoreSchema } from '../schema/mod.js'
 import * as SyncState from './syncstate.js'
 
 /**
@@ -21,6 +30,10 @@ import * as SyncState from './syncstate.js'
  * - We might need to make the rebase behaviour configurable e.g. to let users manually trigger a rebase
  *
  * Longer term we should evalutate whether we can unify the ClientSessionSyncProcessor with the LeaderSyncProcessor.
+ *
+ * The session and leader sync processor are different in the following ways:
+ * - The leader sync processor pulls regular LiveStore events, while the session sync processor pulls SyncState.PayloadUpstream items
+ * - The session sync processor has no downstream nodes.
  */
 export const makeClientSessionSyncProcessor = ({
   schema,
@@ -37,7 +50,7 @@ export const makeClientSessionSyncProcessor = ({
   clientSession: ClientSession
   runtime: Runtime.Runtime<Scope.Scope>
   materializeEvent: (
-    eventDecoded: LiveStoreEvent.PartialAnyDecoded,
+    eventDecoded: LiveStoreEvent.AnyDecoded,
     options: { otelContext: otel.Context; withChangeset: boolean; materializerHashLeader: Option.Option<number> },
   ) => {
     writeTables: Set<string>
@@ -82,7 +95,10 @@ export const makeClientSessionSyncProcessor = ({
     let baseEventSequenceNumber = syncStateRef.current.localHead
     const encodedEventDefs = batch.map(({ name, args }) => {
       const eventDef = getEventDef(schema, name)
-      const nextNumPair = EventSequenceNumber.nextPair(baseEventSequenceNumber, eventDef.eventDef.options.clientOnly)
+      const nextNumPair = EventSequenceNumber.nextPair({
+        seqNum: baseEventSequenceNumber,
+        isClient: eventDef.eventDef.options.clientOnly,
+      })
       baseEventSequenceNumber = nextNumPair.seqNum
       return new LiveStoreEvent.EncodedWithMeta(
         Schema.encodeUnknownSync(eventSchema)({
@@ -103,7 +119,7 @@ export const makeClientSessionSyncProcessor = ({
     })
 
     if (mergeResult._tag === 'unexpected-error') {
-      return shouldNeverHappen('Unexpected error in client-session-sync-processor', mergeResult.cause)
+      return shouldNeverHappen('Unexpected error in client-session-sync-processor', mergeResult.message)
     }
 
     span.addEvent('local-push', {
@@ -155,7 +171,6 @@ export const makeClientSessionSyncProcessor = ({
   const otelContext = otel.trace.setSpan(otel.context.active(), span)
 
   const boot: ClientSessionSyncProcessor['boot'] = Effect.gen(function* () {
-    // eslint-disable-next-line unicorn/prefer-global-this
     if (confirmUnsavedChanges && typeof window !== 'undefined' && typeof window.addEventListener === 'function') {
       const onBeforeUnload = (event: BeforeUnloadEvent) => {
         if (syncStateRef.current.pending.length > 0) {
@@ -184,18 +199,11 @@ export const makeClientSessionSyncProcessor = ({
 
     yield* FiberHandle.run(leaderPushingFiberHandle, backgroundLeaderPushing)
 
-    const getMergeCounter = () =>
-      clientSession.sqliteDb.select<{ mergeCounter: number }>(
-        sql`SELECT mergeCounter FROM ${SystemTables.LEADER_MERGE_COUNTER_TABLE} WHERE id = 0`,
-      )[0]?.mergeCounter ?? 0
-
     // NOTE We need to lazily call `.pull` as we want the cursor to be updated
     yield* Stream.suspend(() =>
-      clientSession.leaderThread.events.pull({
-        cursor: { mergeCounter: getMergeCounter(), eventNum: syncStateRef.current.localHead },
-      }),
+      clientSession.leaderThread.events.pull({ cursor: syncStateRef.current.upstreamHead }),
     ).pipe(
-      Stream.tap(({ payload, mergeCounter: leaderMergeCounter }) =>
+      Stream.tap(({ payload }) =>
         Effect.gen(function* () {
           // yield* Effect.logDebug('ClientSessionSyncProcessor:pull', payload)
 
@@ -211,13 +219,13 @@ export const makeClientSessionSyncProcessor = ({
           })
 
           if (mergeResult._tag === 'unexpected-error') {
-            return yield* Effect.fail(mergeResult.cause)
+            return yield* new SyncError({ cause: mergeResult.message })
           } else if (mergeResult._tag === 'reject') {
             return shouldNeverHappen('Unexpected reject in client-session-sync-processor', mergeResult)
           }
 
           syncStateRef.current = mergeResult.newSyncState
-          syncStateUpdateQueue.offer(mergeResult.newSyncState).pipe(Effect.runSync)
+          yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
 
           if (mergeResult._tag === 'rebase') {
             span.addEvent('merge:pull:rebase', {
@@ -226,7 +234,7 @@ export const makeClientSessionSyncProcessor = ({
               newEventsCount: mergeResult.newEvents.length,
               rollbackCount: mergeResult.rollbackEvents.length,
               res: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
-              leaderMergeCounter,
+              rebaseGeneration: mergeResult.newSyncState.localHead.rebaseGeneration,
             })
 
             debugInfo.rebaseCount++
@@ -243,7 +251,6 @@ export const makeClientSessionSyncProcessor = ({
                 'merge:pull:rebase: rollback',
                 mergeResult.rollbackEvents.length,
                 ...mergeResult.rollbackEvents.slice(0, 10).map((_) => _.toJSON()),
-                { leaderMergeCounter },
               ).pipe(Effect.provide(runtime), Effect.runSync)
             }
 
@@ -263,7 +270,6 @@ export const makeClientSessionSyncProcessor = ({
               payload: TRACE_VERBOSE ? JSON.stringify(payload) : undefined,
               newEventsCount: mergeResult.newEvents.length,
               res: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
-              leaderMergeCounter,
             })
 
             debugInfo.advanceCount++

package/src/sync/index.ts

@@ -1,3 +1,3 @@
+export * from './ClientSessionSyncProcessor.js'
 export * from './sync.js'
 export * from './validate-push-payload.js'
-export * from './ClientSessionSyncProcessor.js'

package/src/sync/next/graphology.ts

@@ -13,17 +13,9 @@ export declare class IGraph<
   constructor(options?: graphologyTypes.GraphOptions)
 }
 
-export const DirectedGraph = class DirectedGraph extends graphology.DirectedGraph {
-  constructor(options?: graphologyTypes.GraphOptions) {
-    super(options)
-  }
-} as typeof IGraph
-
-export const Graph = class Graph extends graphology.Graph {
-  constructor(options?: graphologyTypes.GraphOptions) {
-    super(options)
-  }
-} as typeof IGraph
+export const DirectedGraph = class DirectedGraph extends graphology.DirectedGraph {} as typeof IGraph
+
+export const Graph = class Graph extends graphology.Graph {} as typeof IGraph
 
 // export const graphology = graphology_ as graphologyTypes
 

package/src/sync/next/graphology_.ts

@@ -1,2 +1,2 @@
-export * as graphology from './graphology.js'
 export { default as graphologyDag } from 'graphology-dag'
+export * as graphology from './graphology.js'

package/src/sync/next/mod.ts

@@ -1,5 +1,5 @@
-export * from './history-dag-common.js'
-export * from './history-dag.js'
-export * from './facts.js'
 export * from './compact-events.js'
+export * from './facts.js'
+export * from './history-dag.js'
+export * from './history-dag-common.js'
 export * from './rebase-events.js'

package/src/sync/next/rebase-events.ts

@@ -3,8 +3,8 @@ import * as EventSequenceNumber from '../../schema/EventSequenceNumber.js'
 import type * as LiveStoreEvent from '../../schema/LiveStoreEvent.js'
 import {
   applyFactGroups,
-  factsIntersect,
   type FactValidationResult,
+  factsIntersect,
   getFactsGroupForEventArgs,
   validateFacts,
 } from './facts.js'
@@ -38,6 +38,7 @@ export const defaultRebaseFn: RebaseFn = ({ pendingLocalEvents }) => {
   return { rebasedLocalEvents: pendingLocalEvents }
 }
 
+// TODO replace in favour of current rebase impl
 export const rebaseEvents = ({
   rebaseFn,
   pendingLocalEvents,

package/src/sync/next/test/compact-events.test.ts

@@ -64,7 +64,7 @@ const factsSetToString = (facts: EventDefFacts, prefix: string) =>
 export const customSerializer = {
   test: (val: unknown) => Array.isArray(val),
   print: (val: unknown[], _serialize: (item: unknown) => string) => {
-    return '[\n' + (val as any[]).map((item) => '  ' + customStringify(item)).join('\n') + '\n]'
+    return `[\n${(val as any[]).map((item) => `  ${customStringify(item)}`).join('\n')}\n]`
   },
 } as any
 

package/src/sync/next/test/event-fixtures.ts

@@ -144,7 +144,10 @@ export const toEventNodes = (
 
   const eventNodes = partialEvents.map((partialEvent) => {
     const eventDef = eventDefs[partialEvent.name]!
-    const eventNum = EventSequenceNumber.nextPair(currentEventSequenceNumber, eventDef.options.clientOnly).seqNum
+    const eventNum = EventSequenceNumber.nextPair({
+      seqNum: currentEventSequenceNumber,
+      isClient: eventDef.options.clientOnly,
+    }).seqNum
     currentEventSequenceNumber = eventNum
 
     const factsSnapshot = factsSnapshotForDag(historyDagFromNodes(nodesAcc, { skipFactsCheck: true }), undefined)
@@ -221,8 +224,14 @@ const getParentNum = (eventNum: EventSequenceNumber.EventSequenceNumber): EventS
   const clientParentNum = eventNum.client - 1
 
   if (clientParentNum < 0) {
-    return EventSequenceNumber.make({ global: globalParentNum - 1, client: EventSequenceNumber.clientDefault })
+    return EventSequenceNumber.make({
+      global: globalParentNum - 1,
+      client: EventSequenceNumber.clientDefault,
+    })
   }
 
-  return EventSequenceNumber.make({ global: globalParentNum, client: clientParentNum })
+  return EventSequenceNumber.make({
+    global: globalParentNum,
+    client: clientParentNum,
+  })
 }

package/src/sync/sync.ts

@@ -80,6 +80,7 @@ export type SyncBackend<TSyncMetadata = Schema.JsonValue> = {
 
 export class IsOfflineError extends Schema.TaggedError<IsOfflineError>()('IsOfflineError', {}) {}
 
+// TODO gt rid of this error in favour of SyncError
 export class InvalidPushError extends Schema.TaggedError<InvalidPushError>()('InvalidPushError', {
   reason: Schema.Union(
     Schema.TaggedStruct('Unexpected', {
@@ -92,10 +93,12 @@ export class InvalidPushError extends Schema.TaggedError<InvalidPushError>()('In
   ),
 }) {}
 
+// TODO gt rid of this error in favour of SyncError
 export class InvalidPullError extends Schema.TaggedError<InvalidPullError>()('InvalidPullError', {
   message: Schema.String,
 }) {}
 
+// TODO gt rid of this error in favour of SyncError
 export class LeaderAheadError extends Schema.TaggedError<LeaderAheadError>()('LeaderAheadError', {
   minimumExpectedNum: EventSequenceNumber.EventSequenceNumber,
   providedNum: EventSequenceNumber.EventSequenceNumber,

package/src/sync/syncstate.test.ts

@@ -1,4 +1,3 @@
-/* eslint-disable prefer-arrow/prefer-arrow-functions */
 import { describe, expect, it } from 'vitest'
 
 import * as EventSequenceNumber from '../schema/EventSequenceNumber.js'
@@ -7,10 +6,10 @@ import * as SyncState from './syncstate.js'
 
 class TestEvent extends LiveStoreEvent.EncodedWithMeta {
   constructor(
-    seqNum: EventSequenceNumber.EventSequenceNumber | typeof EventSequenceNumber.EventSequenceNumber.Encoded,
-    parentSeqNum: EventSequenceNumber.EventSequenceNumber,
+    seqNum: EventSequenceNumber.EventSequenceNumberInput,
+    parentSeqNum: EventSequenceNumber.EventSequenceNumberInput,
     public readonly payload: string,
-    public readonly isLocal: boolean,
+    public readonly isClient: boolean,
   ) {
     super({
       seqNum: EventSequenceNumber.make(seqNum),
@@ -22,8 +21,8 @@ class TestEvent extends LiveStoreEvent.EncodedWithMeta {
     })
   }
 
-  rebase_ = (parentSeqNum: EventSequenceNumber.EventSequenceNumber) => {
-    return this.rebase(parentSeqNum, this.isLocal)
+  rebase_ = (parentSeqNum: EventSequenceNumber.EventSequenceNumber, rebaseGeneration: number) => {
+    return this.rebase({ parentSeqNum, isClient: this.isClient, rebaseGeneration })
   }
 
   // Only used for Vitest printing
@@ -41,7 +40,7 @@ const e2_1 = new TestEvent({ global: 2, client: 1 }, e2_0.seqNum, 'a', true)
 
 const isEqualEvent = LiveStoreEvent.isEqualEncoded
 
-const isClientEvent = (event: LiveStoreEvent.EncodedWithMeta) => (event as TestEvent).isLocal
+const isClientEvent = (event: LiveStoreEvent.EncodedWithMeta) => (event as TestEvent).isClient
 
 describe('syncstate', () => {
   describe('merge', () => {
@@ -62,8 +61,8 @@ describe('syncstate', () => {
           upstreamHead: EventSequenceNumber.ROOT,
           localHead: e2_0.seqNum,
         })
-        const e1_0_e2_0 = e1_0.rebase_(e2_0.seqNum)
-        const e1_1_e2_1 = e1_1.rebase_(e1_0_e2_0.seqNum)
+        const e1_0_e2_0 = e1_0.rebase_(e2_0.seqNum, 0)
+        const e1_1_e2_1 = e1_1.rebase_(e1_0_e2_0.seqNum, 0)
         const result = merge({
           syncState,
           payload: SyncState.PayloadUpstreamRebase.make({
@@ -71,7 +70,7 @@ describe('syncstate', () => {
             newEvents: [e1_0_e2_0, e1_1_e2_1],
           }),
         })
-        const e2_0_e3_0 = e2_0.rebase_(e1_0_e2_0.seqNum)
+        const e2_0_e3_0 = e2_0.rebase_(e1_0_e2_0.seqNum, 1)
         expectRebase(result)
         expectEventArraysEqual(result.newSyncState.pending, [e2_0_e3_0])
         expect(result.newSyncState.upstreamHead).toMatchObject(e1_1_e2_1.seqNum)
@@ -86,7 +85,7 @@ describe('syncstate', () => {
           upstreamHead: EventSequenceNumber.ROOT,
           localHead: e2_0.seqNum,
         })
-        const e1_1_e2_0 = e1_1.rebase_(e1_0.seqNum)
+        const e1_1_e2_0 = e1_1.rebase_(e1_0.seqNum, 0)
         const result = merge({
           syncState,
           payload: SyncState.PayloadUpstreamRebase.make({
@@ -94,7 +93,7 @@ describe('syncstate', () => {
             rollbackEvents: [e1_1],
           }),
         })
-        const e2_0_e3_0 = e2_0.rebase_(e1_1_e2_0.seqNum)
+        const e2_0_e3_0 = e2_0.rebase_(e1_1_e2_0.seqNum, 1)
         expectRebase(result)
         expectEventArraysEqual(result.newSyncState.pending, [e2_0_e3_0])
         expect(result.newSyncState.upstreamHead).toMatchObject(e1_1_e2_0.seqNum)
@@ -326,7 +325,7 @@ describe('syncstate', () => {
         })
         const result = merge({ syncState, payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: [e1_1] }) })
 
-        const e1_0_e1_2 = e1_0.rebase_(e1_1.seqNum)
+        const e1_0_e1_2 = e1_0.rebase_(e1_1.seqNum, 1)
 
         expectRebase(result)
         expectEventArraysEqual(result.newSyncState.pending, [e1_0_e1_2])
@@ -344,7 +343,7 @@ describe('syncstate', () => {
           localHead: e2_0_b.seqNum,
         })
         const result = merge({ syncState, payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: [e2_0] }) })
-        const e2_0_e3_0 = e2_0_b.rebase_(e2_0.seqNum)
+        const e2_0_e3_0 = e2_0_b.rebase_(e2_0.seqNum, 1)
 
         expectRebase(result)
         expectEventArraysEqual(result.newSyncState.pending, [e2_0_e3_0])
@@ -365,7 +364,7 @@ describe('syncstate', () => {
           payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: [e1_1, e1_2, e1_3, e2_0] }),
         })
 
-        const e1_0_e3_0 = e1_0.rebase_(e2_0.seqNum)
+        const e1_0_e3_0 = e1_0.rebase_(e2_0.seqNum, 1)
 
         expectRebase(result)
         expectEventArraysEqual(result.newSyncState.pending, [e1_0_e3_0])
@@ -384,7 +383,7 @@ describe('syncstate', () => {
           payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: [e1_0, e1_2, e1_3, e2_0] }),
         })
 
-        const e1_1_e2_1 = e1_1.rebase_(e2_0.seqNum)
+        const e1_1_e2_1 = e1_1.rebase_(e2_0.seqNum, 1)
 
         expectRebase(result)
         expectEventArraysEqual(result.newSyncState.pending, [e1_1_e2_1])
@@ -405,8 +404,8 @@ describe('syncstate', () => {
           payload: SyncState.PayloadUpstreamAdvance.make({ newEvents: [e1_1, e1_2, e1_3, e2_0] }),
         })
 
-        const e1_0_e2_1 = e1_0.rebase_(e2_0.seqNum)
-        const e1_1_e2_2 = e1_1.rebase_(e1_0_e2_1.seqNum)
+        const e1_0_e2_1 = e1_0.rebase_(e2_0.seqNum, 1)
+        const e1_1_e2_2 = e1_1.rebase_(e1_0_e2_1.seqNum, 1)
 
         expectRebase(result)
         expectEventArraysEqual(result.newSyncState.pending, [e1_0_e2_1, e1_1_e2_2])

package/src/sync/syncstate.ts

@@ -1,7 +1,6 @@
 import { casesHandled, LS_DEV, shouldNeverHappen } from '@livestore/utils'
 import { Match, ReadonlyArray, Schema } from '@livestore/utils/effect'
 
-import { UnexpectedError } from '../adapter-types.js'
 import * as EventSequenceNumber from '../schema/EventSequenceNumber.js'
 import * as LiveStoreEvent from '../schema/LiveStoreEvent.js'
 
@@ -162,7 +161,7 @@ export class MergeResultReject extends Schema.Class<MergeResultReject>('MergeRes
 
 export class MergeResultUnexpectedError extends Schema.Class<MergeResultUnexpectedError>('MergeResultUnexpectedError')({
   _tag: Schema.Literal('unexpected-error'),
-  cause: UnexpectedError,
+  message: Schema.String,
 }) {}
 
 export class MergeResult extends Schema.Union(
@@ -172,15 +171,29 @@ export class MergeResult extends Schema.Union(
   MergeResultUnexpectedError,
 ) {}
 
-const unexpectedError = (cause: unknown): MergeResultUnexpectedError => {
+export const payloadFromMergeResult = (
+  mergeResult: typeof MergeResultAdvance.Type | typeof MergeResultRebase.Type,
+): typeof PayloadUpstream.Type =>
+  Match.value(mergeResult).pipe(
+    Match.tag('advance', (result) => ({
+      _tag: 'upstream-advance' as const,
+      newEvents: result.newEvents,
+    })),
+    Match.tag('rebase', (result) => ({
+      _tag: 'upstream-rebase' as const,
+      newEvents: result.newEvents,
+      rollbackEvents: result.rollbackEvents,
+    })),
+    Match.exhaustive,
+  )
+
+const unexpectedError = (message: string): MergeResultUnexpectedError => {
   if (LS_DEV) {
+    // biome-ignore lint/suspicious/noDebugger: debug
     debugger
   }
 
-  return MergeResultUnexpectedError.make({
-    _tag: 'unexpected-error',
-    cause: new UnexpectedError({ cause }),
-  })
+  return MergeResultUnexpectedError.make({ _tag: 'unexpected-error', message })
 }
 
 // TODO Idea: call merge recursively through hierarchy levels
@@ -382,7 +395,10 @@ export const merge = ({
         EventSequenceNumber.isGreaterThan(newEventsFirst.seqNum, syncState.localHead) === false
 
       if (invalidEventSequenceNumber) {
-        const expectedMinimumId = EventSequenceNumber.nextPair(syncState.localHead, true).seqNum
+        const expectedMinimumId = EventSequenceNumber.nextPair({
+          seqNum: syncState.localHead,
+          isClient: true,
+        }).seqNum
         return validateMergeResult(
           MergeResultReject.make({
             _tag: 'reject',
@@ -466,9 +482,14 @@ const rebaseEvents = ({
   isClientEvent: (event: LiveStoreEvent.EncodedWithMeta) => boolean
 }): ReadonlyArray<LiveStoreEvent.EncodedWithMeta> => {
   let prevEventSequenceNumber = baseEventSequenceNumber
+  const rebaseGeneration = baseEventSequenceNumber.rebaseGeneration + 1
   return events.map((event) => {
-    const isLocal = isClientEvent(event)
-    const newEvent = event.rebase(prevEventSequenceNumber, isLocal)
+    const isClient = isClientEvent(event)
+    const newEvent = event.rebase({
+      parentSeqNum: prevEventSequenceNumber,
+      isClient,
+      rebaseGeneration,
+    })
     prevEventSequenceNumber = newEvent.seqNum
     return newEvent
   })

package/src/util.ts

@@ -30,7 +30,6 @@ export const sql = (template: TemplateStringsArray, ...args: unknown[]): string
     str += template[i] + String(arg)
   }
 
-  // eslint-disable-next-line unicorn/prefer-at
   return str + template[template.length - 1]
 }
 

package/src/version.ts

@@ -2,7 +2,7 @@
 // import packageJson from '../package.json' with { type: 'json' }
 // export const liveStoreVersion = packageJson.version
 
-export const liveStoreVersion = '0.0.0-snapshot-f5ece014485b5ab43795c218f7ba7cfa32c81707' as const
+export const liveStoreVersion = '0.0.0-snapshot-057a9e3a18ca69a310d4eb8cf35a34e94fa1841e' as const
 
 /**
  * This version number is incremented whenever the internal storage format changes in a breaking way.