@livestore/common 0.0.0-snapshot-4872718a28e4357bb1acdfb14eb69083ed0a480c → 0.0.0-snapshot-484c9684bac8056d764aa460fd025c45f5856aa5

package/src/leader-thread/apply-mutation.ts

@@ -1,10 +1,11 @@
-import { memoizeByRef, shouldNeverHappen } from '@livestore/utils'
+import { LS_DEV, memoizeByRef, shouldNeverHappen } from '@livestore/utils'
 import type { Scope } from '@livestore/utils/effect'
 import { Effect, Option, Schema } from '@livestore/utils/effect'
 
-import type { SqliteDb, SqliteError, UnexpectedError } from '../index.js'
+import type { PreparedBindValues, SqliteDb, SqliteError, UnexpectedError } from '../index.js'
 import { getExecArgsFromMutation } from '../mutation.js'
 import {
+  EventId,
   type LiveStoreSchema,
   MUTATION_LOG_META_TABLE,
   type MutationEvent,
@@ -127,6 +128,20 @@ const insertIntoMutationLog = (
     const mutationDefSchemaHash =
       mutationDefSchemaHashMap.get(mutationName) ?? shouldNeverHappen(`Unknown mutation: ${mutationName}`)
 
+    if (LS_DEV && mutationEventEncoded.parentId.global !== EventId.ROOT.global) {
+      const parentMutationExists =
+        dbMutationLog.select<{ count: number }>(
+          `SELECT COUNT(*) as count FROM ${MUTATION_LOG_META_TABLE} WHERE idGlobal = ? AND idClient = ?`,
+          [mutationEventEncoded.parentId.global, mutationEventEncoded.parentId.client] as any as PreparedBindValues,
+        )[0]!.count === 1
+
+      if (parentMutationExists === false) {
+        shouldNeverHappen(
+          `Parent mutation ${mutationEventEncoded.parentId.global},${mutationEventEncoded.parentId.client} does not exist`,
+        )
+      }
+    }
+
     // TODO use prepared statements
     yield* execSql(
       dbMutationLog,

package/src/leader-thread/make-leader-thread-layer.ts

@@ -65,6 +65,7 @@ export const makeLeaderThreadLayer = ({
       dbMissing,
       dbMutationLog,
       initialBlockingSyncContext,
+      clientId,
     })
 
     const extraIncomingMessagesQueue = yield* Queue.unbounded<Devtools.Leader.MessageToApp>().pipe(

package/src/leader-thread/pull-queue-set.ts

@@ -2,7 +2,7 @@ import { Effect, Queue } from '@livestore/utils/effect'
 
 import * as MutationEvent from '../schema/MutationEvent.js'
 import { getMutationEventsSince } from './mutationlog.js'
-import { type PullQueueItem, type PullQueueSet } from './types.js'
+import { LeaderThreadCtx, type PullQueueItem, type PullQueueSet } from './types.js'
 
 export const makePullQueueSet = Effect.gen(function* () {
   const set = new Set<Queue.Queue<PullQueueItem>>()
@@ -46,6 +46,15 @@ export const makePullQueueSet = Effect.gen(function* () {
         return
       }
 
+      const { clientId } = yield* LeaderThreadCtx
+      if (clientId === 'client-b') {
+        // console.log(
+        //   'offer',
+        //   item.payload._tag,
+        //   item.payload.newEvents.map((_) => _.toJSON()),
+        // )
+      }
+
       for (const queue of set) {
         yield* Queue.offer(queue, item)
       }

package/src/leader-thread/shutdown-channel.ts

@@ -1,11 +1,9 @@
 import type { WebChannel } from '@livestore/utils/effect'
 import { Schema } from '@livestore/utils/effect'
 
-import { IntentionalShutdownCause } from '../index.js'
+import { IntentionalShutdownCause, UnexpectedError } from '../index.js'
 
-export class DedicatedWorkerDisconnectBroadcast extends Schema.TaggedStruct('DedicatedWorkerDisconnectBroadcast', {}) {}
-
-export class All extends Schema.Union(IntentionalShutdownCause, DedicatedWorkerDisconnectBroadcast) {}
+export class All extends Schema.Union(IntentionalShutdownCause, UnexpectedError) {}
 
 /**
  * Used internally by an adapter to shutdown gracefully.

package/src/leader-thread/types.ts

@@ -14,7 +14,7 @@ import { Context, Schema } from '@livestore/utils/effect'
 import type {
   BootStatus,
   Devtools,
-  InvalidPushError,
+  LeaderAheadError,
   MakeSqliteDb,
   MigrationsReport,
   PersistenceInfo,
@@ -126,13 +126,19 @@ export interface LeaderSyncProcessor {
     /** `batch` needs to follow the same rules as `batch` in `SyncBackend.push` */
     batch: ReadonlyArray<MutationEvent.EncodedWithMeta>,
     options?: {
+      /**
+       * This generation number is used to automatically reject subsequent pushes
+       * of a previously rejected push from a client session. This might occur in
+       * certain concurrent scenarios.
+       */
+      // generation: number
       /**
        * If true, the effect will only finish when the local push has been processed (i.e. succeeded or was rejected).
        * @default false
        */
       waitForProcessing?: boolean
     },
-  ) => Effect.Effect<void, InvalidPushError>
+  ) => Effect.Effect<void, LeaderAheadError>
 
   pushPartial: (args: {
     mutationEvent: MutationEvent.PartialAnyEncoded

package/src/schema/EventId.ts

@@ -33,6 +33,22 @@ export const compare = (a: EventId, b: EventId) => {
   return a.client - b.client
 }
 
+/**
+ * Convert an event id to a string representation.
+ */
+export const toString = (id: EventId) => `(${id.global},${id.client})`
+
+/**
+ * Convert a string representation of an event id to an event id.
+ */
+export const fromString = (str: string): EventId => {
+  const [global, client] = str.slice(1, -1).split(',').map(Number)
+  if (global === undefined || client === undefined) {
+    throw new Error('Invalid event id string')
+  }
+  return { global, client } as EventId
+}
+
 export const isEqual = (a: EventId, b: EventId) => a.global === b.global && a.client === b.client
 
 export type EventIdPair = { id: EventId; parentId: EventId }

package/src/schema/MutationEvent.ts

@@ -169,7 +169,7 @@ export class EncodedWithMeta extends Schema.Class<EncodedWithMeta>('MutationEven
   toJSON = (): any => {
     // Only used for logging/debugging
     // - More readable way to print the id + parentId
-    // - not including `meta`
+    // - not including `meta`, `clientId`, `sessionId`
     return {
       id: `(${this.id.global},${this.id.client}) → (${this.parentId.global},${this.parentId.client})`,
       mutation: this.mutation,

package/src/schema/db-schema/dsl/field-defs.ts

@@ -53,8 +53,7 @@ export type ColDefFn<TColumnType extends FieldColumnType> = {
     const TNullable extends boolean = false,
     const TDefault extends TDecoded | SqlDefaultValue | NoDefault | (TNullable extends true ? null : never) = NoDefault,
     const TPrimaryKey extends boolean = false,
-  >(args: {
-    schema?: Schema.Schema<TDecoded, TEncoded>
+  >(args: {    schema?: Schema.Schema<TDecoded, TEncoded>
     default?: TDefault
     nullable?: TNullable
     primaryKey?: TPrimaryKey

package/src/sync/ClientSessionSyncProcessor.ts

@@ -1,6 +1,6 @@
 import { LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
 import type { Runtime, Scope } from '@livestore/utils/effect'
-import { Effect, Queue, Schema, Stream, Subscribable } from '@livestore/utils/effect'
+import { BucketQueue, Effect, FiberHandle, Queue, Schema, Stream, Subscribable } from '@livestore/utils/effect'
 import * as otel from '@opentelemetry/api'
 
 import type { ClientSession, UnexpectedError } from '../adapter-types.js'
@@ -26,6 +26,7 @@ export const makeClientSessionSyncProcessor = ({
   rollback,
   refreshTables,
   span,
+  params,
 }: {
   schema: LiveStoreSchema
   clientSession: ClientSession
@@ -40,6 +41,9 @@ export const makeClientSessionSyncProcessor = ({
   rollback: (changeset: Uint8Array) => void
   refreshTables: (tables: Set<string>) => void
   span: otel.Span
+  params: {
+    leaderPushBatchSize: number
+  }
 }): ClientSessionSyncProcessor => {
   const mutationEventSchema = MutationEvent.makeMutationEventSchemaMemo(schema)
 
@@ -59,6 +63,9 @@ export const makeClientSessionSyncProcessor = ({
     return mutationDef.options.clientOnly
   }
 
+  /** We're queuing push requests to reduce the number of messages sent to the leader by batching them */
+  const leaderPushQueue = BucketQueue.make<MutationEvent.EncodedWithMeta>().pipe(Effect.runSync)
+
   const push: ClientSessionSyncProcessor['push'] = (batch, { otelContext }) => {
     // TODO validate batch
 
@@ -84,6 +91,10 @@ export const makeClientSessionSyncProcessor = ({
       isEqualEvent: MutationEvent.isEqualEncoded,
     })
 
+    if (updateResult._tag === 'unexpected-error') {
+      return shouldNeverHappen('Unexpected error in client-session-sync-processor', updateResult.cause)
+    }
+
     span.addEvent('local-push', {
       batchSize: encodedMutationEvents.length,
       updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
@@ -108,13 +119,17 @@ export const makeClientSessionSyncProcessor = ({
     }
 
     // console.debug('pushToLeader', encodedMutationEvents.length, ...encodedMutationEvents.map((_) => _.toJSON()))
-    clientSession.leaderThread.mutations
-      .push(encodedMutationEvents)
-      .pipe(Effect.tapCauseLogPretty, Effect.provide(runtime), Effect.runFork)
+    BucketQueue.offerAll(leaderPushQueue, encodedMutationEvents).pipe(Effect.runSync)
 
     return { writeTables }
   }
 
+  const debugInfo = {
+    rebaseCount: 0,
+    advanceCount: 0,
+    rejectCount: 0,
+  }
+
   const otelContext = otel.trace.setSpan(otel.context.active(), span)
 
   const boot: ClientSessionSyncProcessor['boot'] = Effect.gen(function* () {
@@ -133,6 +148,20 @@ export const makeClientSessionSyncProcessor = ({
       )
     }
 
+    const leaderPushingFiberHandle = yield* FiberHandle.make()
+
+    const backgroundLeaderPushing = Effect.gen(function* () {
+      const batch = yield* BucketQueue.takeBetween(leaderPushQueue, 1, params.leaderPushBatchSize)
+      yield* clientSession.leaderThread.mutations.push(batch).pipe(
+        Effect.catchTag('LeaderAheadError', () => {
+          debugInfo.rejectCount++
+          return BucketQueue.clear(leaderPushQueue)
+        }),
+      )
+    }).pipe(Effect.forever, Effect.interruptible, Effect.tapCauseLogPretty)
+
+    yield* FiberHandle.run(leaderPushingFiberHandle, backgroundLeaderPushing)
+
     yield* clientSession.leaderThread.mutations.pull.pipe(
       Stream.tap(({ payload, remaining }) =>
         Effect.gen(function* () {
@@ -148,9 +177,10 @@ export const makeClientSessionSyncProcessor = ({
             isEqualEvent: MutationEvent.isEqualEncoded,
           })
 
-          if (updateResult._tag === 'reject') {
-            debugger
-            throw new Error('TODO: implement reject in client-session-sync-queue for pull')
+          if (updateResult._tag === 'unexpected-error') {
+            return yield* Effect.fail(updateResult.cause)
+          } else if (updateResult._tag === 'reject') {
+            return shouldNeverHappen('Unexpected reject in client-session-sync-processor', updateResult)
           }
 
           syncStateRef.current = updateResult.newSyncState
@@ -165,11 +195,21 @@ export const makeClientSessionSyncProcessor = ({
               res: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
               remaining,
             })
+
+            debugInfo.rebaseCount++
+
+            yield* FiberHandle.clear(leaderPushingFiberHandle)
+
+            // Reset the leader push queue since we're rebasing and will push again
+            yield* BucketQueue.clear(leaderPushQueue)
+
+            yield* FiberHandle.run(leaderPushingFiberHandle, backgroundLeaderPushing)
+
             if (LS_DEV) {
               Effect.logDebug(
                 'pull:rebase: rollback',
                 updateResult.eventsToRollback.length,
-                ...updateResult.eventsToRollback.map((_) => _.toJSON()),
+                ...updateResult.eventsToRollback.slice(0, 10).map((_) => _.toJSON()),
               ).pipe(Effect.provide(runtime), Effect.runSync)
             }
 
@@ -181,9 +221,7 @@ export const makeClientSessionSyncProcessor = ({
               }
             }
 
-            clientSession.leaderThread.mutations
-              .push(updateResult.newSyncState.pending)
-              .pipe(Effect.tapCauseLogPretty, Effect.provide(runtime), Effect.runFork)
+            yield* BucketQueue.offerAll(leaderPushQueue, updateResult.newSyncState.pending)
           } else {
             span.addEvent('pull:advance', {
               payloadTag: payload._tag,
@@ -192,6 +230,8 @@ export const makeClientSessionSyncProcessor = ({
               res: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
               remaining,
             })
+
+            debugInfo.advanceCount++
           }
 
           if (updateResult.newEvents.length === 0) return
@@ -208,10 +248,14 @@ export const makeClientSessionSyncProcessor = ({
           }
 
           refreshTables(writeTables)
-        }),
+        }).pipe(
+          Effect.tapCauseLogPretty,
+          Effect.catchAllCause((cause) => Effect.sync(() => clientSession.shutdown(cause))),
+        ),
       ),
       Stream.runDrain,
       Effect.forever, // NOTE Whenever the leader changes, we need to re-start the stream
+      Effect.withSpan('client-session-sync-processor:pull'),
       Effect.tapCauseLogPretty,
       Effect.forkScoped,
     )
@@ -228,6 +272,21 @@ export const makeClientSessionSyncProcessor = ({
       }),
       changes: Stream.fromQueue(syncStateUpdateQueue),
     }),
+    debug: {
+      print: () =>
+        Effect.gen(function* () {
+          console.log('debugInfo', debugInfo)
+          console.log('syncState', syncStateRef.current)
+          const pushQueueSize = yield* BucketQueue.size(leaderPushQueue)
+          console.log('pushQueueSize', pushQueueSize)
+          const pushQueueItems = yield* BucketQueue.peekAll(leaderPushQueue)
+          console.log(
+            'pushQueueItems',
+            pushQueueItems.map((_) => _.toJSON()),
+          )
+        }).pipe(Effect.provide(runtime), Effect.runSync),
+      debugInfo: () => debugInfo,
+    },
   } satisfies ClientSessionSyncProcessor
 }
 
@@ -239,6 +298,12 @@ export interface ClientSessionSyncProcessor {
     writeTables: Set<string>
   }
   boot: Effect.Effect<void, UnexpectedError, Scope.Scope>
-
   syncState: Subscribable.Subscribable<SyncState.SyncState>
+  debug: {
+    print: () => void
+    debugInfo: () => {
+      rebaseCount: number
+      advanceCount: number
+    }
+  }
 }

package/src/sync/sync.ts

@@ -67,13 +67,16 @@ export class InvalidPushError extends Schema.TaggedError<InvalidPushError>()('In
       minimumExpectedId: Schema.Number,
       providedId: Schema.Number,
     }),
-    Schema.TaggedStruct('LeaderAhead', {
-      minimumExpectedId: EventId.EventId,
-      providedId: EventId.EventId,
-    }),
   ),
 }) {}
 
 export class InvalidPullError extends Schema.TaggedError<InvalidPullError>()('InvalidPullError', {
   message: Schema.String,
 }) {}
+
+export class LeaderAheadError extends Schema.TaggedError<LeaderAheadError>()('LeaderAheadError', {
+  minimumExpectedId: EventId.EventId,
+  providedId: EventId.EventId,
+  /** Generation number the client session should use for subsequent pushes */
+  // nextGeneration: Schema.Number,
+}) {}

package/src/sync/syncstate.test.ts

@@ -17,7 +17,7 @@ class TestEvent extends MutationEvent.EncodedWithMeta {
       parentId: EventId.make(parentId),
       mutation: 'a',
       args: payload,
-      meta: {},
+
       clientId: 'static-local-id',
       sessionId: undefined,
     })
@@ -87,7 +87,7 @@ describe('syncstate', () => {
           }
           expect(result.newSyncState.upstreamHead).toMatchObject(e_0_1_e_1_1.id)
           expect(result.newSyncState.localHead).toMatchObject(e_1_0_e_2_0.id)
-          expectEventArraysEqual(result.newEvents, [e_0_0_e_1_0, e_0_1_e_1_1])
+          expectEventArraysEqual(result.newEvents, [e_0_0_e_1_0, e_0_1_e_1_1, e_1_0_e_2_0])
           expectEventArraysEqual(result.eventsToRollback, [e_0_0, e_0_1, e_1_0])
         })
 
@@ -118,7 +118,7 @@ describe('syncstate', () => {
           }
           expect(result.newSyncState.upstreamHead).toMatchObject(e_0_1_e_1_0.id)
           expect(result.newSyncState.localHead).toMatchObject(e_1_0_e_2_0.id)
-          expectEventArraysEqual(result.newEvents, [e_0_1_e_1_0])
+          expectEventArraysEqual(result.newEvents, [e_0_1_e_1_0, e_1_0_e_2_0])
           expectEventArraysEqual(result.eventsToRollback, [e_0_1, e_1_0])
         })
 
@@ -148,12 +148,11 @@ describe('syncstate', () => {
             upstreamHead: EventId.ROOT,
             localHead: e_0_0.id,
           })
-          expect(() =>
-            run({
-              syncState,
-              payload: { _tag: 'upstream-rebase', rollbackUntil: e_0_0.id, newEvents: [e_1_0] },
-            }),
-          ).toThrow()
+          const result = run({
+            syncState,
+            payload: { _tag: 'upstream-rebase', rollbackUntil: e_0_0.id, newEvents: [e_1_0] },
+          })
+          expect(result).toMatchObject({ _tag: 'unexpected-error' })
         })
 
         it('should work for empty incoming', () => {
@@ -185,7 +184,8 @@ describe('syncstate', () => {
           upstreamHead: EventId.ROOT,
           localHead: e_0_0.id,
         })
-        expect(() => run({ syncState, payload: { _tag: 'upstream-advance', newEvents: [e_0_1, e_0_0] } })).toThrow()
+        const result = run({ syncState, payload: { _tag: 'upstream-advance', newEvents: [e_0_1, e_0_0] } })
+        expect(result).toMatchObject({ _tag: 'unexpected-error' })
       })
 
       it('should throw error if newEvents are not sorted in ascending order by eventId (global)', () => {
@@ -195,7 +195,8 @@ describe('syncstate', () => {
           upstreamHead: EventId.ROOT,
           localHead: e_0_0.id,
         })
-        expect(() => run({ syncState, payload: { _tag: 'upstream-advance', newEvents: [e_1_0, e_0_0] } })).toThrow()
+        const result = run({ syncState, payload: { _tag: 'upstream-advance', newEvents: [e_1_0, e_0_0] } })
+        expect(result).toMatchObject({ _tag: 'unexpected-error' })
       })
 
       it('should acknowledge pending event when receiving matching event', () => {
@@ -329,6 +330,17 @@ describe('syncstate', () => {
         expect(result.newSyncState.localHead).toMatchObject(e_1_0.id)
         expect(result.newEvents).toStrictEqual([e_1_0])
       })
+
+      it('should fail if incoming event is ≤ local head', () => {
+        const syncState = new SyncState.SyncState({
+          pending: [],
+          rollbackTail: [],
+          upstreamHead: e_1_0.id,
+          localHead: e_1_0.id,
+        })
+        const result = run({ syncState, payload: { _tag: 'upstream-advance', newEvents: [e_0_0] } })
+        expect(result).toMatchObject({ _tag: 'unexpected-error' })
+      })
     })
 
     describe('upstream-advance: rebase', () => {
@@ -513,14 +525,20 @@ const expectEventArraysEqual = (
   })
 }
 
-function expectAdvance(result: SyncState.UpdateResult): asserts result is SyncState.UpdateResultAdvance {
+function expectAdvance(
+  result: typeof SyncState.UpdateResult.Type,
+): asserts result is typeof SyncState.UpdateResultAdvance.Type {
   expect(result._tag).toBe('advance')
 }
 
-function expectRebase(result: SyncState.UpdateResult): asserts result is SyncState.UpdateResultRebase {
+function expectRebase(
+  result: typeof SyncState.UpdateResult.Type,
+): asserts result is typeof SyncState.UpdateResultRebase.Type {
   expect(result._tag).toBe('rebase')
 }
 
-function expectReject(result: SyncState.UpdateResult): asserts result is SyncState.UpdateResultReject {
+function expectReject(
+  result: typeof SyncState.UpdateResult.Type,
+): asserts result is typeof SyncState.UpdateResultReject.Type {
   expect(result._tag).toBe('reject')
 }