@livestore/common 0.3.0-dev.16 → 0.3.0-dev.18

package/src/schema/MutationEvent.ts

@@ -124,7 +124,7 @@ export const makeMutationEventSchema = <TSchema extends LiveStoreSchema>(
   schema: TSchema,
 ): ForMutationDefRecord<TSchema['_MutationDefMapType']> =>
   Schema.Union(
-    ...[...schema.mutations.values()].map((def) =>
+    ...[...schema.mutations.map.values()].map((def) =>
       Schema.Struct({
         mutation: Schema.Literal(def.name),
         args: def.schema,
@@ -140,7 +140,7 @@ export const makeMutationEventPartialSchema = <TSchema extends LiveStoreSchema>(
   schema: TSchema,
 ): MutationEventPartialSchema<TSchema['_MutationDefMapType']> =>
   Schema.Union(
-    ...[...schema.mutations.values()].map((def) =>
+    ...[...schema.mutations.map.values()].map((def) =>
       Schema.Struct({
         mutation: Schema.Literal(def.name),
         args: def.schema,
@@ -169,7 +169,7 @@ export class EncodedWithMeta extends Schema.Class<EncodedWithMeta>('MutationEven
   toJSON = (): any => {
     // Only used for logging/debugging
     // - More readable way to print the id + parentId
-    // - not including `meta`
+    // - not including `meta`, `clientId`, `sessionId`
     return {
       id: `(${this.id.global},${this.id.client}) → (${this.parentId.global},${this.parentId.client})`,
       mutation: this.mutation,

package/src/schema/db-schema/dsl/field-defs.ts

@@ -53,8 +53,7 @@ export type ColDefFn<TColumnType extends FieldColumnType> = {
     const TNullable extends boolean = false,
     const TDefault extends TDecoded | SqlDefaultValue | NoDefault | (TNullable extends true ? null : never) = NoDefault,
     const TPrimaryKey extends boolean = false,
-  >(args: {
-    schema?: Schema.Schema<TDecoded, TEncoded>
+  >(args: {    schema?: Schema.Schema<TDecoded, TEncoded>
     default?: TDefault
     nullable?: TNullable
     primaryKey?: TPrimaryKey

package/src/schema/db-schema/dsl/mod.ts

@@ -30,7 +30,7 @@ export type DbSchemaFromInputSchema<TSchemaInput extends DbSchemaInput> =
 export const makeDbSchema = <TDbSchemaInput extends DbSchemaInput>(
   schema: TDbSchemaInput,
 ): DbSchemaFromInputSchema<TDbSchemaInput> => {
-  return Array.isArray(schema) ? Object.fromEntries(schema.map((_) => [_.name, _])) : schema
+  return Array.isArray(schema) ? Object.fromEntries(schema.map((_) => [_.name, _])) : schema as any
 }
 
 export const table = <TTableName extends string, TColumns extends Columns, TIndexes extends Index[]>(

package/src/schema/mutations.ts

@@ -2,7 +2,10 @@ import { Schema } from '@livestore/utils/effect'
 
 import type { BindValues } from '../sql-queries/sql-queries.js'
 
-export type MutationDefMap = Map<string | 'livestore.RawSql', MutationDef.Any>
+export type MutationDefMap = {
+  map: Map<string | 'livestore.RawSql', MutationDef.Any>
+  wasProvided: boolean
+}
 export type MutationDefRecord = {
   'livestore.RawSql': RawSqlMutation
   [name: string]: MutationDef.Any

package/src/schema/schema.ts

@@ -72,29 +72,32 @@ export const makeSchema = <TInputSchema extends InputSchema>(
     tables.set(tableDef.sqliteDef.name, tableDef)
   }
 
-  const mutations: MutationDefMap = new Map()
+  const mutations: MutationDefMap = {
+    map: new Map(),
+    wasProvided: inputSchema.mutations !== undefined,
+  }
 
   if (isReadonlyArray(inputSchema.mutations)) {
     for (const mutation of inputSchema.mutations) {
-      mutations.set(mutation.name, mutation)
+      mutations.map.set(mutation.name, mutation)
     }
   } else {
     for (const mutation of Object.values(inputSchema.mutations ?? {})) {
-      if (mutations.has(mutation.name)) {
+      if (mutations.map.has(mutation.name)) {
         shouldNeverHappen(`Duplicate mutation name: ${mutation.name}. Please use unique names for mutations.`)
       }
-      mutations.set(mutation.name, mutation)
+      mutations.map.set(mutation.name, mutation)
     }
   }
 
-  mutations.set(rawSqlMutation.name, rawSqlMutation)
+  mutations.map.set(rawSqlMutation.name, rawSqlMutation)
 
   for (const tableDef of tables.values()) {
     if (tableHasDerivedMutations(tableDef)) {
       const derivedMutationDefs = makeDerivedMutationDefsForTable(tableDef)
-      mutations.set(derivedMutationDefs.insert.name, derivedMutationDefs.insert)
-      mutations.set(derivedMutationDefs.update.name, derivedMutationDefs.update)
-      mutations.set(derivedMutationDefs.delete.name, derivedMutationDefs.delete)
+      mutations.map.set(derivedMutationDefs.insert.name, derivedMutationDefs.insert)
+      mutations.map.set(derivedMutationDefs.update.name, derivedMutationDefs.update)
+      mutations.map.set(derivedMutationDefs.delete.name, derivedMutationDefs.delete)
     }
   }
 
@@ -114,6 +117,15 @@ export const makeSchema = <TInputSchema extends InputSchema>(
   } satisfies LiveStoreSchema
 }
 
+export const getMutationDef = <TSchema extends LiveStoreSchema>(schema: TSchema, mutationName: string) => {
+  const mutationDef = schema.mutations.map.get(mutationName)
+  if (mutationDef === undefined) {
+    const extraInfo = schema.mutations.wasProvided ? '' : ' Please provide \`mutations\` in the schema options.'
+    return shouldNeverHappen(`No mutation definition found for \`${mutationName}\`.${extraInfo}`)
+  }
+  return mutationDef
+}
+
 export namespace FromInputSchema {
   export type DeriveSchema<TInputSchema extends InputSchema> = LiveStoreSchema<
     DbSchemaFromInputSchemaTables<TInputSchema['tables']>,

package/src/schema-management/validate-mutation-defs.ts

@@ -11,7 +11,7 @@ export const validateSchema = (schema: LiveStoreSchema, schemaManager: SchemaMan
     const registeredMutationDefInfos = schemaManager.getMutationDefInfos()
 
     const missingMutationDefs = registeredMutationDefInfos.filter(
-      (registeredMutationDefInfo) => !schema.mutations.has(registeredMutationDefInfo.mutationName),
+      (registeredMutationDefInfo) => !schema.mutations.map.has(registeredMutationDefInfo.mutationName),
     )
 
     if (missingMutationDefs.length > 0) {
@@ -20,7 +20,7 @@ export const validateSchema = (schema: LiveStoreSchema, schemaManager: SchemaMan
       })
     }
 
-    for (const [, mutationDef] of schema.mutations) {
+    for (const [, mutationDef] of schema.mutations.map) {
       const registeredMutationDefInfo = registeredMutationDefInfos.find(
         (info) => info.mutationName === mutationDef.name,
       )

package/src/sync/ClientSessionSyncProcessor.ts

@@ -1,11 +1,11 @@
 import { LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
 import type { Runtime, Scope } from '@livestore/utils/effect'
-import { Effect, Queue, Schema, Stream, Subscribable } from '@livestore/utils/effect'
+import { BucketQueue, Effect, FiberHandle, Queue, Schema, Stream, Subscribable } from '@livestore/utils/effect'
 import * as otel from '@opentelemetry/api'
 
 import type { ClientSession, UnexpectedError } from '../adapter-types.js'
 import * as EventId from '../schema/EventId.js'
-import { type LiveStoreSchema } from '../schema/mod.js'
+import { getMutationDef, type LiveStoreSchema } from '../schema/mod.js'
 import * as MutationEvent from '../schema/MutationEvent.js'
 import * as SyncState from './syncstate.js'
 
@@ -26,6 +26,7 @@ export const makeClientSessionSyncProcessor = ({
   rollback,
   refreshTables,
   span,
+  params,
 }: {
   schema: LiveStoreSchema
   clientSession: ClientSession
@@ -40,6 +41,9 @@ export const makeClientSessionSyncProcessor = ({
   rollback: (changeset: Uint8Array) => void
   refreshTables: (tables: Set<string>) => void
   span: otel.Span
+  params: {
+    leaderPushBatchSize: number
+  }
 }): ClientSessionSyncProcessor => {
   const mutationEventSchema = MutationEvent.makeMutationEventSchemaMemo(schema)
 
@@ -54,17 +58,18 @@ export const makeClientSessionSyncProcessor = ({
   }
 
   const syncStateUpdateQueue = Queue.unbounded<SyncState.SyncState>().pipe(Effect.runSync)
-  const isLocalEvent = (mutationEventEncoded: MutationEvent.EncodedWithMeta) => {
-    const mutationDef = schema.mutations.get(mutationEventEncoded.mutation)!
-    return mutationDef.options.clientOnly
-  }
+  const isLocalEvent = (mutationEventEncoded: MutationEvent.EncodedWithMeta) =>
+    getMutationDef(schema, mutationEventEncoded.mutation).options.clientOnly
+
+  /** We're queuing push requests to reduce the number of messages sent to the leader by batching them */
+  const leaderPushQueue = BucketQueue.make<MutationEvent.EncodedWithMeta>().pipe(Effect.runSync)
 
   const push: ClientSessionSyncProcessor['push'] = (batch, { otelContext }) => {
     // TODO validate batch
 
     let baseEventId = syncStateRef.current.localHead
     const encodedMutationEvents = batch.map((mutationEvent) => {
-      const mutationDef = schema.mutations.get(mutationEvent.mutation)!
+      const mutationDef = getMutationDef(schema, mutationEvent.mutation)
       const nextIdPair = EventId.nextPair(baseEventId, mutationDef.options.clientOnly)
       baseEventId = nextIdPair.id
       return new MutationEvent.EncodedWithMeta(
@@ -84,6 +89,10 @@ export const makeClientSessionSyncProcessor = ({
       isEqualEvent: MutationEvent.isEqualEncoded,
     })
 
+    if (updateResult._tag === 'unexpected-error') {
+      return shouldNeverHappen('Unexpected error in client-session-sync-processor', updateResult.cause)
+    }
+
     span.addEvent('local-push', {
       batchSize: encodedMutationEvents.length,
       updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
@@ -108,13 +117,17 @@ export const makeClientSessionSyncProcessor = ({
     }
 
     // console.debug('pushToLeader', encodedMutationEvents.length, ...encodedMutationEvents.map((_) => _.toJSON()))
-    clientSession.leaderThread.mutations
-      .push(encodedMutationEvents)
-      .pipe(Effect.tapCauseLogPretty, Effect.provide(runtime), Effect.runFork)
+    BucketQueue.offerAll(leaderPushQueue, encodedMutationEvents).pipe(Effect.runSync)
 
     return { writeTables }
   }
 
+  const debugInfo = {
+    rebaseCount: 0,
+    advanceCount: 0,
+    rejectCount: 0,
+  }
+
   const otelContext = otel.trace.setSpan(otel.context.active(), span)
 
   const boot: ClientSessionSyncProcessor['boot'] = Effect.gen(function* () {
@@ -133,6 +146,20 @@ export const makeClientSessionSyncProcessor = ({
       )
     }
 
+    const leaderPushingFiberHandle = yield* FiberHandle.make()
+
+    const backgroundLeaderPushing = Effect.gen(function* () {
+      const batch = yield* BucketQueue.takeBetween(leaderPushQueue, 1, params.leaderPushBatchSize)
+      yield* clientSession.leaderThread.mutations.push(batch).pipe(
+        Effect.catchTag('LeaderAheadError', () => {
+          debugInfo.rejectCount++
+          return BucketQueue.clear(leaderPushQueue)
+        }),
+      )
+    }).pipe(Effect.forever, Effect.interruptible, Effect.tapCauseLogPretty)
+
+    yield* FiberHandle.run(leaderPushingFiberHandle, backgroundLeaderPushing)
+
     yield* clientSession.leaderThread.mutations.pull.pipe(
       Stream.tap(({ payload, remaining }) =>
         Effect.gen(function* () {
@@ -148,9 +175,10 @@ export const makeClientSessionSyncProcessor = ({
             isEqualEvent: MutationEvent.isEqualEncoded,
           })
 
-          if (updateResult._tag === 'reject') {
-            debugger
-            throw new Error('TODO: implement reject in client-session-sync-queue for pull')
+          if (updateResult._tag === 'unexpected-error') {
+            return yield* Effect.fail(updateResult.cause)
+          } else if (updateResult._tag === 'reject') {
+            return shouldNeverHappen('Unexpected reject in client-session-sync-processor', updateResult)
           }
 
           syncStateRef.current = updateResult.newSyncState
@@ -165,11 +193,21 @@ export const makeClientSessionSyncProcessor = ({
               res: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
               remaining,
             })
+
+            debugInfo.rebaseCount++
+
+            yield* FiberHandle.clear(leaderPushingFiberHandle)
+
+            // Reset the leader push queue since we're rebasing and will push again
+            yield* BucketQueue.clear(leaderPushQueue)
+
+            yield* FiberHandle.run(leaderPushingFiberHandle, backgroundLeaderPushing)
+
             if (LS_DEV) {
               Effect.logDebug(
                 'pull:rebase: rollback',
                 updateResult.eventsToRollback.length,
-                ...updateResult.eventsToRollback.map((_) => _.toJSON()),
+                ...updateResult.eventsToRollback.slice(0, 10).map((_) => _.toJSON()),
               ).pipe(Effect.provide(runtime), Effect.runSync)
             }
 
@@ -181,9 +219,7 @@ export const makeClientSessionSyncProcessor = ({
               }
             }
 
-            clientSession.leaderThread.mutations
-              .push(updateResult.newSyncState.pending)
-              .pipe(Effect.tapCauseLogPretty, Effect.provide(runtime), Effect.runFork)
+            yield* BucketQueue.offerAll(leaderPushQueue, updateResult.newSyncState.pending)
           } else {
             span.addEvent('pull:advance', {
               payloadTag: payload._tag,
@@ -192,6 +228,8 @@ export const makeClientSessionSyncProcessor = ({
               res: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
               remaining,
             })
+
+            debugInfo.advanceCount++
           }
 
           if (updateResult.newEvents.length === 0) return
@@ -208,10 +246,14 @@ export const makeClientSessionSyncProcessor = ({
           }
 
           refreshTables(writeTables)
-        }),
+        }).pipe(
+          Effect.tapCauseLogPretty,
+          Effect.catchAllCause((cause) => Effect.sync(() => clientSession.shutdown(cause))),
+        ),
       ),
       Stream.runDrain,
       Effect.forever, // NOTE Whenever the leader changes, we need to re-start the stream
+      Effect.withSpan('client-session-sync-processor:pull'),
       Effect.tapCauseLogPretty,
       Effect.forkScoped,
     )
@@ -228,6 +270,21 @@ export const makeClientSessionSyncProcessor = ({
       }),
       changes: Stream.fromQueue(syncStateUpdateQueue),
     }),
+    debug: {
+      print: () =>
+        Effect.gen(function* () {
+          console.log('debugInfo', debugInfo)
+          console.log('syncState', syncStateRef.current)
+          const pushQueueSize = yield* BucketQueue.size(leaderPushQueue)
+          console.log('pushQueueSize', pushQueueSize)
+          const pushQueueItems = yield* BucketQueue.peekAll(leaderPushQueue)
+          console.log(
+            'pushQueueItems',
+            pushQueueItems.map((_) => _.toJSON()),
+          )
+        }).pipe(Effect.provide(runtime), Effect.runSync),
+      debugInfo: () => debugInfo,
+    },
   } satisfies ClientSessionSyncProcessor
 }
 
@@ -239,6 +296,12 @@ export interface ClientSessionSyncProcessor {
     writeTables: Set<string>
   }
   boot: Effect.Effect<void, UnexpectedError, Scope.Scope>
-
   syncState: Subscribable.Subscribable<SyncState.SyncState>
+  debug: {
+    print: () => void
+    debugInfo: () => {
+      rebaseCount: number
+      advanceCount: number
+    }
+  }
 }

package/src/sync/sync.ts

@@ -67,13 +67,16 @@ export class InvalidPushError extends Schema.TaggedError<InvalidPushError>()('In
       minimumExpectedId: Schema.Number,
       providedId: Schema.Number,
     }),
-    Schema.TaggedStruct('LeaderAhead', {
-      minimumExpectedId: EventId.EventId,
-      providedId: EventId.EventId,
-    }),
   ),
 }) {}
 
 export class InvalidPullError extends Schema.TaggedError<InvalidPullError>()('InvalidPullError', {
   message: Schema.String,
 }) {}
+
+export class LeaderAheadError extends Schema.TaggedError<LeaderAheadError>()('LeaderAheadError', {
+  minimumExpectedId: EventId.EventId,
+  providedId: EventId.EventId,
+  /** Generation number the client session should use for subsequent pushes */
+  // nextGeneration: Schema.Number,
+}) {}

package/src/sync/syncstate.test.ts

@@ -17,7 +17,7 @@ class TestEvent extends MutationEvent.EncodedWithMeta {
       parentId: EventId.make(parentId),
       mutation: 'a',
       args: payload,
-      meta: {},
+
       clientId: 'static-local-id',
       sessionId: undefined,
     })
@@ -87,7 +87,7 @@ describe('syncstate', () => {
           }
           expect(result.newSyncState.upstreamHead).toMatchObject(e_0_1_e_1_1.id)
           expect(result.newSyncState.localHead).toMatchObject(e_1_0_e_2_0.id)
-          expectEventArraysEqual(result.newEvents, [e_0_0_e_1_0, e_0_1_e_1_1])
+          expectEventArraysEqual(result.newEvents, [e_0_0_e_1_0, e_0_1_e_1_1, e_1_0_e_2_0])
           expectEventArraysEqual(result.eventsToRollback, [e_0_0, e_0_1, e_1_0])
         })
 
@@ -118,7 +118,7 @@ describe('syncstate', () => {
           }
           expect(result.newSyncState.upstreamHead).toMatchObject(e_0_1_e_1_0.id)
           expect(result.newSyncState.localHead).toMatchObject(e_1_0_e_2_0.id)
-          expectEventArraysEqual(result.newEvents, [e_0_1_e_1_0])
+          expectEventArraysEqual(result.newEvents, [e_0_1_e_1_0, e_1_0_e_2_0])
           expectEventArraysEqual(result.eventsToRollback, [e_0_1, e_1_0])
         })
 
@@ -148,12 +148,11 @@ describe('syncstate', () => {
             upstreamHead: EventId.ROOT,
             localHead: e_0_0.id,
           })
-          expect(() =>
-            run({
-              syncState,
-              payload: { _tag: 'upstream-rebase', rollbackUntil: e_0_0.id, newEvents: [e_1_0] },
-            }),
-          ).toThrow()
+          const result = run({
+            syncState,
+            payload: { _tag: 'upstream-rebase', rollbackUntil: e_0_0.id, newEvents: [e_1_0] },
+          })
+          expect(result).toMatchObject({ _tag: 'unexpected-error' })
         })
 
         it('should work for empty incoming', () => {
@@ -185,7 +184,8 @@ describe('syncstate', () => {
           upstreamHead: EventId.ROOT,
           localHead: e_0_0.id,
         })
-        expect(() => run({ syncState, payload: { _tag: 'upstream-advance', newEvents: [e_0_1, e_0_0] } })).toThrow()
+        const result = run({ syncState, payload: { _tag: 'upstream-advance', newEvents: [e_0_1, e_0_0] } })
+        expect(result).toMatchObject({ _tag: 'unexpected-error' })
       })
 
       it('should throw error if newEvents are not sorted in ascending order by eventId (global)', () => {
@@ -195,7 +195,8 @@ describe('syncstate', () => {
           upstreamHead: EventId.ROOT,
           localHead: e_0_0.id,
         })
-        expect(() => run({ syncState, payload: { _tag: 'upstream-advance', newEvents: [e_1_0, e_0_0] } })).toThrow()
+        const result = run({ syncState, payload: { _tag: 'upstream-advance', newEvents: [e_1_0, e_0_0] } })
+        expect(result).toMatchObject({ _tag: 'unexpected-error' })
       })
 
       it('should acknowledge pending event when receiving matching event', () => {
@@ -329,6 +330,17 @@ describe('syncstate', () => {
         expect(result.newSyncState.localHead).toMatchObject(e_1_0.id)
         expect(result.newEvents).toStrictEqual([e_1_0])
       })
+
+      it('should fail if incoming event is ≤ local head', () => {
+        const syncState = new SyncState.SyncState({
+          pending: [],
+          rollbackTail: [],
+          upstreamHead: e_1_0.id,
+          localHead: e_1_0.id,
+        })
+        const result = run({ syncState, payload: { _tag: 'upstream-advance', newEvents: [e_0_0] } })
+        expect(result).toMatchObject({ _tag: 'unexpected-error' })
+      })
     })
 
     describe('upstream-advance: rebase', () => {
@@ -513,14 +525,20 @@ const expectEventArraysEqual = (
   })
 }
 
-function expectAdvance(result: SyncState.UpdateResult): asserts result is SyncState.UpdateResultAdvance {
+function expectAdvance(
+  result: typeof SyncState.UpdateResult.Type,
+): asserts result is typeof SyncState.UpdateResultAdvance.Type {
   expect(result._tag).toBe('advance')
 }
 
-function expectRebase(result: SyncState.UpdateResult): asserts result is SyncState.UpdateResultRebase {
+function expectRebase(
+  result: typeof SyncState.UpdateResult.Type,
+): asserts result is typeof SyncState.UpdateResultRebase.Type {
   expect(result._tag).toBe('rebase')
 }
 
-function expectReject(result: SyncState.UpdateResult): asserts result is SyncState.UpdateResultReject {
+function expectReject(
+  result: typeof SyncState.UpdateResult.Type,
+): asserts result is typeof SyncState.UpdateResultReject.Type {
   expect(result._tag).toBe('reject')
 }