@sanity/sdk 3.0.0-rc.0 → 3.0.0-rc.2

package/src/document/listen.test.ts

@@ -205,4 +205,60 @@ describe('sortListenerEvents operator', () => {
     const events = await lastValueFrom(source$.pipe(sortListenerEvents(), toArray()))
     expect(events).toEqual([other])
   })
+
+  it('should discard replayed events already reflected in the base', async () => {
+    // The snapshot is at rev2; a resumed listener replays the event that
+    // produced rev2. It should be discarded instead of clogging the buffer.
+    const sync = createSyncEvent('rev2')
+    const replayed = createMutationEvent({
+      id: 'replayed',
+      previousRev: 'rev1',
+      resultRev: 'rev2',
+    })
+    const source$ = of(sync, replayed)
+    const events = await lastValueFrom(source$.pipe(sortListenerEvents(), toArray()))
+    expect(events).toEqual([sync])
+  })
+
+  it('should discard the replayed part of a chain and apply the rest', async () => {
+    // The snapshot is at rev2; the listener replays rev1->rev2 followed by a
+    // genuinely new rev2->rev3. Only the new event should be emitted.
+    const sync = createSyncEvent('rev2')
+    const replayed = createMutationEvent({
+      id: 'replayed',
+      previousRev: 'rev1',
+      resultRev: 'rev2',
+    })
+    const fresh = createMutationEvent({
+      id: 'fresh',
+      previousRev: 'rev2',
+      resultRev: 'rev3',
+    })
+    const source$ = of(sync, replayed, fresh)
+    const events = await lastValueFrom(source$.pipe(sortListenerEvents(), toArray()))
+    expect(events).toEqual([sync, fresh])
+  })
+
+  it('should discard a multi-event replayed chain that leads up to the base', async () => {
+    const sync = createSyncEvent('rev3')
+    // two replayed events that together lead up to the base revision
+    const replayedA = createMutationEvent({
+      id: 'replayedA',
+      previousRev: 'rev1',
+      resultRev: 'rev2',
+    })
+    const replayedB = createMutationEvent({
+      id: 'replayedB',
+      previousRev: 'rev2',
+      resultRev: 'rev3',
+    })
+    const fresh = createMutationEvent({
+      id: 'fresh',
+      previousRev: 'rev3',
+      resultRev: 'rev4',
+    })
+    const source$ = of(sync, replayedA, replayedB, fresh)
+    const events = await lastValueFrom(source$.pipe(sortListenerEvents(), toArray()))
+    expect(events).toEqual([sync, fresh])
+  })
 })

package/src/document/listen.ts

@@ -20,7 +20,7 @@ import {type StoreContext} from '../store/defineStore'
 import {type DocumentStoreState} from './documentStore'
 import {processMutations} from './processMutations'
 
-const DEFAULT_MAX_BUFFER_SIZE = 20
+export const DEFAULT_MAX_BUFFER_SIZE = 20
 const DEFAULT_DEADLINE_MS = 30000
 
 export interface RemoteDocument {
@@ -30,6 +30,13 @@ export interface RemoteDocument {
   revision?: string
   previousRev?: string
   timestamp: string
+  /**
+   * The raw mutations from the listener event that produced this document.
+   * Only present for `'mutation'` events. These carry the operational intent
+   * of the remote change (e.g. keyed patches) before it is collapsed into a
+   * whole document snapshot.
+   */
+  mutations?: Mutation[]
 }
 
 export interface SyncEvent {
@@ -90,11 +97,50 @@ interface SortListenerEventsOptions {
   resolveChainDeadline?: number
 }
 
+/**
+ * Orders the given mutation events into chains where each event's
+ * `previousRev` points at another event's `resultRev`. Events whose
+ * `previousRev` matches no other event start a new chain. The buffer may have
+ * multiple holes in it, so multiple chains can exist at once.
+ */
+function toOrderedChains(events: MutationEvent[]): MutationEvent[][] {
+  const chainStarts = events.filter(
+    (event) => !events.some((other) => other.resultRev === event.previousRev),
+  )
+
+  return chainStarts.map((start) => {
+    const chain: MutationEvent[] = []
+    let current: MutationEvent | undefined = start
+    while (current) {
+      chain.push(current)
+      const currentResultRev: string | undefined = current.resultRev
+      current = events.find((event) => event.previousRev === currentResultRev)
+    }
+    return chain
+  })
+}
+
+/**
+ * Discards the leading events of a chain up to and including the event whose
+ * `resultRev` equals the given revision. These events describe changes that
+ * are already reflected in the current base (e.g. events replayed by a
+ * resumed listener after the snapshot was refetched).
+ */
+function discardChainTo(chain: MutationEvent[], revision: string | undefined): MutationEvent[] {
+  const revisionIndex = chain.findIndex((event) => event.resultRev === revision)
+  if (revisionIndex === -1) return chain
+  return chain.slice(revisionIndex + 1)
+}
+
 /**
  * Takes an input observable of listener events that might arrive out of order, and emits them in sequence
  * If we receive mutation events that doesn't line up in [previousRev, resultRev] pairs we'll put them in a buffer and
  * check if we have an unbroken chain every time we receive a new event
  *
+ * Buffered chains that lead up to the current base revision are discarded:
+ * they describe changes already reflected in the base (e.g. events replayed
+ * by a resumed listener after the snapshot was refetched)
+ *
  * If the buffer grows beyond `maxBufferSize`, or if `resolveChainDeadline` milliseconds passes before the chain resolves
  * an OutOfSyncError will be thrown on the stream
  *
@@ -125,38 +171,40 @@ export function sortListenerEvents(options?: SortListenerEventsOptions) {
                 'Invalid state. Cannot process mutation event without a base sync event',
               )
             }
-            // Add the new mutation event into the buffer
-            const buffer = state.buffer.concat(event)
-            const emitEvents: MutationEvent[] = []
-            let baseRevision = state.base.revision
-            let progress = true
+            const baseRevision = state.base.revision
+
+            // Order the buffered events (plus the new one) into chains, then
+            // drop the parts of each chain the base already reflects.
+            const chains = toOrderedChains(state.buffer.concat(event))
+              .map((chain) => discardChainTo(chain, baseRevision))
+              .filter((chain) => chain.length > 0)
+
+            // A chain whose head continues the current base revision can be
+            // applied. There can be at most one; anything else stays buffered.
+            const applicable = chains.find((chain) => chain[0].previousRev === baseRevision)
+            const buffer = chains.filter((chain) => chain !== applicable).flat()
 
-            // Try to apply as many buffered mutations as possible.
-            while (progress) {
-              progress = false
-              // Look for a mutation whose previousRev matches the current base.
-              const idx = buffer.findIndex((e) => e.previousRev === baseRevision)
-              if (idx !== -1) {
-                // Remove the event from the buffer and “apply” it.
-                const [next] = buffer.splice(idx, 1)
-                emitEvents.push(next)
-                // If the mutation is a deletion, the new base revision is undefined.
-                baseRevision = next.transition === 'disappear' ? undefined : next.resultRev
-                progress = true
+            if (applicable) {
+              const last = applicable[applicable.length - 1]
+              return {
+                // If the chain ends in a deletion, the new base revision is undefined.
+                base: {revision: last.transition === 'disappear' ? undefined : last.resultRev},
+                buffer,
+                emitEvents: applicable,
               }
             }
 
             if (buffer.length >= maxBufferSize) {
               throw new MaxBufferExceededError(
                 `Too many unchainable mutation events (${buffer.length}) waiting to resolve.`,
-                {base: {revision: baseRevision}, buffer, emitEvents},
+                {base: {revision: baseRevision}, buffer, emitEvents: []},
               )
             }
 
             return {
               base: {revision: baseRevision},
               buffer,
-              emitEvents,
+              emitEvents: [],
             }
           }
           // Any other event is simply forwarded.
@@ -201,7 +249,11 @@ export const listen = (
 
   return sharedListener.events.pipe(
     concatMap((e) => {
-      if (e.type === 'welcome') {
+      // `welcome` means a fresh listener connection and `reset` means a
+      // resumed listener could not replay what was missed; both require
+      // refetching the snapshot. a `welcomeback` means the resume succeeded
+      // and the missed events were replayed, so no refetch is needed
+      if (e.type === 'welcome' || e.type === 'reset') {
         return fetchDocument(documentId).pipe(
           map((document): SyncEvent => ({type: 'sync', document})),
         )
@@ -251,6 +303,7 @@ export const listen = (
         document: document ?? null,
         revision: transactionId,
         timestamp,
+        mutations: next.mutations as Mutation[],
         ...(previousRev && {previousRev}),
       }
     }),

package/src/document/listenerEventOperators.test.ts

@@ -0,0 +1,311 @@
+import {
+  type ListenEvent,
+  type MutationEvent,
+  type ReconnectEvent,
+  type WelcomeEvent,
+} from '@sanity/client'
+import {type SanityDocument} from '@sanity/types'
+import {from, lastValueFrom, type Observable, Subject} from 'rxjs'
+import {toArray} from 'rxjs/operators'
+import {afterEach, describe, expect, it, vi} from 'vitest'
+
+import {dedupeListenerEvents, groupTransactionEvents} from './listenerEventOperators'
+
+function listenEventsOf(
+  ...events: ListenEvent<SanityDocument>[]
+): Observable<ListenEvent<SanityDocument>> {
+  return from(events)
+}
+
+function createMutationEvent({
+  transactionId,
+  documentId = 'doc1',
+  transactionCurrentEvent = 1,
+  transactionTotalEvents = 1,
+}: {
+  transactionId: string
+  documentId?: string
+  transactionCurrentEvent?: number
+  transactionTotalEvents?: number
+}): MutationEvent {
+  return {
+    type: 'mutation',
+    documentId,
+    eventId: `${transactionId}#${documentId}`,
+    identity: 'user',
+    mutations: [],
+    timestamp: new Date().toISOString(),
+    transactionId,
+    transactionCurrentEvent,
+    transactionTotalEvents,
+    transition: 'update',
+    visibility: 'query',
+    resultRev: transactionId,
+  }
+}
+
+const welcomeEvent: WelcomeEvent = {type: 'welcome', listenerName: 'listener'}
+const reconnectEvent = {type: 'reconnect'} as ReconnectEvent
+
+afterEach(() => {
+  vi.useRealTimers()
+})
+
+describe('dedupeListenerEvents', () => {
+  it('passes unique mutation events through', async () => {
+    const m1 = createMutationEvent({transactionId: 'txn1'})
+    const m2 = createMutationEvent({transactionId: 'txn2'})
+    const events = await lastValueFrom(
+      listenEventsOf(m1, m2).pipe(dedupeListenerEvents(), toArray()),
+    )
+    expect(events).toEqual([m1, m2])
+  })
+
+  it('drops replayed duplicates of the same transaction and document', async () => {
+    const m1 = createMutationEvent({transactionId: 'txn1'})
+    const replay = createMutationEvent({transactionId: 'txn1'})
+    const events = await lastValueFrom(
+      listenEventsOf(m1, replay).pipe(dedupeListenerEvents(), toArray()),
+    )
+    expect(events).toEqual([m1])
+  })
+
+  it('does not dedupe events of the same transaction for different documents', async () => {
+    const draft = createMutationEvent({transactionId: 'txn1', documentId: 'drafts.doc1'})
+    const published = createMutationEvent({transactionId: 'txn1', documentId: 'doc1'})
+    const events = await lastValueFrom(
+      listenEventsOf(draft, published).pipe(dedupeListenerEvents(), toArray()),
+    )
+    expect(events).toEqual([draft, published])
+  })
+
+  it('passes non-mutation events through untouched', async () => {
+    const events = await lastValueFrom(
+      listenEventsOf(welcomeEvent, reconnectEvent).pipe(dedupeListenerEvents(), toArray()),
+    )
+    expect(events).toEqual([welcomeEvent, reconnectEvent])
+  })
+
+  it('allows a key again after its TTL has passed', async () => {
+    vi.useFakeTimers()
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source
+      .pipe(dedupeListenerEvents({ttl: 1000}))
+      .subscribe((e) => received.push(e))
+
+    source.next(createMutationEvent({transactionId: 'txn1'}))
+    source.next(createMutationEvent({transactionId: 'txn1'}))
+    expect(received).toHaveLength(1)
+
+    vi.advanceTimersByTime(1001)
+    source.next(createMutationEvent({transactionId: 'txn1'}))
+    expect(received).toHaveLength(2)
+
+    subscription.unsubscribe()
+  })
+
+  it('evicts the oldest entries when the cache exceeds maxEntries', async () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source
+      .pipe(dedupeListenerEvents({maxEntries: 2}))
+      .subscribe((e) => received.push(e))
+
+    source.next(createMutationEvent({transactionId: 'txn1'}))
+    source.next(createMutationEvent({transactionId: 'txn2'}))
+    source.next(createMutationEvent({transactionId: 'txn3'}))
+    // txn1 was evicted to make room, so a replay of it passes through again
+    source.next(createMutationEvent({transactionId: 'txn1'}))
+    expect(received).toHaveLength(4)
+
+    subscription.unsubscribe()
+  })
+})
+
+describe('groupTransactionEvents', () => {
+  it('passes single-event transactions through immediately', async () => {
+    const m1 = createMutationEvent({transactionId: 'txn1'})
+    const events = await lastValueFrom(listenEventsOf(m1).pipe(groupTransactionEvents(), toArray()))
+    expect(events).toEqual([m1])
+  })
+
+  it('holds multi-document transaction events until the transaction is complete', () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source.pipe(groupTransactionEvents()).subscribe((e) => received.push(e))
+
+    const draftEvent = createMutationEvent({
+      transactionId: 'txn-publish',
+      documentId: 'drafts.doc1',
+      transactionCurrentEvent: 1,
+      transactionTotalEvents: 2,
+    })
+    const publishedEvent = createMutationEvent({
+      transactionId: 'txn-publish',
+      documentId: 'doc1',
+      transactionCurrentEvent: 2,
+      transactionTotalEvents: 2,
+    })
+
+    source.next(draftEvent)
+    expect(received).toEqual([])
+
+    source.next(publishedEvent)
+    expect(received).toEqual([draftEvent, publishedEvent])
+
+    subscription.unsubscribe()
+  })
+
+  it('holds intervening events until the open transaction completes, then releases them after it', () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source.pipe(groupTransactionEvents()).subscribe((e) => received.push(e))
+
+    const draftEvent = createMutationEvent({
+      transactionId: 'txn-publish',
+      documentId: 'drafts.doc1',
+      transactionCurrentEvent: 1,
+      transactionTotalEvents: 2,
+    })
+    // an unrelated single-document mutation interleaved between the two
+    // halves of the publish (legacy listener pipeline behavior)
+    const intervening = createMutationEvent({transactionId: 'txn-other', documentId: 'doc9'})
+    const publishedEvent = createMutationEvent({
+      transactionId: 'txn-publish',
+      documentId: 'doc1',
+      transactionCurrentEvent: 2,
+      transactionTotalEvents: 2,
+    })
+
+    source.next(draftEvent)
+    source.next(intervening)
+    expect(received).toEqual([])
+
+    source.next(publishedEvent)
+    expect(received).toEqual([draftEvent, publishedEvent, intervening])
+
+    subscription.unsubscribe()
+  })
+
+  it('groups a held multi-document transaction after the open one completes', () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source.pipe(groupTransactionEvents()).subscribe((e) => received.push(e))
+
+    const a1 = createMutationEvent({
+      transactionId: 'txn-a',
+      documentId: 'drafts.doc1',
+      transactionTotalEvents: 2,
+    })
+    const b1 = createMutationEvent({
+      transactionId: 'txn-b',
+      documentId: 'drafts.doc2',
+      transactionTotalEvents: 2,
+    })
+    const b2 = createMutationEvent({
+      transactionId: 'txn-b',
+      documentId: 'doc2',
+      transactionTotalEvents: 2,
+    })
+    const a2 = createMutationEvent({
+      transactionId: 'txn-a',
+      documentId: 'doc1',
+      transactionTotalEvents: 2,
+    })
+
+    source.next(a1)
+    source.next(b1)
+    source.next(b2)
+    expect(received).toEqual([])
+
+    source.next(a2)
+    // txn-a completes and is emitted contiguously; the held txn-b events are
+    // re-processed and, being complete, emitted contiguously right after
+    expect(received).toEqual([a1, a2, b1, b2])
+
+    subscription.unsubscribe()
+  })
+
+  it('releases everything in arrival order when the held queue exceeds its cap', () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source
+      .pipe(groupTransactionEvents({maxHeldEvents: 2}))
+      .subscribe((e) => received.push(e))
+
+    const partial = createMutationEvent({
+      transactionId: 'txn-incomplete',
+      documentId: 'drafts.doc1',
+      transactionTotalEvents: 2,
+    })
+    const held1 = createMutationEvent({transactionId: 'txn-h1', documentId: 'doc8'})
+    const held2 = createMutationEvent({transactionId: 'txn-h2', documentId: 'doc9'})
+    const overflow = createMutationEvent({transactionId: 'txn-h3', documentId: 'doc10'})
+
+    source.next(partial)
+    source.next(held1)
+    source.next(held2)
+    expect(received).toEqual([])
+
+    source.next(overflow)
+    expect(received).toEqual([partial, held1, held2, overflow])
+
+    subscription.unsubscribe()
+  })
+
+  it('flushes buffered events before non-mutation events', () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source.pipe(groupTransactionEvents()).subscribe((e) => received.push(e))
+
+    const partial = createMutationEvent({
+      transactionId: 'txn-incomplete',
+      documentId: 'drafts.doc1',
+      transactionTotalEvents: 2,
+    })
+
+    source.next(partial)
+    source.next(reconnectEvent)
+    expect(received).toEqual([partial, reconnectEvent])
+
+    subscription.unsubscribe()
+  })
+
+  it('flushes an incomplete transaction and its held events after the deadline', () => {
+    vi.useFakeTimers()
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source
+      .pipe(groupTransactionEvents({flushDeadline: 500}))
+      .subscribe((e) => received.push(e))
+
+    const partial = createMutationEvent({
+      transactionId: 'txn-incomplete',
+      documentId: 'drafts.doc1',
+      transactionTotalEvents: 2,
+    })
+    const heldBehind = createMutationEvent({transactionId: 'txn-held', documentId: 'doc9'})
+
+    source.next(partial)
+    source.next(heldBehind)
+    expect(received).toEqual([])
+
+    vi.advanceTimersByTime(500)
+    expect(received).toEqual([partial, heldBehind])
+
+    subscription.unsubscribe()
+  })
+
+  it('flushes buffered events on completion', async () => {
+    const partial = createMutationEvent({
+      transactionId: 'txn-incomplete',
+      documentId: 'drafts.doc1',
+      transactionTotalEvents: 2,
+    })
+    const events = await lastValueFrom(
+      listenEventsOf(partial).pipe(groupTransactionEvents(), toArray()),
+    )
+    expect(events).toEqual([partial])
+  })
+})