@sanity/sdk 3.0.0-rc.0 → 3.0.0-rc.1

package/src/document/listen.ts

@@ -20,7 +20,7 @@ import {type StoreContext} from '../store/defineStore'
 import {type DocumentStoreState} from './documentStore'
 import {processMutations} from './processMutations'
 
-const DEFAULT_MAX_BUFFER_SIZE = 20
+export const DEFAULT_MAX_BUFFER_SIZE = 20
 const DEFAULT_DEADLINE_MS = 30000
 
 export interface RemoteDocument {
@@ -30,6 +30,13 @@ export interface RemoteDocument {
   revision?: string
   previousRev?: string
   timestamp: string
+  /**
+   * The raw mutations from the listener event that produced this document.
+   * Only present for `'mutation'` events. These carry the operational intent
+   * of the remote change (e.g. keyed patches) before it is collapsed into a
+   * whole document snapshot.
+   */
+  mutations?: Mutation[]
 }
 
 export interface SyncEvent {
@@ -90,11 +97,50 @@ interface SortListenerEventsOptions {
   resolveChainDeadline?: number
 }
 
+/**
+ * Orders the given mutation events into chains where each event's
+ * `previousRev` points at another event's `resultRev`. Events whose
+ * `previousRev` matches no other event start a new chain. The buffer may have
+ * multiple holes in it, so multiple chains can exist at once.
+ */
+function toOrderedChains(events: MutationEvent[]): MutationEvent[][] {
+  const chainStarts = events.filter(
+    (event) => !events.some((other) => other.resultRev === event.previousRev),
+  )
+
+  return chainStarts.map((start) => {
+    const chain: MutationEvent[] = []
+    let current: MutationEvent | undefined = start
+    while (current) {
+      chain.push(current)
+      const currentResultRev: string | undefined = current.resultRev
+      current = events.find((event) => event.previousRev === currentResultRev)
+    }
+    return chain
+  })
+}
+
+/**
+ * Discards the leading events of a chain up to and including the event whose
+ * `resultRev` equals the given revision. These events describe changes that
+ * are already reflected in the current base (e.g. events replayed by a
+ * resumed listener after the snapshot was refetched).
+ */
+function discardChainTo(chain: MutationEvent[], revision: string | undefined): MutationEvent[] {
+  const revisionIndex = chain.findIndex((event) => event.resultRev === revision)
+  if (revisionIndex === -1) return chain
+  return chain.slice(revisionIndex + 1)
+}
+
 /**
  * Takes an input observable of listener events that might arrive out of order, and emits them in sequence
  * If we receive mutation events that doesn't line up in [previousRev, resultRev] pairs we'll put them in a buffer and
  * check if we have an unbroken chain every time we receive a new event
  *
+ * Buffered chains that lead up to the current base revision are discarded:
+ * they describe changes already reflected in the base (e.g. events replayed
+ * by a resumed listener after the snapshot was refetched)
+ *
  * If the buffer grows beyond `maxBufferSize`, or if `resolveChainDeadline` milliseconds passes before the chain resolves
  * an OutOfSyncError will be thrown on the stream
  *
@@ -125,38 +171,40 @@ export function sortListenerEvents(options?: SortListenerEventsOptions) {
                 'Invalid state. Cannot process mutation event without a base sync event',
               )
             }
-            // Add the new mutation event into the buffer
-            const buffer = state.buffer.concat(event)
-            const emitEvents: MutationEvent[] = []
-            let baseRevision = state.base.revision
-            let progress = true
+            const baseRevision = state.base.revision
+
+            // Order the buffered events (plus the new one) into chains, then
+            // drop the parts of each chain the base already reflects.
+            const chains = toOrderedChains(state.buffer.concat(event))
+              .map((chain) => discardChainTo(chain, baseRevision))
+              .filter((chain) => chain.length > 0)
+
+            // A chain whose head continues the current base revision can be
+            // applied. There can be at most one; anything else stays buffered.
+            const applicable = chains.find((chain) => chain[0].previousRev === baseRevision)
+            const buffer = chains.filter((chain) => chain !== applicable).flat()
 
-            // Try to apply as many buffered mutations as possible.
-            while (progress) {
-              progress = false
-              // Look for a mutation whose previousRev matches the current base.
-              const idx = buffer.findIndex((e) => e.previousRev === baseRevision)
-              if (idx !== -1) {
-                // Remove the event from the buffer and “apply” it.
-                const [next] = buffer.splice(idx, 1)
-                emitEvents.push(next)
-                // If the mutation is a deletion, the new base revision is undefined.
-                baseRevision = next.transition === 'disappear' ? undefined : next.resultRev
-                progress = true
+            if (applicable) {
+              const last = applicable[applicable.length - 1]
+              return {
+                // If the chain ends in a deletion, the new base revision is undefined.
+                base: {revision: last.transition === 'disappear' ? undefined : last.resultRev},
+                buffer,
+                emitEvents: applicable,
               }
             }
 
             if (buffer.length >= maxBufferSize) {
               throw new MaxBufferExceededError(
                 `Too many unchainable mutation events (${buffer.length}) waiting to resolve.`,
-                {base: {revision: baseRevision}, buffer, emitEvents},
+                {base: {revision: baseRevision}, buffer, emitEvents: []},
               )
             }
 
             return {
               base: {revision: baseRevision},
               buffer,
-              emitEvents,
+              emitEvents: [],
             }
           }
           // Any other event is simply forwarded.
@@ -201,7 +249,11 @@ export const listen = (
 
   return sharedListener.events.pipe(
     concatMap((e) => {
-      if (e.type === 'welcome') {
+      // `welcome` means a fresh listener connection and `reset` means a
+      // resumed listener could not replay what was missed; both require
+      // refetching the snapshot. a `welcomeback` means the resume succeeded
+      // and the missed events were replayed, so no refetch is needed
+      if (e.type === 'welcome' || e.type === 'reset') {
         return fetchDocument(documentId).pipe(
           map((document): SyncEvent => ({type: 'sync', document})),
         )
@@ -251,6 +303,7 @@ export const listen = (
         document: document ?? null,
         revision: transactionId,
         timestamp,
+        mutations: next.mutations as Mutation[],
         ...(previousRev && {previousRev}),
       }
     }),

package/src/document/listenerEventOperators.test.ts

@@ -0,0 +1,311 @@
+import {
+  type ListenEvent,
+  type MutationEvent,
+  type ReconnectEvent,
+  type WelcomeEvent,
+} from '@sanity/client'
+import {type SanityDocument} from '@sanity/types'
+import {from, lastValueFrom, type Observable, Subject} from 'rxjs'
+import {toArray} from 'rxjs/operators'
+import {afterEach, describe, expect, it, vi} from 'vitest'
+
+import {dedupeListenerEvents, groupTransactionEvents} from './listenerEventOperators'
+
+function listenEventsOf(
+  ...events: ListenEvent<SanityDocument>[]
+): Observable<ListenEvent<SanityDocument>> {
+  return from(events)
+}
+
+function createMutationEvent({
+  transactionId,
+  documentId = 'doc1',
+  transactionCurrentEvent = 1,
+  transactionTotalEvents = 1,
+}: {
+  transactionId: string
+  documentId?: string
+  transactionCurrentEvent?: number
+  transactionTotalEvents?: number
+}): MutationEvent {
+  return {
+    type: 'mutation',
+    documentId,
+    eventId: `${transactionId}#${documentId}`,
+    identity: 'user',
+    mutations: [],
+    timestamp: new Date().toISOString(),
+    transactionId,
+    transactionCurrentEvent,
+    transactionTotalEvents,
+    transition: 'update',
+    visibility: 'query',
+    resultRev: transactionId,
+  }
+}
+
+const welcomeEvent: WelcomeEvent = {type: 'welcome', listenerName: 'listener'}
+const reconnectEvent = {type: 'reconnect'} as ReconnectEvent
+
+afterEach(() => {
+  vi.useRealTimers()
+})
+
+describe('dedupeListenerEvents', () => {
+  it('passes unique mutation events through', async () => {
+    const m1 = createMutationEvent({transactionId: 'txn1'})
+    const m2 = createMutationEvent({transactionId: 'txn2'})
+    const events = await lastValueFrom(
+      listenEventsOf(m1, m2).pipe(dedupeListenerEvents(), toArray()),
+    )
+    expect(events).toEqual([m1, m2])
+  })
+
+  it('drops replayed duplicates of the same transaction and document', async () => {
+    const m1 = createMutationEvent({transactionId: 'txn1'})
+    const replay = createMutationEvent({transactionId: 'txn1'})
+    const events = await lastValueFrom(
+      listenEventsOf(m1, replay).pipe(dedupeListenerEvents(), toArray()),
+    )
+    expect(events).toEqual([m1])
+  })
+
+  it('does not dedupe events of the same transaction for different documents', async () => {
+    const draft = createMutationEvent({transactionId: 'txn1', documentId: 'drafts.doc1'})
+    const published = createMutationEvent({transactionId: 'txn1', documentId: 'doc1'})
+    const events = await lastValueFrom(
+      listenEventsOf(draft, published).pipe(dedupeListenerEvents(), toArray()),
+    )
+    expect(events).toEqual([draft, published])
+  })
+
+  it('passes non-mutation events through untouched', async () => {
+    const events = await lastValueFrom(
+      listenEventsOf(welcomeEvent, reconnectEvent).pipe(dedupeListenerEvents(), toArray()),
+    )
+    expect(events).toEqual([welcomeEvent, reconnectEvent])
+  })
+
+  it('allows a key again after its TTL has passed', async () => {
+    vi.useFakeTimers()
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source
+      .pipe(dedupeListenerEvents({ttl: 1000}))
+      .subscribe((e) => received.push(e))
+
+    source.next(createMutationEvent({transactionId: 'txn1'}))
+    source.next(createMutationEvent({transactionId: 'txn1'}))
+    expect(received).toHaveLength(1)
+
+    vi.advanceTimersByTime(1001)
+    source.next(createMutationEvent({transactionId: 'txn1'}))
+    expect(received).toHaveLength(2)
+
+    subscription.unsubscribe()
+  })
+
+  it('evicts the oldest entries when the cache exceeds maxEntries', async () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source
+      .pipe(dedupeListenerEvents({maxEntries: 2}))
+      .subscribe((e) => received.push(e))
+
+    source.next(createMutationEvent({transactionId: 'txn1'}))
+    source.next(createMutationEvent({transactionId: 'txn2'}))
+    source.next(createMutationEvent({transactionId: 'txn3'}))
+    // txn1 was evicted to make room, so a replay of it passes through again
+    source.next(createMutationEvent({transactionId: 'txn1'}))
+    expect(received).toHaveLength(4)
+
+    subscription.unsubscribe()
+  })
+})
+
+describe('groupTransactionEvents', () => {
+  it('passes single-event transactions through immediately', async () => {
+    const m1 = createMutationEvent({transactionId: 'txn1'})
+    const events = await lastValueFrom(listenEventsOf(m1).pipe(groupTransactionEvents(), toArray()))
+    expect(events).toEqual([m1])
+  })
+
+  it('holds multi-document transaction events until the transaction is complete', () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source.pipe(groupTransactionEvents()).subscribe((e) => received.push(e))
+
+    const draftEvent = createMutationEvent({
+      transactionId: 'txn-publish',
+      documentId: 'drafts.doc1',
+      transactionCurrentEvent: 1,
+      transactionTotalEvents: 2,
+    })
+    const publishedEvent = createMutationEvent({
+      transactionId: 'txn-publish',
+      documentId: 'doc1',
+      transactionCurrentEvent: 2,
+      transactionTotalEvents: 2,
+    })
+
+    source.next(draftEvent)
+    expect(received).toEqual([])
+
+    source.next(publishedEvent)
+    expect(received).toEqual([draftEvent, publishedEvent])
+
+    subscription.unsubscribe()
+  })
+
+  it('holds intervening events until the open transaction completes, then releases them after it', () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source.pipe(groupTransactionEvents()).subscribe((e) => received.push(e))
+
+    const draftEvent = createMutationEvent({
+      transactionId: 'txn-publish',
+      documentId: 'drafts.doc1',
+      transactionCurrentEvent: 1,
+      transactionTotalEvents: 2,
+    })
+    // an unrelated single-document mutation interleaved between the two
+    // halves of the publish (legacy listener pipeline behavior)
+    const intervening = createMutationEvent({transactionId: 'txn-other', documentId: 'doc9'})
+    const publishedEvent = createMutationEvent({
+      transactionId: 'txn-publish',
+      documentId: 'doc1',
+      transactionCurrentEvent: 2,
+      transactionTotalEvents: 2,
+    })
+
+    source.next(draftEvent)
+    source.next(intervening)
+    expect(received).toEqual([])
+
+    source.next(publishedEvent)
+    expect(received).toEqual([draftEvent, publishedEvent, intervening])
+
+    subscription.unsubscribe()
+  })
+
+  it('groups a held multi-document transaction after the open one completes', () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source.pipe(groupTransactionEvents()).subscribe((e) => received.push(e))
+
+    const a1 = createMutationEvent({
+      transactionId: 'txn-a',
+      documentId: 'drafts.doc1',
+      transactionTotalEvents: 2,
+    })
+    const b1 = createMutationEvent({
+      transactionId: 'txn-b',
+      documentId: 'drafts.doc2',
+      transactionTotalEvents: 2,
+    })
+    const b2 = createMutationEvent({
+      transactionId: 'txn-b',
+      documentId: 'doc2',
+      transactionTotalEvents: 2,
+    })
+    const a2 = createMutationEvent({
+      transactionId: 'txn-a',
+      documentId: 'doc1',
+      transactionTotalEvents: 2,
+    })
+
+    source.next(a1)
+    source.next(b1)
+    source.next(b2)
+    expect(received).toEqual([])
+
+    source.next(a2)
+    // txn-a completes and is emitted contiguously; the held txn-b events are
+    // re-processed and, being complete, emitted contiguously right after
+    expect(received).toEqual([a1, a2, b1, b2])
+
+    subscription.unsubscribe()
+  })
+
+  it('releases everything in arrival order when the held queue exceeds its cap', () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source
+      .pipe(groupTransactionEvents({maxHeldEvents: 2}))
+      .subscribe((e) => received.push(e))
+
+    const partial = createMutationEvent({
+      transactionId: 'txn-incomplete',
+      documentId: 'drafts.doc1',
+      transactionTotalEvents: 2,
+    })
+    const held1 = createMutationEvent({transactionId: 'txn-h1', documentId: 'doc8'})
+    const held2 = createMutationEvent({transactionId: 'txn-h2', documentId: 'doc9'})
+    const overflow = createMutationEvent({transactionId: 'txn-h3', documentId: 'doc10'})
+
+    source.next(partial)
+    source.next(held1)
+    source.next(held2)
+    expect(received).toEqual([])
+
+    source.next(overflow)
+    expect(received).toEqual([partial, held1, held2, overflow])
+
+    subscription.unsubscribe()
+  })
+
+  it('flushes buffered events before non-mutation events', () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source.pipe(groupTransactionEvents()).subscribe((e) => received.push(e))
+
+    const partial = createMutationEvent({
+      transactionId: 'txn-incomplete',
+      documentId: 'drafts.doc1',
+      transactionTotalEvents: 2,
+    })
+
+    source.next(partial)
+    source.next(reconnectEvent)
+    expect(received).toEqual([partial, reconnectEvent])
+
+    subscription.unsubscribe()
+  })
+
+  it('flushes an incomplete transaction and its held events after the deadline', () => {
+    vi.useFakeTimers()
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source
+      .pipe(groupTransactionEvents({flushDeadline: 500}))
+      .subscribe((e) => received.push(e))
+
+    const partial = createMutationEvent({
+      transactionId: 'txn-incomplete',
+      documentId: 'drafts.doc1',
+      transactionTotalEvents: 2,
+    })
+    const heldBehind = createMutationEvent({transactionId: 'txn-held', documentId: 'doc9'})
+
+    source.next(partial)
+    source.next(heldBehind)
+    expect(received).toEqual([])
+
+    vi.advanceTimersByTime(500)
+    expect(received).toEqual([partial, heldBehind])
+
+    subscription.unsubscribe()
+  })
+
+  it('flushes buffered events on completion', async () => {
+    const partial = createMutationEvent({
+      transactionId: 'txn-incomplete',
+      documentId: 'drafts.doc1',
+      transactionTotalEvents: 2,
+    })
+    const events = await lastValueFrom(
+      listenEventsOf(partial).pipe(groupTransactionEvents(), toArray()),
+    )
+    expect(events).toEqual([partial])
+  })
+})

package/src/document/listenerEventOperators.ts

@@ -0,0 +1,217 @@
+import {type ListenEvent, type MutationEvent} from '@sanity/client'
+import {type SanityDocument} from '@sanity/types'
+import {defer, filter, type MonoTypeOperatorFunction, Observable} from 'rxjs'
+
+import {setCleanupTimeout} from '../utils/setCleanupTimeout'
+
+const DEDUPE_TTL = 120_000
+const DEDUPE_MAX_ENTRIES = 1_000
+
+/**
+ * How long to hold an incomplete multi-document transaction (and the events
+ * held behind it) before releasing everything anyway. Only reached when part
+ * of a transaction is lost.
+ */
+const GROUP_FLUSH_DEADLINE = 30_000
+
+/**
+ * How many intervening events to hold behind an incomplete multi-document
+ * transaction before giving up on grouping and releasing everything in
+ * arrival order.
+ */
+const GROUP_MAX_HELD_EVENTS = 100
+
+interface DedupeOptions {
+  ttl?: number
+  maxEntries?: number
+}
+
+/**
+ * Drops mutation events that have already been observed, keyed by
+ * `transactionId#documentId`. With `enableResume`, the listener replays
+ * missed events after a reconnect on an at-least-once basis, so the same
+ * event can be delivered twice. A duplicate would never chain onto the
+ * current base revision and would clog the sequencing buffer until it
+ * overflows into an `OutOfSyncError`.
+ *
+ * Seen keys expire after `ttl` milliseconds and the cache is capped at
+ * `maxEntries`, evicting the oldest entries first.
+ *
+ * @internal
+ */
+export function dedupeListenerEvents(
+  options?: DedupeOptions,
+): MonoTypeOperatorFunction<ListenEvent<SanityDocument>> {
+  const {ttl = DEDUPE_TTL, maxEntries = DEDUPE_MAX_ENTRIES} = options ?? {}
+
+  return (source) =>
+    defer(() => {
+      // insertion-ordered map of dedupe key -> expiry timestamp
+      const seen = new Map<string, number>()
+
+      return source.pipe(
+        filter((event) => {
+          if (event.type !== 'mutation') return true
+
+          const key = `${event.transactionId}#${event.documentId}`
+          const now = Date.now()
+
+          const expiry = seen.get(key)
+          if (expiry !== undefined && expiry > now) return false
+
+          seen.delete(key)
+          seen.set(key, now + ttl)
+
+          for (const [existingKey, existingExpiry] of seen) {
+            if (seen.size <= maxEntries && existingExpiry > now) break
+            seen.delete(existingKey)
+          }
+
+          return true
+        }),
+      )
+    })
+}
+
+interface GroupTransactionEventsOptions {
+  flushDeadline?: number
+  maxHeldEvents?: number
+}
+
+/**
+ * Buffers the mutation events of multi-document transactions (e.g. a publish
+ * touching both the draft and the published document) until every event of
+ * the transaction has arrived, then emits them contiguously. Without this,
+ * a rebase can run between the two halves of a transaction and observe the
+ * dataset in a state that never existed on the server (draft deleted, but
+ * published not yet updated).
+ *
+ * The backend's streaming listener pipeline delivers a transaction's events
+ * contiguously and in order, but the legacy pipeline can interleave
+ * concurrent transactions. So while a multi-document transaction is
+ * incomplete, mutation events from other transactions are held (in arrival
+ * order) rather than emitted, and released right after the transaction
+ * completes; released events are re-processed so a held multi-document
+ * transaction is grouped too. Per-document `previousRev` sequencing
+ * downstream tolerates the reordering this introduces.
+ *
+ * Escape hatches so a lost event degrades to ungrouped delivery instead of
+ * stalling the stream: everything is released in arrival order when a
+ * non-mutation event (e.g. reconnect/welcome) arrives, when more than
+ * `maxHeldEvents` are held, or after `flushDeadline` milliseconds.
+ * Single-event transactions pass through untouched when nothing is buffered.
+ *
+ * @internal
+ */
+export function groupTransactionEvents(
+  options?: GroupTransactionEventsOptions,
+): MonoTypeOperatorFunction<ListenEvent<SanityDocument>> {
+  const {flushDeadline = GROUP_FLUSH_DEADLINE, maxHeldEvents = GROUP_MAX_HELD_EVENTS} =
+    options ?? {}
+
+  return (source) =>
+    new Observable<ListenEvent<SanityDocument>>((subscriber) => {
+      interface OpenTransaction {
+        transactionId: string
+        events: MutationEvent[]
+        total: number
+        timer: ReturnType<typeof setTimeout>
+      }
+
+      let open: OpenTransaction | null = null
+      let held: MutationEvent[] = []
+
+      // emits the open transaction's events (complete or not) without
+      // releasing the held queue; callers decide what happens to `held`
+      const closeOpen = () => {
+        if (!open) return
+        clearTimeout(open.timer)
+        const {events} = open
+        open = null
+        for (const event of events) subscriber.next(event)
+      }
+
+      // re-processes the held queue so a held multi-document transaction
+      // gets grouped in turn. processing may open a new transaction and top
+      // up a new held queue; each drain only touches its own snapshot, so
+      // this terminates
+      const releaseHeld = () => {
+        const queue = held
+        held = []
+        for (const event of queue) processEvent(event)
+      }
+
+      const releaseEverything = () => {
+        closeOpen()
+        const queue = held
+        held = []
+        for (const event of queue) subscriber.next(event)
+      }
+
+      const processEvent = (event: ListenEvent<SanityDocument>) => {
+        if (event.type !== 'mutation') {
+          // connection-level events (welcome/welcomeback/reset/reconnect)
+          // reset the world downstream; release everything before them
+          releaseEverything()
+          subscriber.next(event)
+          return
+        }
+
+        const isMultiDocument = (event.transactionTotalEvents ?? 0) > 1
+
+        if (!open) {
+          if (!isMultiDocument) {
+            subscriber.next(event)
+            return
+          }
+          // isMultiDocument guarantees total >= 2, so a freshly opened
+          // transaction is always incomplete; completion is checked when
+          // subsequent events of the same transaction arrive below
+          open = {
+            transactionId: event.transactionId,
+            events: [event],
+            total: event.transactionTotalEvents,
+            timer: setCleanupTimeout(() => {
+              closeOpen()
+              releaseHeld()
+            }, flushDeadline),
+          }
+          return
+        }
+
+        if (event.transactionId === open.transactionId) {
+          open.events.push(event)
+          if (open.events.length >= open.total) {
+            closeOpen()
+            releaseHeld()
+          }
+          return
+        }
+
+        // an event from another transaction while one is incomplete: hold it
+        // (legacy listener pipeline interleaving) instead of giving up on the
+        // open transaction
+        held.push(event)
+        if (held.length > maxHeldEvents) releaseEverything()
+      }
+
+      const subscription = source.subscribe({
+        next: processEvent,
+        error: (error) => {
+          releaseEverything()
+          subscriber.error(error)
+        },
+        complete: () => {
+          releaseEverything()
+          subscriber.complete()
+        },
+      })
+
+      return () => {
+        if (open) clearTimeout(open.timer)
+        open = null
+        held = []
+        subscription.unsubscribe()
+      }
+    })
+}