@sanity/sdk 2.16.0 → 2.18.0

package/src/document/listenerEventOperators.test.ts

@@ -0,0 +1,311 @@
+import {
+  type ListenEvent,
+  type MutationEvent,
+  type ReconnectEvent,
+  type WelcomeEvent,
+} from '@sanity/client'
+import {type SanityDocument} from '@sanity/types'
+import {from, lastValueFrom, type Observable, Subject} from 'rxjs'
+import {toArray} from 'rxjs/operators'
+import {afterEach, describe, expect, it, vi} from 'vitest'
+
+import {dedupeListenerEvents, groupTransactionEvents} from './listenerEventOperators'
+
+function listenEventsOf(
+  ...events: ListenEvent<SanityDocument>[]
+): Observable<ListenEvent<SanityDocument>> {
+  return from(events)
+}
+
+function createMutationEvent({
+  transactionId,
+  documentId = 'doc1',
+  transactionCurrentEvent = 1,
+  transactionTotalEvents = 1,
+}: {
+  transactionId: string
+  documentId?: string
+  transactionCurrentEvent?: number
+  transactionTotalEvents?: number
+}): MutationEvent {
+  return {
+    type: 'mutation',
+    documentId,
+    eventId: `${transactionId}#${documentId}`,
+    identity: 'user',
+    mutations: [],
+    timestamp: new Date().toISOString(),
+    transactionId,
+    transactionCurrentEvent,
+    transactionTotalEvents,
+    transition: 'update',
+    visibility: 'query',
+    resultRev: transactionId,
+  }
+}
+
+const welcomeEvent: WelcomeEvent = {type: 'welcome', listenerName: 'listener'}
+const reconnectEvent = {type: 'reconnect'} as ReconnectEvent
+
+afterEach(() => {
+  vi.useRealTimers()
+})
+
+describe('dedupeListenerEvents', () => {
+  it('passes unique mutation events through', async () => {
+    const m1 = createMutationEvent({transactionId: 'txn1'})
+    const m2 = createMutationEvent({transactionId: 'txn2'})
+    const events = await lastValueFrom(
+      listenEventsOf(m1, m2).pipe(dedupeListenerEvents(), toArray()),
+    )
+    expect(events).toEqual([m1, m2])
+  })
+
+  it('drops replayed duplicates of the same transaction and document', async () => {
+    const m1 = createMutationEvent({transactionId: 'txn1'})
+    const replay = createMutationEvent({transactionId: 'txn1'})
+    const events = await lastValueFrom(
+      listenEventsOf(m1, replay).pipe(dedupeListenerEvents(), toArray()),
+    )
+    expect(events).toEqual([m1])
+  })
+
+  it('does not dedupe events of the same transaction for different documents', async () => {
+    const draft = createMutationEvent({transactionId: 'txn1', documentId: 'drafts.doc1'})
+    const published = createMutationEvent({transactionId: 'txn1', documentId: 'doc1'})
+    const events = await lastValueFrom(
+      listenEventsOf(draft, published).pipe(dedupeListenerEvents(), toArray()),
+    )
+    expect(events).toEqual([draft, published])
+  })
+
+  it('passes non-mutation events through untouched', async () => {
+    const events = await lastValueFrom(
+      listenEventsOf(welcomeEvent, reconnectEvent).pipe(dedupeListenerEvents(), toArray()),
+    )
+    expect(events).toEqual([welcomeEvent, reconnectEvent])
+  })
+
+  it('allows a key again after its TTL has passed', async () => {
+    vi.useFakeTimers()
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source
+      .pipe(dedupeListenerEvents({ttl: 1000}))
+      .subscribe((e) => received.push(e))
+
+    source.next(createMutationEvent({transactionId: 'txn1'}))
+    source.next(createMutationEvent({transactionId: 'txn1'}))
+    expect(received).toHaveLength(1)
+
+    vi.advanceTimersByTime(1001)
+    source.next(createMutationEvent({transactionId: 'txn1'}))
+    expect(received).toHaveLength(2)
+
+    subscription.unsubscribe()
+  })
+
+  it('evicts the oldest entries when the cache exceeds maxEntries', async () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source
+      .pipe(dedupeListenerEvents({maxEntries: 2}))
+      .subscribe((e) => received.push(e))
+
+    source.next(createMutationEvent({transactionId: 'txn1'}))
+    source.next(createMutationEvent({transactionId: 'txn2'}))
+    source.next(createMutationEvent({transactionId: 'txn3'}))
+    // txn1 was evicted to make room, so a replay of it passes through again
+    source.next(createMutationEvent({transactionId: 'txn1'}))
+    expect(received).toHaveLength(4)
+
+    subscription.unsubscribe()
+  })
+})
+
+describe('groupTransactionEvents', () => {
+  it('passes single-event transactions through immediately', async () => {
+    const m1 = createMutationEvent({transactionId: 'txn1'})
+    const events = await lastValueFrom(listenEventsOf(m1).pipe(groupTransactionEvents(), toArray()))
+    expect(events).toEqual([m1])
+  })
+
+  it('holds multi-document transaction events until the transaction is complete', () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source.pipe(groupTransactionEvents()).subscribe((e) => received.push(e))
+
+    const draftEvent = createMutationEvent({
+      transactionId: 'txn-publish',
+      documentId: 'drafts.doc1',
+      transactionCurrentEvent: 1,
+      transactionTotalEvents: 2,
+    })
+    const publishedEvent = createMutationEvent({
+      transactionId: 'txn-publish',
+      documentId: 'doc1',
+      transactionCurrentEvent: 2,
+      transactionTotalEvents: 2,
+    })
+
+    source.next(draftEvent)
+    expect(received).toEqual([])
+
+    source.next(publishedEvent)
+    expect(received).toEqual([draftEvent, publishedEvent])
+
+    subscription.unsubscribe()
+  })
+
+  it('holds intervening events until the open transaction completes, then releases them after it', () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source.pipe(groupTransactionEvents()).subscribe((e) => received.push(e))
+
+    const draftEvent = createMutationEvent({
+      transactionId: 'txn-publish',
+      documentId: 'drafts.doc1',
+      transactionCurrentEvent: 1,
+      transactionTotalEvents: 2,
+    })
+    // an unrelated single-document mutation interleaved between the two
+    // halves of the publish (legacy listener pipeline behavior)
+    const intervening = createMutationEvent({transactionId: 'txn-other', documentId: 'doc9'})
+    const publishedEvent = createMutationEvent({
+      transactionId: 'txn-publish',
+      documentId: 'doc1',
+      transactionCurrentEvent: 2,
+      transactionTotalEvents: 2,
+    })
+
+    source.next(draftEvent)
+    source.next(intervening)
+    expect(received).toEqual([])
+
+    source.next(publishedEvent)
+    expect(received).toEqual([draftEvent, publishedEvent, intervening])
+
+    subscription.unsubscribe()
+  })
+
+  it('groups a held multi-document transaction after the open one completes', () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source.pipe(groupTransactionEvents()).subscribe((e) => received.push(e))
+
+    const a1 = createMutationEvent({
+      transactionId: 'txn-a',
+      documentId: 'drafts.doc1',
+      transactionTotalEvents: 2,
+    })
+    const b1 = createMutationEvent({
+      transactionId: 'txn-b',
+      documentId: 'drafts.doc2',
+      transactionTotalEvents: 2,
+    })
+    const b2 = createMutationEvent({
+      transactionId: 'txn-b',
+      documentId: 'doc2',
+      transactionTotalEvents: 2,
+    })
+    const a2 = createMutationEvent({
+      transactionId: 'txn-a',
+      documentId: 'doc1',
+      transactionTotalEvents: 2,
+    })
+
+    source.next(a1)
+    source.next(b1)
+    source.next(b2)
+    expect(received).toEqual([])
+
+    source.next(a2)
+    // txn-a completes and is emitted contiguously; the held txn-b events are
+    // re-processed and, being complete, emitted contiguously right after
+    expect(received).toEqual([a1, a2, b1, b2])
+
+    subscription.unsubscribe()
+  })
+
+  it('releases everything in arrival order when the held queue exceeds its cap', () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source
+      .pipe(groupTransactionEvents({maxHeldEvents: 2}))
+      .subscribe((e) => received.push(e))
+
+    const partial = createMutationEvent({
+      transactionId: 'txn-incomplete',
+      documentId: 'drafts.doc1',
+      transactionTotalEvents: 2,
+    })
+    const held1 = createMutationEvent({transactionId: 'txn-h1', documentId: 'doc8'})
+    const held2 = createMutationEvent({transactionId: 'txn-h2', documentId: 'doc9'})
+    const overflow = createMutationEvent({transactionId: 'txn-h3', documentId: 'doc10'})
+
+    source.next(partial)
+    source.next(held1)
+    source.next(held2)
+    expect(received).toEqual([])
+
+    source.next(overflow)
+    expect(received).toEqual([partial, held1, held2, overflow])
+
+    subscription.unsubscribe()
+  })
+
+  it('flushes buffered events before non-mutation events', () => {
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source.pipe(groupTransactionEvents()).subscribe((e) => received.push(e))
+
+    const partial = createMutationEvent({
+      transactionId: 'txn-incomplete',
+      documentId: 'drafts.doc1',
+      transactionTotalEvents: 2,
+    })
+
+    source.next(partial)
+    source.next(reconnectEvent)
+    expect(received).toEqual([partial, reconnectEvent])
+
+    subscription.unsubscribe()
+  })
+
+  it('flushes an incomplete transaction and its held events after the deadline', () => {
+    vi.useFakeTimers()
+    const source = new Subject<ListenEvent<SanityDocument>>()
+    const received: ListenEvent<SanityDocument>[] = []
+    const subscription = source
+      .pipe(groupTransactionEvents({flushDeadline: 500}))
+      .subscribe((e) => received.push(e))
+
+    const partial = createMutationEvent({
+      transactionId: 'txn-incomplete',
+      documentId: 'drafts.doc1',
+      transactionTotalEvents: 2,
+    })
+    const heldBehind = createMutationEvent({transactionId: 'txn-held', documentId: 'doc9'})
+
+    source.next(partial)
+    source.next(heldBehind)
+    expect(received).toEqual([])
+
+    vi.advanceTimersByTime(500)
+    expect(received).toEqual([partial, heldBehind])
+
+    subscription.unsubscribe()
+  })
+
+  it('flushes buffered events on completion', async () => {
+    const partial = createMutationEvent({
+      transactionId: 'txn-incomplete',
+      documentId: 'drafts.doc1',
+      transactionTotalEvents: 2,
+    })
+    const events = await lastValueFrom(
+      listenEventsOf(partial).pipe(groupTransactionEvents(), toArray()),
+    )
+    expect(events).toEqual([partial])
+  })
+})

package/src/document/listenerEventOperators.ts

@@ -0,0 +1,217 @@
+import {type ListenEvent, type MutationEvent} from '@sanity/client'
+import {type SanityDocument} from '@sanity/types'
+import {defer, filter, type MonoTypeOperatorFunction, Observable} from 'rxjs'
+
+import {setCleanupTimeout} from '../utils/setCleanupTimeout'
+
+const DEDUPE_TTL = 120_000
+const DEDUPE_MAX_ENTRIES = 1_000
+
+/**
+ * How long to hold an incomplete multi-document transaction (and the events
+ * held behind it) before releasing everything anyway. Only reached when part
+ * of a transaction is lost.
+ */
+const GROUP_FLUSH_DEADLINE = 30_000
+
+/**
+ * How many intervening events to hold behind an incomplete multi-document
+ * transaction before giving up on grouping and releasing everything in
+ * arrival order.
+ */
+const GROUP_MAX_HELD_EVENTS = 100
+
+interface DedupeOptions {
+  ttl?: number
+  maxEntries?: number
+}
+
+/**
+ * Drops mutation events that have already been observed, keyed by
+ * `transactionId#documentId`. With `enableResume`, the listener replays
+ * missed events after a reconnect on an at-least-once basis, so the same
+ * event can be delivered twice. A duplicate would never chain onto the
+ * current base revision and would clog the sequencing buffer until it
+ * overflows into an `OutOfSyncError`.
+ *
+ * Seen keys expire after `ttl` milliseconds and the cache is capped at
+ * `maxEntries`, evicting the oldest entries first.
+ *
+ * @internal
+ */
+export function dedupeListenerEvents(
+  options?: DedupeOptions,
+): MonoTypeOperatorFunction<ListenEvent<SanityDocument>> {
+  const {ttl = DEDUPE_TTL, maxEntries = DEDUPE_MAX_ENTRIES} = options ?? {}
+
+  return (source) =>
+    defer(() => {
+      // insertion-ordered map of dedupe key -> expiry timestamp
+      const seen = new Map<string, number>()
+
+      return source.pipe(
+        filter((event) => {
+          if (event.type !== 'mutation') return true
+
+          const key = `${event.transactionId}#${event.documentId}`
+          const now = Date.now()
+
+          const expiry = seen.get(key)
+          if (expiry !== undefined && expiry > now) return false
+
+          seen.delete(key)
+          seen.set(key, now + ttl)
+
+          for (const [existingKey, existingExpiry] of seen) {
+            if (seen.size <= maxEntries && existingExpiry > now) break
+            seen.delete(existingKey)
+          }
+
+          return true
+        }),
+      )
+    })
+}
+
+interface GroupTransactionEventsOptions {
+  flushDeadline?: number
+  maxHeldEvents?: number
+}
+
+/**
+ * Buffers the mutation events of multi-document transactions (e.g. a publish
+ * touching both the draft and the published document) until every event of
+ * the transaction has arrived, then emits them contiguously. Without this,
+ * a rebase can run between the two halves of a transaction and observe the
+ * dataset in a state that never existed on the server (draft deleted, but
+ * published not yet updated).
+ *
+ * The backend's streaming listener pipeline delivers a transaction's events
+ * contiguously and in order, but the legacy pipeline can interleave
+ * concurrent transactions. So while a multi-document transaction is
+ * incomplete, mutation events from other transactions are held (in arrival
+ * order) rather than emitted, and released right after the transaction
+ * completes; released events are re-processed so a held multi-document
+ * transaction is grouped too. Per-document `previousRev` sequencing
+ * downstream tolerates the reordering this introduces.
+ *
+ * Escape hatches so a lost event degrades to ungrouped delivery instead of
+ * stalling the stream: everything is released in arrival order when a
+ * non-mutation event (e.g. reconnect/welcome) arrives, when more than
+ * `maxHeldEvents` are held, or after `flushDeadline` milliseconds.
+ * Single-event transactions pass through untouched when nothing is buffered.
+ *
+ * @internal
+ */
+export function groupTransactionEvents(
+  options?: GroupTransactionEventsOptions,
+): MonoTypeOperatorFunction<ListenEvent<SanityDocument>> {
+  const {flushDeadline = GROUP_FLUSH_DEADLINE, maxHeldEvents = GROUP_MAX_HELD_EVENTS} =
+    options ?? {}
+
+  return (source) =>
+    new Observable<ListenEvent<SanityDocument>>((subscriber) => {
+      interface OpenTransaction {
+        transactionId: string
+        events: MutationEvent[]
+        total: number
+        timer: ReturnType<typeof setTimeout>
+      }
+
+      let open: OpenTransaction | null = null
+      let held: MutationEvent[] = []
+
+      // emits the open transaction's events (complete or not) without
+      // releasing the held queue; callers decide what happens to `held`
+      const closeOpen = () => {
+        if (!open) return
+        clearTimeout(open.timer)
+        const {events} = open
+        open = null
+        for (const event of events) subscriber.next(event)
+      }
+
+      // re-processes the held queue so a held multi-document transaction
+      // gets grouped in turn. processing may open a new transaction and top
+      // up a new held queue; each drain only touches its own snapshot, so
+      // this terminates
+      const releaseHeld = () => {
+        const queue = held
+        held = []
+        for (const event of queue) processEvent(event)
+      }
+
+      const releaseEverything = () => {
+        closeOpen()
+        const queue = held
+        held = []
+        for (const event of queue) subscriber.next(event)
+      }
+
+      const processEvent = (event: ListenEvent<SanityDocument>) => {
+        if (event.type !== 'mutation') {
+          // connection-level events (welcome/welcomeback/reset/reconnect)
+          // reset the world downstream; release everything before them
+          releaseEverything()
+          subscriber.next(event)
+          return
+        }
+
+        const isMultiDocument = (event.transactionTotalEvents ?? 0) > 1
+
+        if (!open) {
+          if (!isMultiDocument) {
+            subscriber.next(event)
+            return
+          }
+          // isMultiDocument guarantees total >= 2, so a freshly opened
+          // transaction is always incomplete; completion is checked when
+          // subsequent events of the same transaction arrive below
+          open = {
+            transactionId: event.transactionId,
+            events: [event],
+            total: event.transactionTotalEvents,
+            timer: setCleanupTimeout(() => {
+              closeOpen()
+              releaseHeld()
+            }, flushDeadline),
+          }
+          return
+        }
+
+        if (event.transactionId === open.transactionId) {
+          open.events.push(event)
+          if (open.events.length >= open.total) {
+            closeOpen()
+            releaseHeld()
+          }
+          return
+        }
+
+        // an event from another transaction while one is incomplete: hold it
+        // (legacy listener pipeline interleaving) instead of giving up on the
+        // open transaction
+        held.push(event)
+        if (held.length > maxHeldEvents) releaseEverything()
+      }
+
+      const subscription = source.subscribe({
+        next: processEvent,
+        error: (error) => {
+          releaseEverything()
+          subscriber.error(error)
+        },
+        complete: () => {
+          releaseEverything()
+          subscriber.complete()
+        },
+      })
+
+      return () => {
+        if (open) clearTimeout(open.timer)
+        open = null
+        held = []
+        subscription.unsubscribe()
+      }
+    })
+}

package/src/document/patchOperations.test.ts

@@ -274,10 +274,51 @@ describe('set', () => {
     expect(output).toEqual({a: 1, nonexistent: {path: 999}})
   })
 
-  it('creates an item from a key constraint if the key is not present', () => {
+  // Content Lake never creates array items when setting: a keyed or indexed
+  // segment that does not resolve is a silent no-op server-side. The applier
+  // used to append a degenerate item (without the key, even), which
+  // fabricated array items locally that would never exist on the server.
+  it('does not create an item from a key constraint if the key is not present', () => {
     const input = {items: [{_key: 'item1'}]}
     const output = set(input, {'items[_key=="item2"]': {_key: 'item2'}})
-    expect(output).toEqual({items: [{_key: 'item1'}, {_key: 'item2'}]})
+    expect(output).toEqual({items: [{_key: 'item1'}]})
+  })
+
+  it('does not create an item when setting a property under an unresolvable key constraint', () => {
+    const input = {
+      blocks: [
+        {
+          _key: 'b1',
+          children: [{_key: 's1', _type: 'span', text: 'hello', marks: []}],
+        },
+      ],
+    }
+    const output = set(input, {'blocks[_key=="b1"].children[_key=="missing"].marks': ['mark1']})
+    expect(output).toEqual(input)
+  })
+
+  it('does not create an item when setting through an out-of-bounds index', () => {
+    const input = {items: [{_key: 'item1', value: 1}]}
+    const output = set(input, {'items[3].value': 99})
+    expect(output).toEqual(input)
+  })
+
+  it('still sets a missing property on an existing keyed item', () => {
+    const input = {items: [{_key: 'item1'}]}
+    const output = set(input, {'items[_key=="item1"].value': 42})
+    expect(output).toEqual({items: [{_key: 'item1', value: 42}]})
+  })
+
+  it('does not create anything when an array-addressing segment targets a non-array', () => {
+    const input = {items: {nested: true}}
+    const output = set(input, {'items[_key=="item1"].value': 42})
+    expect(output).toEqual(input)
+  })
+
+  it('does not create an item when setting through a negative out-of-bounds index', () => {
+    const input = {items: [{_key: 'item1', value: 1}]}
+    const output = set(input, {'items[-5].value': 99})
+    expect(output).toEqual(input)
   })
 })
 
@@ -299,6 +340,12 @@ describe('setIfMissing', () => {
     const output = setIfMissing(input, {'a.b': 42})
     expect(output).toEqual({a: {b: 1}})
   })
+
+  it('does not create an item under an unresolvable key constraint', () => {
+    const input = {items: [{_key: 'item1'}]}
+    const output = setIfMissing(input, {'items[_key=="missing"].value': 1})
+    expect(output).toEqual(input)
+  })
 })
 
 describe('unset', () => {

package/src/document/patchOperations.ts

@@ -175,6 +175,54 @@ export const ensureArrayKeysDeep = memoize(<R>(input: R): R => {
   return Object.fromEntries(entries) as R
 })
 
+/**
+ * Whether every array-addressing segment (keyed `[_key=="…"]` or numeric
+ * index) in the given match path resolves to an existing array item.
+ *
+ * This mirrors Content Lake, which auto-creates missing *object* properties
+ * when setting but never creates *array items*:
+ *
+ * ```js
+ * // object properties are created, even nested ones:
+ * set({}, {'meta.author.name': 'Ada'})
+ * // -> {meta: {author: {name: 'Ada'}}}                    (applies)
+ *
+ * // array items are not:
+ * set({items: [{_key: 'a'}]}, {'items[_key=="b"].title': 'x'})
+ * // -> {items: [{_key: 'a'}]}                             (no-op)
+ * ```
+ *
+ * Without this filter, the second example fabricated a degenerate item
+ * (`{title: 'x'}`, no `_key`) that would never exist on the server, which
+ * is how concurrent Portable Text edits grew phantom span-less children.
+ */
+function resolvesArraySegments(input: unknown, path: SingleValuePath): boolean {
+  let current: unknown = input
+  for (const segment of path) {
+    if (typeof segment === 'string') {
+      // missing object properties (nested ones included) are created
+      // server-side, so a string segment may descend into undefined
+      current =
+        typeof current === 'object' && current !== null && !Array.isArray(current)
+          ? (current as Record<string, unknown>)[segment]
+          : undefined
+      continue
+    }
+    // keyed and numeric segments address array items, which are never created
+    if (!Array.isArray(current)) return false
+    if (isKeySegment(segment)) {
+      const index = getIndexForKey(current, segment._key)
+      if (index === undefined) return false
+      current = current[index]
+    } else {
+      const index = segment < 0 ? current.length + segment : segment
+      if (!(index in current)) return false
+      current = current[index]
+    }
+  }
+  return true
+}
+
 /**
  * Given an input object and a record of path expressions to values, this
  * function will set each match with the given value.
@@ -198,6 +246,7 @@ export function set(input: unknown, pathExpressionValues: Record<string, unknown
         replacementValue,
       })),
     )
+    .filter(({path}) => resolvesArraySegments(input, path))
     .reduce((acc, {path, replacementValue}) => setDeep(acc, path, replacementValue), input)
 
   return ensureArrayKeysDeep(result)
@@ -231,6 +280,7 @@ export function setIfMissing(
       }))
     })
     .filter((matchEntry) => matchEntry.value === null || matchEntry.value === undefined)
+    .filter(({path}) => resolvesArraySegments(input, path))
     .reduce((acc, {path, replacementValue}) => setDeep(acc, path, replacementValue), input)
 
   return ensureArrayKeysDeep(result)