@sanity/sdk 3.0.0-rc.0 → 3.0.0-rc.2

package/src/document/listenerEventOperators.ts

@@ -0,0 +1,217 @@
+import {type ListenEvent, type MutationEvent} from '@sanity/client'
+import {type SanityDocument} from '@sanity/types'
+import {defer, filter, type MonoTypeOperatorFunction, Observable} from 'rxjs'
+
+import {setCleanupTimeout} from '../utils/setCleanupTimeout'
+
+const DEDUPE_TTL = 120_000
+const DEDUPE_MAX_ENTRIES = 1_000
+
+/**
+ * How long to hold an incomplete multi-document transaction (and the events
+ * held behind it) before releasing everything anyway. Only reached when part
+ * of a transaction is lost.
+ */
+const GROUP_FLUSH_DEADLINE = 30_000
+
+/**
+ * How many intervening events to hold behind an incomplete multi-document
+ * transaction before giving up on grouping and releasing everything in
+ * arrival order.
+ */
+const GROUP_MAX_HELD_EVENTS = 100
+
+interface DedupeOptions {
+  ttl?: number
+  maxEntries?: number
+}
+
+/**
+ * Drops mutation events that have already been observed, keyed by
+ * `transactionId#documentId`. With `enableResume`, the listener replays
+ * missed events after a reconnect on an at-least-once basis, so the same
+ * event can be delivered twice. A duplicate would never chain onto the
+ * current base revision and would clog the sequencing buffer until it
+ * overflows into an `OutOfSyncError`.
+ *
+ * Seen keys expire after `ttl` milliseconds and the cache is capped at
+ * `maxEntries`, evicting the oldest entries first.
+ *
+ * @internal
+ */
+export function dedupeListenerEvents(
+  options?: DedupeOptions,
+): MonoTypeOperatorFunction<ListenEvent<SanityDocument>> {
+  const {ttl = DEDUPE_TTL, maxEntries = DEDUPE_MAX_ENTRIES} = options ?? {}
+
+  return (source) =>
+    defer(() => {
+      // insertion-ordered map of dedupe key -> expiry timestamp
+      const seen = new Map<string, number>()
+
+      return source.pipe(
+        filter((event) => {
+          if (event.type !== 'mutation') return true
+
+          const key = `${event.transactionId}#${event.documentId}`
+          const now = Date.now()
+
+          const expiry = seen.get(key)
+          if (expiry !== undefined && expiry > now) return false
+
+          seen.delete(key)
+          seen.set(key, now + ttl)
+
+          for (const [existingKey, existingExpiry] of seen) {
+            if (seen.size <= maxEntries && existingExpiry > now) break
+            seen.delete(existingKey)
+          }
+
+          return true
+        }),
+      )
+    })
+}
+
+interface GroupTransactionEventsOptions {
+  flushDeadline?: number
+  maxHeldEvents?: number
+}
+
+/**
+ * Buffers the mutation events of multi-document transactions (e.g. a publish
+ * touching both the draft and the published document) until every event of
+ * the transaction has arrived, then emits them contiguously. Without this,
+ * a rebase can run between the two halves of a transaction and observe the
+ * dataset in a state that never existed on the server (draft deleted, but
+ * published not yet updated).
+ *
+ * The backend's streaming listener pipeline delivers a transaction's events
+ * contiguously and in order, but the legacy pipeline can interleave
+ * concurrent transactions. So while a multi-document transaction is
+ * incomplete, mutation events from other transactions are held (in arrival
+ * order) rather than emitted, and released right after the transaction
+ * completes; released events are re-processed so a held multi-document
+ * transaction is grouped too. Per-document `previousRev` sequencing
+ * downstream tolerates the reordering this introduces.
+ *
+ * Escape hatches so a lost event degrades to ungrouped delivery instead of
+ * stalling the stream: everything is released in arrival order when a
+ * non-mutation event (e.g. reconnect/welcome) arrives, when more than
+ * `maxHeldEvents` are held, or after `flushDeadline` milliseconds.
+ * Single-event transactions pass through untouched when nothing is buffered.
+ *
+ * @internal
+ */
+export function groupTransactionEvents(
+  options?: GroupTransactionEventsOptions,
+): MonoTypeOperatorFunction<ListenEvent<SanityDocument>> {
+  const {flushDeadline = GROUP_FLUSH_DEADLINE, maxHeldEvents = GROUP_MAX_HELD_EVENTS} =
+    options ?? {}
+
+  return (source) =>
+    new Observable<ListenEvent<SanityDocument>>((subscriber) => {
+      interface OpenTransaction {
+        transactionId: string
+        events: MutationEvent[]
+        total: number
+        timer: ReturnType<typeof setTimeout>
+      }
+
+      let open: OpenTransaction | null = null
+      let held: MutationEvent[] = []
+
+      // emits the open transaction's events (complete or not) without
+      // releasing the held queue; callers decide what happens to `held`
+      const closeOpen = () => {
+        if (!open) return
+        clearTimeout(open.timer)
+        const {events} = open
+        open = null
+        for (const event of events) subscriber.next(event)
+      }
+
+      // re-processes the held queue so a held multi-document transaction
+      // gets grouped in turn. processing may open a new transaction and top
+      // up a new held queue; each drain only touches its own snapshot, so
+      // this terminates
+      const releaseHeld = () => {
+        const queue = held
+        held = []
+        for (const event of queue) processEvent(event)
+      }
+
+      const releaseEverything = () => {
+        closeOpen()
+        const queue = held
+        held = []
+        for (const event of queue) subscriber.next(event)
+      }
+
+      const processEvent = (event: ListenEvent<SanityDocument>) => {
+        if (event.type !== 'mutation') {
+          // connection-level events (welcome/welcomeback/reset/reconnect)
+          // reset the world downstream; release everything before them
+          releaseEverything()
+          subscriber.next(event)
+          return
+        }
+
+        const isMultiDocument = (event.transactionTotalEvents ?? 0) > 1
+
+        if (!open) {
+          if (!isMultiDocument) {
+            subscriber.next(event)
+            return
+          }
+          // isMultiDocument guarantees total >= 2, so a freshly opened
+          // transaction is always incomplete; completion is checked when
+          // subsequent events of the same transaction arrive below
+          open = {
+            transactionId: event.transactionId,
+            events: [event],
+            total: event.transactionTotalEvents,
+            timer: setCleanupTimeout(() => {
+              closeOpen()
+              releaseHeld()
+            }, flushDeadline),
+          }
+          return
+        }
+
+        if (event.transactionId === open.transactionId) {
+          open.events.push(event)
+          if (open.events.length >= open.total) {
+            closeOpen()
+            releaseHeld()
+          }
+          return
+        }
+
+        // an event from another transaction while one is incomplete: hold it
+        // (legacy listener pipeline interleaving) instead of giving up on the
+        // open transaction
+        held.push(event)
+        if (held.length > maxHeldEvents) releaseEverything()
+      }
+
+      const subscription = source.subscribe({
+        next: processEvent,
+        error: (error) => {
+          releaseEverything()
+          subscriber.error(error)
+        },
+        complete: () => {
+          releaseEverything()
+          subscriber.complete()
+        },
+      })
+
+      return () => {
+        if (open) clearTimeout(open.timer)
+        open = null
+        held = []
+        subscription.unsubscribe()
+      }
+    })
+}

package/src/document/patchOperations.test.ts

@@ -274,10 +274,51 @@ describe('set', () => {
     expect(output).toEqual({a: 1, nonexistent: {path: 999}})
   })
 
-  it('creates an item from a key constraint if the key is not present', () => {
+  // Content Lake never creates array items when setting: a keyed or indexed
+  // segment that does not resolve is a silent no-op server-side. The applier
+  // used to append a degenerate item (without the key, even), which
+  // fabricated array items locally that would never exist on the server.
+  it('does not create an item from a key constraint if the key is not present', () => {
     const input = {items: [{_key: 'item1'}]}
     const output = set(input, {'items[_key=="item2"]': {_key: 'item2'}})
-    expect(output).toEqual({items: [{_key: 'item1'}, {_key: 'item2'}]})
+    expect(output).toEqual({items: [{_key: 'item1'}]})
+  })
+
+  it('does not create an item when setting a property under an unresolvable key constraint', () => {
+    const input = {
+      blocks: [
+        {
+          _key: 'b1',
+          children: [{_key: 's1', _type: 'span', text: 'hello', marks: []}],
+        },
+      ],
+    }
+    const output = set(input, {'blocks[_key=="b1"].children[_key=="missing"].marks': ['mark1']})
+    expect(output).toEqual(input)
+  })
+
+  it('does not create an item when setting through an out-of-bounds index', () => {
+    const input = {items: [{_key: 'item1', value: 1}]}
+    const output = set(input, {'items[3].value': 99})
+    expect(output).toEqual(input)
+  })
+
+  it('still sets a missing property on an existing keyed item', () => {
+    const input = {items: [{_key: 'item1'}]}
+    const output = set(input, {'items[_key=="item1"].value': 42})
+    expect(output).toEqual({items: [{_key: 'item1', value: 42}]})
+  })
+
+  it('does not create anything when an array-addressing segment targets a non-array', () => {
+    const input = {items: {nested: true}}
+    const output = set(input, {'items[_key=="item1"].value': 42})
+    expect(output).toEqual(input)
+  })
+
+  it('does not create an item when setting through a negative out-of-bounds index', () => {
+    const input = {items: [{_key: 'item1', value: 1}]}
+    const output = set(input, {'items[-5].value': 99})
+    expect(output).toEqual(input)
   })
 })
 
@@ -299,6 +340,12 @@ describe('setIfMissing', () => {
     const output = setIfMissing(input, {'a.b': 42})
     expect(output).toEqual({a: {b: 1}})
   })
+
+  it('does not create an item under an unresolvable key constraint', () => {
+    const input = {items: [{_key: 'item1'}]}
+    const output = setIfMissing(input, {'items[_key=="missing"].value': 1})
+    expect(output).toEqual(input)
+  })
 })
 
 describe('unset', () => {
@@ -518,6 +565,33 @@ describe('diffMatchPatch', () => {
     expect(output).toEqual(input)
   })
 
+  it('applies the patch when its offsets miss a multi-byte boundary', () => {
+    // `\u00f8` is two bytes, so this patch's offsets land inside it and the byte
+    // walk overshoots. The default is to throw, which escapes the rebase loop in
+    // `reducers.ts` and kills the whole outgoing-actions pipeline: the instance
+    // then silently stops writing every document. The patch itself is fine and
+    // still applies.
+    const input = {foo: 'abcdefg\u00f8'}
+    const patch = '@@ -9,1 +9,2 @@\n \u00f8\n+!\n'
+
+    expect(() => diffMatchPatch(input, {foo: patch})).not.toThrow()
+    expect(diffMatchPatch(input, {foo: patch})).toEqual({foo: 'abcdefg\u00f8!'})
+  })
+
+  it('still patches the other properties in the same mutation', () => {
+    const input = {multibyte: 'abcdefg\u00f8', ascii: 'the quick brown fox'}
+
+    const output = diffMatchPatch(input, {
+      multibyte: '@@ -9,1 +9,2 @@\n \u00f8\n+!\n',
+      ascii: '@@ -13,7 +13,7 @@\n own \n-fox\n+cat\n',
+    })
+
+    expect(output).toEqual({
+      multibyte: 'abcdefg\u00f8!',
+      ascii: 'the quick brown cat',
+    })
+  })
+
   it('applies a diff-match-patch that makes no changes', () => {
     const input = {foo: 'unchanged'}
     const patch = '@@ -1,9 +1,9 @@\n unchanged\n'

package/src/document/patchOperations.ts

@@ -175,6 +175,54 @@ export const ensureArrayKeysDeep = memoize(<R>(input: R): R => {
   return Object.fromEntries(entries) as R
 })
 
+/**
+ * Whether every array-addressing segment (keyed `[_key=="…"]` or numeric
+ * index) in the given match path resolves to an existing array item.
+ *
+ * This mirrors Content Lake, which auto-creates missing *object* properties
+ * when setting but never creates *array items*:
+ *
+ * ```js
+ * // object properties are created, even nested ones:
+ * set({}, {'meta.author.name': 'Ada'})
+ * // -> {meta: {author: {name: 'Ada'}}}                    (applies)
+ *
+ * // array items are not:
+ * set({items: [{_key: 'a'}]}, {'items[_key=="b"].title': 'x'})
+ * // -> {items: [{_key: 'a'}]}                             (no-op)
+ * ```
+ *
+ * Without this filter, the second example fabricated a degenerate item
+ * (`{title: 'x'}`, no `_key`) that would never exist on the server, which
+ * is how concurrent Portable Text edits grew phantom span-less children.
+ */
+function resolvesArraySegments(input: unknown, path: SingleValuePath): boolean {
+  let current: unknown = input
+  for (const segment of path) {
+    if (typeof segment === 'string') {
+      // missing object properties (nested ones included) are created
+      // server-side, so a string segment may descend into undefined
+      current =
+        typeof current === 'object' && current !== null && !Array.isArray(current)
+          ? (current as Record<string, unknown>)[segment]
+          : undefined
+      continue
+    }
+    // keyed and numeric segments address array items, which are never created
+    if (!Array.isArray(current)) return false
+    if (isKeySegment(segment)) {
+      const index = getIndexForKey(current, segment._key)
+      if (index === undefined) return false
+      current = current[index]
+    } else {
+      const index = segment < 0 ? current.length + segment : segment
+      if (!(index in current)) return false
+      current = current[index]
+    }
+  }
+  return true
+}
+
 /**
  * Given an input object and a record of path expressions to values, this
  * function will set each match with the given value.
@@ -198,6 +246,7 @@ export function set(input: unknown, pathExpressionValues: Record<string, unknown
         replacementValue,
       })),
     )
+    .filter(({path}) => resolvesArraySegments(input, path))
     .reduce((acc, {path, replacementValue}) => setDeep(acc, path, replacementValue), input)
 
   return ensureArrayKeysDeep(result)
@@ -231,6 +280,7 @@ export function setIfMissing(
       }))
     })
     .filter((matchEntry) => matchEntry.value === null || matchEntry.value === undefined)
+    .filter(({path}) => resolvesArraySegments(input, path))
     .reduce((acc, {path, replacementValue}) => setDeep(acc, path, replacementValue), input)
 
   return ensureArrayKeysDeep(result)
@@ -567,7 +617,16 @@ export function diffMatchPatch(
         )
       }
 
-      const [nextValue] = applyPatches(parsePatch(dmp), value)
+      // Offsets arrive as UTF-8 byte positions and can overshoot when they
+      // land inside a multi-byte character, which happens once the string a
+      // patch was computed against has diverged. The default is to throw, and
+      // that throw escapes the rebase loop in `reducers.ts`, kills the
+      // outgoing-actions pipeline, and leaves the instance silently dropping
+      // every write. Clamping to the nearest index applies the patch instead,
+      // which is what the Studio and the Portable Text editor already do.
+      const [nextValue] = applyPatches(parsePatch(dmp), value, {
+        allowExceedingIndices: true,
+      })
       return {path, value: nextValue}
     })
     .reduce((acc, {path, value}) => setDeep(acc, path, value), input)