@sanity/sdk 2.17.0 → 2.18.0

package/src/document/listenerEventOperators.ts

@@ -0,0 +1,217 @@
+import {type ListenEvent, type MutationEvent} from '@sanity/client'
+import {type SanityDocument} from '@sanity/types'
+import {defer, filter, type MonoTypeOperatorFunction, Observable} from 'rxjs'
+
+import {setCleanupTimeout} from '../utils/setCleanupTimeout'
+
+const DEDUPE_TTL = 120_000
+const DEDUPE_MAX_ENTRIES = 1_000
+
+/**
+ * How long to hold an incomplete multi-document transaction (and the events
+ * held behind it) before releasing everything anyway. Only reached when part
+ * of a transaction is lost.
+ */
+const GROUP_FLUSH_DEADLINE = 30_000
+
+/**
+ * How many intervening events to hold behind an incomplete multi-document
+ * transaction before giving up on grouping and releasing everything in
+ * arrival order.
+ */
+const GROUP_MAX_HELD_EVENTS = 100
+
+interface DedupeOptions {
+  ttl?: number
+  maxEntries?: number
+}
+
+/**
+ * Drops mutation events that have already been observed, keyed by
+ * `transactionId#documentId`. With `enableResume`, the listener replays
+ * missed events after a reconnect on an at-least-once basis, so the same
+ * event can be delivered twice. A duplicate would never chain onto the
+ * current base revision and would clog the sequencing buffer until it
+ * overflows into an `OutOfSyncError`.
+ *
+ * Seen keys expire after `ttl` milliseconds and the cache is capped at
+ * `maxEntries`, evicting the oldest entries first.
+ *
+ * @internal
+ */
+export function dedupeListenerEvents(
+  options?: DedupeOptions,
+): MonoTypeOperatorFunction<ListenEvent<SanityDocument>> {
+  const {ttl = DEDUPE_TTL, maxEntries = DEDUPE_MAX_ENTRIES} = options ?? {}
+
+  return (source) =>
+    defer(() => {
+      // insertion-ordered map of dedupe key -> expiry timestamp
+      const seen = new Map<string, number>()
+
+      return source.pipe(
+        filter((event) => {
+          if (event.type !== 'mutation') return true
+
+          const key = `${event.transactionId}#${event.documentId}`
+          const now = Date.now()
+
+          const expiry = seen.get(key)
+          if (expiry !== undefined && expiry > now) return false
+
+          seen.delete(key)
+          seen.set(key, now + ttl)
+
+          for (const [existingKey, existingExpiry] of seen) {
+            if (seen.size <= maxEntries && existingExpiry > now) break
+            seen.delete(existingKey)
+          }
+
+          return true
+        }),
+      )
+    })
+}
+
+interface GroupTransactionEventsOptions {
+  flushDeadline?: number
+  maxHeldEvents?: number
+}
+
+/**
+ * Buffers the mutation events of multi-document transactions (e.g. a publish
+ * touching both the draft and the published document) until every event of
+ * the transaction has arrived, then emits them contiguously. Without this,
+ * a rebase can run between the two halves of a transaction and observe the
+ * dataset in a state that never existed on the server (draft deleted, but
+ * published not yet updated).
+ *
+ * The backend's streaming listener pipeline delivers a transaction's events
+ * contiguously and in order, but the legacy pipeline can interleave
+ * concurrent transactions. So while a multi-document transaction is
+ * incomplete, mutation events from other transactions are held (in arrival
+ * order) rather than emitted, and released right after the transaction
+ * completes; released events are re-processed so a held multi-document
+ * transaction is grouped too. Per-document `previousRev` sequencing
+ * downstream tolerates the reordering this introduces.
+ *
+ * Escape hatches so a lost event degrades to ungrouped delivery instead of
+ * stalling the stream: everything is released in arrival order when a
+ * non-mutation event (e.g. reconnect/welcome) arrives, when more than
+ * `maxHeldEvents` are held, or after `flushDeadline` milliseconds.
+ * Single-event transactions pass through untouched when nothing is buffered.
+ *
+ * @internal
+ */
+export function groupTransactionEvents(
+  options?: GroupTransactionEventsOptions,
+): MonoTypeOperatorFunction<ListenEvent<SanityDocument>> {
+  const {flushDeadline = GROUP_FLUSH_DEADLINE, maxHeldEvents = GROUP_MAX_HELD_EVENTS} =
+    options ?? {}
+
+  return (source) =>
+    new Observable<ListenEvent<SanityDocument>>((subscriber) => {
+      interface OpenTransaction {
+        transactionId: string
+        events: MutationEvent[]
+        total: number
+        timer: ReturnType<typeof setTimeout>
+      }
+
+      let open: OpenTransaction | null = null
+      let held: MutationEvent[] = []
+
+      // emits the open transaction's events (complete or not) without
+      // releasing the held queue; callers decide what happens to `held`
+      const closeOpen = () => {
+        if (!open) return
+        clearTimeout(open.timer)
+        const {events} = open
+        open = null
+        for (const event of events) subscriber.next(event)
+      }
+
+      // re-processes the held queue so a held multi-document transaction
+      // gets grouped in turn. processing may open a new transaction and top
+      // up a new held queue; each drain only touches its own snapshot, so
+      // this terminates
+      const releaseHeld = () => {
+        const queue = held
+        held = []
+        for (const event of queue) processEvent(event)
+      }
+
+      const releaseEverything = () => {
+        closeOpen()
+        const queue = held
+        held = []
+        for (const event of queue) subscriber.next(event)
+      }
+
+      const processEvent = (event: ListenEvent<SanityDocument>) => {
+        if (event.type !== 'mutation') {
+          // connection-level events (welcome/welcomeback/reset/reconnect)
+          // reset the world downstream; release everything before them
+          releaseEverything()
+          subscriber.next(event)
+          return
+        }
+
+        const isMultiDocument = (event.transactionTotalEvents ?? 0) > 1
+
+        if (!open) {
+          if (!isMultiDocument) {
+            subscriber.next(event)
+            return
+          }
+          // isMultiDocument guarantees total >= 2, so a freshly opened
+          // transaction is always incomplete; completion is checked when
+          // subsequent events of the same transaction arrive below
+          open = {
+            transactionId: event.transactionId,
+            events: [event],
+            total: event.transactionTotalEvents,
+            timer: setCleanupTimeout(() => {
+              closeOpen()
+              releaseHeld()
+            }, flushDeadline),
+          }
+          return
+        }
+
+        if (event.transactionId === open.transactionId) {
+          open.events.push(event)
+          if (open.events.length >= open.total) {
+            closeOpen()
+            releaseHeld()
+          }
+          return
+        }
+
+        // an event from another transaction while one is incomplete: hold it
+        // (legacy listener pipeline interleaving) instead of giving up on the
+        // open transaction
+        held.push(event)
+        if (held.length > maxHeldEvents) releaseEverything()
+      }
+
+      const subscription = source.subscribe({
+        next: processEvent,
+        error: (error) => {
+          releaseEverything()
+          subscriber.error(error)
+        },
+        complete: () => {
+          releaseEverything()
+          subscriber.complete()
+        },
+      })
+
+      return () => {
+        if (open) clearTimeout(open.timer)
+        open = null
+        held = []
+        subscription.unsubscribe()
+      }
+    })
+}

package/src/document/patchOperations.test.ts

@@ -274,10 +274,51 @@ describe('set', () => {
     expect(output).toEqual({a: 1, nonexistent: {path: 999}})
   })
 
-  it('creates an item from a key constraint if the key is not present', () => {
+  // Content Lake never creates array items when setting: a keyed or indexed
+  // segment that does not resolve is a silent no-op server-side. The applier
+  // used to append a degenerate item (without the key, even), which
+  // fabricated array items locally that would never exist on the server.
+  it('does not create an item from a key constraint if the key is not present', () => {
     const input = {items: [{_key: 'item1'}]}
     const output = set(input, {'items[_key=="item2"]': {_key: 'item2'}})
-    expect(output).toEqual({items: [{_key: 'item1'}, {_key: 'item2'}]})
+    expect(output).toEqual({items: [{_key: 'item1'}]})
+  })
+
+  it('does not create an item when setting a property under an unresolvable key constraint', () => {
+    const input = {
+      blocks: [
+        {
+          _key: 'b1',
+          children: [{_key: 's1', _type: 'span', text: 'hello', marks: []}],
+        },
+      ],
+    }
+    const output = set(input, {'blocks[_key=="b1"].children[_key=="missing"].marks': ['mark1']})
+    expect(output).toEqual(input)
+  })
+
+  it('does not create an item when setting through an out-of-bounds index', () => {
+    const input = {items: [{_key: 'item1', value: 1}]}
+    const output = set(input, {'items[3].value': 99})
+    expect(output).toEqual(input)
+  })
+
+  it('still sets a missing property on an existing keyed item', () => {
+    const input = {items: [{_key: 'item1'}]}
+    const output = set(input, {'items[_key=="item1"].value': 42})
+    expect(output).toEqual({items: [{_key: 'item1', value: 42}]})
+  })
+
+  it('does not create anything when an array-addressing segment targets a non-array', () => {
+    const input = {items: {nested: true}}
+    const output = set(input, {'items[_key=="item1"].value': 42})
+    expect(output).toEqual(input)
+  })
+
+  it('does not create an item when setting through a negative out-of-bounds index', () => {
+    const input = {items: [{_key: 'item1', value: 1}]}
+    const output = set(input, {'items[-5].value': 99})
+    expect(output).toEqual(input)
   })
 })
 
@@ -299,6 +340,12 @@ describe('setIfMissing', () => {
     const output = setIfMissing(input, {'a.b': 42})
     expect(output).toEqual({a: {b: 1}})
   })
+
+  it('does not create an item under an unresolvable key constraint', () => {
+    const input = {items: [{_key: 'item1'}]}
+    const output = setIfMissing(input, {'items[_key=="missing"].value': 1})
+    expect(output).toEqual(input)
+  })
 })
 
 describe('unset', () => {

package/src/document/patchOperations.ts

@@ -175,6 +175,54 @@ export const ensureArrayKeysDeep = memoize(<R>(input: R): R => {
   return Object.fromEntries(entries) as R
 })
 
+/**
+ * Whether every array-addressing segment (keyed `[_key=="…"]` or numeric
+ * index) in the given match path resolves to an existing array item.
+ *
+ * This mirrors Content Lake, which auto-creates missing *object* properties
+ * when setting but never creates *array items*:
+ *
+ * ```js
+ * // object properties are created, even nested ones:
+ * set({}, {'meta.author.name': 'Ada'})
+ * // -> {meta: {author: {name: 'Ada'}}}                    (applies)
+ *
+ * // array items are not:
+ * set({items: [{_key: 'a'}]}, {'items[_key=="b"].title': 'x'})
+ * // -> {items: [{_key: 'a'}]}                             (no-op)
+ * ```
+ *
+ * Without this filter, the second example fabricated a degenerate item
+ * (`{title: 'x'}`, no `_key`) that would never exist on the server, which
+ * is how concurrent Portable Text edits grew phantom span-less children.
+ */
+function resolvesArraySegments(input: unknown, path: SingleValuePath): boolean {
+  let current: unknown = input
+  for (const segment of path) {
+    if (typeof segment === 'string') {
+      // missing object properties (nested ones included) are created
+      // server-side, so a string segment may descend into undefined
+      current =
+        typeof current === 'object' && current !== null && !Array.isArray(current)
+          ? (current as Record<string, unknown>)[segment]
+          : undefined
+      continue
+    }
+    // keyed and numeric segments address array items, which are never created
+    if (!Array.isArray(current)) return false
+    if (isKeySegment(segment)) {
+      const index = getIndexForKey(current, segment._key)
+      if (index === undefined) return false
+      current = current[index]
+    } else {
+      const index = segment < 0 ? current.length + segment : segment
+      if (!(index in current)) return false
+      current = current[index]
+    }
+  }
+  return true
+}
+
 /**
  * Given an input object and a record of path expressions to values, this
  * function will set each match with the given value.
@@ -198,6 +246,7 @@ export function set(input: unknown, pathExpressionValues: Record<string, unknown
         replacementValue,
       })),
     )
+    .filter(({path}) => resolvesArraySegments(input, path))
     .reduce((acc, {path, replacementValue}) => setDeep(acc, path, replacementValue), input)
 
   return ensureArrayKeysDeep(result)
@@ -231,6 +280,7 @@ export function setIfMissing(
       }))
     })
     .filter((matchEntry) => matchEntry.value === null || matchEntry.value === undefined)
+    .filter(({path}) => resolvesArraySegments(input, path))
     .reduce((acc, {path, replacementValue}) => setDeep(acc, path, replacementValue), input)
 
   return ensureArrayKeysDeep(result)

package/src/document/processActions/edit.ts

@@ -13,6 +13,7 @@ import {
   checkGrant,
   createMutationApplier,
   PermissionActionError,
+  preserveNumericOperations,
 } from './shared'
 
 export function handleEdit(
@@ -100,10 +101,16 @@ export function handleEdit(
   // when the caller asked us to preserve their operations, forward their
   // patches verbatim instead of re-deriving them from a snapshot diff. this
   // keeps keyed array operations addressable so concurrent edits from other
-  // clients interleave instead of being overwritten
+  // clients interleave instead of being overwritten. otherwise, diff the
+  // snapshots but swap back any `inc`/`dec` the diff collapsed into a `set`,
+  // so concurrent increments accumulate instead of last-write-winning
   const patches = action.preserveOperations
     ? (action.patches ?? [])
-    : diffValue(baseBefore, baseAfter)
+    : preserveNumericOperations(
+        baseBefore,
+        action.patches,
+        diffValue(baseBefore, baseAfter) as PatchOperations[],
+      )
 
   const workingMutations: Mutation[] = []
   if (!isReleasePerspective(action.perspective) && !working[draftId] && working[publishedId]) {

package/src/document/processActions/shared.ts

@@ -1,4 +1,5 @@
 import {diffValue} from '@sanity/diff-patch'
+import {jsonMatch, stringifyPath} from '@sanity/json-match'
 import {type Mutation, type PatchOperations, type SanityDocument} from '@sanity/types'
 import {evaluateSync, type ExprNode} from 'groq-js'
 
@@ -92,6 +93,111 @@ export function createMutationApplier(options: {
   }
 }
 
+/**
+ * Re-applies the operational intent of user-supplied `inc`/`dec` patches to a
+ * set of snapshot-diffed patches.
+ *
+ * The outgoing patches sent to Content Lake are re-derived by diffing the
+ * base document before and after the user's patches were applied. That diff
+ * collapses an `inc` into a `set` of the resulting number, which turns
+ * concurrent increments from different clients into last-write-wins. This
+ * helper finds diffed `set` operations that are exactly explained by the
+ * user's `inc`/`dec` operations and swaps them back, so the server applies
+ * the increment against whatever value is current at commit time.
+ *
+ * A `set` is only swapped when its value equals the base value plus the
+ * accumulated user deltas for that exact path; anything else (e.g. an `inc`
+ * combined with a `set` of the same field) keeps the diffed `set`.
+ *
+ * Content Lake fails the whole transaction when `inc`/`dec` targets a missing
+ * path (unlike `set`, which always succeeds). To avoid turning a concurrent
+ * field removal into a transaction revert, the emitted patch seeds the target
+ * with a `setIfMissing` of the base value; the server executes `setIfMissing`
+ * before `inc`/`dec` within a single patch, so a concurrently removed field
+ * degrades to the same result the diffed `set` would have produced. A path
+ * into a concurrently removed keyed array item can still fail the
+ * transaction (`setIfMissing` cannot create array items); the transaction is
+ * then reverted and reported, which is preferable to silently resurrecting
+ * the item.
+ */
+export function preserveNumericOperations(
+  baseBefore: SanityDocument | null | undefined,
+  userPatches: PatchOperations[] | undefined,
+  diffedPatches: PatchOperations[],
+): PatchOperations[] {
+  if (!baseBefore || !userPatches?.length) return diffedPatches
+
+  // accumulate the user's inc/dec deltas per concrete (stringified) path,
+  // tracking the expected final value by replaying the additions in order
+  const targets = new Map<string, {baseValue: number; expectedValue: number; delta: number}>()
+  for (const patch of userPatches) {
+    for (const [op, sign] of [
+      ['inc', 1],
+      ['dec', -1],
+    ] as const) {
+      const record = patch[op]
+      if (!record) continue
+      for (const [pathExpression, amount] of Object.entries(record)) {
+        if (typeof amount !== 'number') continue
+        for (const match of jsonMatch(baseBefore, pathExpression)) {
+          const path = stringifyPath(match.path)
+          const existing = targets.get(path)
+          if (existing) {
+            existing.expectedValue += sign * amount
+            existing.delta += sign * amount
+          } else if (typeof match.value === 'number') {
+            targets.set(path, {
+              baseValue: match.value,
+              expectedValue: match.value + sign * amount,
+              delta: sign * amount,
+            })
+          }
+        }
+      }
+    }
+  }
+  if (!targets.size) return diffedPatches
+
+  const preserved = new Map<string, {delta: number; baseValue: number}>()
+  const next = diffedPatches.flatMap((patch) => {
+    if (!patch.set) return [patch]
+
+    const keptSet: Record<string, unknown> = {}
+    for (const [path, value] of Object.entries(patch.set)) {
+      const target = targets.get(path)
+      if (target && typeof value === 'number' && value === target.expectedValue) {
+        preserved.set(path, {delta: target.delta, baseValue: target.baseValue})
+      } else {
+        keptSet[path] = value
+      }
+    }
+
+    if (Object.keys(keptSet).length === Object.keys(patch.set).length) return [patch]
+    const {set: _set, ...rest} = patch
+    const withKept = Object.keys(keptSet).length ? {...rest, set: keptSet} : rest
+    return Object.keys(withKept).length ? [withKept] : []
+  })
+
+  if (!preserved.size) return next
+
+  const setIfMissing: Record<string, unknown> = {}
+  const inc: Record<string, number> = {}
+  const dec: Record<string, number> = {}
+  for (const [path, {delta, baseValue}] of preserved) {
+    setIfMissing[path] = baseValue
+    if (delta >= 0) inc[path] = delta
+    else dec[path] = -delta
+  }
+  return [
+    ...next,
+    {
+      setIfMissing,
+      ...(Object.keys(inc).length && {inc}),
+      ...(Object.keys(dec).length && {dec}),
+    },
+  ]
+}
+
 interface ApplySingleDocPatchOptions {
   base: DocumentSet
   working: DocumentSet
@@ -181,7 +287,11 @@ export function applySingleDocPatch({
   const baseAfter = base[documentId]
   const diffedPatches = preserveOperations
     ? (patches as PatchOperations[])
-    : (diffValue(baseBefore, baseAfter) as PatchOperations[])
+    : preserveNumericOperations(
+        baseBefore,
+        patches,
+        diffValue(baseBefore, baseAfter) as PatchOperations[],
+      )
 
   const workingBefore = working[documentId] as SanityDocument
   if (!checkGrant(grants.update, workingBefore, identity)) {