@tanstack/db 0.5.23 → 0.5.25

package/src/query/live/collection-subscriber.ts

@@ -1,4 +1,4 @@
-import { MultiSet } from '@tanstack/db-ivm'
+import { MultiSet, serializeValue } from '@tanstack/db-ivm'
 import {
   normalizeExpressionPaths,
   normalizeOrderByPaths,
@@ -26,6 +26,11 @@ export class CollectionSubscriber<
   // Keep track of the biggest value we've sent so far (needed for orderBy optimization)
   private biggest: any = undefined
 
+  // Track the most recent ordered load request key (cursor + window).
+  // This avoids infinite loops from cached data re-writes while still allowing
+  // window moves or new keys at the same cursor value to trigger new requests.
+  private lastLoadRequestKey: string | undefined
+
   // Track deferred promises for subscription loading states
   private subscriptionLoadingPromises = new Map<
     CollectionSubscription,
@@ -37,6 +42,11 @@ export class CollectionSubscriber<
   // can potentially send the same item to D2 multiple times.
   private sentToD2Keys = new Set<string | number>()
 
+  // Direct load tracking callback for ordered path (set during subscribeToOrderedChanges,
+  // used by loadNextItems for subsequent requestLimitedSnapshot calls)
+  private orderedLoadSubsetResult?: (result: Promise<void> | true) => void
+  private pendingOrderedLoadPromise: Promise<void> | undefined
+
   constructor(
     private alias: string,
     private collectionId: string,
@@ -58,35 +68,29 @@ export class CollectionSubscriber<
   private subscribeToChanges(whereExpression?: BasicExpression<boolean>) {
     const orderByInfo = this.getOrderByInfo()
 
-    // Track load promises using subscription from the event (avoids circular dependency)
-    const trackLoadPromise = (subscription: CollectionSubscription) => {
-      // Guard against duplicate transitions
-      if (!this.subscriptionLoadingPromises.has(subscription)) {
-        let resolve: () => void
-        const promise = new Promise<void>((res) => {
-          resolve = res
-        })
-
-        this.subscriptionLoadingPromises.set(subscription, {
-          resolve: resolve!,
-        })
+    // Direct load promise tracking: pipes loadSubset results straight to the
+    // live query collection, avoiding the multi-hop deferred promise chain that
+    // can break under microtask timing (e.g., queueMicrotask in TanStack Query).
+    const trackLoadResult = (result: Promise<void> | true) => {
+      if (result instanceof Promise) {
         this.collectionConfigBuilder.liveQueryCollection!._sync.trackLoadPromise(
-          promise,
+          result,
         )
       }
     }
 
     // Status change handler - passed to subscribeChanges so it's registered
-    // BEFORE any snapshot is requested, preventing race conditions
+    // BEFORE any snapshot is requested, preventing race conditions.
+    // Used as a fallback for status transitions not covered by direct tracking
+    // (e.g., truncate-triggered reloads that call trackLoadSubsetPromise directly).
     const onStatusChange = (event: SubscriptionStatusChangeEvent) => {
       const subscription = event.subscription as CollectionSubscription
       if (event.status === `loadingSubset`) {
-        trackLoadPromise(subscription)
+        this.ensureLoadingPromise(subscription)
       } else {
         // status is 'ready'
         const deferred = this.subscriptionLoadingPromises.get(subscription)
         if (deferred) {
-          // Clear the map entry FIRST (before resolving)
           this.subscriptionLoadingPromises.delete(subscription)
           deferred.resolve()
         }
@@ -100,6 +104,7 @@ export class CollectionSubscriber<
         whereExpression,
         orderByInfo,
         onStatusChange,
+        trackLoadResult,
       )
     } else {
       // If the source alias is lazy then we should not include the initial state
@@ -117,14 +122,13 @@ export class CollectionSubscriber<
     // Check current status after subscribing - if status is 'loadingSubset', track it.
     // The onStatusChange listener will catch the transition to 'ready'.
     if (subscription.status === `loadingSubset`) {
-      trackLoadPromise(subscription)
+      this.ensureLoadingPromise(subscription)
     }
 
     const unsubscribe = () => {
       // If subscription has a pending promise, resolve it before unsubscribing
       const deferred = this.subscriptionLoadingPromises.get(subscription)
       if (deferred) {
-        // Clear the map entry FIRST (before resolving)
         this.subscriptionLoadingPromises.delete(subscription)
         deferred.resolve()
       }
@@ -197,13 +201,49 @@ export class CollectionSubscriber<
       this.sendChangesToPipeline(changes)
     }
 
-    // Create subscription with onStatusChange - listener is registered before snapshot
-    // Note: For non-ordered queries (no limit/offset), we use trackLoadSubsetPromise: false
-    // which is the default behavior in subscribeChanges
+    // Get the query's orderBy and limit to pass to loadSubset.
+    // Only include orderBy when it is scoped to this alias and uses simple refs,
+    // to avoid leaking cross-collection paths into backend-specific compilers.
+    const { orderBy, limit, offset } = this.collectionConfigBuilder.query
+    const effectiveLimit =
+      limit !== undefined && offset !== undefined ? limit + offset : limit
+    const normalizedOrderBy = orderBy
+      ? normalizeOrderByPaths(orderBy, this.alias)
+      : undefined
+    const canPassOrderBy =
+      normalizedOrderBy?.every((clause) => {
+        const exp = clause.expression
+        if (exp.type !== `ref`) {
+          return false
+        }
+        const path = exp.path
+        return Array.isArray(path) && path.length === 1
+      }) ?? false
+    const orderByForSubscription = canPassOrderBy
+      ? normalizedOrderBy
+      : undefined
+    const limitForSubscription = canPassOrderBy ? effectiveLimit : undefined
+
+    // Track loading via the loadSubset promise directly.
+    // requestSnapshot uses trackLoadSubsetPromise: false (needed for truncate handling),
+    // so we use onLoadSubsetResult to get the promise and track it ourselves.
+    const onLoadSubsetResult = includeInitialState
+      ? (result: Promise<void> | true) => {
+          if (result instanceof Promise) {
+            this.collectionConfigBuilder.liveQueryCollection!._sync.trackLoadPromise(
+              result,
+            )
+          }
+        }
+      : undefined
+
     const subscription = this.collection.subscribeChanges(sendChanges, {
       ...(includeInitialState && { includeInitialState }),
       whereExpression,
       onStatusChange,
+      orderBy: orderByForSubscription,
+      limit: limitForSubscription,
+      onLoadSubsetResult,
     })
 
     return subscription
@@ -213,17 +253,39 @@ export class CollectionSubscriber<
     whereExpression: BasicExpression<boolean> | undefined,
     orderByInfo: OrderByOptimizationInfo,
     onStatusChange: (event: SubscriptionStatusChangeEvent) => void,
+    onLoadSubsetResult: (result: Promise<void> | true) => void,
   ): CollectionSubscription {
     const { orderBy, offset, limit, index } = orderByInfo
 
+    // Store the callback so loadNextItems can also use direct tracking.
+    // Track in-flight ordered loads to avoid issuing redundant requests while
+    // a previous snapshot is still pending.
+    const handleLoadSubsetResult = (result: Promise<void> | true) => {
+      if (result instanceof Promise) {
+        this.pendingOrderedLoadPromise = result
+        result.finally(() => {
+          if (this.pendingOrderedLoadPromise === result) {
+            this.pendingOrderedLoadPromise = undefined
+          }
+        })
+      }
+      onLoadSubsetResult(result)
+    }
+
+    this.orderedLoadSubsetResult = handleLoadSubsetResult
+
     // Use a holder to forward-reference subscription in the callback
     const subscriptionHolder: { current?: CollectionSubscription } = {}
 
     const sendChangesInRange = (
       changes: Iterable<ChangeMessage<any, string | number>>,
     ) => {
+      const changesArray = Array.isArray(changes) ? changes : [...changes]
+
+      this.trackSentValues(changesArray, orderByInfo.comparator)
+
       // Split live updates into a delete of the old value and an insert of the new value
-      const splittedChanges = splitUpdates(changes)
+      const splittedChanges = splitUpdates(changesArray)
       this.sendChangesToPipelineWithTracking(
         splittedChanges,
         subscriptionHolder.current!,
@@ -243,6 +305,8 @@ export class CollectionSubscriber<
     // and allow re-inserts of previously sent keys
     const truncateUnsubscribe = this.collection.on(`truncate`, () => {
       this.biggest = undefined
+      this.lastLoadRequestKey = undefined
+      this.pendingOrderedLoadPromise = undefined
       this.sentToD2Keys.clear()
     })
 
@@ -254,26 +318,27 @@ export class CollectionSubscriber<
     // Normalize the orderBy clauses such that the references are relative to the collection
     const normalizedOrderBy = normalizeOrderByPaths(orderBy, this.alias)
 
-    // Trigger the snapshot request - onStatusChange listener is already registered
+    // Trigger the snapshot request — use direct load tracking (trackLoadSubsetPromise: false)
+    // to pipe the loadSubset result straight to the live query collection. This bypasses
+    // the subscription status → onStatusChange → deferred promise chain which is fragile
+    // under microtask timing (e.g., queueMicrotask delays in TanStack Query observers).
     if (index) {
       // We have an index on the first orderBy column - use lazy loading optimization
-      // This works for both single-column and multi-column orderBy:
-      // - Single-column: index provides exact ordering
-      // - Multi-column: index provides ordering on first column, secondary sort in memory
       subscription.setOrderByIndex(index)
 
-      // Load the first `offset + limit` values from the index
-      // i.e. the K items from the collection that fall into the requested range: [offset, offset + limit[
       subscription.requestLimitedSnapshot({
         limit: offset + limit,
         orderBy: normalizedOrderBy,
+        trackLoadSubsetPromise: false,
+        onLoadSubsetResult: handleLoadSubsetResult,
       })
     } else {
       // No index available (e.g., non-ref expression): pass orderBy/limit to loadSubset
-      // so the sync layer can optimize if the backend supports it
       subscription.requestSnapshot({
         orderBy: normalizedOrderBy,
         limit: offset + limit,
+        trackLoadSubsetPromise: false,
+        onLoadSubsetResult: handleLoadSubsetResult,
       })
     }
 
@@ -301,6 +366,11 @@ export class CollectionSubscriber<
       return true
     }
 
+    if (this.pendingOrderedLoadPromise) {
+      // Wait for in-flight ordered loads to resolve before issuing another request.
+      return true
+    }
+
     // `dataNeeded` probes the orderBy operator to see if it needs more data
     // if it needs more data, it returns the number of items it needs
     const n = dataNeeded()
@@ -320,8 +390,6 @@ export class CollectionSubscriber<
       return
     }
 
-    const trackedChanges = this.trackSentValues(changes, orderByInfo.comparator)
-
     // Cache the loadMoreIfNeeded callback on the subscription using a symbol property.
     // This ensures we pass the same function instance to the scheduler each time,
     // allowing it to deduplicate callbacks when multiple changes arrive during a transaction.
@@ -335,7 +403,7 @@ export class CollectionSubscriber<
       this.loadMoreIfNeeded.bind(this, subscription)
 
     this.sendChangesToPipeline(
-      trackedChanges,
+      changes,
       subscriptionWithLoader[loadMoreCallbackSymbol],
     )
   }
@@ -357,12 +425,25 @@ export class CollectionSubscriber<
       : undefined
 
     // Normalize to array format for minValues
-    const minValues =
-      extractedValues !== undefined
-        ? Array.isArray(extractedValues)
-          ? extractedValues
-          : [extractedValues]
-        : undefined
+    let minValues: Array<unknown> | undefined
+    if (extractedValues !== undefined) {
+      minValues = Array.isArray(extractedValues)
+        ? extractedValues
+        : [extractedValues]
+    }
+
+    const loadRequestKey = this.getLoadRequestKey({
+      minValues,
+      offset,
+      limit: n,
+    })
+
+    // Skip if we already requested a load for this cursor+window.
+    // This prevents infinite loops from cached data re-writes while still allowing
+    // window moves (offset/limit changes) to trigger new requests.
+    if (this.lastLoadRequestKey === loadRequestKey) {
+      return
+    }
 
     // Normalize the orderBy clauses such that the references are relative to the collection
     const normalizedOrderBy = normalizeOrderByPaths(orderBy, this.alias)
@@ -373,8 +454,13 @@ export class CollectionSubscriber<
       orderBy: normalizedOrderBy,
       limit: n,
       minValues,
-      offset,
+      // Omit offset so requestLimitedSnapshot can advance the offset based on
+      // the number of rows already loaded (supports offset-based backends).
+      trackLoadSubsetPromise: false,
+      onLoadSubsetResult: this.orderedLoadSubsetResult,
     })
+
+    this.lastLoadRequestKey = loadRequestKey
   }
 
   private getWhereClauseForAlias(): BasicExpression<boolean> | undefined {
@@ -397,22 +483,59 @@ export class CollectionSubscriber<
     return undefined
   }
 
-  private *trackSentValues(
-    changes: Iterable<ChangeMessage<any, string | number>>,
+  private trackSentValues(
+    changes: Array<ChangeMessage<any, string | number>>,
     comparator: (a: any, b: any) => number,
-  ) {
+  ): void {
     for (const change of changes) {
+      if (change.type === `delete`) {
+        continue
+      }
+
+      const isNewKey = !this.sentToD2Keys.has(change.key)
+
       // Only track inserts/updates for cursor positioning, not deletes
-      if (change.type !== `delete`) {
-        if (!this.biggest) {
-          this.biggest = change.value
-        } else if (comparator(this.biggest, change.value) < 0) {
-          this.biggest = change.value
-        }
+      if (!this.biggest) {
+        this.biggest = change.value
+        this.lastLoadRequestKey = undefined
+      } else if (comparator(this.biggest, change.value) < 0) {
+        this.biggest = change.value
+        this.lastLoadRequestKey = undefined
+      } else if (isNewKey) {
+        // New key with same orderBy value - allow another load if needed
+        this.lastLoadRequestKey = undefined
       }
+    }
+  }
 
-      yield change
+  private ensureLoadingPromise(subscription: CollectionSubscription) {
+    if (this.subscriptionLoadingPromises.has(subscription)) {
+      return
     }
+
+    let resolve: () => void
+    const promise = new Promise<void>((res) => {
+      resolve = res
+    })
+
+    this.subscriptionLoadingPromises.set(subscription, {
+      resolve: resolve!,
+    })
+    this.collectionConfigBuilder.liveQueryCollection!._sync.trackLoadPromise(
+      promise,
+    )
+  }
+
+  private getLoadRequestKey(options: {
+    minValues: Array<unknown> | undefined
+    offset: number
+    limit: number
+  }): string {
+    return serializeValue({
+      minValues: options.minValues ?? null,
+      offset: options.offset,
+      limit: options.limit,
+    })
   }
 }
 

package/src/types.ts

@@ -809,6 +809,22 @@ export interface SubscribeChangesOptions<
    * @internal
    */
   onStatusChange?: (event: SubscriptionStatusChangeEvent) => void
+  /**
+   * Optional orderBy to include in loadSubset for query-specific cache keys.
+   * @internal
+   */
+  orderBy?: OrderBy
+  /**
+   * Optional limit to include in loadSubset for query-specific cache keys.
+   * @internal
+   */
+  limit?: number
+  /**
+   * Callback that receives the loadSubset result (Promise or true) from requestSnapshot.
+   * Allows the caller to directly track the loading promise for isReady status.
+   * @internal
+   */
+  onLoadSubsetResult?: (result: Promise<void> | true) => void
 }
 
 export interface SubscribeChangesSnapshotOptions<

package/src/utils/comparison.ts

@@ -134,10 +134,20 @@ function areUint8ArraysEqual(a: Uint8Array, b: Uint8Array): boolean {
  */
 const UINT8ARRAY_NORMALIZE_THRESHOLD = 128
 
+/**
+ * Sentinel value representing undefined in normalized form.
+ * This allows distinguishing between "start from beginning" (undefined parameter)
+ * and "start from the key undefined" (actual undefined value in the tree).
+ */
+export const UNDEFINED_SENTINEL = `__TS_DB_BTREE_UNDEFINED_VALUE__`
+
 /**
  * Normalize a value for comparison and Map key usage
  * Converts values that can't be directly compared or used as Map keys
  * into comparable primitive representations
+ *
+ * Note: This does NOT convert undefined to a sentinel. Use normalizeForBTree
+ * for BTree index operations that need to distinguish undefined values.
  */
 export function normalizeValue(value: any): any {
   if (value instanceof Date) {
@@ -164,6 +174,30 @@ export function normalizeValue(value: any): any {
   return value
 }
 
+/**
+ * Normalize a value for BTree index usage.
+ * Extends normalizeValue to also convert undefined to a sentinel value.
+ * This is needed because the BTree does not properly support `undefined` as a key
+ * (it interprets undefined as "start from beginning" in nextHigherPair/nextLowerPair).
+ */
+export function normalizeForBTree(value: any): any {
+  if (value === undefined) {
+    return UNDEFINED_SENTINEL
+  }
+  return normalizeValue(value)
+}
+
+/**
+ * Converts the `UNDEFINED_SENTINEL` back to `undefined`.
+ * Needed such that the sentinel is converted back to `undefined` before comparison.
+ */
+export function denormalizeUndefined(value: any): any {
+  if (value === UNDEFINED_SENTINEL) {
+    return undefined
+  }
+  return value
+}
+
 /**
  * Compare two values for equality, with special handling for Uint8Arrays and Buffers
  */