@tanstack/db 0.5.32 → 0.5.33

package/src/query/live/collection-config-builder.ts

@@ -1,6 +1,5 @@
 import { D2, output } from '@tanstack/db-ivm'
 import { compileQuery } from '../compiler/index.js'
-import { buildQuery, getQueryIR } from '../builder/index.js'
 import {
   MissingAliasInputsError,
   SetWindowRequiresOrderByError,
@@ -10,6 +9,12 @@ import { getActiveTransaction } from '../../transactions.js'
 import { CollectionSubscriber } from './collection-subscriber.js'
 import { getCollectionBuilder } from './collection-registry.js'
 import { LIVE_QUERY_INTERNAL } from './internal.js'
+import {
+  buildQueryFromConfig,
+  extractCollectionAliases,
+  extractCollectionFromSource,
+  extractCollectionsFromQuery,
+} from './utils.js'
 import type { LiveQueryInternalUtils } from './internal.js'
 import type { WindowOptions } from '../compiler/index.js'
 import type { SchedulerContextId } from '../../scheduler.js'
@@ -984,16 +989,6 @@ export class CollectionConfigBuilder<
   }
 }
 
-function buildQueryFromConfig<TContext extends Context>(
-  config: LiveQueryCollectionConfig<any, any>,
-) {
-  // Build the query using the provided query builder function or instance
-  if (typeof config.query === `function`) {
-    return buildQuery<TContext>(config.query)
-  }
-  return getQueryIR(config.query)
-}
-
 function createOrderByComparator<T extends object>(
   orderByIndices: WeakMap<object, string>,
 ) {
@@ -1018,127 +1013,6 @@ function createOrderByComparator<T extends object>(
   }
 }
 
-/**
- * Helper function to extract collections from a compiled query
- * Traverses the query IR to find all collection references
- * Maps collections by their ID (not alias) as expected by the compiler
- */
-function extractCollectionsFromQuery(
-  query: any,
-): Record<string, Collection<any, any, any>> {
-  const collections: Record<string, any> = {}
-
-  // Helper function to recursively extract collections from a query or source
-  function extractFromSource(source: any) {
-    if (source.type === `collectionRef`) {
-      collections[source.collection.id] = source.collection
-    } else if (source.type === `queryRef`) {
-      // Recursively extract from subquery
-      extractFromQuery(source.query)
-    }
-  }
-
-  // Helper function to recursively extract collections from a query
-  function extractFromQuery(q: any) {
-    // Extract from FROM clause
-    if (q.from) {
-      extractFromSource(q.from)
-    }
-
-    // Extract from JOIN clauses
-    if (q.join && Array.isArray(q.join)) {
-      for (const joinClause of q.join) {
-        if (joinClause.from) {
-          extractFromSource(joinClause.from)
-        }
-      }
-    }
-  }
-
-  // Start extraction from the root query
-  extractFromQuery(query)
-
-  return collections
-}
-
-/**
- * Helper function to extract the collection that is referenced in the query's FROM clause.
- * The FROM clause may refer directly to a collection or indirectly to a subquery.
- */
-function extractCollectionFromSource(query: any): Collection<any, any, any> {
-  const from = query.from
-
-  if (from.type === `collectionRef`) {
-    return from.collection
-  } else if (from.type === `queryRef`) {
-    // Recursively extract from subquery
-    return extractCollectionFromSource(from.query)
-  }
-
-  throw new Error(
-    `Failed to extract collection. Invalid FROM clause: ${JSON.stringify(query)}`,
-  )
-}
-
-/**
- * Extracts all aliases used for each collection across the entire query tree.
- *
- * Traverses the QueryIR recursively to build a map from collection ID to all aliases
- * that reference that collection. This is essential for self-join support, where the
- * same collection may be referenced multiple times with different aliases.
- *
- * For example, given a query like:
- * ```ts
- * q.from({ employee: employeesCollection })
- *   .join({ manager: employeesCollection }, ({ employee, manager }) =>
- *     eq(employee.managerId, manager.id)
- *   )
- * ```
- *
- * This function would return:
- * ```
- * Map { "employees" => Set { "employee", "manager" } }
- * ```
- *
- * @param query - The query IR to extract aliases from
- * @returns A map from collection ID to the set of all aliases referencing that collection
- */
-function extractCollectionAliases(query: QueryIR): Map<string, Set<string>> {
-  const aliasesById = new Map<string, Set<string>>()
-
-  function recordAlias(source: any) {
-    if (!source) return
-
-    if (source.type === `collectionRef`) {
-      const { id } = source.collection
-      const existing = aliasesById.get(id)
-      if (existing) {
-        existing.add(source.alias)
-      } else {
-        aliasesById.set(id, new Set([source.alias]))
-      }
-    } else if (source.type === `queryRef`) {
-      traverse(source.query)
-    }
-  }
-
-  function traverse(q?: QueryIR) {
-    if (!q) return
-
-    recordAlias(q.from)
-
-    if (q.join) {
-      for (const joinClause of q.join) {
-        recordAlias(joinClause.from)
-      }
-    }
-  }
-
-  traverse(query)
-
-  return aliasesById
-}
-
 function accumulateChanges<T>(
   acc: Map<unknown, Changes<T>>,
   [[key, tupleData], multiplicity]: [

package/src/query/live/collection-subscriber.ts

@@ -1,9 +1,15 @@
-import { MultiSet, serializeValue } from '@tanstack/db-ivm'
 import {
   normalizeExpressionPaths,
   normalizeOrderByPaths,
 } from '../compiler/expressions.js'
-import type { MultiSetArray, RootStreamBuilder } from '@tanstack/db-ivm'
+import {
+  computeOrderedLoadCursor,
+  computeSubscriptionOrderByHints,
+  filterDuplicateInserts,
+  sendChangesToInput,
+  splitUpdates,
+  trackBiggestSentValue,
+} from './utils.js'
 import type { Collection } from '../../collection/index.js'
 import type {
   ChangeMessage,
@@ -147,25 +153,11 @@ export class CollectionSubscriber<
     changes: Iterable<ChangeMessage<any, string | number>>,
     callback?: () => boolean,
   ) {
-    // Filter changes to prevent duplicate inserts to D2 pipeline.
-    // This ensures D2 multiplicity stays at 1 for visible items, so deletes
-    // properly reduce multiplicity to 0 (triggering DELETE output).
     const changesArray = Array.isArray(changes) ? changes : [...changes]
-    const filteredChanges: Array<ChangeMessage<any, string | number>> = []
-    for (const change of changesArray) {
-      if (change.type === `insert`) {
-        if (this.sentToD2Keys.has(change.key)) {
-          // Skip duplicate insert - already sent to D2
-          continue
-        }
-        this.sentToD2Keys.add(change.key)
-      } else if (change.type === `delete`) {
-        // Remove from tracking so future re-inserts are allowed
-        this.sentToD2Keys.delete(change.key)
-      }
-      // Updates are handled as delete+insert by splitUpdates, so no special handling needed
-      filteredChanges.push(change)
-    }
+    const filteredChanges = filterDuplicateInserts(
+      changesArray,
+      this.sentToD2Keys,
+    )
 
     // currentSyncState and input are always defined when this method is called
     // (only called from active subscriptions during a sync session)
@@ -202,27 +194,10 @@ export class CollectionSubscriber<
     }
 
     // Get the query's orderBy and limit to pass to loadSubset.
-    // Only include orderBy when it is scoped to this alias and uses simple refs,
-    // to avoid leaking cross-collection paths into backend-specific compilers.
-    const { orderBy, limit, offset } = this.collectionConfigBuilder.query
-    const effectiveLimit =
-      limit !== undefined && offset !== undefined ? limit + offset : limit
-    const normalizedOrderBy = orderBy
-      ? normalizeOrderByPaths(orderBy, this.alias)
-      : undefined
-    const canPassOrderBy =
-      normalizedOrderBy?.every((clause) => {
-        const exp = clause.expression
-        if (exp.type !== `ref`) {
-          return false
-        }
-        const path = exp.path
-        return Array.isArray(path) && path.length === 1
-      }) ?? false
-    const orderByForSubscription = canPassOrderBy
-      ? normalizedOrderBy
-      : undefined
-    const limitForSubscription = canPassOrderBy ? effectiveLimit : undefined
+    const hints = computeSubscriptionOrderByHints(
+      this.collectionConfigBuilder.query,
+      this.alias,
+    )
 
     // Track loading via the loadSubset promise directly.
     // requestSnapshot uses trackLoadSubsetPromise: false (needed for truncate handling),
@@ -241,8 +216,8 @@ export class CollectionSubscriber<
       ...(includeInitialState && { includeInitialState }),
       whereExpression,
       onStatusChange,
-      orderBy: orderByForSubscription,
-      limit: limitForSubscription,
+      orderBy: hints.orderBy,
+      limit: hints.limit,
       onLoadSubsetResult,
     })
 
@@ -415,52 +390,28 @@ export class CollectionSubscriber<
     if (!orderByInfo) {
       return
     }
-    const { orderBy, valueExtractorForRawRow, offset } = orderByInfo
-    const biggestSentRow = this.biggest
-
-    // Extract all orderBy column values from the biggest sent row
-    // For single-column: returns single value, for multi-column: returns array
-    const extractedValues = biggestSentRow
-      ? valueExtractorForRawRow(biggestSentRow)
-      : undefined
-
-    // Normalize to array format for minValues
-    let minValues: Array<unknown> | undefined
-    if (extractedValues !== undefined) {
-      minValues = Array.isArray(extractedValues)
-        ? extractedValues
-        : [extractedValues]
-    }
-
-    const loadRequestKey = this.getLoadRequestKey({
-      minValues,
-      offset,
-      limit: n,
-    })
 
-    // Skip if we already requested a load for this cursor+window.
-    // This prevents infinite loops from cached data re-writes while still allowing
-    // window moves (offset/limit changes) to trigger new requests.
-    if (this.lastLoadRequestKey === loadRequestKey) {
-      return
-    }
+    const cursor = computeOrderedLoadCursor(
+      orderByInfo,
+      this.biggest,
+      this.lastLoadRequestKey,
+      this.alias,
+      n,
+    )
+    if (!cursor) return // Duplicate request — skip
 
-    // Normalize the orderBy clauses such that the references are relative to the collection
-    const normalizedOrderBy = normalizeOrderByPaths(orderBy, this.alias)
+    this.lastLoadRequestKey = cursor.loadRequestKey
 
     // Take the `n` items after the biggest sent value
-    // Pass the current window offset to ensure proper deduplication
+    // Omit offset so requestLimitedSnapshot can advance based on
+    // the number of rows already loaded (supports offset-based backends).
     subscription.requestLimitedSnapshot({
-      orderBy: normalizedOrderBy,
+      orderBy: cursor.normalizedOrderBy,
       limit: n,
-      minValues,
-      // Omit offset so requestLimitedSnapshot can advance the offset based on
-      // the number of rows already loaded (supports offset-based backends).
+      minValues: cursor.minValues,
       trackLoadSubsetPromise: false,
       onLoadSubsetResult: this.orderedLoadSubsetResult,
     })
-
-    this.lastLoadRequestKey = loadRequestKey
   }
 
   private getWhereClauseForAlias(): BasicExpression<boolean> | undefined {
@@ -487,24 +438,15 @@ export class CollectionSubscriber<
     changes: Array<ChangeMessage<any, string | number>>,
     comparator: (a: any, b: any) => number,
   ): void {
-    for (const change of changes) {
-      if (change.type === `delete`) {
-        continue
-      }
-
-      const isNewKey = !this.sentToD2Keys.has(change.key)
-
-      // Only track inserts/updates for cursor positioning, not deletes
-      if (!this.biggest) {
-        this.biggest = change.value
-        this.lastLoadRequestKey = undefined
-      } else if (comparator(this.biggest, change.value) < 0) {
-        this.biggest = change.value
-        this.lastLoadRequestKey = undefined
-      } else if (isNewKey) {
-        // New key with same orderBy value - allow another load if needed
-        this.lastLoadRequestKey = undefined
-      }
+    const result = trackBiggestSentValue(
+      changes,
+      this.biggest,
+      this.sentToD2Keys,
+      comparator,
+    )
+    this.biggest = result.biggest
+    if (result.shouldResetLoadKey) {
+      this.lastLoadRequestKey = undefined
     }
   }
 
@@ -525,62 +467,4 @@ export class CollectionSubscriber<
       promise,
     )
   }
-
-  private getLoadRequestKey(options: {
-    minValues: Array<unknown> | undefined
-    offset: number
-    limit: number
-  }): string {
-    return serializeValue({
-      minValues: options.minValues ?? null,
-      offset: options.offset,
-      limit: options.limit,
-    })
-  }
-}
-
-/**
- * Helper function to send changes to a D2 input stream
- */
-function sendChangesToInput(
-  input: RootStreamBuilder<unknown>,
-  changes: Iterable<ChangeMessage>,
-  getKey: (item: ChangeMessage[`value`]) => any,
-): number {
-  const multiSetArray: MultiSetArray<unknown> = []
-  for (const change of changes) {
-    const key = getKey(change.value)
-    if (change.type === `insert`) {
-      multiSetArray.push([[key, change.value], 1])
-    } else if (change.type === `update`) {
-      multiSetArray.push([[key, change.previousValue], -1])
-      multiSetArray.push([[key, change.value], 1])
-    } else {
-      // change.type === `delete`
-      multiSetArray.push([[key, change.value], -1])
-    }
-  }
-
-  if (multiSetArray.length !== 0) {
-    input.sendData(new MultiSet(multiSetArray))
-  }
-
-  return multiSetArray.length
-}
-
-/** Splits updates into a delete of the old value and an insert of the new value */
-function* splitUpdates<
-  T extends object = Record<string, unknown>,
-  TKey extends string | number = string | number,
->(
-  changes: Iterable<ChangeMessage<T, TKey>>,
-): Generator<ChangeMessage<T, TKey>> {
-  for (const change of changes) {
-    if (change.type === `update`) {
-      yield { type: `delete`, key: change.key, value: change.previousValue! }
-      yield { type: `insert`, key: change.key, value: change.value }
-    } else {
-      yield change
-    }
-  }
 }